增加dockerfile配置

feat: 添加 Meilisearch 排名分数支持
feat: 添加分词工具支持并更新字段命名
2026-04-09 16:10:11 +08:00 · 2026-04-09 13:58:36 +08:00 · 2026-04-08 14:21:12 +08:00 · 2026-04-03 17:59:04 +08:00 · 2026-04-03 15:21:23 +08:00 · 2026-04-03 15:21:01 +08:00
18 changed files with 54 additions and 1301 deletions
--- a/config/config.go
+++ b/config/config.go
@@ -0,0 +1,16 @@
+package config
+
+import (
+	"fmt"
+
+	"github.com/gogf/gf/v2/frame/g"
+	"github.com/gogf/gf/v2/os/gcfg"
+	"github.com/gogf/gf/v2/os/genv"
+)
+
+func init() {
+	env := genv.Get("APP_ENV", "").String()
+	if env != "" {
+		g.Cfg().GetAdapter().(*gcfg.AdapterFile).SetFileName(fmt.Sprintf("config-%s.yml", env))
+	}
+}
--- a/db/gfdb/gfdb.go
+++ b/db/gfdb/gfdb.go
@@ -53,7 +53,7 @@ func getFromCache(ctx context.Context, key string) ([]byte, bool) {
 		if err == nil && !result.IsEmpty() {
 			data := result.Bytes()
 			// 写入本地缓存
-			err = getLocalCache().Set(ctx, key, data, time.Duration(g.Cfg().MustGet(ctx, "cache.localTTL").Int64())*time.Second)
+			err = getLocalCache().Set(ctx, key, data, time.Duration(g.Cfg().MustGet(ctx, "cache.localTTL", 60).Int64())*time.Second)
 			if err != nil {
 				return nil, false
 			}
@@ -71,7 +71,7 @@ func setToCache(ctx context.Context, key string, data []byte) (err error) {
 	}

 	// 1. 写入本地缓存
-	if err = getLocalCache().Set(ctx, key, data, time.Duration(g.Cfg().MustGet(ctx, "cache.localTTL").Int64())*time.Second); err != nil {
+	if err = getLocalCache().Set(ctx, key, data, time.Duration(g.Cfg().MustGet(ctx, "cache.localTTL", 60).Int64())*time.Second); err != nil {
 		return
 	}

@@ -79,7 +79,7 @@ func setToCache(ctx context.Context, key string, data []byte) (err error) {
 	if g.Redis() != nil {
 		_, err = g.Redis().Set(ctx, key, data, gredis.SetOption{
 			TTLOption: gredis.TTLOption{
-				EX: gconv.PtrInt64(g.Cfg().MustGet(ctx, "cache.redisTTL")),
+				EX: gconv.PtrInt64(g.Cfg().MustGet(ctx, "cache.redisTTL", 300)),
 			},
 		})
 		if err != nil {
@@ -405,7 +405,7 @@ func getSelectTypeString(selectType gdb.SelectType) string {
 }

 // ==================== 调用方法 ====================
-
+var TablePrefix string
 var (
 	schemaPrefix        = "tenant-"
 	cacheKeyPrefix      = "cache-"
@@ -414,6 +414,7 @@ var (

 type Gfdb interface {
 	Exec(ctx context.Context, sql string, args ...any) (sql.Result, error)
+	GetAll(ctx context.Context, sql string, args ...any) (gdb.Result, error)
 	Model(ctx context.Context, tableNameOrStruct ...any) *model
 	Transaction(ctx context.Context, f func(ctx context.Context, tx gdb.TX) error) error
 }
@@ -441,7 +442,6 @@ type modelCache struct {
 func checkSchemaConfig(ctx context.Context) (uint64, bool) {
 	user, err := utils.GetUserInfo(ctx)
 	if err != nil {
-		glog.Errorf(ctx, "[DB] GetUserInfo error: %v", err)
 		return 0, false
 	}
 	var schema = fmt.Sprintf("%s%v", schemaPrefix, user.TenantId)
@@ -452,51 +452,26 @@ func checkSchemaConfig(ctx context.Context) (uint64, bool) {
 	return user.TenantId, false
 }

-func DB(ctx context.Context) Gfdb {
-	tenantId, config := checkSchemaConfig(ctx)
-
-	var schema = fmt.Sprintf("%s%v", schemaPrefix, tenantId)
-
-	var dbName []string
-	if config {
-		dbName = append(dbName, schema)
+func DB(ctx context.Context, name ...string) Gfdb {
+	var groupName = gdb.DefaultGroupName
+	if len(name) > 0 && name[0] != "" {
+		groupName = name[0]
 	} else {
-		dbName = append(dbName, "default")
-		// 配置文件中 default 是数组格式，需要通过索引 0 访问
-		defaultConfig := g.Cfg().MustGet(ctx, "database.default")
-		if defaultConfig.IsSlice() {
-			schema = g.Cfg().MustGet(ctx, "database.default.0.name").String()
-		} else {
-			schema = g.Cfg().MustGet(ctx, "database.default.name").String()
+		tenantId, config := checkSchemaConfig(ctx)
+		if config {
+			groupName = fmt.Sprintf("%s%v", schemaPrefix, tenantId)
 		}
 	}
-
+	db := g.DB(groupName)
+	TablePrefix = db.GetConfig().Prefix
 	return &dataBase{
-		DB: g.DB(dbName...).Schema(schema),
+		DB: db,
 	}
 }

 func (d *dataBase) Model(ctx context.Context, tableNameOrStruct ...any) *model {
-
-	m := d.DB.Model(tableNameOrStruct...).Ctx(ctx)
-
-	tenantId, config := checkSchemaConfig(ctx)
-
-	if config {
-		// 创建按地区分库的配置
-		shardingConfig := gdb.ShardingConfig{
-			Schema: gdb.ShardingSchemaConfig{
-				Enable: true,                                         // 启用分库
-				Prefix: schemaPrefix,                                 // 分库前缀
-				Rule:   &RegionShardingRule{RegionMapping: tenantId}, // 自定义分库规则
-			},
-		}
-		m.Sharding(shardingConfig).ShardingValue(tenantId)
-	}
-
-	m.OmitNil().Hook(catchSQLHook())
 	return &model{
-		Model: m,
+		Model: d.DB.Model(tableNameOrStruct...).Ctx(ctx).OmitNil().Hook(catchSQLHook()),
 	}
 }

@@ -542,42 +517,3 @@ func getTraceID(ctx context.Context, prefix string) string {
 	}
 	return ""
 }
-
-type RegionShardingRule struct {
-	RegionMapping uint64
-}
-
-func (r *RegionShardingRule) SchemaName(ctx context.Context, config gdb.ShardingSchemaConfig, value any) (string, error) {
-	region, ok := value.(uint64)
-	if !ok {
-		return "", fmt.Errorf("sharding value must be string for RegionShardingRule")
-	}
-
-	if r.RegionMapping == region {
-		return config.Prefix + gconv.String(region), nil
-	}
-
-	return "default", nil
-}
-
-// TableName 实现分表规则接口
-func (r *RegionShardingRule) TableName(ctx context.Context, config gdb.ShardingTableConfig, value any) (string, error) {
-	// 这里不实现分表，返回空字符串
-	return "", nil
-}
-
-func GetTablePrefix(ctx context.Context) (prefix string, err error) {
-	tenantId, config := checkSchemaConfig(ctx)
-	if config {
-		sprintf := fmt.Sprintf("database.%s%v.0.prefix", schemaPrefix, tenantId)
-		prefix = g.Cfg().MustGet(ctx, sprintf).String()
-		return
-	}
-	defaultConfig := g.Cfg().MustGet(ctx, "database.default")
-	if defaultConfig.IsSlice() {
-		prefix = g.Cfg().MustGet(ctx, "database.default.0.prefix").String()
-	} else {
-		prefix = g.Cfg().MustGet(ctx, "database.default.prefix").String()
-	}
-	return
-}
--- a/full-text-search/meilisearch/meilisearch.go
+++ b/full-text-search/meilisearch/meilisearch.go
@@ -118,7 +118,7 @@ func (m *meilisearchDB) ensureIndexExists(client ms.ServiceManager, indexName st
 		return err
 	}

-	requiredFilterable := []string{"tenantId", "isDeleted", "datasetId", "creator", "updater"}
+	requiredFilterable := []string{"tenantId", "isDeleted", "dataset_id", "creator", "updater"}
 	needUpdate := false

 	// 检查是否缺少必要的 filterable attributes
@@ -226,6 +226,8 @@ func (m *meilisearchDB) buildSearchRequest(ctx context.Context, searchParams *Se
 		req.AttributesToRetrieve = searchParams.AttributesToRetrieve
 	}

+	req.ShowRankingScore = searchParams.ShowRankingScore
+
 	return req, nil
 }

--- a/full-text-search/meilisearch/types.go
+++ b/full-text-search/meilisearch/types.go
@@ -15,6 +15,7 @@ type SearchParams struct {
 	AttributesToRetrieve []string // 返回字段
 	Facets               []string // 聚合字段
 	HitsPerPage          int      // 每页命中数
+	ShowRankingScore     bool     // 是否显示排名分数
 }

 // SearchResult 搜索结果
--- a/go.mod
+++ b/go.mod
@@ -5,24 +5,10 @@ go 1.26.0
 require (
 	github.com/alibaba/sentinel-golang v1.0.4
 	github.com/bwmarrin/snowflake v0.3.0
-	github.com/cloudwego/eino v0.7.26
-	github.com/cloudwego/eino-ext/components/document/loader/url v0.0.0-20260323112355-f061db7e8419
-	github.com/cloudwego/eino-ext/components/document/parser/docx v0.0.0-20260323112355-f061db7e8419
-	github.com/cloudwego/eino-ext/components/document/parser/pdf v0.0.0-20260323112355-f061db7e8419
-	github.com/cloudwego/eino-ext/components/document/parser/xlsx v0.0.0-20260323112355-f061db7e8419
-	github.com/cloudwego/eino-ext/components/document/transformer/splitter/recursive v0.0.0-20260323112355-f061db7e8419
-	github.com/cloudwego/eino-ext/components/document/transformer/splitter/semantic v0.0.0-20260323112355-f061db7e8419
-	github.com/cloudwego/eino-ext/components/embedding/ark v0.1.1
-	github.com/cloudwego/eino-ext/components/embedding/dashscope v0.0.0-20260323112355-f061db7e8419
-	github.com/cloudwego/eino-ext/components/embedding/openai v0.0.0-20260323112355-f061db7e8419
-	github.com/cloudwego/eino-ext/components/indexer/es8 v0.0.0-20260331071634-4f359694d2d9
-	github.com/cloudwego/eino-ext/components/retriever/es8 v0.0.0-20260331071634-4f359694d2d9
-	github.com/elastic/go-elasticsearch/v8 v8.16.0
 	github.com/go-ego/gse v1.0.2
 	github.com/gogf/gf/contrib/registry/consul/v2 v2.9.5
 	github.com/gogf/gf/contrib/trace/otlphttp/v2 v2.9.5
 	github.com/gogf/gf/v2 v2.9.5
-	github.com/golang/glog v1.2.5
 	github.com/google/uuid v1.6.0
 	github.com/meilisearch/meilisearch-go v0.36.1
 	github.com/minio/minio-go/v7 v7.0.97
@@ -39,42 +25,25 @@ require (

 require (
 	github.com/BurntSushi/toml v1.5.0 // indirect
-	github.com/PuerkitoBio/goquery v1.8.1 // indirect
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
 	github.com/akutz/memconn v0.1.0 // indirect
 	github.com/alitto/pond v1.9.2 // indirect
 	github.com/andybalholm/brotli v1.1.1 // indirect
-	github.com/andybalholm/cascadia v1.3.1 // indirect
 	github.com/apache/thrift v0.21.0 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
-	github.com/aymerick/douceur v0.2.0 // indirect
-	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/buger/jsonparser v1.1.1 // indirect
-	github.com/bytedance/gopkg v0.1.3 // indirect
-	github.com/bytedance/sonic v1.14.1 // indirect
-	github.com/bytedance/sonic/loader v0.3.0 // indirect
 	github.com/cenk/backoff v2.2.1+incompatible // indirect
 	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
-	github.com/cenkalti/backoff/v4 v4.1.2 // indirect
 	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/clbanning/mxj/v2 v2.7.0 // indirect
-	github.com/cloudwego/base64x v0.1.6 // indirect
-	github.com/cloudwego/eino-ext/components/document/parser/html v0.0.0-20241224063832-9fbcc0e56c28 // indirect
-	github.com/cloudwego/eino-ext/libs/acl/openai v0.1.14 // indirect
 	github.com/dgraph-io/badger/v4 v4.2.0 // indirect
 	github.com/dgraph-io/ristretto v0.1.1 // indirect
 	github.com/dgryski/go-jump v0.0.0-20211018200510-ba001c3ffce0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	github.com/dslipak/pdf v0.0.2 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/edwingeng/doublejump v1.0.1 // indirect
-	github.com/eino-contrib/docx2md v0.0.1 // indirect
-	github.com/eino-contrib/jsonschema v1.0.3 // indirect
-	github.com/elastic/elastic-transport-go/v8 v8.8.0 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
-	github.com/evanphx/json-patch v0.5.2 // indirect
 	github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect
 	github.com/fatih/color v1.18.0 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
@@ -88,13 +57,12 @@ require (
 	github.com/gogf/gf/contrib/nosql/redis/v2 v2.9.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
+	github.com/golang/glog v1.2.5 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/golang/snappy v1.0.0 // indirect
 	github.com/google/flatbuffers v1.12.1 // indirect
 	github.com/google/pprof v0.0.0-20250128161936-077ca0a936bf // indirect
-	github.com/goph/emperror v0.17.2 // indirect
-	github.com/gorilla/css v1.0.1 // indirect
 	github.com/gorilla/websocket v1.5.3 // indirect
 	github.com/grandcat/zeroconf v1.0.0 // indirect
 	github.com/grokify/html-strip-tags-go v0.1.0 // indirect
@@ -108,9 +76,7 @@ require (
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
 	github.com/hashicorp/golang-lru v1.0.2 // indirect
 	github.com/hashicorp/serf v0.10.1 // indirect
-	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/juju/ratelimit v1.0.2 // indirect
 	github.com/julienschmidt/httprouter v1.3.0 // indirect
 	github.com/kavu/go_reuseport v1.5.0 // indirect
@@ -124,24 +90,17 @@ require (
 	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect
-	github.com/meguminnnnnnnnn/go-openai v0.1.1 // indirect
-	github.com/microcosm-cc/bluemonday v1.0.27 // indirect
 	github.com/miekg/dns v1.1.63 // indirect
 	github.com/minio/crc64nvme v1.1.0 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
 	github.com/nats-io/nkeys v0.4.11 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
-	github.com/nikolalohinski/gonja v1.5.3 // indirect
 	github.com/olekukonko/errors v1.1.0 // indirect
 	github.com/olekukonko/ll v0.0.9 // indirect
 	github.com/olekukonko/tablewriter v1.1.0 // indirect
 	github.com/onsi/ginkgo/v2 v2.22.2 // indirect
-	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
 	github.com/philhofer/fwd v1.2.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_golang v1.19.1 // indirect
@@ -151,16 +110,12 @@ require (
 	github.com/quic-go/quic-go v0.49.0 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/redis/go-redis/v9 v9.12.1 // indirect
-	github.com/richardlehane/mscfb v1.0.4 // indirect
-	github.com/richardlehane/msoleps v1.0.4 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/rpcxio/libkv v0.5.1 // indirect
 	github.com/rs/cors v1.11.1 // indirect
 	github.com/rs/xid v1.6.0 // indirect
 	github.com/rubyist/circuitbreaker v2.2.1+incompatible // indirect
 	github.com/shirou/gopsutil/v3 v3.21.6 // indirect
-	github.com/sirupsen/logrus v1.9.3 // indirect
-	github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f // indirect
 	github.com/smallnest/quick v0.2.0 // indirect
 	github.com/smallnest/rsocket v0.0.0-20241130031020-4a72eb6ff62a // indirect
 	github.com/soheilhy/cmux v0.1.5 // indirect
@@ -170,22 +125,14 @@ require (
 	github.com/tjfoc/gmsm v1.4.1 // indirect
 	github.com/tklauser/go-sysconf v0.3.6 // indirect
 	github.com/tklauser/numcpus v0.2.2 // indirect
-	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/valyala/fastrand v1.1.0 // indirect
 	github.com/vcaesar/cedar v0.30.0 // indirect
 	github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
-	github.com/volcengine/volc-sdk-golang v1.0.199 // indirect
-	github.com/volcengine/volcengine-go-sdk v1.0.181 // indirect
-	github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/xtaci/kcp-go v5.4.20+incompatible // indirect
-	github.com/xuri/efp v0.0.0-20240408161823-9ad904a10d6d // indirect
-	github.com/xuri/excelize/v2 v2.9.0 // indirect
-	github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 // indirect
-	github.com/yargevad/filepathx v1.0.0 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
 	go.opencensus.io v0.23.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
@@ -195,7 +142,6 @@ require (
 	go.opentelemetry.io/otel/sdk v1.38.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.7.1 // indirect
 	go.uber.org/mock v0.5.0 // indirect
-	golang.org/x/arch v0.15.0 // indirect
 	golang.org/x/crypto v0.41.0 // indirect
 	golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect
 	golang.org/x/mod v0.26.0 // indirect
--- a/go.sum
+++ b/go.sum
--- a/middleware/rate_limiter.go
+++ b/middleware/rate_limiter.go
@@ -62,7 +62,7 @@ func IPLimiter(r *ghttp.Request) {

 // UserLimiter 用户维度限流中间件（防止单用户滥用）
 func UserLimiter(r *ghttp.Request) {
-	if r.RequestURI == "/swagger" || r.RequestURI == "/admin-go/api/v1/pub/captcha/get" || r.RequestURI == "/admin-go/api/v1/system/login" {
+	if strings.Contains(r.RequestURI, "/swagger") || strings.Contains(r.RequestURI, "/pub/captcha/get") || strings.Contains(r.RequestURI, "/login") || strings.Contains(r.RequestURI, "/web/socket/") {
 		r.Middleware.Next()
 		return
 	}
--- a/rag/eino/base_task.go
+++ b/rag/eino/base_task.go
@@ -1,49 +0,0 @@
-package eino
-
-import (
-	"time"
-
-	"gitea.com/red-future/common/beans"
-)
-
-// BaseTask 任务基类 - MongoDB版本
-type BaseTask struct {
-	beans.MongoBaseDO `bson:",inline"`
-	// 任务信息
-	TaskType TaskType     `bson:"taskType" json:"taskType"`
-	Status   TaskStatus   `bson:"status" json:"status"`
-	Priority TaskPriority `bson:"priority,omitempty" json:"priority,omitempty"`
-	// 进度
-	TotalItems     int64   `bson:"totalItems" json:"totalItems"`
-	ProcessedItems int64   `bson:"processedItems" json:"processedItems"`
-	Progress       float64 `bson:"progress" json:"progress"`
-	// 结果
-	StartTime    *time.Time `bson:"startTime" json:"startTime"`
-	EndTime      *time.Time `bson:"endTime,omitempty" json:"endTime,omitempty"`
-	Duration     int64      `bson:"duration,omitempty" json:"duration,omitempty"`
-	SuccessCount int64      `bson:"successCount" json:"successCount"`
-	FailCount    int64      `bson:"failCount" json:"failCount"`
-	// 其他
-	Executor string `bson:"executor,omitempty" json:"executor,omitempty"`
-}
-
-// SQLBaseTask 任务基类 - SQL版本
-type SQLBaseTask struct {
-	beans.SQLBaseDO
-	// 任务信息
-	TaskType TaskType     `json:"taskType"`
-	Status   TaskStatus   `json:"status"`
-	Priority TaskPriority `json:"priority,omitempty"`
-	// 进度
-	TotalItems     int64   `json:"totalItems"`
-	ProcessedItems int64   `json:"processedItems"`
-	Progress       float64 `json:"progress"`
-	// 结果
-	StartTime    *time.Time `json:"startTime"`
-	EndTime      *time.Time `json:"endTime,omitempty"`
-	Duration     int64      `json:"duration,omitempty"`
-	SuccessCount int64      `json:"successCount"`
-	FailCount    int64      `json:"failCount"`
-	// 其他
-	Executor string `json:"executor,omitempty"`
-}
--- a/rag/eino/consts.go
+++ b/rag/eino/consts.go
@@ -1,8 +0,0 @@
-package eino
-
-const (
-	providerArk       = "ark"
-	providerOpenai    = "openai"
-	providerQianfan   = "qianfan"
-	providerDashscope = "dashscope"
-)
--- a/rag/eino/document_loader.go
+++ b/rag/eino/document_loader.go
@@ -1,51 +0,0 @@
-package eino
-
-import (
-	"context"
-	"fmt"
-
-	"gitea.com/red-future/common/utils"
-	"github.com/cloudwego/eino-ext/components/document/loader/url"
-	"github.com/cloudwego/eino-ext/components/document/parser/docx"
-	"github.com/cloudwego/eino-ext/components/document/parser/pdf"
-	"github.com/cloudwego/eino-ext/components/document/parser/xlsx"
-	"github.com/cloudwego/eino/components/document"
-	"github.com/cloudwego/eino/components/document/parser"
-	"github.com/cloudwego/eino/schema"
-)
-
-// LoadDocument 业务函数：加载文件
-func LoadDocument(ctx context.Context, filePath, fileFormat string) (docs []*schema.Document, err error) {
-	p, err := docsParser(ctx, fileFormat)
-	if err != nil {
-		return
-	}
-	loader, err := url.NewLoader(ctx, &url.LoaderConfig{
-		Parser: p,
-	})
-	imageUrl, err := utils.GetFileAddressPrefix(ctx)
-	if err != nil {
-		return
-	}
-	docs, err = loader.Load(context.Background(), document.Source{
-		URI: fmt.Sprintf("%s%s", imageUrl, filePath),
-	})
-	return
-}
-
-func docsParser(ctx context.Context, fileFormat string) (p parser.Parser, err error) {
-	switch fileFormat {
-	case "docx":
-		p, err = docx.NewDocxParser(ctx, &docx.Config{
-			ToSections:     true,
-			IncludeHeaders: true,
-			IncludeFooters: true,
-			IncludeTables:  true,
-		})
-	case "pdf":
-		p, err = pdf.NewPDFParser(ctx, &pdf.Config{})
-	case "xlsx":
-		p, err = xlsx.NewXlsxParser(ctx, &xlsx.Config{})
-	}
-	return
-}
--- a/rag/eino/document_semantic.go
+++ b/rag/eino/document_semantic.go
@@ -1,64 +0,0 @@
-package eino
-
-import (
-	"context"
-
-	"github.com/cloudwego/eino-ext/components/document/transformer/splitter/recursive"
-	"github.com/cloudwego/eino-ext/components/document/transformer/splitter/semantic"
-	"github.com/cloudwego/eino/schema"
-	"github.com/gogf/gf/v2/frame/g"
-)
-
-// SemanticSplitDocument 语义分割文档
-func SemanticSplitDocument(ctx context.Context, docs []*schema.Document) (res []*schema.Document, err error) {
-	// 默认分隔符（支持中英文）
-	separators := []string{"\n\n", "\n", "。", "！", "？", "；", ".", "!", "?", ";"}
-	// 读取配置，使用合理的默认值
-	bufferSize := g.Cfg().MustGet(ctx, "eino.splitter.bufferSize").Int()
-	minChunkSize := g.Cfg().MustGet(ctx, "eino.splitter.minChunkSize").Int()
-	percentile := g.Cfg().MustGet(ctx, "eino.splitter.percentile").Float64()
-	batchSize := g.Cfg().MustGet(ctx, "eino.splitter.batchSize").Int()
-	if batchSize <= 0 {
-		batchSize = 10 // doubao-embedding-vision 限制每批最多 10 个
-	}
-
-	// 使用批量包装器
-	var batchEmbedder *BatchEmbedder
-	provider := g.Cfg().MustGet(ctx, "eino.embedding.provider").String()
-	switch provider {
-	case providerArk:
-		batchEmbedder = NewBatchEmbedder(EmbedderArk, batchSize)
-	case providerOpenai:
-		batchEmbedder = NewBatchEmbedder(EmbedderOpenAI, batchSize)
-	case providerDashscope:
-		batchEmbedder = NewBatchEmbedder(EmbedderDashscope, batchSize)
-	}
-
-	splitter, err := semantic.NewSplitter(ctx, &semantic.Config{
-		Embedding:    batchEmbedder,
-		BufferSize:   bufferSize,
-		MinChunkSize: minChunkSize,
-		Percentile:   percentile,
-		Separators:   separators,
-	})
-	if err != nil {
-		return
-	}
-	return splitter.Transform(ctx, docs)
-}
-
-// RecursiveSplitDocument 递归分割文档
-func RecursiveSplitDocument(ctx context.Context, docs []*schema.Document) (res []*schema.Document, err error) {
-	// 默认分隔符（支持中英文）
-	separators := []string{"\n\n", "\n", "。", "！", "？", "；", ".", "!", "?", ";"}
-	splitter, err := recursive.NewSplitter(ctx, &recursive.Config{
-		ChunkSize:   512,
-		OverlapSize: 100,
-		KeepType:    recursive.KeepTypeNone,
-		Separators:  separators,
-	})
-	if err != nil {
-		return
-	}
-	return splitter.Transform(ctx, docs)
-}
--- a/rag/eino/embedding.go
+++ b/rag/eino/embedding.go
@@ -1,69 +0,0 @@
-package eino
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/cloudwego/eino-ext/components/embedding/ark"
-	"github.com/cloudwego/eino-ext/components/embedding/dashscope"
-	"github.com/cloudwego/eino-ext/components/embedding/openai"
-	"github.com/gogf/gf/v2/frame/g"
-	"github.com/golang/glog"
-)
-
-// 全局只初始化一次
-var (
-	EmbedderArk       *ark.Embedder
-	EmbedderDashscope *dashscope.Embedder
-	EmbedderOpenAI    *openai.Embedder
-)
-
-func init() {
-	ctx := context.Background()
-	if !g.Cfg().MustGet(ctx, "eino.embedding").IsEmpty() {
-		var err error
-		provider := g.Cfg().MustGet(ctx, "eino.embedding.provider").String()
-		switch provider {
-		case providerArk:
-			cfg := &ark.EmbeddingConfig{
-				APIKey: g.Cfg().MustGet(ctx, "eino.embedding.apiKey").String(),
-				Model:  g.Cfg().MustGet(ctx, "eino.embedding.model").String(),
-			}
-			if apiType := g.Cfg().MustGet(ctx, "eino.embedding.apiType").String(); apiType != "" {
-				apiTypeVal := ark.APIType(apiType)
-				cfg.APIType = &apiTypeVal
-			}
-			EmbedderArk, err = ark.NewEmbedder(ctx, cfg)
-		case providerOpenai:
-			chatModelConfig := &openai.EmbeddingConfig{
-				APIKey: g.Cfg().MustGet(ctx, "eino.embedding.apiKey").String(),
-				Model:  g.Cfg().MustGet(ctx, "eino.embedding.model").String(),
-			}
-			EmbedderOpenAI, err = openai.NewEmbedder(ctx, chatModelConfig)
-		case providerDashscope:
-			cfg := &dashscope.EmbeddingConfig{
-				APIKey: g.Cfg().MustGet(ctx, "eino.embedding.apiKey").String(),
-				Model:  g.Cfg().MustGet(ctx, "eino.embedding.model").String(),
-			}
-			EmbedderDashscope, err = dashscope.NewEmbedder(ctx, cfg)
-		}
-		if err != nil {
-			glog.Fatalf("NewEmbedder of %v error: %v", provider, err)
-		}
-	}
-
-	return
-}
-
-func EmbedStrings(ctx context.Context, texts []string) (embeddings [][]float64, err error) {
-	provider := g.Cfg().MustGet(ctx, "eino.embedding.provider").String()
-	switch provider {
-	case providerArk:
-		return EmbedderArk.EmbedStrings(ctx, texts)
-	case providerOpenai:
-		return EmbedderOpenAI.EmbedStrings(ctx, texts)
-	case providerDashscope:
-		return EmbedderDashscope.EmbedStrings(ctx, texts)
-	}
-	return nil, fmt.Errorf("unsupported provider: %v", provider)
-}
--- a/rag/eino/embedding_batch.go
+++ b/rag/eino/embedding_batch.go
@@ -1,47 +0,0 @@
-package eino
-
-import (
-	"context"
-
-	"github.com/cloudwego/eino/components/embedding"
-)
-
-// BatchEmbedder 包装器，支持批量限制
-type BatchEmbedder struct {
-	embedder  embedding.Embedder
-	batchSize int
-}
-
-// NewBatchEmbedder 创建支持批量限制的 embedding 包装器
-func NewBatchEmbedder(embedder embedding.Embedder, batchSize int) *BatchEmbedder {
-	if batchSize <= 0 {
-		batchSize = 10 // 默认每批 10 个
-	}
-	return &BatchEmbedder{
-		embedder:  embedder,
-		batchSize: batchSize,
-	}
-}
-
-// EmbedStrings 分批调用 embedding
-func (b *BatchEmbedder) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) {
-	if len(texts) <= b.batchSize {
-		return b.embedder.EmbedStrings(ctx, texts, opts...)
-	}
-
-	var allEmbeddings [][]float64
-	for i := 0; i < len(texts); i += b.batchSize {
-		end := i + b.batchSize
-		if end > len(texts) {
-			end = len(texts)
-		}
-
-		batch := texts[i:end]
-		embeddings, err := b.embedder.EmbedStrings(ctx, batch, opts...)
-		if err != nil {
-			return nil, err
-		}
-		allEmbeddings = append(allEmbeddings, embeddings...)
-	}
-	return allEmbeddings, nil
-}
--- a/rag/eino/embedding_qwen.go
+++ b/rag/eino/embedding_qwen.go
@@ -1,273 +0,0 @@
-/*
- * Copyright 2024 Red Future Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package eino
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"time"
-
-	"github.com/cloudwego/eino/callbacks"
-	"github.com/cloudwego/eino/components"
-	"github.com/cloudwego/eino/components/embedding"
-	"github.com/gogf/gf/v2/frame/g"
-	"github.com/gogf/gf/v2/net/gclient"
-	"github.com/gogf/gf/v2/util/gconv"
-)
-
-var (
-	// 千问API默认配置
-	defaultBaseURL    = "https://dashscope.aliyuncs.com/api/v1/services/embeddings/text-embedding"
-	defaultTimeout    = 10 * time.Minute
-	defaultRetryTimes = 2
-)
-
-type QwenEmbeddingConfig struct {
-	// Timeout specifies the maximum duration to wait for API responses
-	// Optional. Default: 10 minutes
-	Timeout *time.Duration `json:"timeout"`
-
-	// HTTPClient specifies the client to send HTTP requests.
-	// Optional. Default &http.Client{Timeout: Timeout}
-	HTTPClient *http.Client `json:"http_client"`
-
-	// RetryTimes specifies the number of retry attempts for failed API calls
-	// Optional. Default: 2
-	RetryTimes *int `json:"retry_times"`
-
-	// BaseURL specifies the base URL for Qwen DashScope service
-	// Optional. Default: "https://dashscope.aliyuncs.com/api/v1/services/embeddings/text-embedding"
-	BaseURL string `json:"base_url"`
-
-	// APIKey specifies the API Key for authentication
-	// Required
-	APIKey string `json:"api_key"`
-
-	// Model specifies the model name for Qwen embedding
-	// Required. Examples: "text-embedding-v2", "text-embedding-v3"
-	Model string `json:"model"`
-
-	// TextType specifies the type of text: "document" or "query"
-	// Optional. Default: "document"
-	TextType string `json:"text_type"`
-
-	// MaxConcurrentRequests specifies the maximum number of concurrent requests allowed
-	// Optional. Default: 5
-	MaxConcurrentRequests *int `json:"max_concurrent_requests"`
-}
-
-type QwenEmbedder struct {
-	client *gclient.Client
-	conf   *QwenEmbeddingConfig
-}
-
-// EmbeddingRequest 千问embedding请求结构
-type EmbeddingRequest struct {
-	Model string `json:"model"`
-	Input struct {
-		Texts []string `json:"texts"`
-	} `json:"input"`
-	Parameters struct {
-		TextType string `json:"text_type,omitempty"`
-	} `json:"parameters,omitempty"`
-}
-
-// EmbeddingResponse 千问embedding响应结构
-type EmbeddingResponse struct {
-	Output struct {
-		Embeddings []struct {
-			TextIndex int       `json:"text_index"`
-			Embedding []float64 `json:"embedding"`
-		} `json:"embeddings"`
-	} `json:"output"`
-	Usage struct {
-		TotalTokens int `json:"total_tokens"`
-	} `json:"usage"`
-	RequestID string `json:"request_id"`
-}
-
-type APIError struct {
-	Code      string `json:"code"`
-	Message   string `json:"message"`
-	RequestID string `json:"request_id"`
-}
-
-func (e *APIError) Error() string {
-	return fmt.Sprintf("API Error: %s - %s (RequestID: %s)", e.Code, e.Message, e.RequestID)
-}
-
-func buildQwenClient(config *QwenEmbeddingConfig) *gclient.Client {
-	if len(config.BaseURL) == 0 {
-		config.BaseURL = defaultBaseURL
-	}
-	if config.Timeout == nil {
-		config.Timeout = &defaultTimeout
-	}
-	if config.RetryTimes == nil {
-		defaultRetryTimes := 2
-		config.RetryTimes = &defaultRetryTimes
-	}
-	if len(config.TextType) == 0 {
-		config.TextType = "document"
-	}
-	if config.MaxConcurrentRequests == nil {
-		defaultMaxConcurrentRequests := 5
-		config.MaxConcurrentRequests = &defaultMaxConcurrentRequests
-	}
-
-	client := g.Client()
-	client.SetTimeout(*config.Timeout)
-
-	return client
-}
-
-func NewQwenEmbedder(ctx context.Context, config *QwenEmbeddingConfig) (*QwenEmbedder, error) {
-	if len(config.APIKey) == 0 {
-		return nil, fmt.Errorf("[Qwen] APIKey is required")
-	}
-	if len(config.Model) == 0 {
-		return nil, fmt.Errorf("[Qwen] Model is required")
-	}
-
-	client := buildQwenClient(config)
-
-	return &QwenEmbedder{
-		client: client,
-		conf:   config,
-	}, nil
-}
-
-func (e *QwenEmbedder) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) (
-	[][]float64, error) {
-
-	if len(texts) == 0 {
-		return nil, fmt.Errorf("[Qwen] texts cannot be empty")
-	}
-
-	options := embedding.GetCommonOptions(&embedding.Options{
-		Model: &e.conf.Model,
-	}, opts...)
-
-	conf := &embedding.Config{
-		Model: dereferenceOrZero(options.Model),
-	}
-
-	ctx = callbacks.EnsureRunInfo(ctx, e.GetType(), components.ComponentOfEmbedding)
-	ctx = callbacks.OnStart(ctx, &embedding.CallbackInput{
-		Texts:  texts,
-		Config: conf,
-	})
-	defer func() {
-		if err := recover(); err != nil {
-			callbacks.OnError(ctx, fmt.Errorf("[Qwen] panic: %v", err))
-		}
-	}()
-
-	var usage *embedding.TokenUsage
-	var embeddings [][]float64
-	var err error
-
-	// 调用千问API获取embedding
-	embeddings, usage, err = e.callEmbeddingAPI(ctx, texts)
-	if err != nil {
-		callbacks.OnError(ctx, err)
-		return nil, err
-	}
-
-	callbacks.OnEnd(ctx, &embedding.CallbackOutput{
-		Embeddings: embeddings,
-		Config:     conf,
-		TokenUsage: usage,
-	})
-
-	return embeddings, nil
-}
-
-func (e *QwenEmbedder) callEmbeddingAPI(ctx context.Context, texts []string) ([][]float64, *embedding.TokenUsage, error) {
-	// 构建请求
-	var req EmbeddingRequest
-	req.Model = e.conf.Model
-	req.Input.Texts = texts
-	req.Parameters.TextType = e.conf.TextType
-
-	// 调用API
-	client := e.client.Clone()
-	client.SetHeader("Authorization", "Bearer "+e.conf.APIKey)
-	client.SetHeader("Content-Type", "application/json")
-	client.SetTimeout(*e.conf.Timeout)
-
-	resp, err := client.Post(ctx, e.conf.BaseURL, req)
-	if err != nil {
-		return nil, nil, fmt.Errorf("[Qwen] HTTP request error: %w", err)
-	}
-
-	defer resp.Close()
-
-	// 检查状态码
-	if resp.StatusCode != http.StatusOK {
-		var errResp APIError
-		result := resp.ReadAll()
-		if err = gconv.Struct(result, &errResp); err == nil && errResp.Code != "" {
-			return nil, nil, &errResp
-		}
-		return nil, nil, fmt.Errorf("[Qwen] HTTP status error: %d", resp.StatusCode)
-	}
-
-	// 解析响应
-	var apiResp EmbeddingResponse
-	result := resp.ReadAll()
-	if err = gconv.Struct(result, &apiResp); err != nil {
-		return nil, nil, fmt.Errorf("[Qwen] parse response error: %w", err)
-	}
-
-	// 解析响应结果
-	embeddings := make([][]float64, len(texts))
-	for _, emb := range apiResp.Output.Embeddings {
-		if emb.TextIndex >= 0 && emb.TextIndex < len(embeddings) {
-			embeddings[emb.TextIndex] = emb.Embedding
-		}
-	}
-
-	usage := &embedding.TokenUsage{
-		TotalTokens: apiResp.Usage.TotalTokens,
-	}
-
-	g.Log().Debugf(ctx, "[Qwen] Embedding success: request_id=%s, total_tokens=%d", apiResp.RequestID, usage.TotalTokens)
-
-	return embeddings, usage, nil
-}
-
-func (e *QwenEmbedder) GetType() string {
-	return getType()
-}
-
-func (e *QwenEmbedder) IsCallbacksEnabled() bool {
-	return true
-}
-
-func getType() string {
-	return "Qwen"
-}
-
-func dereferenceOrZero[T any](v *T) T {
-	if v == nil {
-		var t T
-		return t
-	}
-	return *v
-}
--- a/rag/eino/priority_enum.go
+++ b/rag/eino/priority_enum.go
@@ -1,11 +0,0 @@
-package eino
-
-// TaskPriority 任务优先级
-type TaskPriority string
-
-const (
-	TaskPriorityLow    TaskPriority = "low"    // 低优先级
-	TaskPriorityMedium TaskPriority = "medium" // 中优先级
-	TaskPriorityHigh   TaskPriority = "high"   // 高优先级
-	TaskPriorityUrgent TaskPriority = "urgent" // 紧急
-)
--- a/rag/eino/status_enum.go
+++ b/rag/eino/status_enum.go
@@ -1,12 +0,0 @@
-package eino
-
-// TaskStatus 任务状态
-type TaskStatus string
-
-const (
-	TaskStatusPending   TaskStatus = "pending"   // 待处理
-	TaskStatusRunning   TaskStatus = "running"   // 运行中
-	TaskStatusCompleted TaskStatus = "completed" // 已完成
-	TaskStatusFailed    TaskStatus = "failed"    // 失败
-	TaskStatusCancelled TaskStatus = "cancelled" // 已取消
-)
--- a/rag/eino/task_type.go
+++ b/rag/eino/task_type.go
@@ -1,14 +0,0 @@
-package eino
-
-// TaskType 任务类型
-type TaskType string
-
-const (
-	TaskTypeDocumentIngestion     TaskType = "document_ingestion"     // 文档摄入任务
-	TaskTypeVectorIngestion       TaskType = "vector_ingestion"       // 向量摄入任务
-	TaskTypeIndexCreation         TaskType = "index_creation"         // 索引创建任务
-	TaskTypeQAProcessing          TaskType = "qa_processing"          // 问答处理任务
-	TaskTypeKnowledgeConstruction TaskType = "knowledge_construction" // 知识库构建任务
-	TaskTypeGraphBuilding         TaskType = "graph_building"         // 图谱构建任务
-	TaskTypeKnowledgeSync         TaskType = "knowledge_sync"         // 知识同步任务
-)
--- a/rag/gse/utils.go
+++ b/rag/gse/utils.go
@@ -1,8 +1,9 @@
-package gse
+package utils

 import (
 	"context"
 	"sort"
+	"sync"

 	"github.com/go-ego/gse"
 	"github.com/go-ego/gse/hmm/extracker"
@@ -10,15 +11,22 @@ import (
 	"github.com/gogf/gf/v2/os/glog"
 )

-var GseTool *gseTool
+// 全局工具实例（不再自动初始化）
+var (
+	GseTool *gseTool
+	once    sync.Once // 保证只初始化一次，线程安全
+)

-// 初始化函数：程序启动时执行一次
-func init() {
+func InitGseTool(ctx context.Context) error {
 	var err error
-	GseTool, err = newGseTool()
+	once.Do(func() {
+		// 只执行一次初始化
+		GseTool, err = newGseTool()
+	})
 	if err != nil {
-		glog.Error(context.Background(), err)
+		glog.Error(ctx, "gse 分词工具初始化失败:", err)
 	}
+	return err
 }

 // gseTool 关键词提取工具（gse v1.0.2 标准）
Author	SHA1	Message	Date
张斌	45e6bf653f	增加dockerfile配置	2026-04-09 16:10:11 +08:00
qhd	8eb992e658	feat: 添加 Meilisearch 排名分数支持	2026-04-09 13:58:36 +08:00
qhd	5022e9115d	feat: 添加分词工具支持并更新字段命名	2026-04-08 14:21:12 +08:00
qhd	7acee87023	feat: 支持多数据库配置与PGVector检索	2026-04-03 17:59:04 +08:00
qhd	3adb14132b	Merge branch 'dev' of http://116.204.74.41:3000/red-future/common into dev	2026-04-03 15:21:23 +08:00
qhd	d7d039a423	refactor: 移除 eino 相关代码及依赖	2026-04-03 15:21:01 +08:00
张斌	2934edf8a9	golang版本升级1.26.0	2026-04-03 11:30:13 +08:00