feat: rag初始版

common/eino/a.go

@@ -0,0 +1,177 @@
+package eino
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"rag/dao"
+	"rag/model/dto"
+	"rag/model/entity"
+
+	"gitea.com/red-future/common/beans"
+	"github.com/cloudwego/eino/callbacks"
+	"github.com/cloudwego/eino/components/indexer"
+	"github.com/cloudwego/eino/schema"
+	"github.com/gogf/gf/v2/os/glog"
+	"github.com/gogf/gf/v2/util/gconv"
+	"github.com/pgvector/pgvector-go"
+)
+
+type PGVectorIndexerOptions struct {
+	BatchSize int // 每批处理多少条
+}
+
+type PGVectorIndexer struct {
+	opts *PGVectorIndexerOptions
+}
+
+func NewPGVectorIndexer(opts *PGVectorIndexerOptions) *PGVectorIndexer {
+	// 默认值
+	if opts.BatchSize <= 0 {
+		opts.BatchSize = 5
+	}
+	return &PGVectorIndexer{opts: opts}
+}
+
+func (i *PGVectorIndexer) Store(ctx context.Context, docs []*schema.Document, opts ...indexer.Option) (rows int64, err error) {
+	commonOpts := indexer.GetCommonOptions(&indexer.Options{}, opts...)
+
+	if commonOpts.Embedding == nil {
+		return 0, errors.New("embedding model not set")
+	}
+
+	// 回调
+	ctx = callbacks.OnStart(ctx, &indexer.CallbackInput{Docs: docs})
+
+	ids, err := i.bulkStore(ctx, docs, commonOpts)
+	if err != nil {
+		callbacks.OnError(ctx, err)
+		return
+	}
+
+	callbacks.OnEnd(ctx, &indexer.CallbackOutput{IDs: gconv.Strings(ids)})
+	return
+}
+
+func (i *PGVectorIndexer) bulkStore(ctx context.Context, docs []*schema.Document, opts *indexer.Options) (rows int64, err error) {
+	var batchDocs []*schema.Document
+
+	// 官方ES同款逻辑：满 BatchSize 就处理一批
+	for _, doc := range docs {
+		batchDocs = append(batchDocs, doc)
+
+		// 满了 → 处理
+		if len(batchDocs) >= i.opts.BatchSize {
+			var r int64
+			r, err = i.doStore(ctx, batchDocs, opts)
+			if err != nil {
+				return
+			}
+			rows = rows + r
+			batchDocs = nil
+		}
+	}
+
+	// 最后一批
+	if len(batchDocs) > 0 {
+		var r int64
+		r, err = i.doStore(ctx, batchDocs, opts)
+		if err != nil {
+			return
+		}
+		rows = rows + r
+	}
+
+	return
+}
+
+func (i *PGVectorIndexer) doStore(ctx context.Context, docs []*schema.Document, opts *indexer.Options) (rows int64, err error) {
+
+	texts := make([]string, len(docs))
+	for i, d := range docs {
+		texts[i] = d.Content
+	}
+
+	// 向量化（官方ES也没有重试！）
+	vectors, err := opts.Embedding.EmbedStrings(ctx, texts)
+	if err != nil {
+		return
+	}
+
+	// 转成业务实体
+	var chunks []*dto.VectorDocumentChunkMsg
+	for idx, doc := range docs {
+		ck := new(dto.VectorDocumentChunkMsg)
+		err = gconv.Struct(doc.MetaData, ck)
+		if err != nil {
+			glog.Errorf(ctx, "doStore err: %v", err)
+			continue
+		}
+		ck.Content = doc.Content
+		ck.Vector = pgvector.NewVector(gconv.Float32s(vectors[idx]))
+		ck.VectorStatus = gconv.PtrInt8(1)
+		ck.Status = gconv.PtrInt8(1)
+		chunks = append(chunks, ck)
+	}
+	if len(chunks) == 0 {
+		return
+	}
+	ctx = context.WithValue(ctx, "user", &beans.User{
+		TenantId: chunks[0].TenantId,
+		UserName: chunks[0].Creator,
+	})
+	// 创建索引
+	if err = i.createOrUpdateDatasetIndex(ctx, chunks[0].DatasetId, len(vectors[0]), int64(len(chunks))); err != nil {
+		return
+	}
+	// 入库
+	rows, err = dao.DocumentChunk.BatchInsert(ctx, chunks)
+
+	return
+}
+
+func (i *PGVectorIndexer) createOrUpdateDatasetIndex(ctx context.Context, datasetId int64, dimension int, vectorCount int64) error {
+	exist, err := dao.DatasetIndex.GetByDatasetId(ctx, datasetId)
+	if err != nil && !errors.Is(err, sql.ErrNoRows) {
+		return err
+	}
+	if exist != nil {
+		_ = dao.DatasetIndex.IncVectorCount(ctx, exist.Id, vectorCount)
+		return nil
+	}
+
+	indexName := fmt.Sprintf("idx_dataset_%d_vector", datasetId)
+	idx := &entity.DatasetIndex{
+		DatasetId:   datasetId,
+		Name:        indexName,
+		Dimension:   dimension,
+		FieldType:   "float",
+		MetricType:  "COSINE",
+		Status:      gconv.PtrInt8(1),
+		VectorCount: vectorCount,
+		Description: fmt.Sprintf("数据集%d向量索引", datasetId),
+	}
+	_, err = dao.DatasetIndex.Insert(ctx, idx)
+	if err != nil {
+		return err
+	}
+	return i.createRealPGVectorIndex(ctx, indexName)
+}
+
+func (i *PGVectorIndexer) createRealPGVectorIndex(ctx context.Context, indexName string) error {
+	if err := dao.DatasetIndex.InsertIndex(ctx, indexName); err != nil {
+		glog.Errorf(ctx, "create vector index failed: %v", err)
+		return err
+	}
+	glog.Infof(ctx, "created pgvector index: %s", indexName)
+	return nil
+}
+
+func (i *PGVectorIndexer) GetType() string {
+	return "pgvector_indexer"
+}
+
+func (i *PGVectorIndexer) IsCallbacksEnabled() bool {
+	return true
+}

common/eino/b.go

@@ -0,0 +1,107 @@
+package eino
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/cloudwego/eino/schema"
+	"github.com/elastic/go-elasticsearch/v8"
+
+	"github.com/cloudwego/eino-ext/components/indexer/es8"
+)
+
+const (
+	indexName          = "eino_example"
+	fieldContent       = "content"
+	fieldContentVector = "content_vector"
+	fieldExtraLocation = "location"
+	docExtraLocation   = "location"
+)
+
+func TestIndexer() {
+	ctx := context.Background()
+
+	// 1. 创建 ES 客户端
+	client, err := elasticsearch.NewClient(elasticsearch.Config{
+		Addresses: []string{"http://localhost:9200"},
+	})
+
+	if err != nil {
+		fmt.Printf("create client error: %v\n", err)
+		return
+	}
+
+	// 2. 定义 Index Spec（选填：如果索引不存在，将自动创建）
+	indexSpec := &es8.IndexSpec{
+		Settings: map[string]any{
+			"number_of_shards":   1,
+			"number_of_replicas": 0,
+		},
+		Mappings: map[string]any{
+			"properties": map[string]any{
+				fieldContentVector: map[string]any{
+					"type":       "dense_vector",
+					"dims":       1024,
+					"index":      true,
+					"similarity": "l2_norm",
+				},
+			},
+		},
+	}
+
+	// 4. 准备文档
+	// 文档通常包含 ID 和 Content
+	// 也可以包含额外的 Metadata 用于过滤或其他用途
+	docs := []*schema.Document{
+		{
+			ID:      "1",
+			Content: "Eiffel Tower: Located in Paris, France.",
+			MetaData: map[string]any{
+				docExtraLocation: "France",
+			},
+		},
+		{
+			ID:      "2",
+			Content: "The Great Wall: Located in China.",
+			MetaData: map[string]any{
+				docExtraLocation: "China",
+			},
+		},
+	}
+
+	// 5. 创建 ES 索引器组件
+	indexer, err := es8.NewIndexer(ctx, &es8.IndexerConfig{
+		Client:    client,
+		Index:     indexName,
+		IndexSpec: indexSpec, // 添加此项以启用自动索引创建
+		BatchSize: 10,
+		// DocumentToFields 指定如何将文档字段映射到 ES 字段
+		DocumentToFields: func(ctx context.Context, doc *schema.Document) (field2Value map[string]es8.FieldValue, err error) {
+			return map[string]es8.FieldValue{
+				fieldContent: {
+					Value:    doc.Content,
+					EmbedKey: fieldContentVector, // 对文档内容进行向量化并保存到 "content_vector" 字段
+				},
+				fieldExtraLocation: {
+					// 额外的 metadata 字段
+					Value: doc.MetaData[docExtraLocation],
+				},
+			}, nil
+		},
+		// 提供 embedding 组件用于向量化
+		Embedding: EmbedderDashscope,
+	})
+
+	if err != nil {
+		fmt.Printf("create indexer error: %v\n", err)
+		return
+	}
+
+	// 6. 索引文档
+	ids, err := indexer.Store(ctx, docs)
+	if err != nil {
+		fmt.Printf("index error: %v\n", err)
+		return
+	}
+	fmt.Println("indexed ids:", ids)
+}

common/eino/base_task.go

@@ -0,0 +1,49 @@
+package eino
+
+import (
+	"time"
+
+	"gitea.com/red-future/common/beans"
+)
+
+// BaseTask 任务基类 - MongoDB版本
+type BaseTask struct {
+	beans.MongoBaseDO `bson:",inline"`
+	// 任务信息
+	TaskType TaskType     `bson:"taskType" json:"taskType"`
+	Status   TaskStatus   `bson:"status" json:"status"`
+	Priority TaskPriority `bson:"priority,omitempty" json:"priority,omitempty"`
+	// 进度
+	TotalItems     int64   `bson:"totalItems" json:"totalItems"`
+	ProcessedItems int64   `bson:"processedItems" json:"processedItems"`
+	Progress       float64 `bson:"progress" json:"progress"`
+	// 结果
+	StartTime    *time.Time `bson:"startTime" json:"startTime"`
+	EndTime      *time.Time `bson:"endTime,omitempty" json:"endTime,omitempty"`
+	Duration     int64      `bson:"duration,omitempty" json:"duration,omitempty"`
+	SuccessCount int64      `bson:"successCount" json:"successCount"`
+	FailCount    int64      `bson:"failCount" json:"failCount"`
+	// 其他
+	Executor string `bson:"executor,omitempty" json:"executor,omitempty"`
+}
+
+// SQLBaseTask 任务基类 - SQL版本
+type SQLBaseTask struct {
+	beans.SQLBaseDO
+	// 任务信息
+	TaskType TaskType     `json:"taskType"`
+	Status   TaskStatus   `json:"status"`
+	Priority TaskPriority `json:"priority,omitempty"`
+	// 进度
+	TotalItems     int64   `json:"totalItems"`
+	ProcessedItems int64   `json:"processedItems"`
+	Progress       float64 `json:"progress"`
+	// 结果
+	StartTime    *time.Time `json:"startTime"`
+	EndTime      *time.Time `json:"endTime,omitempty"`
+	Duration     int64      `json:"duration,omitempty"`
+	SuccessCount int64      `json:"successCount"`
+	FailCount    int64      `json:"failCount"`
+	// 其他
+	Executor string `json:"executor,omitempty"`
+}

common/eino/c.go

@@ -0,0 +1,94 @@
+package eino
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/cloudwego/eino/schema"
+	"github.com/elastic/go-elasticsearch/v8"
+	"github.com/elastic/go-elasticsearch/v8/typedapi/types"
+
+	"github.com/cloudwego/eino-ext/components/retriever/es8"
+	"github.com/cloudwego/eino-ext/components/retriever/es8/search_mode"
+)
+
+func TestRetriever() {
+	ctx := context.Background()
+
+	client, _ := elasticsearch.NewClient(elasticsearch.Config{
+		Addresses: []string{"http://localhost:9200"},
+	})
+
+	// 创建 retriever 组件
+	retriever, _ := es8.NewRetriever(ctx, &es8.RetrieverConfig{
+		Client: client,
+		Index:  indexName,
+		TopK:   5,
+		SearchMode: search_mode.SearchModeApproximate(&search_mode.ApproximateConfig{
+			QueryFieldName:  fieldContent,
+			VectorFieldName: fieldContentVector,
+			Hybrid:          false,
+			// RRF 仅在特定许可证下可用
+			// 参见: https://www.elastic.co/subscriptions
+			RRF:             false,
+			RRFRankConstant: nil,
+			RRFWindowSize:   nil,
+		}),
+		ResultParser: func(ctx context.Context, hit types.Hit) (doc *schema.Document, err error) {
+			doc = &schema.Document{
+				ID:       *hit.Id_,
+				Content:  "",
+				MetaData: map[string]any{},
+			}
+
+			var src map[string]any
+			if err = json.Unmarshal(hit.Source_, &src); err != nil {
+				return nil, err
+			}
+
+			for field, val := range src {
+				switch field {
+				case fieldContent:
+					doc.Content = val.(string)
+				case fieldContentVector:
+					var v []float64
+					for _, item := range val.([]interface{}) {
+						v = append(v, item.(float64))
+					}
+					doc.WithDenseVector(v)
+				case fieldExtraLocation:
+					doc.MetaData[docExtraLocation] = val.(string)
+				}
+			}
+
+			if hit.Score_ != nil {
+				doc.WithScore(float64(*hit.Score_))
+			}
+
+			return doc, nil
+		},
+		Embedding: EmbedderDashscope,
+	})
+
+	// 不带过滤器的搜索
+	docs, _ := retriever.Retrieve(ctx, "tourist attraction")
+
+	// 带过滤器的搜索
+	docs, _ = retriever.Retrieve(ctx, "tourist attraction",
+		es8.WithFilters([]types.Query{{
+			Term: map[string]types.TermQuery{
+				fieldExtraLocation: {
+					CaseInsensitive: of(true),
+					Value:           "China",
+				},
+			},
+		}}),
+	)
+
+	fmt.Printf("retrieved docs: %+v\n", docs)
+}
+
+func of[T any](v T) *T {
+	return &v
+}

common/eino/consts.go

@@ -0,0 +1,8 @@
+package eino
+
+const (
+	providerArk       = "ark"
+	providerOpenai    = "openai"
+	providerQianfan   = "qianfan"
+	providerDashscope = "dashscope"
+)

common/eino/document_loader.go

@@ -0,0 +1,51 @@
+package eino
+
+import (
+	"context"
+	"fmt"
+
+	"gitea.com/red-future/common/utils"
+	"github.com/cloudwego/eino-ext/components/document/loader/url"
+	"github.com/cloudwego/eino-ext/components/document/parser/docx"
+	"github.com/cloudwego/eino-ext/components/document/parser/pdf"
+	"github.com/cloudwego/eino-ext/components/document/parser/xlsx"
+	"github.com/cloudwego/eino/components/document"
+	"github.com/cloudwego/eino/components/document/parser"
+	"github.com/cloudwego/eino/schema"
+)
+
+// LoadDocument 业务函数：加载文件
+func LoadDocument(ctx context.Context, filePath, fileFormat string) (docs []*schema.Document, err error) {
+	p, err := docsParser(ctx, fileFormat)
+	if err != nil {
+		return
+	}
+	loader, err := url.NewLoader(ctx, &url.LoaderConfig{
+		Parser: p,
+	})
+	imageUrl, err := utils.GetFileAddressPrefix(ctx)
+	if err != nil {
+		return
+	}
+	docs, err = loader.Load(context.Background(), document.Source{
+		URI: fmt.Sprintf("%s%s", imageUrl, filePath),
+	})
+	return
+}
+
+func docsParser(ctx context.Context, fileFormat string) (p parser.Parser, err error) {
+	switch fileFormat {
+	case "docx":
+		p, err = docx.NewDocxParser(ctx, &docx.Config{
+			ToSections:     true,
+			IncludeHeaders: true,
+			IncludeFooters: true,
+			IncludeTables:  true,
+		})
+	case "pdf":
+		p, err = pdf.NewPDFParser(ctx, &pdf.Config{})
+	case "xlsx":
+		p, err = xlsx.NewXlsxParser(ctx, &xlsx.Config{})
+	}
+	return
+}

common/eino/document_semantic.go

@@ -0,0 +1,64 @@
+package eino
+
+import (
+	"context"
+
+	"github.com/cloudwego/eino-ext/components/document/transformer/splitter/recursive"
+	"github.com/cloudwego/eino-ext/components/document/transformer/splitter/semantic"
+	"github.com/cloudwego/eino/schema"
+	"github.com/gogf/gf/v2/frame/g"
+)
+
+// SemanticSplitDocument 语义分割文档
+func SemanticSplitDocument(ctx context.Context, docs []*schema.Document) (res []*schema.Document, err error) {
+	// 默认分隔符（支持中英文）
+	separators := []string{"\n\n", "\n", "。", "！", "？", "；", ".", "!", "?", ";"}
+	// 读取配置，使用合理的默认值
+	bufferSize := g.Cfg().MustGet(ctx, "eino.splitter.bufferSize").Int()
+	minChunkSize := g.Cfg().MustGet(ctx, "eino.splitter.minChunkSize").Int()
+	percentile := g.Cfg().MustGet(ctx, "eino.splitter.percentile").Float64()
+	batchSize := g.Cfg().MustGet(ctx, "eino.splitter.batchSize").Int()
+	if batchSize <= 0 {
+		batchSize = 10 // doubao-embedding-vision 限制每批最多 10 个
+	}
+
+	// 使用批量包装器
+	var batchEmbedder *BatchEmbedder
+	provider := g.Cfg().MustGet(ctx, "eino.embedding.provider").String()
+	switch provider {
+	case providerArk:
+		batchEmbedder = NewBatchEmbedder(EmbedderArk, batchSize)
+	case providerOpenai:
+		batchEmbedder = NewBatchEmbedder(EmbedderOpenAI, batchSize)
+	case providerDashscope:
+		batchEmbedder = NewBatchEmbedder(EmbedderDashscope, batchSize)
+	}
+
+	splitter, err := semantic.NewSplitter(ctx, &semantic.Config{
+		Embedding:    batchEmbedder,
+		BufferSize:   bufferSize,
+		MinChunkSize: minChunkSize,
+		Percentile:   percentile,
+		Separators:   separators,
+	})
+	if err != nil {
+		return
+	}
+	return splitter.Transform(ctx, docs)
+}
+
+// RecursiveSplitDocument 递归分割文档
+func RecursiveSplitDocument(ctx context.Context, docs []*schema.Document) (res []*schema.Document, err error) {
+	// 默认分隔符（支持中英文）
+	separators := []string{"\n\n", "\n", "。", "！", "？", "；", ".", "!", "?", ";"}
+	splitter, err := recursive.NewSplitter(ctx, &recursive.Config{
+		ChunkSize:   512,
+		OverlapSize: 100,
+		KeepType:    recursive.KeepTypeNone,
+		Separators:  separators,
+	})
+	if err != nil {
+		return
+	}
+	return splitter.Transform(ctx, docs)
+}

common/eino/embedding.go

@@ -0,0 +1,69 @@
+package eino
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/cloudwego/eino-ext/components/embedding/ark"
+	"github.com/cloudwego/eino-ext/components/embedding/dashscope"
+	"github.com/cloudwego/eino-ext/components/embedding/openai"
+	"github.com/gogf/gf/v2/frame/g"
+	"github.com/golang/glog"
+)
+
+// 全局只初始化一次
+var (
+	EmbedderArk       *ark.Embedder
+	EmbedderDashscope *dashscope.Embedder
+	EmbedderOpenAI    *openai.Embedder
+)
+
+func init() {
+	ctx := context.Background()
+	if !g.Cfg().MustGet(ctx, "eino.embedding").IsEmpty() {
+		var err error
+		provider := g.Cfg().MustGet(ctx, "eino.embedding.provider").String()
+		switch provider {
+		case providerArk:
+			cfg := &ark.EmbeddingConfig{
+				APIKey: g.Cfg().MustGet(ctx, "eino.embedding.apiKey").String(),
+				Model:  g.Cfg().MustGet(ctx, "eino.embedding.model").String(),
+			}
+			if apiType := g.Cfg().MustGet(ctx, "eino.embedding.apiType").String(); apiType != "" {
+				apiTypeVal := ark.APIType(apiType)
+				cfg.APIType = &apiTypeVal
+			}
+			EmbedderArk, err = ark.NewEmbedder(ctx, cfg)
+		case providerOpenai:
+			chatModelConfig := &openai.EmbeddingConfig{
+				APIKey: g.Cfg().MustGet(ctx, "eino.embedding.apiKey").String(),
+				Model:  g.Cfg().MustGet(ctx, "eino.embedding.model").String(),
+			}
+			EmbedderOpenAI, err = openai.NewEmbedder(ctx, chatModelConfig)
+		case providerDashscope:
+			cfg := &dashscope.EmbeddingConfig{
+				APIKey: g.Cfg().MustGet(ctx, "eino.embedding.apiKey").String(),
+				Model:  g.Cfg().MustGet(ctx, "eino.embedding.model").String(),
+			}
+			EmbedderDashscope, err = dashscope.NewEmbedder(ctx, cfg)
+		}
+		if err != nil {
+			glog.Fatalf("NewEmbedder of %v error: %v", provider, err)
+		}
+	}
+
+	return
+}
+
+func EmbedStrings(ctx context.Context, texts []string) (embeddings [][]float64, err error) {
+	provider := g.Cfg().MustGet(ctx, "eino.embedding.provider").String()
+	switch provider {
+	case providerArk:
+		return EmbedderArk.EmbedStrings(ctx, texts)
+	case providerOpenai:
+		return EmbedderOpenAI.EmbedStrings(ctx, texts)
+	case providerDashscope:
+		return EmbedderDashscope.EmbedStrings(ctx, texts)
+	}
+	return nil, fmt.Errorf("unsupported provider: %v", provider)
+}

common/eino/embedding_batch.go

@@ -0,0 +1,47 @@
+package eino
+
+import (
+	"context"
+
+	"github.com/cloudwego/eino/components/embedding"
+)
+
+// BatchEmbedder 包装器，支持批量限制
+type BatchEmbedder struct {
+	embedder  embedding.Embedder
+	batchSize int
+}
+
+// NewBatchEmbedder 创建支持批量限制的 embedding 包装器
+func NewBatchEmbedder(embedder embedding.Embedder, batchSize int) *BatchEmbedder {
+	if batchSize <= 0 {
+		batchSize = 10 // 默认每批 10 个
+	}
+	return &BatchEmbedder{
+		embedder:  embedder,
+		batchSize: batchSize,
+	}
+}
+
+// EmbedStrings 分批调用 embedding
+func (b *BatchEmbedder) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) {
+	if len(texts) <= b.batchSize {
+		return b.embedder.EmbedStrings(ctx, texts, opts...)
+	}
+
+	var allEmbeddings [][]float64
+	for i := 0; i < len(texts); i += b.batchSize {
+		end := i + b.batchSize
+		if end > len(texts) {
+			end = len(texts)
+		}
+
+		batch := texts[i:end]
+		embeddings, err := b.embedder.EmbedStrings(ctx, batch, opts...)
+		if err != nil {
+			return nil, err
+		}
+		allEmbeddings = append(allEmbeddings, embeddings...)
+	}
+	return allEmbeddings, nil
+}

common/eino/embedding_qwen.go

@@ -0,0 +1,273 @@
+/*
+ * Copyright 2024 Red Future Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package eino
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/cloudwego/eino/callbacks"
+	"github.com/cloudwego/eino/components"
+	"github.com/cloudwego/eino/components/embedding"
+	"github.com/gogf/gf/v2/frame/g"
+	"github.com/gogf/gf/v2/net/gclient"
+	"github.com/gogf/gf/v2/util/gconv"
+)
+
+var (
+	// 千问API默认配置
+	defaultBaseURL    = "https://dashscope.aliyuncs.com/api/v1/services/embeddings/text-embedding"
+	defaultTimeout    = 10 * time.Minute
+	defaultRetryTimes = 2
+)
+
+type QwenEmbeddingConfig struct {
+	// Timeout specifies the maximum duration to wait for API responses
+	// Optional. Default: 10 minutes
+	Timeout *time.Duration `json:"timeout"`
+
+	// HTTPClient specifies the client to send HTTP requests.
+	// Optional. Default &http.Client{Timeout: Timeout}
+	HTTPClient *http.Client `json:"http_client"`
+
+	// RetryTimes specifies the number of retry attempts for failed API calls
+	// Optional. Default: 2
+	RetryTimes *int `json:"retry_times"`
+
+	// BaseURL specifies the base URL for Qwen DashScope service
+	// Optional. Default: "https://dashscope.aliyuncs.com/api/v1/services/embeddings/text-embedding"
+	BaseURL string `json:"base_url"`
+
+	// APIKey specifies the API Key for authentication
+	// Required
+	APIKey string `json:"api_key"`
+
+	// Model specifies the model name for Qwen embedding
+	// Required. Examples: "text-embedding-v2", "text-embedding-v3"
+	Model string `json:"model"`
+
+	// TextType specifies the type of text: "document" or "query"
+	// Optional. Default: "document"
+	TextType string `json:"text_type"`
+
+	// MaxConcurrentRequests specifies the maximum number of concurrent requests allowed
+	// Optional. Default: 5
+	MaxConcurrentRequests *int `json:"max_concurrent_requests"`
+}
+
+type QwenEmbedder struct {
+	client *gclient.Client
+	conf   *QwenEmbeddingConfig
+}
+
+// EmbeddingRequest 千问embedding请求结构
+type EmbeddingRequest struct {
+	Model string `json:"model"`
+	Input struct {
+		Texts []string `json:"texts"`
+	} `json:"input"`
+	Parameters struct {
+		TextType string `json:"text_type,omitempty"`
+	} `json:"parameters,omitempty"`
+}
+
+// EmbeddingResponse 千问embedding响应结构
+type EmbeddingResponse struct {
+	Output struct {
+		Embeddings []struct {
+			TextIndex int       `json:"text_index"`
+			Embedding []float64 `json:"embedding"`
+		} `json:"embeddings"`
+	} `json:"output"`
+	Usage struct {
+		TotalTokens int `json:"total_tokens"`
+	} `json:"usage"`
+	RequestID string `json:"request_id"`
+}
+
+type APIError struct {
+	Code      string `json:"code"`
+	Message   string `json:"message"`
+	RequestID string `json:"request_id"`
+}
+
+func (e *APIError) Error() string {
+	return fmt.Sprintf("API Error: %s - %s (RequestID: %s)", e.Code, e.Message, e.RequestID)
+}
+
+func buildQwenClient(config *QwenEmbeddingConfig) *gclient.Client {
+	if len(config.BaseURL) == 0 {
+		config.BaseURL = defaultBaseURL
+	}
+	if config.Timeout == nil {
+		config.Timeout = &defaultTimeout
+	}
+	if config.RetryTimes == nil {
+		defaultRetryTimes := 2
+		config.RetryTimes = &defaultRetryTimes
+	}
+	if len(config.TextType) == 0 {
+		config.TextType = "document"
+	}
+	if config.MaxConcurrentRequests == nil {
+		defaultMaxConcurrentRequests := 5
+		config.MaxConcurrentRequests = &defaultMaxConcurrentRequests
+	}
+
+	client := g.Client()
+	client.SetTimeout(*config.Timeout)
+
+	return client
+}
+
+func NewQwenEmbedder(ctx context.Context, config *QwenEmbeddingConfig) (*QwenEmbedder, error) {
+	if len(config.APIKey) == 0 {
+		return nil, fmt.Errorf("[Qwen] APIKey is required")
+	}
+	if len(config.Model) == 0 {
+		return nil, fmt.Errorf("[Qwen] Model is required")
+	}
+
+	client := buildQwenClient(config)
+
+	return &QwenEmbedder{
+		client: client,
+		conf:   config,
+	}, nil
+}
+
+func (e *QwenEmbedder) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) (
+	[][]float64, error) {
+
+	if len(texts) == 0 {
+		return nil, fmt.Errorf("[Qwen] texts cannot be empty")
+	}
+
+	options := embedding.GetCommonOptions(&embedding.Options{
+		Model: &e.conf.Model,
+	}, opts...)
+
+	conf := &embedding.Config{
+		Model: dereferenceOrZero(options.Model),
+	}
+
+	ctx = callbacks.EnsureRunInfo(ctx, e.GetType(), components.ComponentOfEmbedding)
+	ctx = callbacks.OnStart(ctx, &embedding.CallbackInput{
+		Texts:  texts,
+		Config: conf,
+	})
+	defer func() {
+		if err := recover(); err != nil {
+			callbacks.OnError(ctx, fmt.Errorf("[Qwen] panic: %v", err))
+		}
+	}()
+
+	var usage *embedding.TokenUsage
+	var embeddings [][]float64
+	var err error
+
+	// 调用千问API获取embedding
+	embeddings, usage, err = e.callEmbeddingAPI(ctx, texts)
+	if err != nil {
+		callbacks.OnError(ctx, err)
+		return nil, err
+	}
+
+	callbacks.OnEnd(ctx, &embedding.CallbackOutput{
+		Embeddings: embeddings,
+		Config:     conf,
+		TokenUsage: usage,
+	})
+
+	return embeddings, nil
+}
+
+func (e *QwenEmbedder) callEmbeddingAPI(ctx context.Context, texts []string) ([][]float64, *embedding.TokenUsage, error) {
+	// 构建请求
+	var req EmbeddingRequest
+	req.Model = e.conf.Model
+	req.Input.Texts = texts
+	req.Parameters.TextType = e.conf.TextType
+
+	// 调用API
+	client := e.client.Clone()
+	client.SetHeader("Authorization", "Bearer "+e.conf.APIKey)
+	client.SetHeader("Content-Type", "application/json")
+	client.SetTimeout(*e.conf.Timeout)
+
+	resp, err := client.Post(ctx, e.conf.BaseURL, req)
+	if err != nil {
+		return nil, nil, fmt.Errorf("[Qwen] HTTP request error: %w", err)
+	}
+
+	defer resp.Close()
+
+	// 检查状态码
+	if resp.StatusCode != http.StatusOK {
+		var errResp APIError
+		result := resp.ReadAll()
+		if err = gconv.Struct(result, &errResp); err == nil && errResp.Code != "" {
+			return nil, nil, &errResp
+		}
+		return nil, nil, fmt.Errorf("[Qwen] HTTP status error: %d", resp.StatusCode)
+	}
+
+	// 解析响应
+	var apiResp EmbeddingResponse
+	result := resp.ReadAll()
+	if err = gconv.Struct(result, &apiResp); err != nil {
+		return nil, nil, fmt.Errorf("[Qwen] parse response error: %w", err)
+	}
+
+	// 解析响应结果
+	embeddings := make([][]float64, len(texts))
+	for _, emb := range apiResp.Output.Embeddings {
+		if emb.TextIndex >= 0 && emb.TextIndex < len(embeddings) {
+			embeddings[emb.TextIndex] = emb.Embedding
+		}
+	}
+
+	usage := &embedding.TokenUsage{
+		TotalTokens: apiResp.Usage.TotalTokens,
+	}
+
+	g.Log().Debugf(ctx, "[Qwen] Embedding success: request_id=%s, total_tokens=%d", apiResp.RequestID, usage.TotalTokens)
+
+	return embeddings, usage, nil
+}
+
+func (e *QwenEmbedder) GetType() string {
+	return getType()
+}
+
+func (e *QwenEmbedder) IsCallbacksEnabled() bool {
+	return true
+}
+
+func getType() string {
+	return "Qwen"
+}
+
+func dereferenceOrZero[T any](v *T) T {
+	if v == nil {
+		var t T
+		return t
+	}
+	return *v
+}

common/eino/priority_enum.go

@@ -0,0 +1,11 @@
+package eino
+
+// TaskPriority 任务优先级
+type TaskPriority string
+
+const (
+	TaskPriorityLow    TaskPriority = "low"    // 低优先级
+	TaskPriorityMedium TaskPriority = "medium" // 中优先级
+	TaskPriorityHigh   TaskPriority = "high"   // 高优先级
+	TaskPriorityUrgent TaskPriority = "urgent" // 紧急
+)

common/eino/status_enum.go

@@ -0,0 +1,12 @@
+package eino
+
+// TaskStatus 任务状态
+type TaskStatus string
+
+const (
+	TaskStatusPending   TaskStatus = "pending"   // 待处理
+	TaskStatusRunning   TaskStatus = "running"   // 运行中
+	TaskStatusCompleted TaskStatus = "completed" // 已完成
+	TaskStatusFailed    TaskStatus = "failed"    // 失败
+	TaskStatusCancelled TaskStatus = "cancelled" // 已取消
+)

common/eino/task_type.go

@@ -0,0 +1,14 @@
+package eino
+
+// TaskType 任务类型
+type TaskType string
+
+const (
+	TaskTypeDocumentIngestion     TaskType = "document_ingestion"     // 文档摄入任务
+	TaskTypeVectorIngestion       TaskType = "vector_ingestion"       // 向量摄入任务
+	TaskTypeIndexCreation         TaskType = "index_creation"         // 索引创建任务
+	TaskTypeQAProcessing          TaskType = "qa_processing"          // 问答处理任务
+	TaskTypeKnowledgeConstruction TaskType = "knowledge_construction" // 知识库构建任务
+	TaskTypeGraphBuilding         TaskType = "graph_building"         // 图谱构建任务
+	TaskTypeKnowledgeSync         TaskType = "knowledge_sync"         // 知识同步任务
+)