feat: 支持多租户多模型对话及文档去重优化
This commit is contained in:
243
common/eino/chat.go
Normal file
243
common/eino/chat.go
Normal file
@@ -0,0 +1,243 @@
|
||||
package eino
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"rag/consts/model"
|
||||
"rag/dao"
|
||||
"rag/model/dto"
|
||||
"rag/model/entity"
|
||||
|
||||
"gitea.com/red-future/common/jaeger"
|
||||
"gitea.com/red-future/common/utils"
|
||||
"github.com/cloudwego/eino-ext/components/model/ark"
|
||||
"github.com/cloudwego/eino-ext/components/model/arkbot"
|
||||
"github.com/cloudwego/eino-ext/components/model/claude"
|
||||
"github.com/cloudwego/eino-ext/components/model/deepseek"
|
||||
"github.com/cloudwego/eino-ext/components/model/ollama"
|
||||
"github.com/cloudwego/eino-ext/components/model/openai"
|
||||
"github.com/cloudwego/eino-ext/components/model/qianfan"
|
||||
"github.com/cloudwego/eino-ext/components/model/qwen"
|
||||
modelChat "github.com/cloudwego/eino/components/model"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/gogf/gf/v2/util/gconv"
|
||||
)
|
||||
|
||||
type ChatModelSet struct {
|
||||
Ark *ark.ChatModel
|
||||
ArkBot *arkbot.ChatModel
|
||||
Claude *claude.ChatModel
|
||||
DeepSeek *deepseek.ChatModel
|
||||
Ollama *ollama.ChatModel
|
||||
OpenAI *openai.ChatModel
|
||||
Qianfan *qianfan.ChatModel
|
||||
Qwen *qwen.ChatModel
|
||||
}
|
||||
|
||||
// 全局租户容器：key=tenantId，value=该租户的对话模型
|
||||
var tenantChatModels = make(map[uint64]*ChatModelSet)
|
||||
|
||||
func init() {
|
||||
ctx := context.Background()
|
||||
ctx, span := jaeger.NewSpan(ctx, "InitAllChat")
|
||||
defer span.End()
|
||||
InitAllChat(ctx)
|
||||
return
|
||||
}
|
||||
|
||||
// ===================== 1. 服务启动时：初始化所有租户对话模型 =====================
|
||||
func InitAllChat(ctx context.Context) {
|
||||
list, err := dao.Model.GetNoTenantId(ctx, &dto.GetModelReq{
|
||||
ModelType: model.ModelTypeChat.Code(),
|
||||
})
|
||||
if err != nil {
|
||||
g.Log().Errorf(ctx, "获取所有租户对话模型失败: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, l := range list {
|
||||
err = InitChat(ctx, l)
|
||||
if err != nil {
|
||||
g.Log().Errorf(ctx, "初始化租户[%v]的对话模型失败: %v", l.TenantId, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func InitChat(ctx context.Context, modelDO *entity.Model) (err error) {
|
||||
set := &ChatModelSet{}
|
||||
switch *modelDO.ConfigType {
|
||||
case *model.ModelConfigTypeChatArk.Code():
|
||||
var cfg entity.ChatModelConfigArk
|
||||
if err = gconv.Struct(modelDO.ConfigContent, &cfg); err != nil {
|
||||
return fmt.Errorf("解析Ark配置失败: %v", err)
|
||||
}
|
||||
set.Ark, err = ark.NewChatModel(ctx, &ark.ChatModelConfig{
|
||||
APIKey: cfg.APIKey,
|
||||
Model: cfg.Model,
|
||||
Temperature: gconv.PtrFloat32(0.7),
|
||||
MaxTokens: gconv.PtrInt(1024),
|
||||
TopP: gconv.PtrFloat32(1.0),
|
||||
})
|
||||
|
||||
case *model.ModelConfigTypeChatArkBot.Code():
|
||||
var cfg entity.ChatModelConfigArkBot
|
||||
if err = gconv.Struct(modelDO.ConfigContent, &cfg); err != nil {
|
||||
return fmt.Errorf("解析ArkBot配置失败: %v", err)
|
||||
}
|
||||
set.ArkBot, err = arkbot.NewChatModel(ctx, &arkbot.Config{
|
||||
APIKey: cfg.APIKey,
|
||||
Model: cfg.Model,
|
||||
Temperature: gconv.PtrFloat32(0.7),
|
||||
MaxTokens: gconv.PtrInt(1024),
|
||||
TopP: gconv.PtrFloat32(1.0),
|
||||
})
|
||||
|
||||
case *model.ModelConfigTypeChatClaude.Code():
|
||||
var cfg entity.ChatModelConfigClaude
|
||||
if err = gconv.Struct(modelDO.ConfigContent, &cfg); err != nil {
|
||||
return fmt.Errorf("解析Claude配置失败: %v", err)
|
||||
}
|
||||
claudeCfg := claude.Config{
|
||||
APIKey: cfg.APIKey,
|
||||
BaseURL: gconv.PtrString(cfg.BaseURL),
|
||||
Model: cfg.Model,
|
||||
Temperature: gconv.PtrFloat32(0.7),
|
||||
MaxTokens: gconv.Int(1024),
|
||||
TopP: gconv.PtrFloat32(1.0),
|
||||
ByBedrock: cfg.ByBedrock,
|
||||
AccessKey: cfg.AccessKey,
|
||||
SecretAccessKey: cfg.SecretAccessKey,
|
||||
Region: cfg.Region,
|
||||
}
|
||||
set.Claude, err = claude.NewChatModel(ctx, &claudeCfg)
|
||||
|
||||
case *model.ModelConfigTypeChatDeepSeek.Code():
|
||||
var cfg entity.ChatModelConfigDeepSeek
|
||||
if err = gconv.Struct(modelDO.ConfigContent, &cfg); err != nil {
|
||||
return fmt.Errorf("解析DeepSeek配置失败: %v", err)
|
||||
}
|
||||
set.DeepSeek, err = deepseek.NewChatModel(ctx, &deepseek.ChatModelConfig{
|
||||
APIKey: cfg.APIKey,
|
||||
Model: cfg.Model,
|
||||
BaseURL: cfg.BaseURL,
|
||||
Temperature: gconv.Float32(0.7),
|
||||
MaxTokens: gconv.Int(1024),
|
||||
TopP: gconv.Float32(1.0),
|
||||
})
|
||||
|
||||
case *model.ModelConfigTypeChatOllama.Code():
|
||||
var cfg entity.ChatModelConfigOllama
|
||||
if err = gconv.Struct(modelDO.ConfigContent, &cfg); err != nil {
|
||||
return fmt.Errorf("解析Ollama配置失败: %v", err)
|
||||
}
|
||||
set.Ollama, err = ollama.NewChatModel(ctx, &ollama.ChatModelConfig{
|
||||
BaseURL: cfg.BaseURL,
|
||||
Model: cfg.Model,
|
||||
})
|
||||
|
||||
case *model.ModelConfigTypeChatOpenAI.Code():
|
||||
var cfg entity.ChatModelConfigOpenAI
|
||||
if err = gconv.Struct(modelDO.ConfigContent, &cfg); err != nil {
|
||||
return fmt.Errorf("解析OpenAI配置失败: %v", err)
|
||||
}
|
||||
openAiCfg := openai.ChatModelConfig{
|
||||
APIKey: cfg.APIKey,
|
||||
Model: cfg.Model,
|
||||
ByAzure: cfg.ByAzure,
|
||||
BaseURL: cfg.BaseURL,
|
||||
APIVersion: cfg.APIVersion,
|
||||
Temperature: gconv.PtrFloat32(0.7),
|
||||
MaxCompletionTokens: gconv.PtrInt(1024),
|
||||
TopP: gconv.PtrFloat32(1.0),
|
||||
}
|
||||
set.OpenAI, err = openai.NewChatModel(ctx, &openAiCfg)
|
||||
|
||||
case *model.ModelConfigTypeChatQianfan.Code():
|
||||
var cfg entity.ChatModelConfigQianfan
|
||||
if err = gconv.Struct(modelDO.ConfigContent, &cfg); err != nil {
|
||||
return fmt.Errorf("解析千帆配置失败: %v", err)
|
||||
}
|
||||
qcfg := qianfan.GetQianfanSingletonConfig()
|
||||
qcfg.AccessKey = cfg.AccessKey
|
||||
qcfg.SecretKey = cfg.SecretKey
|
||||
set.Qianfan, err = qianfan.NewChatModel(ctx, &qianfan.ChatModelConfig{
|
||||
Model: cfg.Model,
|
||||
Temperature: gconv.PtrFloat32(0.7),
|
||||
MaxCompletionTokens: gconv.PtrInt(1024),
|
||||
TopP: gconv.PtrFloat32(1.0),
|
||||
})
|
||||
|
||||
case *model.ModelConfigTypeChatQwen.Code():
|
||||
var cfg entity.ChatModelConfigQwen
|
||||
if err = gconv.Struct(modelDO.ConfigContent, &cfg); err != nil {
|
||||
return fmt.Errorf("解析Qwen配置失败: %v", err)
|
||||
}
|
||||
set.Qwen, err = qwen.NewChatModel(ctx, &qwen.ChatModelConfig{
|
||||
APIKey: cfg.APIKey,
|
||||
Model: cfg.Model,
|
||||
BaseURL: cfg.BaseURL,
|
||||
Temperature: gconv.PtrFloat32(0.7),
|
||||
MaxTokens: gconv.PtrInt(1024),
|
||||
TopP: gconv.PtrFloat32(1.0),
|
||||
})
|
||||
|
||||
default:
|
||||
return fmt.Errorf("不支持的对话模型类型: %v", *modelDO.ConfigType)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("初始化对话模型失败: %v", err)
|
||||
}
|
||||
|
||||
// 无锁存入租户 map
|
||||
tenantChatModels[modelDO.TenantId] = set
|
||||
g.Log().Infof(ctx, "租户[%v]对话模型[%v]初始化成功", modelDO.TenantId, *modelDO.ConfigType)
|
||||
return
|
||||
}
|
||||
|
||||
func GetTenantChatModel(tenantId uint64) (*ChatModelSet, error) {
|
||||
set := tenantChatModels[tenantId]
|
||||
if set == nil {
|
||||
return nil, fmt.Errorf("租户[%v]对话模型未初始化", tenantId)
|
||||
}
|
||||
return set, nil
|
||||
}
|
||||
|
||||
func GetTenantChatModelByType(ctx context.Context, configType model.ModelConfigType) (modelChat.BaseChatModel, error) {
|
||||
userInfo, err := utils.GetUserInfo(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
set, err := GetTenantChatModel(userInfo.TenantId)
|
||||
if set == nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
switch *configType {
|
||||
case *model.ModelConfigTypeChatArk.Code():
|
||||
return set.Ark, nil
|
||||
case *model.ModelConfigTypeChatArkBot.Code():
|
||||
return set.ArkBot, nil
|
||||
case *model.ModelConfigTypeChatClaude.Code():
|
||||
return set.Claude, nil
|
||||
case *model.ModelConfigTypeChatDeepSeek.Code():
|
||||
return set.DeepSeek, nil
|
||||
case *model.ModelConfigTypeChatOllama.Code():
|
||||
return set.Ollama, nil
|
||||
case *model.ModelConfigTypeChatOpenAI.Code():
|
||||
return set.OpenAI, nil
|
||||
case *model.ModelConfigTypeChatQianfan.Code():
|
||||
return set.Qianfan, nil
|
||||
case *model.ModelConfigTypeChatQwen.Code():
|
||||
return set.Qwen, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("不支持的对话模型类型: %v", configType)
|
||||
}
|
||||
}
|
||||
|
||||
func RefreshTenantChatModel(ctx context.Context, modelDO *entity.Model) error {
|
||||
delete(tenantChatModels, modelDO.TenantId)
|
||||
return InitChat(ctx, modelDO)
|
||||
}
|
||||
@@ -5,13 +5,10 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"rag/consts/model"
|
||||
|
||||
"github.com/cloudwego/eino-ext/components/model/qwen"
|
||||
"github.com/cloudwego/eino/components/prompt"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/gogf/gf/v2/os/glog"
|
||||
"github.com/gogf/gf/v2/util/gconv"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -19,48 +16,15 @@ const (
|
||||
)
|
||||
|
||||
var (
|
||||
globalChatModel *qwen.ChatModel
|
||||
ragPromptTemplate prompt.ChatTemplate // EINO 官方模板
|
||||
)
|
||||
|
||||
func init() {
|
||||
ctx := context.Background()
|
||||
// 初始化大模型
|
||||
if err := initChatModel(ctx); err != nil {
|
||||
glog.Errorf(ctx, "初始化大模型失败: %v", err)
|
||||
}
|
||||
// 初始化 EINO 提示词模板
|
||||
initRAGPromptTemplate()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// 初始化通义千问
|
||||
func initChatModel(ctx context.Context) error {
|
||||
if globalChatModel != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
apiKey := g.Cfg().MustGet(ctx, "eino.chatmodel.apiKey").String()
|
||||
model := g.Cfg().MustGet(ctx, "eino.chatmodel.model").String()
|
||||
|
||||
cm, err := qwen.NewChatModel(ctx, &qwen.ChatModelConfig{
|
||||
APIKey: apiKey,
|
||||
Model: model,
|
||||
BaseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1",
|
||||
Timeout: 60 * 1e9,
|
||||
Temperature: gconv.PtrFloat32(0.7), // 客服最佳
|
||||
MaxTokens: gconv.PtrInt(1024), // 最长回答
|
||||
TopP: gconv.PtrFloat32(1.0),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
globalChatModel = cm
|
||||
return nil
|
||||
}
|
||||
|
||||
// 初始化 EINO 官方提示词模板（最关键！）
|
||||
func initRAGPromptTemplate() {
|
||||
ragPromptTemplate = prompt.FromMessages(
|
||||
@@ -69,7 +33,7 @@ func initRAGPromptTemplate() {
|
||||
&schema.Message{
|
||||
Role: schema.System,
|
||||
Content: `你是专业客服，语气友好简洁。
|
||||
请严格依据参考知识回答，不知道就说：抱歉，我暂时无法回答这个问题。
|
||||
请依据参考知识回答，不知道就说：抱歉，我暂时无法回答这个问题。
|
||||
|
||||
参考知识：
|
||||
{knowledge}`,
|
||||
@@ -83,7 +47,7 @@ func initRAGPromptTemplate() {
|
||||
}
|
||||
|
||||
// NewChatModel 只处理逻辑，不复用创建模型
|
||||
func NewChatModel(ctx context.Context, question string, docs []*schema.Document, history []*schema.Message) (replyMsg *schema.Message, err error) {
|
||||
func NewChatModel(ctx context.Context, question string, docs []*schema.Document, history []*schema.Message, chatModel model.ModelConfigType) (replyMsg *schema.Message, err error) {
|
||||
// 1. 构建参考知识
|
||||
knowledge := buildKnowledgeAndSources(docs)
|
||||
// 2. 历史精简
|
||||
@@ -101,7 +65,7 @@ func NewChatModel(ctx context.Context, question string, docs []*schema.Document,
|
||||
msgs = append(msgs[:1], append(history, msgs[1:]...)...)
|
||||
}
|
||||
// 5. 🔥 直接使用全局单例，不重复创建
|
||||
replyMsg, err = streamGenerateAnswer(ctx, globalChatModel, msgs)
|
||||
replyMsg, err = streamGenerateAnswer(ctx, msgs, chatModel)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -133,9 +97,14 @@ func buildKnowledgeAndSources(docs []*schema.Document) string {
|
||||
}
|
||||
|
||||
// streamGenerateAnswer 流式生成
|
||||
func streamGenerateAnswer(ctx context.Context, chatModel *qwen.ChatModel, msgs []*schema.Message) (reply *schema.Message, err error) {
|
||||
func streamGenerateAnswer(ctx context.Context, msgs []*schema.Message, chatModel model.ModelConfigType) (reply *schema.Message, err error) {
|
||||
|
||||
sr, err := chatModel.Stream(ctx, msgs)
|
||||
cm, err := GetTenantChatModelByType(ctx, chatModel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sr, err := cm.Stream(ctx, msgs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stream failed: %w", err)
|
||||
}
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
package eino
|
||||
|
||||
const (
|
||||
providerArk = "ark"
|
||||
providerOpenai = "openai"
|
||||
providerQianfan = "qianfan"
|
||||
providerDashscope = "dashscope"
|
||||
)
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
|
||||
"gitea.com/red-future/common/utils"
|
||||
"github.com/cloudwego/eino-ext/components/document/loader/file"
|
||||
"github.com/cloudwego/eino-ext/components/document/loader/url"
|
||||
"github.com/cloudwego/eino-ext/components/document/parser/docx"
|
||||
"github.com/cloudwego/eino-ext/components/document/parser/pdf"
|
||||
@@ -15,7 +16,7 @@ import (
|
||||
)
|
||||
|
||||
// LoadDocument 业务函数：加载文件
|
||||
func LoadDocument(ctx context.Context, filePath, fileFormat string) (docs []*schema.Document, err error) {
|
||||
func a(ctx context.Context, filePath, fileFormat string) (docs []*schema.Document, err error) {
|
||||
p, err := docsParser(ctx, fileFormat)
|
||||
if err != nil {
|
||||
return
|
||||
@@ -27,12 +28,34 @@ func LoadDocument(ctx context.Context, filePath, fileFormat string) (docs []*sch
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
docs, err = loader.Load(context.Background(), document.Source{
|
||||
docs, err = loader.Load(ctx, document.Source{
|
||||
URI: fmt.Sprintf("%s%s", imageUrl, filePath),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
func LoadDocument(ctx context.Context, filePath, fileFormat string) (docs []*schema.Document, err error) {
|
||||
p, err := docsParser(ctx, fileFormat)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// 1. 创建文件加载器
|
||||
loader, err := file.NewFileLoader(ctx, &file.FileLoaderConfig{
|
||||
UseNameAsID: false, // 使用文件名作为文档ID
|
||||
Parser: p,
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// 2. 加载本地文件
|
||||
docs, err = loader.Load(ctx, document.Source{
|
||||
URI: "C:\\Users\\AI\\Desktop\\手机发展史.txt",
|
||||
})
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func docsParser(ctx context.Context, fileFormat string) (p parser.Parser, err error) {
|
||||
switch fileFormat {
|
||||
case "docx":
|
||||
|
||||
@@ -2,6 +2,7 @@ package eino
|
||||
|
||||
import (
|
||||
"context"
|
||||
"rag/consts/model"
|
||||
|
||||
"github.com/cloudwego/eino-ext/components/document/transformer/splitter/recursive"
|
||||
"github.com/cloudwego/eino-ext/components/document/transformer/splitter/semantic"
|
||||
@@ -10,7 +11,7 @@ import (
|
||||
)
|
||||
|
||||
// SemanticSplitDocument 语义分割文档
|
||||
func SemanticSplitDocument(ctx context.Context, docs []*schema.Document) (res []*schema.Document, err error) {
|
||||
func SemanticSplitDocument(ctx context.Context, docs []*schema.Document, vectorModel model.ModelConfigType) (res []*schema.Document, err error) {
|
||||
// 默认分隔符（支持中英文）
|
||||
separators := []string{"\n\n", "\n", "。", "！", "？", "；", ".", "!", "?", ";"}
|
||||
// 读取配置，使用合理的默认值
|
||||
@@ -18,24 +19,14 @@ func SemanticSplitDocument(ctx context.Context, docs []*schema.Document) (res []
|
||||
minChunkSize := g.Cfg().MustGet(ctx, "eino.splitter.minChunkSize").Int()
|
||||
percentile := g.Cfg().MustGet(ctx, "eino.splitter.percentile").Float64()
|
||||
batchSize := g.Cfg().MustGet(ctx, "eino.splitter.batchSize").Int()
|
||||
if batchSize <= 0 {
|
||||
batchSize = 10 // doubao-embedding-vision 限制每批最多 10 个
|
||||
}
|
||||
|
||||
// 使用批量包装器
|
||||
var batchEmbedder *BatchEmbedder
|
||||
provider := g.Cfg().MustGet(ctx, "eino.embedding.provider").String()
|
||||
switch provider {
|
||||
case providerArk:
|
||||
batchEmbedder = NewBatchEmbedder(EmbedderArk, batchSize)
|
||||
case providerOpenai:
|
||||
batchEmbedder = NewBatchEmbedder(EmbedderOpenAI, batchSize)
|
||||
case providerDashscope:
|
||||
batchEmbedder = NewBatchEmbedder(EmbedderDashscope, batchSize)
|
||||
embedder, err := GetTenantEmbedderByType(ctx, vectorModel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
splitter, err := semantic.NewSplitter(ctx, &semantic.Config{
|
||||
Embedding: batchEmbedder,
|
||||
Embedding: NewBatchEmbedder(embedder, batchSize),
|
||||
BufferSize: bufferSize,
|
||||
MinChunkSize: minChunkSize,
|
||||
Percentile: percentile,
|
||||
|
||||
@@ -3,67 +3,211 @@ package eino
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"rag/consts/model"
|
||||
"rag/model/entity"
|
||||
|
||||
"gitea.com/red-future/common/jaeger"
|
||||
"gitea.com/red-future/common/utils"
|
||||
"github.com/cloudwego/eino-ext/components/embedding/ark"
|
||||
"github.com/cloudwego/eino-ext/components/embedding/dashscope"
|
||||
"github.com/cloudwego/eino-ext/components/embedding/ollama"
|
||||
"github.com/cloudwego/eino-ext/components/embedding/openai"
|
||||
"github.com/cloudwego/eino-ext/components/embedding/qianfan"
|
||||
"github.com/cloudwego/eino-ext/components/embedding/tencentcloud"
|
||||
"github.com/cloudwego/eino/components/embedding"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/golang/glog"
|
||||
"github.com/gogf/gf/v2/util/gconv"
|
||||
)
|
||||
|
||||
// 全局只初始化一次
|
||||
var (
|
||||
EmbedderArk *ark.Embedder
|
||||
EmbedderDashscope *dashscope.Embedder
|
||||
EmbedderOpenAI *openai.Embedder
|
||||
)
|
||||
type EmbedderSet struct {
|
||||
Ark *ark.Embedder
|
||||
Ollama *ollama.Embedder
|
||||
OpenAI *openai.Embedder
|
||||
Qianfan *qianfan.Embedder
|
||||
TencentCloud *tencentcloud.Embedder
|
||||
DashScope *dashscope.Embedder
|
||||
}
|
||||
|
||||
// 全局租户容器：key=tenantId，value=该租户的向量模型
|
||||
var tenantEmbedders = make(map[uint64]*EmbedderSet)
|
||||
|
||||
func init() {
|
||||
ctx := context.Background()
|
||||
if !g.Cfg().MustGet(ctx, "eino.embedding").IsEmpty() {
|
||||
var err error
|
||||
provider := g.Cfg().MustGet(ctx, "eino.embedding.provider").String()
|
||||
switch provider {
|
||||
case providerArk:
|
||||
cfg := &ark.EmbeddingConfig{
|
||||
APIKey: g.Cfg().MustGet(ctx, "eino.embedding.apiKey").String(),
|
||||
Model: g.Cfg().MustGet(ctx, "eino.embedding.model").String(),
|
||||
}
|
||||
if apiType := g.Cfg().MustGet(ctx, "eino.embedding.apiType").String(); apiType != "" {
|
||||
apiTypeVal := ark.APIType(apiType)
|
||||
cfg.APIType = &apiTypeVal
|
||||
}
|
||||
EmbedderArk, err = ark.NewEmbedder(ctx, cfg)
|
||||
case providerOpenai:
|
||||
chatModelConfig := &openai.EmbeddingConfig{
|
||||
APIKey: g.Cfg().MustGet(ctx, "eino.embedding.apiKey").String(),
|
||||
Model: g.Cfg().MustGet(ctx, "eino.embedding.model").String(),
|
||||
}
|
||||
EmbedderOpenAI, err = openai.NewEmbedder(ctx, chatModelConfig)
|
||||
case providerDashscope:
|
||||
cfg := &dashscope.EmbeddingConfig{
|
||||
APIKey: g.Cfg().MustGet(ctx, "eino.embedding.apiKey").String(),
|
||||
Model: g.Cfg().MustGet(ctx, "eino.embedding.model").String(),
|
||||
}
|
||||
EmbedderDashscope, err = dashscope.NewEmbedder(ctx, cfg)
|
||||
}
|
||||
if err != nil {
|
||||
glog.Fatalf("NewEmbedder of %v error: %v", provider, err)
|
||||
}
|
||||
}
|
||||
|
||||
ctx, span := jaeger.NewSpan(ctx, "InitAllVector")
|
||||
defer span.End()
|
||||
InitAllVector(ctx)
|
||||
return
|
||||
}
|
||||
|
||||
func EmbedStrings(ctx context.Context, texts []string) (embeddings [][]float64, err error) {
|
||||
provider := g.Cfg().MustGet(ctx, "eino.embedding.provider").String()
|
||||
switch provider {
|
||||
case providerArk:
|
||||
return EmbedderArk.EmbedStrings(ctx, texts)
|
||||
case providerOpenai:
|
||||
return EmbedderOpenAI.EmbedStrings(ctx, texts)
|
||||
case providerDashscope:
|
||||
return EmbedderDashscope.EmbedStrings(ctx, texts)
|
||||
// ===================== 1. 服务启动时调用：初始化所有租户 =====================
|
||||
func InitAllVector(ctx context.Context) {
|
||||
//list, err := dao.Model.GetNoTenantId(ctx, &dto.GetModelReq{
|
||||
// ModelType: model.ModelTypeVector.Code(),
|
||||
//})
|
||||
//if err != nil {
|
||||
// g.Log().Errorf(ctx, "获取所有租户ID失败: %v", err)
|
||||
// return
|
||||
//}
|
||||
//
|
||||
//for _, l := range list {
|
||||
// err = InitVector(ctx, l)
|
||||
// if err != nil {
|
||||
// g.Log().Errorf(ctx, "初始化租户[%v]的向量模型失败: %v", l.TenantId, err)
|
||||
// continue
|
||||
// }
|
||||
//}
|
||||
modelDO := new(entity.Model)
|
||||
modelDO.TenantId = 1
|
||||
modelDO.ConfigType = model.ModelConfigTypeVectorDashScope.Code()
|
||||
var cfg entity.VectorModelConfigDashScope
|
||||
cfg.APIKey = "sk-4a8b82770bf74bc490eb3e4c5a8e2be9"
|
||||
cfg.Model = "text-embedding-v3"
|
||||
modelDO.ConfigContent = gconv.Map(&cfg)
|
||||
err := InitVector(ctx, modelDO)
|
||||
if err != nil {
|
||||
g.Log().Errorf(ctx, "初始化向量模型失败: %v", err)
|
||||
return
|
||||
}
|
||||
return nil, fmt.Errorf("unsupported provider: %v", provider)
|
||||
}
|
||||
|
||||
func InitVector(ctx context.Context, modelDO *entity.Model) (err error) {
|
||||
set := &EmbedderSet{}
|
||||
switch *modelDO.ConfigType {
|
||||
case *model.ModelConfigTypeVectorArk.Code():
|
||||
// 解析 Ark 向量配置
|
||||
var cfg entity.VectorModelConfigArk
|
||||
err = gconv.Struct(modelDO.ConfigContent, &cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("解析Ark向量配置失败: %v", err)
|
||||
}
|
||||
arkCfg := &ark.EmbeddingConfig{
|
||||
APIKey: cfg.APIKey,
|
||||
Model: cfg.Model,
|
||||
}
|
||||
if !g.IsEmpty(cfg.APIType) {
|
||||
arkCfg.APIType = new(ark.APIType(cfg.APIType))
|
||||
}
|
||||
set.Ark, err = ark.NewEmbedder(ctx, arkCfg)
|
||||
|
||||
case *model.ModelConfigTypeVectorOllama.Code():
|
||||
// 解析 Ollama 向量配置
|
||||
var cfg entity.VectorModelConfigOllama
|
||||
err = gconv.Struct(modelDO.ConfigContent, &cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("解析Ollama向量配置失败: %v", err)
|
||||
}
|
||||
set.Ollama, err = ollama.NewEmbedder(ctx, &ollama.EmbeddingConfig{
|
||||
BaseURL: cfg.BaseURL,
|
||||
Model: cfg.Model,
|
||||
})
|
||||
|
||||
case *model.ModelConfigTypeVectorOpenAI.Code():
|
||||
// 解析 OpenAI 向量配置
|
||||
var cfg entity.VectorModelConfigOpenAI
|
||||
err = gconv.Struct(modelDO.ConfigContent, &cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("解析OpenAI向量配置失败: %v", err)
|
||||
}
|
||||
openaiCfg := &openai.EmbeddingConfig{
|
||||
APIKey: cfg.APIKey,
|
||||
Model: cfg.Model,
|
||||
ByAzure: cfg.ByAzure,
|
||||
BaseURL: cfg.BaseURL,
|
||||
APIVersion: cfg.APIVersion,
|
||||
}
|
||||
set.OpenAI, err = openai.NewEmbedder(ctx, openaiCfg)
|
||||
|
||||
case *model.ModelConfigTypeVectorQianfan.Code():
|
||||
// 解析 千帆 向量配置
|
||||
var cfg entity.VectorModelConfigQianfan
|
||||
err = gconv.Struct(modelDO.ConfigContent, &cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("解析千帆向量配置失败: %v", err)
|
||||
}
|
||||
qcfg := qianfan.GetQianfanSingletonConfig()
|
||||
qcfg.AccessKey = cfg.AccessKey
|
||||
qcfg.SecretKey = cfg.SecretKey
|
||||
set.Qianfan, err = qianfan.NewEmbedder(ctx, &qianfan.EmbeddingConfig{
|
||||
Model: cfg.Model,
|
||||
})
|
||||
|
||||
case *model.ModelConfigTypeVectorTencentCloud.Code():
|
||||
// 解析 腾讯云 向量配置
|
||||
var cfg entity.VectorModelConfigTencentCloud
|
||||
err = gconv.Struct(modelDO.ConfigContent, &cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("解析腾讯云向量配置失败: %v", err)
|
||||
}
|
||||
set.TencentCloud, err = tencentcloud.NewEmbedder(ctx, &tencentcloud.EmbeddingConfig{
|
||||
SecretID: cfg.SecretID,
|
||||
SecretKey: cfg.SecretKey,
|
||||
Region: cfg.Region,
|
||||
})
|
||||
|
||||
case *model.ModelConfigTypeVectorDashScope.Code():
|
||||
// 解析 阿里 dashscope 向量配置
|
||||
var cfg entity.VectorModelConfigDashScope
|
||||
err = gconv.Struct(modelDO.ConfigContent, &cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("解析阿里dashscope向量配置失败: %v", err)
|
||||
}
|
||||
set.DashScope, err = dashscope.NewEmbedder(ctx, &dashscope.EmbeddingConfig{
|
||||
APIKey: cfg.APIKey,
|
||||
Model: cfg.Model,
|
||||
})
|
||||
|
||||
default:
|
||||
return fmt.Errorf("不支持的向量模型配置类型: %v", *modelDO.ConfigType)
|
||||
}
|
||||
|
||||
// 统一错误处理
|
||||
if err != nil {
|
||||
return fmt.Errorf("初始化向量模型失败: %v", err)
|
||||
}
|
||||
// 直接存入 map（无锁，重复初始化会直接覆盖）
|
||||
tenantEmbedders[modelDO.TenantId] = set
|
||||
g.Log().Infof(ctx, "向量模型[%v]初始化成功", modelDO.ConfigType)
|
||||
return
|
||||
}
|
||||
|
||||
func GetTenantEmbedder(tenantId uint64) (*EmbedderSet, error) {
|
||||
set := tenantEmbedders[tenantId]
|
||||
if set == nil {
|
||||
return nil, fmt.Errorf("租户[%v]的向量模型未初始化", tenantId)
|
||||
}
|
||||
return set, nil
|
||||
}
|
||||
|
||||
func GetTenantEmbedderByType(ctx context.Context, configType model.ModelConfigType) (embedding.Embedder, error) {
|
||||
userInfo, err := utils.GetUserInfo(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
set, err := GetTenantEmbedder(userInfo.TenantId)
|
||||
if set == nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
switch *configType {
|
||||
case *model.ModelConfigTypeVectorArk.Code():
|
||||
return set.Ark, nil
|
||||
case *model.ModelConfigTypeVectorOllama.Code():
|
||||
return set.Ollama, nil
|
||||
case *model.ModelConfigTypeVectorOpenAI.Code():
|
||||
return set.OpenAI, nil
|
||||
case *model.ModelConfigTypeVectorQianfan.Code():
|
||||
return set.Qianfan, nil
|
||||
case *model.ModelConfigTypeVectorTencentCloud.Code():
|
||||
return set.TencentCloud, nil
|
||||
case *model.ModelConfigTypeVectorDashScope.Code():
|
||||
return set.DashScope, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("不支持的向量模型配置类型: %v", *configType)
|
||||
}
|
||||
}
|
||||
|
||||
func RefreshTenantEmbedder(ctx context.Context, modelDO *entity.Model) error {
|
||||
delete(tenantEmbedders, modelDO.TenantId)
|
||||
return InitVector(ctx, modelDO)
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"rag/consts/model"
|
||||
"rag/dao"
|
||||
"rag/model/dto"
|
||||
"rag/model/entity"
|
||||
@@ -34,7 +35,14 @@ func NewPGVectorIndexer(opts *PGVectorIndexerOptions) *PGVectorIndexer {
|
||||
return &PGVectorIndexer{opts: opts}
|
||||
}
|
||||
|
||||
func (i *PGVectorIndexer) Store(ctx context.Context, docs []*schema.Document, opts ...indexer.Option) (rows int64, err error) {
|
||||
func (i *PGVectorIndexer) Store(ctx context.Context, docs []*schema.Document, configType model.ModelConfigType, opts ...indexer.Option) (rows int64, err error) {
|
||||
|
||||
embedderByType, err := GetTenantEmbedderByType(ctx, configType)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
indexer.WithEmbedding(embedderByType)
|
||||
|
||||
commonOpts := indexer.GetCommonOptions(&indexer.Options{}, opts...)
|
||||
|
||||
if commonOpts.Embedding == nil {
|
||||
|
||||
116
common/eino/rerank.go
Normal file
116
common/eino/rerank.go
Normal file
@@ -0,0 +1,116 @@
|
||||
package eino
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
)
|
||||
|
||||
// DashScopeReranker 通义百炼 Rerank 精排（Cross-Encoder）
|
||||
type DashScopeReranker struct {
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
func NewDashScopeReranker() *DashScopeReranker {
|
||||
return &DashScopeReranker{
|
||||
httpClient: &http.Client{
|
||||
Timeout: 10 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Rerank 对文档进行精排（Cross-Encoder 核心）
|
||||
func (d *DashScopeReranker) Rerank(ctx context.Context, query string, docs []*schema.Document) ([]*schema.Document, error) {
|
||||
if len(docs) == 0 {
|
||||
return docs, nil
|
||||
}
|
||||
|
||||
// 官方必过 URL
|
||||
url := "https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank"
|
||||
apiKey := g.Cfg().MustGet(ctx, "eino.rerank.apiKey").String()
|
||||
model := g.Cfg().MustGet(ctx, "eino.rerank.model").String()
|
||||
|
||||
documents := make([]string, len(docs))
|
||||
for i, doc := range docs {
|
||||
documents[i] = doc.Content
|
||||
}
|
||||
|
||||
reqBody := map[string]any{
|
||||
"model": model,
|
||||
"input": map[string]any{
|
||||
"query": query,
|
||||
"documents": documents,
|
||||
},
|
||||
"parameters": map[string]any{
|
||||
"top_n": len(docs),
|
||||
},
|
||||
}
|
||||
bs, _ := json.Marshal(reqBody)
|
||||
|
||||
req, _ := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(bs))
|
||||
req.Header.Set("Authorization", "Bearer "+apiKey)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := d.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("rerank api error: status=%d, body=%s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
// 解析结果
|
||||
var result struct {
|
||||
Output struct {
|
||||
Results []struct {
|
||||
Index int `json:"index"`
|
||||
RelevanceScore float64 `json:"relevance_score"`
|
||||
} `json:"results"`
|
||||
} `json:"output"`
|
||||
}
|
||||
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 按分数排序
|
||||
type scoredDoc struct {
|
||||
doc *schema.Document
|
||||
score float64
|
||||
}
|
||||
scored := make([]scoredDoc, len(docs))
|
||||
for i, doc := range docs {
|
||||
scored[i] = scoredDoc{doc: doc, score: 0}
|
||||
}
|
||||
for _, res := range result.Output.Results {
|
||||
scored[res.Index].score = res.RelevanceScore
|
||||
}
|
||||
|
||||
// 分数从高到低排序
|
||||
for i := 0; i < len(scored); i++ {
|
||||
for j := i + 1; j < len(scored); j++ {
|
||||
if scored[j].score > scored[i].score {
|
||||
scored[i], scored[j] = scored[j], scored[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 输出最终排好的文档
|
||||
ranked := make([]*schema.Document, 0, len(scored))
|
||||
for _, s := range scored {
|
||||
s.doc.MetaData["rerank_score"] = s.score
|
||||
ranked = append(ranked, s.doc)
|
||||
}
|
||||
|
||||
return ranked, nil
|
||||
}
|
||||
@@ -3,6 +3,8 @@ package eino
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"rag/consts/model"
|
||||
"rag/dao"
|
||||
"sort"
|
||||
"time"
|
||||
@@ -29,21 +31,25 @@ type PGVectorRetriever struct {
|
||||
topK int
|
||||
index string
|
||||
dslInfo map[string]any
|
||||
reranker *DashScopeReranker // 通义精排
|
||||
}
|
||||
|
||||
func NewPGVectorRetriever(config *PGVectorRetrieverConfig) (*PGVectorRetriever, error) {
|
||||
if config.Embedder == nil {
|
||||
return nil, errors.New("embedder is required")
|
||||
}
|
||||
func NewPGVectorRetriever(ctx context.Context, config *PGVectorRetrieverConfig, configType model.ModelConfigType) (*PGVectorRetriever, error) {
|
||||
if config.DefaultTopK <= 0 {
|
||||
config.DefaultTopK = 5
|
||||
}
|
||||
|
||||
e, err := GetTenantEmbedderByType(ctx, configType)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &PGVectorRetriever{
|
||||
embedder: config.Embedder,
|
||||
embedder: e,
|
||||
topK: config.DefaultTopK,
|
||||
index: config.DefaultIndex,
|
||||
dslInfo: config.DSLInfo,
|
||||
//reranker: NewDashScopeReranker(), // 👈 直接初始化你的精排
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -138,48 +144,37 @@ func (r *PGVectorRetriever) Retrieve(ctx context.Context, query string, opts ...
|
||||
}
|
||||
|
||||
// 合并 + 智能去重（保留最优分数）
|
||||
docs := mergeAndDeduplicate(docsVector, docsFulltext)
|
||||
mergedDocs := mergeAndDeduplicate(docsVector, docsFulltext)
|
||||
|
||||
// 排序：向量优先，同类型按距离升序
|
||||
sort.Slice(docs, func(i, j int) bool {
|
||||
//byI, okI := docs[i].MetaData["retrieve_by"].(string)
|
||||
//byJ, okJ := docs[j].MetaData["retrieve_by"].(string)
|
||||
//
|
||||
//// 有类型标记的优先
|
||||
//if okI && !okJ {
|
||||
// return true
|
||||
//}
|
||||
//if !okI && okJ {
|
||||
// return false
|
||||
//}
|
||||
//
|
||||
//// 向量永远排前面
|
||||
//if byI == "vector" && byJ == "fulltext" {
|
||||
// return true
|
||||
//}
|
||||
//if byI == "fulltext" && byJ == "vector" {
|
||||
// return false
|
||||
//}
|
||||
|
||||
// 同类型按 distance 升序（越小越相似）
|
||||
d1 := gconv.Float64(docs[i].MetaData["distance"])
|
||||
d2 := gconv.Float64(docs[j].MetaData["distance"])
|
||||
return d1 < d2
|
||||
})
|
||||
|
||||
// 在Retrieve方法末尾，增加相关性校验
|
||||
validDocs := make([]*schema.Document, 0)
|
||||
for i, d := range docs {
|
||||
// 过滤distance过大的垃圾结果（比如distance>0.8的直接丢弃）
|
||||
if gconv.Float64(docs[i].MetaData["distance"]) < 0.8 {
|
||||
validDocs = append(validDocs, d)
|
||||
// =========================
|
||||
// 🔥 Cross-Encoder 精排
|
||||
// =========================
|
||||
var finalDocs []*schema.Document
|
||||
if r.reranker != nil {
|
||||
ranked, err := r.reranker.Rerank(ctx, query, mergedDocs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("rerank failed: %w", err)
|
||||
}
|
||||
finalDocs = ranked
|
||||
} else {
|
||||
sort.Slice(mergedDocs, func(i, j int) bool {
|
||||
d1 := gconv.Float64(mergedDocs[i].MetaData["distance"])
|
||||
d2 := gconv.Float64(mergedDocs[j].MetaData["distance"])
|
||||
return d1 < d2
|
||||
})
|
||||
finalDocs = mergedDocs
|
||||
}
|
||||
|
||||
// 如果没有有效结果，返回空，让LLM回答「暂无相关信息」
|
||||
if len(validDocs) == 0 {
|
||||
callbacks.OnEnd(ctx, &retriever.CallbackOutput{Docs: validDocs})
|
||||
return validDocs, nil
|
||||
// =========================
|
||||
// 过滤无效文档
|
||||
// =========================
|
||||
const maxDistance = 0.8
|
||||
validDocs := make([]*schema.Document, 0, len(finalDocs))
|
||||
for _, doc := range finalDocs {
|
||||
dist := gconv.Float64(doc.MetaData["distance"])
|
||||
if dist <= maxDistance {
|
||||
validDocs = append(validDocs, doc)
|
||||
}
|
||||
}
|
||||
|
||||
// 最多保留 topK
|
||||
@@ -208,9 +203,15 @@ func (r *PGVectorRetriever) doRetrieveVector(ctx context.Context, query string,
|
||||
if opts.TopK != nil {
|
||||
topK = *opts.TopK
|
||||
}
|
||||
datasetIds := gconv.Int64s(opts.DSLInfo["dataset_ids"])
|
||||
var datasetIds, documentIds []int64
|
||||
if g.IsEmpty(opts.DSLInfo["dataset_ids"]) {
|
||||
datasetIds = gconv.Int64s(opts.DSLInfo["dataset_ids"])
|
||||
}
|
||||
if g.IsEmpty(opts.DSLInfo["document_ids"]) {
|
||||
documentIds = gconv.Int64s(opts.DSLInfo["document_ids"])
|
||||
}
|
||||
|
||||
rows, err := dao.DocumentVector.GetAllByVector(ctx, datasetIds, queryVec, topK)
|
||||
rows, err := dao.DocumentVector.GetAllByVector(ctx, datasetIds, documentIds, queryVec, topK)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -236,10 +237,17 @@ func (r *PGVectorRetriever) doRetrieveVector(ctx context.Context, query string,
|
||||
// ==========================================
|
||||
func (r *PGVectorRetriever) doRetrieveMeilisearch(ctx context.Context, query string, opts *retriever.Options) ([]*schema.Document, error) {
|
||||
topK := *opts.TopK
|
||||
datasetIds := gconv.Int64s(opts.DSLInfo["dataset_ids"])
|
||||
|
||||
var datasetIds, documentIds []int64
|
||||
if g.IsEmpty(opts.DSLInfo["dataset_ids"]) {
|
||||
datasetIds = gconv.Int64s(opts.DSLInfo["dataset_ids"])
|
||||
}
|
||||
if g.IsEmpty(opts.DSLInfo["document_ids"]) {
|
||||
documentIds = gconv.Int64s(opts.DSLInfo["document_ids"])
|
||||
}
|
||||
|
||||
// 调用你已有的 Meilisearch DAO
|
||||
rows, err := dao.DocumentVector.SearchByKeywords(ctx, query, datasetIds, topK)
|
||||
rows, err := dao.DocumentVector.SearchByKeywords(ctx, query, datasetIds, documentIds, topK)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
132
consts/model/config_type.go
Normal file
132
consts/model/config_type.go
Normal file
@@ -0,0 +1,132 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"github.com/gogf/gf/v2/util/gconv"
|
||||
)
|
||||
|
||||
var (
|
||||
ModelConfigTypeVectorArk = newModelConfigType(gconv.PtrString("ark"), "字节跳动火山引擎方舟大模型服务")
|
||||
ModelConfigTypeVectorOllama = newModelConfigType(gconv.PtrString("ollama"), "Ollama 本地大模型运行工具")
|
||||
ModelConfigTypeVectorOpenAI = newModelConfigType(gconv.PtrString("openAI"), "OpenAI 官方大模型服务")
|
||||
ModelConfigTypeVectorQianfan = newModelConfigType(gconv.PtrString("qianfan"), "百度文心一言千帆大模型平台")
|
||||
ModelConfigTypeVectorTencentCloud = newModelConfigType(gconv.PtrString("tencentCloud"), "腾讯云大模型服务")
|
||||
ModelConfigTypeVectorDashScope = newModelConfigType(gconv.PtrString("dashScope"), "阿里云通义千问 DashScope 平台")
|
||||
|
||||
ModelConfigTypeChatArk = newModelConfigType(gconv.PtrString("ark"), "字节跳动火山引擎方舟大模型服务")
|
||||
ModelConfigTypeChatArkBot = newModelConfigType(gconv.PtrString("arkBot"), "火山引擎 ARK Bot 智能体服务")
|
||||
ModelConfigTypeChatClaude = newModelConfigType(gconv.PtrString("claude"), "Anthropic Claude 系列大模型")
|
||||
ModelConfigTypeChatDeepSeek = newModelConfigType(gconv.PtrString("deepSeek"), "DeepSeek 深度求索大模型")
|
||||
ModelConfigTypeChatOllama = newModelConfigType(gconv.PtrString("ollama"), "Ollama 本地大模型运行工具")
|
||||
ModelConfigTypeChatOpenAI = newModelConfigType(gconv.PtrString("openAI"), "OpenAI 官方大模型服务")
|
||||
ModelConfigTypeChatQianfan = newModelConfigType(gconv.PtrString("qianfan"), "百度文心一言千帆大模型平台")
|
||||
ModelConfigTypeChatQwen = newModelConfigType(gconv.PtrString("qwen"), "腾讯文心千问大模型平台")
|
||||
)
|
||||
|
||||
type ModelConfigType *string
|
||||
|
||||
type modelConfigType struct {
|
||||
code ModelConfigType
|
||||
desc string
|
||||
}
|
||||
|
||||
func (s modelConfigType) Code() ModelConfigType {
|
||||
return s.code
|
||||
}
|
||||
func (s modelConfigType) Desc() string {
|
||||
return s.desc
|
||||
}
|
||||
|
||||
func newModelConfigType(code ModelConfigType, desc string) modelConfigType {
|
||||
return modelConfigType{code: code, desc: desc}
|
||||
}
|
||||
|
||||
func GetVectorDescByCode(code ModelConfigType) string {
|
||||
switch *code {
|
||||
case *ModelConfigTypeVectorArk.Code():
|
||||
return ModelConfigTypeVectorArk.Desc()
|
||||
case *ModelConfigTypeVectorOllama.Code():
|
||||
return ModelConfigTypeVectorOllama.Desc()
|
||||
case *ModelConfigTypeVectorOpenAI.Code():
|
||||
return ModelConfigTypeVectorOpenAI.Desc()
|
||||
case *ModelConfigTypeVectorQianfan.Code():
|
||||
return ModelConfigTypeVectorQianfan.Desc()
|
||||
case *ModelConfigTypeVectorTencentCloud.Code():
|
||||
return ModelConfigTypeVectorTencentCloud.Desc()
|
||||
case *ModelConfigTypeVectorDashScope.Code():
|
||||
return ModelConfigTypeVectorDashScope.Desc()
|
||||
}
|
||||
return "未知类型"
|
||||
}
|
||||
|
||||
func GetChatDescByCode(code ModelConfigType) string {
|
||||
switch *code {
|
||||
case *ModelConfigTypeChatArk.Code():
|
||||
return ModelConfigTypeChatArk.Desc()
|
||||
case *ModelConfigTypeChatArkBot.Code():
|
||||
return ModelConfigTypeChatArkBot.Desc()
|
||||
case *ModelConfigTypeChatClaude.Code():
|
||||
return ModelConfigTypeChatClaude.Desc()
|
||||
case *ModelConfigTypeChatDeepSeek.Code():
|
||||
return ModelConfigTypeChatDeepSeek.Desc()
|
||||
case *ModelConfigTypeChatOllama.Code():
|
||||
return ModelConfigTypeChatOllama.Desc()
|
||||
case *ModelConfigTypeChatOpenAI.Code():
|
||||
return ModelConfigTypeChatOpenAI.Desc()
|
||||
case *ModelConfigTypeChatQianfan.Code():
|
||||
return ModelConfigTypeChatQianfan.Desc()
|
||||
case *ModelConfigTypeChatQwen.Code():
|
||||
return ModelConfigTypeChatQwen.Desc()
|
||||
}
|
||||
return "未知类型"
|
||||
}
|
||||
|
||||
type GetModelConfigTypeEnumRes struct {
|
||||
Options []ConfigTypeKeyValue `json:"options"`
|
||||
}
|
||||
|
||||
type ConfigTypeKeyValue struct {
|
||||
Key interface{} `json:"key"` // 对应原有常量值
|
||||
Value interface{} `json:"value"` // 对应描述信息
|
||||
}
|
||||
|
||||
func GetAllModelConfigTypeEnums(modelType ModelType) *GetModelConfigTypeEnumRes {
|
||||
// 枚举列表
|
||||
var list []modelConfigType
|
||||
|
||||
if *modelType == *ModelTypeVector.Code() {
|
||||
list = []modelConfigType{
|
||||
ModelConfigTypeVectorArk,
|
||||
ModelConfigTypeVectorOllama,
|
||||
ModelConfigTypeVectorOpenAI,
|
||||
ModelConfigTypeVectorQianfan,
|
||||
ModelConfigTypeVectorTencentCloud,
|
||||
ModelConfigTypeVectorDashScope,
|
||||
}
|
||||
}
|
||||
|
||||
if *modelType == *ModelTypeChat.Code() {
|
||||
list = []modelConfigType{
|
||||
ModelConfigTypeChatArk,
|
||||
ModelConfigTypeChatArkBot,
|
||||
ModelConfigTypeChatClaude,
|
||||
ModelConfigTypeChatDeepSeek,
|
||||
ModelConfigTypeChatOllama,
|
||||
ModelConfigTypeChatOpenAI,
|
||||
ModelConfigTypeChatQianfan,
|
||||
ModelConfigTypeChatQwen,
|
||||
}
|
||||
}
|
||||
|
||||
// 组装返回格式
|
||||
options := make([]ConfigTypeKeyValue, 0, len(list))
|
||||
for _, item := range list {
|
||||
options = append(options, ConfigTypeKeyValue{
|
||||
Key: *item.Code(),
|
||||
Value: item.Desc(),
|
||||
})
|
||||
}
|
||||
|
||||
return &GetModelConfigTypeEnumRes{
|
||||
Options: options,
|
||||
}
|
||||
}
|
||||
66
consts/model/type.go
Normal file
66
consts/model/type.go
Normal file
@@ -0,0 +1,66 @@
|
||||
package model
|
||||
|
||||
import "github.com/gogf/gf/v2/util/gconv"
|
||||
|
||||
var (
|
||||
ModelTypeVector = newModelType(gconv.PtrString("vector"), "向量")
|
||||
ModelTypeChat = newModelType(gconv.PtrString("chat"), "对话")
|
||||
)
|
||||
|
||||
type ModelType *string
|
||||
|
||||
type modelType struct {
|
||||
code ModelType
|
||||
desc string
|
||||
}
|
||||
|
||||
func (s modelType) Code() ModelType {
|
||||
return s.code
|
||||
}
|
||||
func (s modelType) Desc() string {
|
||||
return s.desc
|
||||
}
|
||||
|
||||
func newModelType(code ModelType, desc string) modelType {
|
||||
return modelType{code: code, desc: desc}
|
||||
}
|
||||
|
||||
func GetModelTypeDescByCode(code ModelType) string {
|
||||
switch *code {
|
||||
case *ModelTypeVector.Code():
|
||||
return ModelTypeVector.Desc()
|
||||
case *ModelTypeChat.Code():
|
||||
return ModelTypeChat.Desc()
|
||||
}
|
||||
return "未知类型"
|
||||
}
|
||||
|
||||
type GetModelTypeEnumRes struct {
|
||||
Options []TypeKeyValue `json:"options"`
|
||||
}
|
||||
|
||||
type TypeKeyValue struct {
|
||||
Key interface{} `json:"key"` // 对应原有常量值
|
||||
Value interface{} `json:"value"` // 对应描述信息
|
||||
}
|
||||
|
||||
func GetAllModelTypeEnums() *GetModelTypeEnumRes {
|
||||
// 枚举列表
|
||||
list := []modelType{
|
||||
//ModelTypeVector,
|
||||
ModelTypeChat,
|
||||
}
|
||||
|
||||
// 组装返回格式
|
||||
options := make([]TypeKeyValue, 0, len(list))
|
||||
for _, item := range list {
|
||||
options = append(options, TypeKeyValue{
|
||||
Key: *item.Code(),
|
||||
Value: item.Desc(),
|
||||
})
|
||||
}
|
||||
|
||||
return &GetModelTypeEnumRes{
|
||||
Options: options,
|
||||
}
|
||||
}
|
||||
@@ -2,17 +2,10 @@ package public
|
||||
|
||||
const GmqMsgPluginsName = "gmq_msg"
|
||||
|
||||
const KnowledgeLockEsKey = "rag_binary:knowledge:lock:knowledgeIdEs-%v"
|
||||
const KnowledgeLockSqlKey = "rag_binary:knowledge:lock:knowledgeIdSql-%v"
|
||||
const KnowledgeContentHashEsKey = "rag_binary:knowledge:knowledgeId:contentHashEs-%v"
|
||||
const KnowledgeContentHashSqlKey = "rag_binary:knowledge:knowledgeId:contentHashSql-%v"
|
||||
|
||||
const (
|
||||
KnowledgeDocumentVectorStatusTopic = "knowledge:document:vector:status:stream"
|
||||
KnowledgeDocumentVectorStatusConsumer = "knowledge-document-vector-status-consumer"
|
||||
KnowledgeDocumentVectorStatusBatchSize = 1
|
||||
KnowledgeDocumentVectorStatusAutoAck = false
|
||||
)
|
||||
const KnowledgeLockEsKey = "knowledge:lock:knowledgeIdEs-%v"
|
||||
const KnowledgeLockSqlKey = "knowledge:lock:knowledgeIdSql-%v"
|
||||
const KnowledgeContentHashEsKey = "knowledge:knowledgeId:contentHashEs-%v"
|
||||
const KnowledgeContentHashSqlKey = "knowledge:knowledgeId:contentHashSql-%v"
|
||||
|
||||
const (
|
||||
KnowledgeDocumentVectorTopic = "knowledge:document:vector:stream" // 请求 Stream 键名（与发消息的key一致）
|
||||
|
||||
@@ -12,6 +12,7 @@ const (
|
||||
TableNameDataset = "dataset"
|
||||
TableNameKeyword = "keyword"
|
||||
TableNameTask = "task"
|
||||
TableNameModel = "model"
|
||||
TableNameDatasetIndex = "dataset_index"
|
||||
TableNameDocumentVector = "document_vector"
|
||||
)
|
||||
|
||||
52
controller/model.go
Normal file
52
controller/model.go
Normal file
@@ -0,0 +1,52 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"rag/model/dto"
|
||||
"rag/service"
|
||||
|
||||
"gitea.com/red-future/common/beans"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
)
|
||||
|
||||
type model struct{}
|
||||
|
||||
var Model = new(model)
|
||||
|
||||
func (c *model) GetModelTypeEnums(ctx context.Context, req *dto.GetModelAllEnumsReq) (res *dto.GetModelEnumRes, err error) {
|
||||
res, err = service.ModelService.GetModelAllEnums(ctx, req)
|
||||
return
|
||||
}
|
||||
|
||||
func (c *model) GetModelConfigFormFields(ctx context.Context, req *dto.GetModelConfigFormFieldsReq) (res *dto.GetModelConfigFormFieldsRes, err error) {
|
||||
res, err = service.ModelService.GetModelConfigFormFields(ctx, req)
|
||||
return
|
||||
}
|
||||
|
||||
func (c *model) Create(ctx context.Context, req *dto.CreateModelReq) (res *dto.CreateModelRes, err error) {
|
||||
res, err = service.ModelService.Create(ctx, req)
|
||||
return
|
||||
}
|
||||
|
||||
func (c *model) Update(ctx context.Context, req *dto.UpdateModelReq) (res *beans.ResponseEmpty, err error) {
|
||||
err = service.ModelService.Update(ctx, req)
|
||||
return
|
||||
}
|
||||
|
||||
func (c *model) Delete(ctx context.Context, req *dto.DeleteModelReq) (res *beans.ResponseEmpty, err error) {
|
||||
err = service.ModelService.Delete(ctx, req)
|
||||
return
|
||||
}
|
||||
|
||||
func (c *model) Get(ctx context.Context, req *dto.GetModelReq) (res *dto.ModelVO, err error) {
|
||||
res, err = service.ModelService.Get(ctx, req)
|
||||
return
|
||||
}
|
||||
|
||||
func (c *model) List(ctx context.Context, req *dto.ListModelReq) (res *dto.ListModelRes, err error) {
|
||||
if !g.IsEmpty(req.Page) {
|
||||
req.Page = &beans.Page{PageNum: 1, PageSize: 20}
|
||||
}
|
||||
res, err = service.ModelService.List(ctx, req)
|
||||
return
|
||||
}
|
||||
16
controller/task.go
Normal file
16
controller/task.go
Normal file
@@ -0,0 +1,16 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"rag/model/dto"
|
||||
"rag/service"
|
||||
)
|
||||
|
||||
type task struct{}
|
||||
|
||||
var Task = new(task)
|
||||
|
||||
func (c *task) Get(ctx context.Context, req *dto.GetTaskReq) (res *dto.ListTaskRes, err error) {
|
||||
res, err = service.Task.Get(ctx, req)
|
||||
return
|
||||
}
|
||||
@@ -48,16 +48,28 @@ func (d *documentDao) Update(ctx context.Context, req *dto.UpdateDocumentReq) (r
|
||||
|
||||
// Delete 删除文件
|
||||
func (d *documentDao) Delete(ctx context.Context, req *dto.DeleteDocumentReq) (rows int64, err error) {
|
||||
r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameDocument).Where(entity.DocumentCol.Id, req.Id).Delete()
|
||||
r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameDocument).OmitEmpty().
|
||||
Where(entity.DocumentCol.Id, req.Id).
|
||||
Delete()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return r.RowsAffected()
|
||||
}
|
||||
|
||||
// GetByID 根据ID获取文件
|
||||
func (d *documentDao) GetByID(ctx context.Context, req *dto.GetDocumentReq, fields ...string) (res *entity.Document, err error) {
|
||||
r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameDocument).Where(entity.DocumentCol.Id, req.Id).Fields(fields).One()
|
||||
func (d *documentDao) Count(ctx context.Context, req *dto.ListDocumentReq) (count int, err error) {
|
||||
count, err = gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameDocument).OmitEmpty().
|
||||
Where(entity.DocumentCol.DatasetId, req.DatasetId).
|
||||
Where(entity.DocumentCol.Title, req.Title).Count()
|
||||
return
|
||||
}
|
||||
|
||||
// Get 根据ID获取文件
|
||||
func (d *documentDao) Get(ctx context.Context, req *dto.GetDocumentReq, fields ...string) (res *entity.Document, err error) {
|
||||
r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameDocument).Fields(fields).OmitEmpty().
|
||||
Where(entity.DocumentCol.Id, req.Id).
|
||||
Where(entity.DocumentCol.Title, req.Title).
|
||||
Where(entity.DocumentCol.DatasetId, req.DatasetId).One()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -43,17 +43,29 @@ func (d *documentVectorDao) Update(ctx context.Context, req *dto.UpdateDocumentV
|
||||
return r.RowsAffected()
|
||||
}
|
||||
|
||||
func (d *documentVectorDao) Delete(ctx context.Context, req *dto.DeleteDocumentVectorReq) (rows int64, err error) {
|
||||
result, err := gfdb.DB(ctx, public.DbNameVector).Model(ctx, public.TableNameDocumentVector).OmitEmpty().
|
||||
Where(entity.DocumentVectorCol.Id, req.Id).
|
||||
Where(entity.DocumentVectorCol.DocumentId, req.DocumentId).
|
||||
Delete()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return result.RowsAffected()
|
||||
}
|
||||
|
||||
// List 文件块列表
|
||||
func (d *documentVectorDao) List(ctx context.Context, req *dto.ListDocumentVectorReq, fields ...string) (res []*entity.DocumentVector, total int, err error) {
|
||||
model := gfdb.DB(ctx, public.DbNameVector).Model(ctx, public.TableNameDocumentVector).Fields(fields).OmitEmpty().
|
||||
Where(entity.DocumentVectorCol.DatasetId, req.DatasetId).
|
||||
Where(entity.DocumentVectorCol.DocumentId, req.DocumentId).
|
||||
Where(entity.DocumentVectorCol.Status, req.Status).
|
||||
Where(entity.DocumentVectorCol.VectorStatus, req.VectorStatus)
|
||||
Where(entity.DocumentVectorCol.VectorStatus, req.VectorStatus).
|
||||
WhereIn(entity.DocumentVectorCol.DocumentId, req.DocumentIds)
|
||||
if !g.IsEmpty(req.Keyword) {
|
||||
model.WhereLike(entity.DocumentVectorCol.Content, "%"+req.Keyword+"%")
|
||||
}
|
||||
model.OrderDesc(entity.DocumentVectorCol.CreatedAt)
|
||||
model.OrderAsc(entity.DocumentVectorCol.ChunkIndex)
|
||||
if req.Page != nil {
|
||||
model.Page(int(req.Page.PageNum), int(req.Page.PageSize))
|
||||
}
|
||||
@@ -65,28 +77,32 @@ func (d *documentVectorDao) List(ctx context.Context, req *dto.ListDocumentVecto
|
||||
return
|
||||
}
|
||||
|
||||
func (d *documentVectorDao) GetAllByVector(ctx context.Context, datasetIds []int64, vector pgvector.Vector, topK int) (list gdb.List, err error) {
|
||||
//result, err := gfdb.DB(ctx, public.DbNameVector).Model(ctx, public.TableNameDocumentVector).
|
||||
// Fields("id, content, dataset_id, document_id, vector <=> ? AS distance").
|
||||
// WhereIn(entity.DocumentVectorCol.DatasetId, datasetIds).
|
||||
// WhereNotNull(entity.DocumentVectorCol.Vector).
|
||||
// OrderAsc("distance").
|
||||
// Limit(topK).
|
||||
// All()
|
||||
//if err != nil {
|
||||
// return nil, err
|
||||
//}
|
||||
func (d *documentVectorDao) GetAllByVector(ctx context.Context, datasetIds, documentIds []int64, vector pgvector.Vector, topK int) (list gdb.List, err error) {
|
||||
// 动态拼接 WHERE 条件
|
||||
var whereCondition string
|
||||
var queryParams []interface{}
|
||||
|
||||
// 优先使用 documentIds 查询
|
||||
if len(documentIds) > 0 {
|
||||
whereCondition = fmt.Sprintf(" AND %s IN (?) ", entity.DocumentVectorCol.DocumentId)
|
||||
queryParams = append(queryParams, documentIds)
|
||||
}
|
||||
if len(datasetIds) > 0 {
|
||||
whereCondition = fmt.Sprintf(" AND %s IN (?) ", entity.DocumentVectorCol.DatasetId)
|
||||
queryParams = append(queryParams, datasetIds)
|
||||
}
|
||||
|
||||
// 完整 SQL
|
||||
sql := `
|
||||
SELECT id, content, dataset_id, document_id,
|
||||
vector <=> ? AS distance
|
||||
SELECT id, content, dataset_id, document_id,vector <=> ? AS distance
|
||||
FROM rag_vector_document_vector
|
||||
WHERE dataset_id IN (?)
|
||||
AND vector IS NOT NULL
|
||||
ORDER BY distance ASC
|
||||
LIMIT ?
|
||||
`
|
||||
// 顺序：vector, dataset_id, topK
|
||||
result, err := gfdb.DB(ctx, public.DbNameVector).GetAll(ctx, sql, vector, datasetIds, topK)
|
||||
WHERE 1=1 ` + whereCondition + ` AND vector IS NOT NULL ORDER BY distance ASC LIMIT ?`
|
||||
// 拼接参数：vector + 条件参数 + topK
|
||||
queryParams = append([]interface{}{vector}, queryParams...)
|
||||
queryParams = append(queryParams, topK)
|
||||
|
||||
// 执行查询
|
||||
result, err := gfdb.DB(ctx, public.DbNameVector).GetAll(ctx, sql, queryParams...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -94,7 +110,7 @@ func (d *documentVectorDao) GetAllByVector(ctx context.Context, datasetIds []int
|
||||
}
|
||||
|
||||
// SearchByKeywords 通过关键词全文检索文档块
|
||||
func (d *documentVectorDao) SearchByKeywords(ctx context.Context, query string, datasetIds []int64, topK int) (list gdb.List, err error) {
|
||||
func (d *documentVectorDao) SearchByKeywords(ctx context.Context, query string, datasetIds, documentIds []int64, topK int) (list gdb.List, err error) {
|
||||
// 构建 meilisearch 查询参数
|
||||
searchParams := &meilisearch.SearchParams{
|
||||
Query: query,
|
||||
@@ -102,14 +118,22 @@ func (d *documentVectorDao) SearchByKeywords(ctx context.Context, query string,
|
||||
ShowRankingScore: true,
|
||||
}
|
||||
|
||||
// 构建 datasetIds 过滤条件
|
||||
if len(datasetIds) > 0 {
|
||||
datasetIdStrs := gconv.Strings(datasetIds)
|
||||
quotedIds := make([]string, len(datasetIdStrs))
|
||||
for i, id := range datasetIdStrs {
|
||||
quotedIds[i] = fmt.Sprintf("%s", id)
|
||||
}
|
||||
searchParams.Filter = fmt.Sprintf("dataset_id IN [%s]", gstr.Implode(", ", quotedIds))
|
||||
searchParams.Filter = fmt.Sprintf("%s IN [%s]", entity.DocumentVectorCol.DatasetId, gstr.Implode(", ", quotedIds))
|
||||
}
|
||||
|
||||
if len(documentIds) > 0 {
|
||||
documentIdStrs := gconv.Strings(documentIds)
|
||||
quotedIds := make([]string, len(documentIdStrs))
|
||||
for i, id := range documentIdStrs {
|
||||
quotedIds[i] = fmt.Sprintf("%s", id)
|
||||
}
|
||||
searchParams.Filter = fmt.Sprintf("%s IN [%s]", entity.DocumentVectorCol.DocumentId, gstr.Implode(", ", quotedIds))
|
||||
}
|
||||
|
||||
// 执行搜索
|
||||
@@ -124,6 +148,5 @@ func (d *documentVectorDao) SearchByKeywords(ctx context.Context, query string,
|
||||
for _, hit := range hits {
|
||||
resultList = append(resultList, hit)
|
||||
}
|
||||
|
||||
return resultList, nil
|
||||
}
|
||||
|
||||
@@ -53,7 +53,10 @@ func (d *keywordDao) Update(ctx context.Context, req *dto.UpdateKeywordReq) (row
|
||||
}
|
||||
|
||||
func (d *keywordDao) Delete(ctx context.Context, req *dto.DeleteKeywordReq) (rows int64, err error) {
|
||||
r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameKeyword).Where(entity.KeywordCol.Id, req.Id).Delete()
|
||||
r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameKeyword).OmitEmpty().
|
||||
Where(entity.KeywordCol.Id, req.Id).
|
||||
Where(entity.KeywordCol.DocumentId, req.DocumentId).
|
||||
Delete()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
88
dao/model.go
Normal file
88
dao/model.go
Normal file
@@ -0,0 +1,88 @@
|
||||
package dao
|
||||
|
||||
import (
|
||||
"context"
|
||||
"rag/consts/public"
|
||||
"rag/model/dto"
|
||||
"rag/model/entity"
|
||||
|
||||
"gitea.com/red-future/common/db/gfdb"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/gogf/gf/v2/util/gconv"
|
||||
)
|
||||
|
||||
var Model = new(modelDao)
|
||||
|
||||
type modelDao struct{}
|
||||
|
||||
func (d *modelDao) Insert(ctx context.Context, req *dto.CreateModelReq) (id int64, err error) {
|
||||
var res *entity.Model
|
||||
if err = gconv.Struct(req, &res); err != nil {
|
||||
return
|
||||
}
|
||||
r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameModel).Data(&res).Insert()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return r.LastInsertId()
|
||||
}
|
||||
|
||||
func (d *modelDao) Update(ctx context.Context, req *dto.UpdateModelReq) (rows int64, err error) {
|
||||
model := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameModel).OmitEmpty()
|
||||
r, err := model.Data(&req).Where(entity.ModelCol.Id, req.Id).Update()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return r.RowsAffected()
|
||||
}
|
||||
|
||||
func (d *modelDao) Delete(ctx context.Context, req *dto.DeleteModelReq) (rows int64, err error) {
|
||||
r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameModel).Where(entity.ModelCol.Id, req.Id).Delete()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return r.RowsAffected()
|
||||
}
|
||||
|
||||
func (d *modelDao) Count(ctx context.Context, req *dto.GetModelReq) (count int, err error) {
|
||||
count, err = gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameModel).OmitEmpty().Where(entity.ModelCol.ModelType, req.ModelType).Count()
|
||||
return
|
||||
}
|
||||
|
||||
func (d *modelDao) Get(ctx context.Context, req *dto.GetModelReq, fields ...string) (res *entity.Model, err error) {
|
||||
r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameModel).Fields(fields).OmitEmpty().
|
||||
Where(entity.ModelCol.Id, req.Id).
|
||||
Where(entity.ModelCol.ModelType, req.ModelType).One()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = r.Struct(&res)
|
||||
return
|
||||
}
|
||||
|
||||
func (d *modelDao) GetNoTenantId(ctx context.Context, req *dto.GetModelReq, fields ...string) (res []*entity.Model, err error) {
|
||||
r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameModel).NoTenantId(ctx).Where(entity.ModelCol.ModelType, req.ModelType).Fields(fields).All()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = r.Structs(&res)
|
||||
return
|
||||
}
|
||||
|
||||
func (d *modelDao) List(ctx context.Context, req *dto.ListModelReq, fields ...string) (res []*entity.Model, total int, err error) {
|
||||
model := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameModel).Fields(fields).OmitEmpty()
|
||||
if !g.IsEmpty(req.ModelName) {
|
||||
model.WhereLike(entity.ModelCol.ModelName, "%"+req.ModelName+"%")
|
||||
}
|
||||
model.Where(entity.ModelCol.ModelType, req.ModelType)
|
||||
model.OrderDesc(entity.KeywordCol.CreatedAt)
|
||||
if req.Page != nil {
|
||||
model.Page(int(req.Page.PageNum), int(req.Page.PageSize))
|
||||
}
|
||||
r, total, err := model.AllAndCount(false)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = r.Structs(&res)
|
||||
return
|
||||
}
|
||||
@@ -37,6 +37,15 @@ func (d *taskDao) Update(ctx context.Context, req *dto.UpdateTaskReq) (rows int6
|
||||
return r.RowsAffected()
|
||||
}
|
||||
|
||||
func (d *taskDao) Count(ctx context.Context, req *dto.GetTaskReq) (count int, err error) {
|
||||
count, err = gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameTask).OmitEmpty().
|
||||
Where(entity.TaskCol.TaskId, req.TaskId).
|
||||
Where(entity.TaskCol.TaskType, req.TaskType).
|
||||
Where(entity.TaskCol.Status, req.TaskStatus).
|
||||
Count()
|
||||
return
|
||||
}
|
||||
|
||||
func (d *taskDao) Get(ctx context.Context, req *dto.GetTaskReq) (res []*entity.Task, total int, err error) {
|
||||
r, total, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameTask).OmitEmpty().
|
||||
Where(entity.TaskCol.Id, req.Id).
|
||||
|
||||
73
go.mod
73
go.mod
@@ -6,6 +6,7 @@ require (
|
||||
gitea.com/red-future/common v0.0.12
|
||||
github.com/bjang03/gmq v0.0.0-00010101000000-000000000000
|
||||
github.com/cloudwego/eino v0.8.6
|
||||
github.com/cloudwego/eino-ext/components/document/loader/file v0.0.0-20250519091007-282cc7eb18d3
|
||||
github.com/cloudwego/eino-ext/components/document/loader/url v0.0.0-20260323112355-f061db7e8419
|
||||
github.com/cloudwego/eino-ext/components/document/parser/docx v0.0.0-20260323112355-f061db7e8419
|
||||
github.com/cloudwego/eino-ext/components/document/parser/pdf v0.0.0-20260323112355-f061db7e8419
|
||||
@@ -14,11 +15,20 @@ require (
|
||||
github.com/cloudwego/eino-ext/components/document/transformer/splitter/semantic v0.0.0-20260323112355-f061db7e8419
|
||||
github.com/cloudwego/eino-ext/components/embedding/ark v0.1.1
|
||||
github.com/cloudwego/eino-ext/components/embedding/dashscope v0.0.0-20260323112355-f061db7e8419
|
||||
github.com/cloudwego/eino-ext/components/embedding/ollama v0.0.0-20260413110502-8d10f059c9a4
|
||||
github.com/cloudwego/eino-ext/components/embedding/openai v0.0.0-20260323112355-f061db7e8419
|
||||
github.com/cloudwego/eino-ext/components/embedding/qianfan v0.0.0-20260413110502-8d10f059c9a4
|
||||
github.com/cloudwego/eino-ext/components/embedding/tencentcloud v0.0.0-20260413110502-8d10f059c9a4
|
||||
github.com/cloudwego/eino-ext/components/model/ark v0.1.65
|
||||
github.com/cloudwego/eino-ext/components/model/arkbot v0.1.2
|
||||
github.com/cloudwego/eino-ext/components/model/claude v0.1.17
|
||||
github.com/cloudwego/eino-ext/components/model/deepseek v0.1.2
|
||||
github.com/cloudwego/eino-ext/components/model/ollama v0.1.9
|
||||
github.com/cloudwego/eino-ext/components/model/openai v0.1.12
|
||||
github.com/cloudwego/eino-ext/components/model/qianfan v0.1.4
|
||||
github.com/cloudwego/eino-ext/components/model/qwen v0.1.7
|
||||
github.com/gogf/gf/contrib/drivers/pgsql/v2 v2.10.0
|
||||
github.com/gogf/gf/v2 v2.10.0
|
||||
github.com/golang/glog v1.2.5
|
||||
github.com/pgvector/pgvector-go v0.3.0
|
||||
)
|
||||
|
||||
@@ -27,13 +37,33 @@ replace gitea.com/red-future/common v0.0.12 => ../common
|
||||
replace github.com/bjang03/gmq => ../gmq
|
||||
|
||||
require (
|
||||
cloud.google.com/go/auth v0.7.2 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.7.0 // indirect
|
||||
github.com/BurntSushi/toml v1.6.0 // indirect
|
||||
github.com/PuerkitoBio/goquery v1.8.1 // indirect
|
||||
github.com/andybalholm/brotli v1.1.1 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.1 // indirect
|
||||
github.com/anthropics/anthropic-sdk-go v1.26.0 // indirect
|
||||
github.com/armon/go-metrics v0.4.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.33.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.29.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.54 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.28 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.28 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.24.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.33.9 // indirect
|
||||
github.com/aws/smithy-go v1.22.1 // indirect
|
||||
github.com/aymerick/douceur v0.2.0 // indirect
|
||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||
github.com/baidubce/bce-qianfan-sdk/go/qianfan v0.0.14 // indirect
|
||||
github.com/baidubce/bce-sdk-go v0.9.164 // indirect
|
||||
github.com/buger/jsonparser v1.1.1 // indirect
|
||||
github.com/bwmarrin/snowflake v0.3.0 // indirect
|
||||
github.com/bytedance/gopkg v0.1.3 // indirect
|
||||
@@ -47,7 +77,8 @@ require (
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
|
||||
github.com/cloudwego/base64x v0.1.6 // indirect
|
||||
github.com/cloudwego/eino-ext/components/document/parser/html v0.0.0-20241224063832-9fbcc0e56c28 // indirect
|
||||
github.com/cloudwego/eino-ext/libs/acl/openai v0.1.15 // indirect
|
||||
github.com/cloudwego/eino-ext/libs/acl/openai v0.1.16 // indirect
|
||||
github.com/cohesion-org/deepseek-go v1.3.2 // indirect
|
||||
github.com/dgraph-io/badger/v4 v4.2.0 // indirect
|
||||
github.com/dgraph-io/ristretto v0.1.1 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
@@ -55,9 +86,11 @@ require (
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/eino-contrib/docx2md v0.0.1 // indirect
|
||||
github.com/eino-contrib/jsonschema v1.0.3 // indirect
|
||||
github.com/eino-contrib/ollama v0.1.0 // indirect
|
||||
github.com/emirpasic/gods/v2 v2.0.0-alpha // indirect
|
||||
github.com/evanphx/json-patch v0.5.2 // indirect
|
||||
github.com/fatih/color v1.19.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.13 // indirect
|
||||
github.com/go-ego/gse v1.0.2 // indirect
|
||||
@@ -72,14 +105,18 @@ require (
|
||||
github.com/gogf/gf/contrib/trace/otlphttp/v2 v2.9.5 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
|
||||
github.com/golang/glog v1.2.5 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/golang/protobuf v1.5.4 // indirect
|
||||
github.com/golang/snappy v1.0.0 // indirect
|
||||
github.com/google/flatbuffers v1.12.1 // indirect
|
||||
github.com/google/flatbuffers v24.3.25+incompatible // indirect
|
||||
github.com/google/s2a-go v0.1.7 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
|
||||
github.com/goph/emperror v0.17.2 // indirect
|
||||
github.com/gorilla/css v1.0.1 // indirect
|
||||
github.com/gorilla/websocket v1.5.3 // indirect
|
||||
github.com/gorilla/mux v1.8.1 // indirect
|
||||
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect
|
||||
github.com/grokify/html-strip-tags-go v0.1.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect
|
||||
github.com/hashicorp/consul/api v1.26.1 // indirect
|
||||
@@ -90,8 +127,10 @@ require (
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
|
||||
github.com/hashicorp/golang-lru v1.0.2 // indirect
|
||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
github.com/hashicorp/serf v0.10.1 // indirect
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/joho/godotenv v1.5.1 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/compress v1.18.4 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
||||
@@ -99,6 +138,7 @@ require (
|
||||
github.com/lib/pq v1.12.1 // indirect
|
||||
github.com/magiconair/properties v1.8.10 // indirect
|
||||
github.com/mailru/easyjson v0.9.0 // indirect
|
||||
github.com/matoous/go-nanoid v1.5.1 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.21 // indirect
|
||||
@@ -108,7 +148,7 @@ require (
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
|
||||
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
|
||||
github.com/nats-io/nats.go v1.49.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.15 // indirect
|
||||
@@ -118,6 +158,7 @@ require (
|
||||
github.com/olekukonko/errors v1.2.0 // indirect
|
||||
github.com/olekukonko/ll v0.1.8 // indirect
|
||||
github.com/olekukonko/tablewriter v1.1.4 // indirect
|
||||
github.com/ollama/ollama v0.9.6 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/r3labs/diff/v2 v2.15.1 // indirect
|
||||
@@ -125,9 +166,22 @@ require (
|
||||
github.com/redis/go-redis/v9 v9.18.0 // indirect
|
||||
github.com/richardlehane/mscfb v1.0.4 // indirect
|
||||
github.com/richardlehane/msoleps v1.0.4 // indirect
|
||||
github.com/sagikazarmark/locafero v0.4.0 // indirect
|
||||
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
|
||||
github.com/sirupsen/logrus v1.9.3 // indirect
|
||||
github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f // indirect
|
||||
github.com/sourcegraph/conc v0.3.0 // indirect
|
||||
github.com/spf13/afero v1.11.0 // indirect
|
||||
github.com/spf13/cast v1.10.0 // indirect
|
||||
github.com/spf13/pflag v1.0.9 // indirect
|
||||
github.com/spf13/viper v1.18.2 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.1093 // indirect
|
||||
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/hunyuan v1.0.1093 // indirect
|
||||
github.com/tidwall/gjson v1.18.0 // indirect
|
||||
github.com/tidwall/match v1.1.1 // indirect
|
||||
github.com/tidwall/pretty v1.2.1 // indirect
|
||||
github.com/tidwall/sjson v1.2.5 // indirect
|
||||
github.com/tiger1103/gfast-token v1.0.10 // indirect
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/vcaesar/cedar v0.30.0 // indirect
|
||||
@@ -140,8 +194,10 @@ require (
|
||||
github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 // indirect
|
||||
github.com/yargevad/filepathx v1.0.0 // indirect
|
||||
go.mongodb.org/mongo-driver/v2 v2.4.0 // indirect
|
||||
go.opencensus.io v0.23.0 // indirect
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
|
||||
go.opentelemetry.io/otel v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 // indirect
|
||||
@@ -150,18 +206,23 @@ require (
|
||||
go.opentelemetry.io/otel/trace v1.42.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.7.1 // indirect
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
go.uber.org/multierr v1.9.0 // indirect
|
||||
golang.org/x/arch v0.15.0 // indirect
|
||||
golang.org/x/crypto v0.49.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90 // indirect
|
||||
golang.org/x/net v0.52.0 // indirect
|
||||
golang.org/x/oauth2 v0.30.0 // indirect
|
||||
golang.org/x/sync v0.20.0 // indirect
|
||||
golang.org/x/sys v0.42.0 // indirect
|
||||
golang.org/x/text v0.35.0 // indirect
|
||||
golang.org/x/time v0.9.0 // indirect
|
||||
google.golang.org/api v0.189.0 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect
|
||||
google.golang.org/grpc v1.75.0 // indirect
|
||||
google.golang.org/protobuf v1.36.11 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
151
go.sum
151
go.sum
@@ -13,12 +13,18 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV
|
||||
cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
|
||||
cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
|
||||
cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
|
||||
cloud.google.com/go/auth v0.7.2 h1:uiha352VrCDMXg+yoBtaD0tUF4Kv9vrtrWPYXwutnDE=
|
||||
cloud.google.com/go/auth v0.7.2/go.mod h1:VEc4p5NNxycWQTMQEDQF0bd6aTMb6VgYDXEwiJJQAbs=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.3 h1:MlxF+Pd3OmSudg/b1yZ5lJwoXCEaeedAguodky1PcKI=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.3/go.mod h1:tMQXOfZzFuNuUxOypHlQEXgdfX5cuhwU+ffUuXRJE8I=
|
||||
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
|
||||
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
|
||||
cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
|
||||
cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
|
||||
cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
|
||||
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
|
||||
cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU=
|
||||
cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo=
|
||||
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
|
||||
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
|
||||
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
|
||||
@@ -62,6 +68,8 @@ github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7X
|
||||
github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
|
||||
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
|
||||
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
|
||||
github.com/anthropics/anthropic-sdk-go v1.26.0 h1:oUTzFaUpAevfuELAP1sjL6CQJ9HHAfT7CoSYSac11PY=
|
||||
github.com/anthropics/anthropic-sdk-go v1.26.0/go.mod h1:qUKmaW+uuPB64iy1l+4kOSvaLqPXnHTTBKH6RVZ7q5Q=
|
||||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
|
||||
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
|
||||
@@ -73,12 +81,44 @@ github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgI
|
||||
github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
|
||||
github.com/aws/aws-sdk-go v1.40.45/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
|
||||
github.com/aws/aws-sdk-go-v2 v1.9.1/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4=
|
||||
github.com/aws/aws-sdk-go-v2 v1.33.0 h1:Evgm4DI9imD81V0WwD+TN4DCwjUMdc94TrduMLbgZJs=
|
||||
github.com/aws/aws-sdk-go-v2 v1.33.0/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 h1:tW1/Rkad38LA15X4UQtjXZXNKsCgkshC3EbmcUmghTg=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3/go.mod h1:UbnqO+zjqk3uIt9yCACHJ9IVNhyhOCnYk8yA19SAWrM=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.29.1 h1:JZhGawAyZ/EuJeBtbQYnaoftczcb2drR2Iq36Wgz4sQ=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.29.1/go.mod h1:7bR2YD5euaxBhzt2y/oDkt3uNRb6tjFp98GlTFueRwk=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.54 h1:4UmqeOqJPvdvASZWrKlhzpRahAulBfyTJQUaYy4+hEI=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.54/go.mod h1:RTdfo0P0hbbTxIhmQrOsC/PquBZGabEPnCaxxKRPSnI=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24 h1:5grmdTdMsovn9kPZPI23Hhvp0ZyNm5cRO+IZFIYiAfw=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24/go.mod h1:zqi7TVKTswH3Ozq28PkmBmgzG1tona7mo9G2IJg4Cis=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.28 h1:igORFSiH3bfq4lxKFkTSYDhJEUCYo6C8VKiWJjYwQuQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.28/go.mod h1:3So8EA/aAYm36L7XIvCVwLa0s5N0P7o2b1oqnx/2R4g=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.28 h1:1mOW9zAUMhTSrMDssEHS/ajx8JcAj/IcftzcmNlmVLI=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.28/go.mod h1:kGlXVIWDfvt2Ox5zEaNglmq0hXPHgQFNMix33Tw22jA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
|
||||
github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.8.1/go.mod h1:CM+19rL1+4dFWnOQKwDc7H1KwXTz+h61oUSHyhV0b3o=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9 h1:TQmKDyETFGiXVhZfQ/I0cCFziqqX58pi4tKJGYGFSz0=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9/go.mod h1:HVLPK2iHQBUx7HfZeOQSEu3v2ubZaAY2YPbAm5/WUyY=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.24.11 h1:kuIyu4fTT38Kj7YCC7ouNbVZSSpqkZ+LzIfhCr6Dg+I=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.24.11/go.mod h1:Ro744S4fKiCCuZECXgOi760TiYylUM8ZBf6OGiZzJtY=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10 h1:l+dgv/64iVlQ3WsBbnn+JSbkj01jIi+SM0wYsj3y/hY=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10/go.mod h1:Fzsj6lZEb8AkTE5S68OhcbBqeWPsR8RnGuKPr8Todl8=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.33.9 h1:BRVDbewN6VZcwr+FBOszDKvYeXY1kJ+GGMCcpghlw0U=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.33.9/go.mod h1:f6vjfZER1M17Fokn0IzssOTMT2N8ZSq+7jnNF0tArvw=
|
||||
github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E=
|
||||
github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro=
|
||||
github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
|
||||
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
|
||||
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
|
||||
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
|
||||
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
|
||||
github.com/baidubce/bce-qianfan-sdk/go/qianfan v0.0.14 h1:XNP24illv5CWTLinpdF8Xo73YWQ2ZWbmlNT0BTWFCGg=
|
||||
github.com/baidubce/bce-qianfan-sdk/go/qianfan v0.0.14/go.mod h1:f/kIWWvAHAcU7bzgkfN30SkpN0I4lLvsJkljVK6v5YY=
|
||||
github.com/baidubce/bce-sdk-go v0.9.164 h1:7gswLMsdQyarovMKuv3i6wxFQ3BQgvc5CmyGXb/D/xA=
|
||||
github.com/baidubce/bce-sdk-go v0.9.164/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg=
|
||||
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
|
||||
@@ -134,6 +174,8 @@ github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI
|
||||
github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU=
|
||||
github.com/cloudwego/eino v0.8.6 h1:Rc9/ElXNrTrSCv68t/U0yUmNVu5uMmpPyMCb+WyFIQQ=
|
||||
github.com/cloudwego/eino v0.8.6/go.mod h1:+2N4nsMPxA6kGBHpH+75JuTfEcGprAMTdsZESrShKpU=
|
||||
github.com/cloudwego/eino-ext/components/document/loader/file v0.0.0-20250519091007-282cc7eb18d3 h1:ykb5Nz6WZR6U3CgffUIxdPWi8lLttvhOeA3gYqbXOpY=
|
||||
github.com/cloudwego/eino-ext/components/document/loader/file v0.0.0-20250519091007-282cc7eb18d3/go.mod h1:I4vbBCIMMKeF436Lc+L3DSPQ3f1nmiHD0JS+LhMYCdQ=
|
||||
github.com/cloudwego/eino-ext/components/document/loader/url v0.0.0-20260323112355-f061db7e8419 h1:dMr31rw5pjZMKMPEvNvpy+1RI3HnqVUWmk2abNkb+yM=
|
||||
github.com/cloudwego/eino-ext/components/document/loader/url v0.0.0-20260323112355-f061db7e8419/go.mod h1:/IeSk52Hhym5AUjCs3ESTF5Nb0RLYFWW8llSpWuc/JA=
|
||||
github.com/cloudwego/eino-ext/components/document/parser/docx v0.0.0-20260323112355-f061db7e8419 h1:NkEyzkMvYj1imCNclbL0OeesosinmbU10uW+ZlC0vtA=
|
||||
@@ -152,15 +194,39 @@ github.com/cloudwego/eino-ext/components/embedding/ark v0.1.1 h1:PM/+XAvJtrBqFlB
|
||||
github.com/cloudwego/eino-ext/components/embedding/ark v0.1.1/go.mod h1:6O6x0fHfM3uCLr3lX1DnB/my7fC3WRUA5hpkCkrkZrg=
|
||||
github.com/cloudwego/eino-ext/components/embedding/dashscope v0.0.0-20260323112355-f061db7e8419 h1:gGnohcgEaHqp5V826Ay0H3fi4TpK8ReWlUPePAnzvA4=
|
||||
github.com/cloudwego/eino-ext/components/embedding/dashscope v0.0.0-20260323112355-f061db7e8419/go.mod h1:ekJmA+GLD9vJyZNeODZDBFMiJ92Suy6nF0OY42X3sao=
|
||||
github.com/cloudwego/eino-ext/components/embedding/ollama v0.0.0-20260413110502-8d10f059c9a4 h1:j17fEjj9dsLFIVYnh7lLyxrGNiuQF2E9iyDAIZ6Ohf8=
|
||||
github.com/cloudwego/eino-ext/components/embedding/ollama v0.0.0-20260413110502-8d10f059c9a4/go.mod h1:mI8QMT4DtgLGUuMTVFDNIgRFmirA//do8UnLmZg0DZ4=
|
||||
github.com/cloudwego/eino-ext/components/embedding/openai v0.0.0-20260323112355-f061db7e8419 h1:eM29lyMShtFZNoAhE5g96+zHg9PBLckRyd2HtVeeY4E=
|
||||
github.com/cloudwego/eino-ext/components/embedding/openai v0.0.0-20260323112355-f061db7e8419/go.mod h1:SajSFFRIXJXIbxadAAlSUIS5KTY8R/jzJg9RNSOXCCI=
|
||||
github.com/cloudwego/eino-ext/components/embedding/qianfan v0.0.0-20260413110502-8d10f059c9a4 h1:NsZBgdDn2Qmga8B81z+Rbxz0ouy/CDDoMUW5rq8Im/o=
|
||||
github.com/cloudwego/eino-ext/components/embedding/qianfan v0.0.0-20260413110502-8d10f059c9a4/go.mod h1:Ga/ANlFO4JGsNPk8vAC+v7hiGYfLmXpYqrihiOpR26I=
|
||||
github.com/cloudwego/eino-ext/components/embedding/tencentcloud v0.0.0-20260413110502-8d10f059c9a4 h1:F1l0lHLLErXnVAKho7or9NTM6RIdT3UWBFQlCEUHmUE=
|
||||
github.com/cloudwego/eino-ext/components/embedding/tencentcloud v0.0.0-20260413110502-8d10f059c9a4/go.mod h1:SM49B6gK7V7nR/7/dXSp0Z2pAy38uu/ZaREbs/9S5Vw=
|
||||
github.com/cloudwego/eino-ext/components/model/ark v0.1.65 h1:52ukXVU9ntToTa36SwI8be81qskGkpUEZraIFOf0wqk=
|
||||
github.com/cloudwego/eino-ext/components/model/ark v0.1.65/go.mod h1:aabMR15RTXBSi9Eu13CWavzE+no5BQO4FJUEEdqImbg=
|
||||
github.com/cloudwego/eino-ext/components/model/arkbot v0.1.2 h1:dpVEadipJKrsQ8JBSpNcosQrhis+lOIivnSmGla9+v4=
|
||||
github.com/cloudwego/eino-ext/components/model/arkbot v0.1.2/go.mod h1:sEsPSnIiB+zi7MjG++MLOPTJd7cT2gqfcK4pMpAkleI=
|
||||
github.com/cloudwego/eino-ext/components/model/claude v0.1.17 h1:QcK41lNtNfWBAXnaEGjAUtrbC+t8n1PMH9OSl9qdVu4=
|
||||
github.com/cloudwego/eino-ext/components/model/claude v0.1.17/go.mod h1:2sGGgwpR60LW+RdG/hcjdGBVEVEJ6EkKxlva8mju9BI=
|
||||
github.com/cloudwego/eino-ext/components/model/deepseek v0.1.2 h1:PSHIDLUOv3ZCO7G6ZXnuJWb5pvRZV6xnfLLbwbfY704=
|
||||
github.com/cloudwego/eino-ext/components/model/deepseek v0.1.2/go.mod h1:beCP+L7CsxDz4+DvBjo8iR/v/ZBPpmQfJtrqG280rjw=
|
||||
github.com/cloudwego/eino-ext/components/model/ollama v0.1.9 h1:+eZbquy5lF3WHvK9+T7UUqI0CTRqDEniP7fzL85lJuk=
|
||||
github.com/cloudwego/eino-ext/components/model/ollama v0.1.9/go.mod h1:C3rf3yy2nEoXFP/CQJne4gbiu1pREKplHKmFlhuOzPE=
|
||||
github.com/cloudwego/eino-ext/components/model/openai v0.1.12 h1:vcwNXeT7bpaXMNwUhtcHZwMYY8II2jAihuooyivmEZ0=
|
||||
github.com/cloudwego/eino-ext/components/model/openai v0.1.12/go.mod h1:ve/+/hLZMvxD5AieQ355xHIFhAZVlsG4rdwTnE16aQU=
|
||||
github.com/cloudwego/eino-ext/components/model/qianfan v0.1.4 h1:wCl6EDT4eacyBAopQ0N6bQVwRy6wW9HNmIag9zBljkI=
|
||||
github.com/cloudwego/eino-ext/components/model/qianfan v0.1.4/go.mod h1:ShVCwEhltA7hyc4jYfxMS5TbF6N/RATlNHt/jpGgzWI=
|
||||
github.com/cloudwego/eino-ext/components/model/qwen v0.1.7 h1:8c1LB5lH+dERbf2twp18B1Y822JOQSsS6x7Vnksehk0=
|
||||
github.com/cloudwego/eino-ext/components/model/qwen v0.1.7/go.mod h1:n4iuIUQeL3D8GRsGAhkgceRZpoyPQbqOXFMXM2Q4hNY=
|
||||
github.com/cloudwego/eino-ext/libs/acl/openai v0.1.15 h1:LbdSG9+qWzzp9RFW6dSFkaUW171JvCoYn/K63zX6dQE=
|
||||
github.com/cloudwego/eino-ext/libs/acl/openai v0.1.15/go.mod h1:p+l0zBB0GjjX8HTlbTs3g3KfUFwZC11bsCGZOXW/3L0=
|
||||
github.com/cloudwego/eino-ext/devops v0.1.8 h1:qBg5vjZSDnd9tHzCHG8YsjnGB5vKG2EoZuuQCI8qrGs=
|
||||
github.com/cloudwego/eino-ext/devops v0.1.8/go.mod h1:8yjvPNTaB5Ve4aJmJ0ysFgB10y3YbIuqMh0/Uwt5Fnw=
|
||||
github.com/cloudwego/eino-ext/libs/acl/openai v0.1.16 h1:q242n5P5Tx3a2QLaBmkfEpfRs/o17Ac6u3EAgItEEOc=
|
||||
github.com/cloudwego/eino-ext/libs/acl/openai v0.1.16/go.mod h1:p+l0zBB0GjjX8HTlbTs3g3KfUFwZC11bsCGZOXW/3L0=
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||
github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/cohesion-org/deepseek-go v1.3.2 h1:WTZ/2346KFYca+n+DL5p+Ar1RQxF2w/wGkU4jDvyXaQ=
|
||||
github.com/cohesion-org/deepseek-go v1.3.2/go.mod h1:bOVyKj38r90UEYZFrmJOzJKPxuAh8sIzHOCnLOpiXeI=
|
||||
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
|
||||
github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||
@@ -177,6 +243,8 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczC
|
||||
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
|
||||
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
|
||||
github.com/dslipak/pdf v0.0.2 h1:djAvcM5neg9Ush+zR6QXB+VMJzR6TdnX766HPIg1JmI=
|
||||
github.com/dslipak/pdf v0.0.2/go.mod h1:2L3SnkI9cQwnAS9gfPz2iUoLC0rUZwbucpbKi5R1mUo=
|
||||
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
||||
@@ -191,6 +259,8 @@ github.com/eino-contrib/docx2md v0.0.1 h1:Clz0sF8jiQRYAIZAUTuTAjh0vF/1KqHQqsMha1
|
||||
github.com/eino-contrib/docx2md v0.0.1/go.mod h1:b1dupA9cF5yExHjVMCcP6feyE6mwZjsY7Cc9ESO5Y14=
|
||||
github.com/eino-contrib/jsonschema v1.0.3 h1:2Kfsm1xlMV0ssY2nuxshS4AwbLFuqmPmzIjLVJ1Fsp0=
|
||||
github.com/eino-contrib/jsonschema v1.0.3/go.mod h1:cpnX4SyKjWjGC7iN2EbhxaTdLqGjCi0e9DxpLYxddD4=
|
||||
github.com/eino-contrib/ollama v0.1.0 h1:z1NaMdKW6X1ftP8g5xGGR5zDRPUtuTKFq35vBQgxsN4=
|
||||
github.com/eino-contrib/ollama v0.1.0/go.mod h1:mYsQ7b3DeqY8bHPuD3MZJYTqkgyL6LoemxoP/B7ZNhA=
|
||||
github.com/emirpasic/gods/v2 v2.0.0-alpha h1:dwFlh8pBg1VMOXWGipNMRt8v96dKAIvBehtCt6OtunU=
|
||||
github.com/emirpasic/gods/v2 v2.0.0-alpha/go.mod h1:W0y4M2dtBB9U5z3YlghmpuUhiaZT2h6yoeE+C1sCp6A=
|
||||
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
@@ -208,6 +278,8 @@ github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGE
|
||||
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
|
||||
github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w=
|
||||
github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
|
||||
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
|
||||
github.com/franela/goblin v0.0.0-20210519012713-85d372ac71e2/go.mod h1:VzmDKDJVZI3aJmnRI9VjAn9nJ8qPPsN1fqzr9dqInIo=
|
||||
@@ -322,8 +394,8 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4=
|
||||
github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
|
||||
github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw=
|
||||
github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
|
||||
github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI=
|
||||
github.com/google/flatbuffers v24.3.25+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
@@ -350,10 +422,14 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf
|
||||
github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o=
|
||||
github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw=
|
||||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/goph/emperror v0.17.2 h1:yLapQcmEsO0ipe9p5TaN22djm3OFV/TfM/fcYP0/J18=
|
||||
@@ -366,10 +442,12 @@ github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
|
||||
github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0=
|
||||
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
|
||||
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
|
||||
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
|
||||
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
|
||||
github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
|
||||
github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
|
||||
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
||||
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo=
|
||||
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA=
|
||||
github.com/grokify/html-strip-tags-go v0.1.0 h1:03UrQLjAny8xci+R+qjCce/MYnpNXCtgzltlQbOBae4=
|
||||
github.com/grokify/html-strip-tags-go v0.1.0/go.mod h1:ZdzgfHEzAfz9X6Xe5eBLVblWIxXfYSQ40S/VKrAOGpc=
|
||||
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
|
||||
@@ -422,6 +500,8 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
|
||||
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
|
||||
github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c=
|
||||
github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
|
||||
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
|
||||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
||||
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
|
||||
github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY=
|
||||
github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc=
|
||||
@@ -460,6 +540,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
||||
github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
|
||||
github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
|
||||
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
|
||||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
@@ -504,6 +586,8 @@ github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8S
|
||||
github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
|
||||
github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
|
||||
github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
|
||||
github.com/matoous/go-nanoid v1.5.1 h1:aCjdvTyO9LLnTIi0fgdXhOPPvOHjpXN6Ik9DaNjIct4=
|
||||
github.com/matoous/go-nanoid v1.5.1/go.mod h1:zyD2a71IubI24efhpvkJz+ZwfwagzgSO6UNiFsZKN7U=
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
||||
github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
|
||||
@@ -553,8 +637,9 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8=
|
||||
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
|
||||
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
@@ -584,6 +669,8 @@ github.com/olekukonko/ll v0.1.8 h1:ysHCJRGHYKzmBSdz9w5AySztx7lG8SQY+naTGYUbsz8=
|
||||
github.com/olekukonko/ll v0.1.8/go.mod h1:RPRC6UcscfFZgjo1nulkfMH5IM0QAYim0LfnMvUuozw=
|
||||
github.com/olekukonko/tablewriter v1.1.4 h1:ORUMI3dXbMnRlRggJX3+q7OzQFDdvgbN9nVWj1drm6I=
|
||||
github.com/olekukonko/tablewriter v1.1.4/go.mod h1:+kedxuyTtgoZLwif3P1Em4hARJs+mVnzKxmsCL/C5RY=
|
||||
github.com/ollama/ollama v0.9.6 h1:HZNJmB52pMt6zLkGkkheBuXBXM5478eiSAj7GR75AMc=
|
||||
github.com/ollama/ollama v0.9.6/go.mod h1:zLwx3iZ3AI4Rc/egsrx3u1w4RU2MHQ/Ylxse48jvyt4=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
@@ -657,6 +744,10 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7
|
||||
github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ=
|
||||
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ=
|
||||
github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
|
||||
github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
|
||||
github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||
@@ -675,9 +766,17 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9
|
||||
github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY=
|
||||
github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60=
|
||||
github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
|
||||
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
|
||||
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
|
||||
github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
|
||||
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
|
||||
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
|
||||
github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
|
||||
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ=
|
||||
github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk=
|
||||
github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
|
||||
github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
|
||||
github.com/streadway/handy v0.0.0-20200128134331-0f66f006fb2e/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=
|
||||
@@ -696,10 +795,27 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.1093 h1:pkz4SrPMy3TLKwqwCH6gIgv6TqEvJKkyiq0sEFl8wb0=
|
||||
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.1093/go.mod h1:r5r4xbfxSaeR04b166HGsBa/R4U3SueirEUpXGuw+Q0=
|
||||
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/hunyuan v1.0.1093 h1:YlJETpB0b4KtK3Km8ak+Td1WqY8kQYaF+r3LId/hOc0=
|
||||
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/hunyuan v1.0.1093/go.mod h1:IUc23LMmefQegnwF1l9DlvIbgFY5xD9AzpWVaBiBYBk=
|
||||
github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
|
||||
github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
|
||||
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
|
||||
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
|
||||
github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
|
||||
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
|
||||
github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
|
||||
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
|
||||
github.com/tiger1103/gfast-token v1.0.10 h1:fNiBE/Dq5iTHvTGlCx3DmXa2o4hr0NtumFpffZ39k6s=
|
||||
github.com/tiger1103/gfast-token v1.0.10/go.mod h1:a/21mxmj7zFeNvjhZSC0XpEAFHfb1aT2k6DXnufFU1s=
|
||||
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo=
|
||||
@@ -771,10 +887,15 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
|
||||
go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
|
||||
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
|
||||
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw=
|
||||
go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho=
|
||||
go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24=
|
||||
@@ -803,6 +924,8 @@ go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU=
|
||||
go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM=
|
||||
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
|
||||
go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
|
||||
go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
|
||||
go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
|
||||
go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
|
||||
go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI=
|
||||
golang.org/x/arch v0.15.0 h1:QtOrQd0bTUnhNVNndMpLHNWrDmYzZ2KDqSrEymqInZw=
|
||||
@@ -843,8 +966,8 @@ golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90/go.mod h1:xE1HEv6b+1SCZ5/usc
|
||||
golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ=
|
||||
golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E=
|
||||
golang.org/x/image v0.22.0 h1:UtK5yLUzilVrkjMAZAZ34DXGpASN8i8pj8g+O+yd10g=
|
||||
golang.org/x/image v0.22.0/go.mod h1:9hPFhljd4zZ1GNSIZJ49sqbp45GKK9t6w+iXvGqZUz4=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
@@ -924,6 +1047,8 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr
|
||||
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
|
||||
golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@@ -1035,6 +1160,8 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY=
|
||||
golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
|
||||
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
@@ -1114,6 +1241,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
|
||||
google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
|
||||
google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
|
||||
google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
|
||||
google.golang.org/api v0.189.0 h1:equMo30LypAkdkLMBqfeIqtyAnlyig1JSZArl4XPwdI=
|
||||
google.golang.org/api v0.189.0/go.mod h1:FLWGJKb0hb+pU2j+rJqwbnsF+ym+fQs73rbJ+KAUgy8=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
@@ -1204,6 +1333,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
|
||||
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
|
||||
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
|
||||
|
||||
2
main.go
2
main.go
@@ -29,7 +29,9 @@ func main() {
|
||||
controller.Dataset,
|
||||
controller.Document,
|
||||
controller.DocumentVector,
|
||||
controller.Model,
|
||||
controller.Keyword,
|
||||
controller.Task,
|
||||
})
|
||||
|
||||
if err := utils.InitGseTool(ctx); err != nil {
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"gitea.com/red-future/common/beans"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/gogf/gf/v2/os/gtime"
|
||||
"github.com/pgvector/pgvector-go"
|
||||
)
|
||||
|
||||
// CreateDocumentReq 创建文件请求
|
||||
@@ -45,7 +46,9 @@ type DeleteDocumentReq struct {
|
||||
type GetDocumentReq struct {
|
||||
g.Meta `path:"/get" method:"get" tags:"文件管理" summary:"获取文件详情" dc:"获取文件详情"`
|
||||
|
||||
Id int64 `json:"id" v:"required#ID不能为空"`
|
||||
Id int64 `json:"id" v:"required#ID不能为空"`
|
||||
DatasetId int64 `json:"datasetId"`
|
||||
Title string `json:"title"`
|
||||
}
|
||||
|
||||
type GetDocumentRes struct {
|
||||
@@ -60,6 +63,7 @@ type ListDocumentReq struct {
|
||||
Page *beans.Page `json:"page"`
|
||||
DatasetId int64 `json:"datasetId"`
|
||||
Keyword string `json:"keyword" dc:"关键词搜索"`
|
||||
Title string `json:"title" dc:"文件标题"`
|
||||
Status document.Status `json:"status"`
|
||||
}
|
||||
|
||||
@@ -92,7 +96,9 @@ type DocumentVectorReq struct {
|
||||
}
|
||||
|
||||
type DocumentVectorRPC struct {
|
||||
Id int64 `json:"id" dc:"id"`
|
||||
DatasetId int64 `json:"datasetId" dc:"所属数据集ID"`
|
||||
ContentHash string `json:"contentHash" dc:"内容hash"`
|
||||
Id int64 `json:"id" dc:"id"`
|
||||
DatasetId int64 `json:"datasetId" dc:"所属数据集ID"`
|
||||
DocumentId int64 `json:"documentId" dc:"文件ID"`
|
||||
ContentHash string `json:"contentHash" dc:"内容hash"`
|
||||
Vector pgvector.Vector `json:"vector" dc:"向量"`
|
||||
}
|
||||
|
||||
@@ -13,10 +13,11 @@ import (
|
||||
type RAGQueryReq struct {
|
||||
g.Meta `path:"/ragQuery" method:"post" tags:"RAG查询" summary:"执行RAG查询" dc:"执行RAG查询"`
|
||||
|
||||
Content string `json:"content" v:"required#查询内容不能为空" dc:"用户问题"`
|
||||
DatasetIds []int64 `json:"datasetIds" dc:"数据集ID"`
|
||||
History []*Message `json:"history" dc:"历史对话"`
|
||||
TopK int `json:"topK" d:"5" dc:"检索topK，默认5"`
|
||||
Content string `json:"content" v:"required#查询内容不能为空" dc:"用户问题"`
|
||||
DatasetIds []int64 `json:"datasetIds" dc:"数据集ID"`
|
||||
DocumentIds []int64 `json:"documentIds" dc:"文档ID"`
|
||||
History []*Message `json:"history" dc:"历史对话"`
|
||||
TopK int `json:"topK" d:"5" dc:"检索topK，默认5"`
|
||||
}
|
||||
|
||||
type Message struct {
|
||||
@@ -37,6 +38,13 @@ type UpdateDocumentVectorReq struct {
|
||||
Status document.Status `json:"status"`
|
||||
}
|
||||
|
||||
type DeleteDocumentVectorReq struct {
|
||||
g.Meta `path:"/delete" method:"put" tags:"文件块向量管理" summary:"删除文件块" dc:"删除文件块"`
|
||||
|
||||
Id int64 `json:"id"`
|
||||
DocumentId int64 `json:"documentId"`
|
||||
}
|
||||
|
||||
// ListDocumentVectorReq 文件块向量列表请求
|
||||
type ListDocumentVectorReq struct {
|
||||
g.Meta `path:"/list" method:"get" tags:"文件块向量管理" summary:"获取文件块向量列表" dc:"分页查询文件块向量列表，支持多条件筛选"`
|
||||
@@ -45,6 +53,8 @@ type ListDocumentVectorReq struct {
|
||||
Keyword string `json:"keyword" dc:"关键词搜索"`
|
||||
DatasetId int64 `json:"datasetId"`
|
||||
DocumentId int64 `json:"documentId"`
|
||||
DocumentIds []int64 `json:"documentIds"`
|
||||
ContentHashs []string `json:"contentHash"`
|
||||
Status document.Status `json:"status"`
|
||||
VectorStatus document.VectorStatus `json:"vectorStatus"`
|
||||
}
|
||||
|
||||
@@ -37,7 +37,8 @@ type UpdateKeywordReq struct {
|
||||
type DeleteKeywordReq struct {
|
||||
g.Meta `path:"/delete" method:"delete" tags:"关键词管理" summary:"删除关键词" dc:"删除关键词"`
|
||||
|
||||
Id int64 `json:"id" v:"required#ID不能为空"`
|
||||
Id int64 `json:"id"`
|
||||
DocumentId int64 `json:"documentId"`
|
||||
}
|
||||
|
||||
// GetKeywordReq 获取关键词请求
|
||||
|
||||
114
model/dto/model.go
Normal file
114
model/dto/model.go
Normal file
@@ -0,0 +1,114 @@
|
||||
package dto
|
||||
|
||||
import (
|
||||
"rag/consts/model"
|
||||
"time"
|
||||
|
||||
"gitea.com/red-future/common/beans"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
)
|
||||
|
||||
type GetModelAllEnumsReq struct {
|
||||
g.Meta `path:"/getAllEnums" method:"get" tags:"模型配置管理" summary:"获取全量模型枚举（类型+配置）"`
|
||||
}
|
||||
|
||||
type GetModelEnumRes struct {
|
||||
Options []ModelEnumOption `json:"options"`
|
||||
}
|
||||
|
||||
// ModelEnumOption 主类型：模型类型（vector/chat）
|
||||
type ModelEnumOption struct {
|
||||
Key interface{} `json:"key"`
|
||||
Value interface{} `json:"value"`
|
||||
ConfigTypes []ModelKeyValue `json:"configTypes"` // 这里统一！
|
||||
}
|
||||
|
||||
// ModelKeyValue 统一的 KV 结构 → 给模型类型 + 配置类型共用
|
||||
type ModelKeyValue struct {
|
||||
Key interface{} `json:"key"`
|
||||
Value interface{} `json:"value"`
|
||||
}
|
||||
|
||||
// GetModelConfigFormFieldsReq 获取模型配置表单请求
|
||||
type GetModelConfigFormFieldsReq struct {
|
||||
g.Meta `path:"/getModelFormField" method:"get" tags:"模型配置管理" summary:"获取模型表单" dc:"获取模型表单列表"`
|
||||
|
||||
ModelType model.ModelType `json:"modelType"` // 模型类型 vector/chat
|
||||
ConfigType model.ModelConfigType `json:"configType"` // 配置类型 ark/ollama/openai...
|
||||
}
|
||||
|
||||
// GetModelConfigFormFieldsRes 获取模型配置表单响应
|
||||
type GetModelConfigFormFieldsRes struct {
|
||||
ModelType model.ModelType `json:"modelType"`
|
||||
ConfigType model.ModelConfigType `json:"configType"`
|
||||
Fields []map[string]interface{} `json:"fields"`
|
||||
}
|
||||
|
||||
// CreateModelReq 创建模型请求
|
||||
type CreateModelReq struct {
|
||||
g.Meta `path:"/create" method:"post" tags:"模型配置管理" summary:"创建模型配置" dc:"创建模型配置"`
|
||||
|
||||
ModelType model.ModelType `json:"modelType" v:"required#模型类型不能为空"`
|
||||
ModelName string `json:"modelName" v:"required#模型名称不能为空"`
|
||||
ModelDesc string `json:"modelDesc"`
|
||||
ConfigType model.ModelConfigType `json:"configType"`
|
||||
ConfigContent map[string]interface{} `json:"configContent"`
|
||||
}
|
||||
|
||||
// CreateModelRes 创建模型响应
|
||||
type CreateModelRes struct {
|
||||
Id int64 `json:"id,string"`
|
||||
}
|
||||
|
||||
// UpdateModelReq 更新模型请求
|
||||
type UpdateModelReq struct {
|
||||
g.Meta `path:"/update" method:"put" tags:"模型配置管理" summary:"更新模型配置" dc:"更新模型配置"`
|
||||
|
||||
Id int64 `json:"id" v:"required#ID不能为空"`
|
||||
ModelType model.ModelType `json:"modelType"`
|
||||
ModelName string `json:"modelName"`
|
||||
ModelDesc string `json:"modelDesc"`
|
||||
ConfigType model.ModelConfigType `json:"configType"`
|
||||
ConfigContent map[string]interface{} `json:"configContent"`
|
||||
}
|
||||
|
||||
// DeleteModelReq 删除模型请求
|
||||
type DeleteModelReq struct {
|
||||
g.Meta `path:"/delete" method:"delete" tags:"模型配置管理" summary:"删除模型配置" dc:"删除模型配置"`
|
||||
|
||||
Id int64 `json:"id" v:"required#ID不能为空"`
|
||||
}
|
||||
|
||||
// GetModelReq 获取模型请求
|
||||
type GetModelReq struct {
|
||||
g.Meta `path:"/get" method:"get" tags:"模型配置管理" summary:"获取模型配置详情" dc:"获取模型配置详情"`
|
||||
|
||||
Id int64 `json:"id"`
|
||||
ModelType model.ModelType `json:"modelType"`
|
||||
}
|
||||
|
||||
// ListModelReq 获取模型列表请求
|
||||
type ListModelReq struct {
|
||||
g.Meta `path:"/list" method:"get" tags:"模型配置管理" summary:"获取模型配置列表" dc:"分页查询模型配置列表，支持多条件筛选"`
|
||||
|
||||
Page *beans.Page `json:"page"`
|
||||
ModelType model.ModelType `json:"modelType"`
|
||||
ModelName string `json:"modelName"`
|
||||
}
|
||||
|
||||
// ListModelRes 获取模型列表响应
|
||||
type ListModelRes struct {
|
||||
List []*ModelVO `json:"list"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
|
||||
type ModelVO struct {
|
||||
Id int64 `json:"id,string"`
|
||||
ModelType model.ModelType `json:"modelType"`
|
||||
ModelName string `json:"modelName"`
|
||||
ModelDesc string `json:"modelDesc"`
|
||||
ConfigType model.ModelConfigType `json:"configType"`
|
||||
ConfigContent map[string]interface{} `json:"configContent"`
|
||||
CreateTime time.Time `json:"createTime"`
|
||||
UpdateTime time.Time `json:"updateTime"`
|
||||
}
|
||||
@@ -2,6 +2,8 @@ package dto
|
||||
|
||||
import (
|
||||
"rag/consts/task"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
)
|
||||
|
||||
// WriteTaskProgressReq 写入任务进度请求
|
||||
@@ -35,9 +37,17 @@ type DeleteTaskByTaskIdReq struct {
|
||||
|
||||
// GetTaskReq 获取任务请求
|
||||
type GetTaskReq struct {
|
||||
Id int64 `json:"id" dc:"任务ID"`
|
||||
TaskId int64 `json:"taskId" dc:"任务ID"`
|
||||
TaskType task.TaskType `json:"taskType" dc:"任务类型"`
|
||||
g.Meta `path:"/get" method:"get" tags:"任务管理" summary:"获取任务详情" dc:"获取任务详情"`
|
||||
|
||||
Id int64 `json:"id" dc:"任务ID"`
|
||||
TaskId int64 `json:"taskId" dc:"任务ID"`
|
||||
TaskType task.TaskType `json:"taskType" dc:"任务类型"`
|
||||
TaskStatus task.TaskStatus `json:"taskStatus" dc:"任务状态"`
|
||||
}
|
||||
|
||||
type ListTaskRes struct {
|
||||
List []*TaskVO `json:"list"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
|
||||
// TaskVO 任务视图对象
|
||||
|
||||
119
model/entity/model.go
Normal file
119
model/entity/model.go
Normal file
@@ -0,0 +1,119 @@
|
||||
package entity
|
||||
|
||||
import (
|
||||
"rag/consts/model"
|
||||
|
||||
"gitea.com/red-future/common/beans"
|
||||
)
|
||||
|
||||
type modelCol struct {
|
||||
beans.SQLBaseCol
|
||||
DatasetId string
|
||||
ModelType string
|
||||
ModelName string
|
||||
ModelDesc string
|
||||
ConfigType string
|
||||
ConfigContent string
|
||||
}
|
||||
|
||||
var ModelCol = modelCol{
|
||||
SQLBaseCol: beans.DefSQLBaseCol,
|
||||
DatasetId: "dataset_id",
|
||||
ModelType: "model_type",
|
||||
ModelName: "model_name",
|
||||
ModelDesc: "model_desc",
|
||||
ConfigType: "config_type",
|
||||
ConfigContent: "config_content",
|
||||
}
|
||||
|
||||
type Model struct {
|
||||
beans.SQLBaseDO `orm:",inline"`
|
||||
DatasetId int64 `orm:"dataset_id" json:"datasetId" dc:"数据集ID"`
|
||||
ModelType model.ModelType `orm:"model_type" json:"modelType" dc:"模型类型"` // 向量/对话
|
||||
ModelName string `orm:"model_name" json:"modelName" dc:"模型名称"`
|
||||
ModelDesc string `orm:"model_desc" json:"modelDesc" dc:"模型描述"`
|
||||
ConfigType model.ModelConfigType `orm:"config_type" json:"configType" dc:"配置类型"` // ark/ollama等
|
||||
ConfigContent map[string]interface{} `orm:"config_content" json:"configContent" dc:"配置详情"` // 存JSON
|
||||
}
|
||||
|
||||
// -------------------------- 通用配置结构体（抽离重复字段）--------------------------
|
||||
|
||||
// OllamaConfig 通用配置（向量/对话完全一致）
|
||||
type OllamaConfig struct {
|
||||
BaseURL string `json:"base_url"`
|
||||
Model string `json:"model"`
|
||||
}
|
||||
|
||||
// OpenAIConfig 通用配置
|
||||
type OpenAIConfig struct {
|
||||
APIKey string `json:"api_key"`
|
||||
Model string `json:"model"`
|
||||
ByAzure bool `json:"by_azure"`
|
||||
BaseURL string `json:"base_url"`
|
||||
APIVersion string `json:"api_version"`
|
||||
}
|
||||
|
||||
// QianfanConfig 千帆通用配置
|
||||
type QianfanConfig struct {
|
||||
AccessKey string `json:"access_key"`
|
||||
SecretKey string `json:"secret_key"`
|
||||
Model string `json:"model"`
|
||||
}
|
||||
|
||||
// ArkConfig 通用配置
|
||||
type ArkConfig struct {
|
||||
APIKey string `json:"api_key"`
|
||||
Model string `json:"model"`
|
||||
}
|
||||
|
||||
// -------------------------- 向量模型配置 --------------------------
|
||||
|
||||
type VectorModelConfigOllama = OllamaConfig // 直接复用
|
||||
type VectorModelConfigOpenAI = OpenAIConfig // 直接复用
|
||||
type VectorModelConfigQianfan = QianfanConfig // 直接复用
|
||||
|
||||
type VectorModelConfigArk struct {
|
||||
ArkConfig
|
||||
APIType string `json:"api_type"`
|
||||
}
|
||||
|
||||
type VectorModelConfigTencentCloud struct {
|
||||
SecretID string `json:"secret_id"`
|
||||
SecretKey string `json:"secret_key"`
|
||||
Region string `json:"region"`
|
||||
}
|
||||
|
||||
type VectorModelConfigDashScope struct {
|
||||
APIKey string `json:"api_key"`
|
||||
Model string `json:"model"`
|
||||
}
|
||||
|
||||
// -------------------------- 对话模型配置 --------------------------
|
||||
|
||||
type ChatModelConfigArk = ArkConfig // 直接复用
|
||||
type ChatModelConfigArkBot = ArkConfig // 直接复用
|
||||
type ChatModelConfigOllama = OllamaConfig // 直接复用
|
||||
type ChatModelConfigOpenAI = OpenAIConfig // 直接复用
|
||||
type ChatModelConfigQianfan = QianfanConfig // 直接复用
|
||||
|
||||
type ChatModelConfigClaude struct {
|
||||
ByBedrock bool `json:"by_bedrock"`
|
||||
AccessKey string `json:"access_key"`
|
||||
SecretAccessKey string `json:"secret_access_key"`
|
||||
Region string `json:"region"`
|
||||
APIKey string `json:"api_key"`
|
||||
Model string `json:"model"`
|
||||
BaseURL string `json:"base_url"`
|
||||
}
|
||||
|
||||
type ChatModelConfigDeepSeek struct {
|
||||
APIKey string `json:"api_key"`
|
||||
Model string `json:"model"`
|
||||
BaseURL string `json:"base_url"`
|
||||
}
|
||||
|
||||
type ChatModelConfigQwen struct {
|
||||
APIKey string `json:"api_key"`
|
||||
Model string `json:"model"`
|
||||
BaseURL string `json:"base_url"`
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"rag/common/eino"
|
||||
"rag/consts/document"
|
||||
"rag/consts/keyword"
|
||||
"rag/consts/model"
|
||||
"rag/consts/public"
|
||||
"rag/consts/task"
|
||||
"rag/dao"
|
||||
@@ -22,10 +23,8 @@ import (
|
||||
"github.com/bjang03/gmq/mq"
|
||||
"github.com/bjang03/gmq/types"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"github.com/gogf/gf/v2/container/gvar"
|
||||
"github.com/gogf/gf/v2/crypto/gmd5"
|
||||
"github.com/gogf/gf/v2/database/gdb"
|
||||
"github.com/gogf/gf/v2/database/gredis"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/gogf/gf/v2/os/grpool"
|
||||
"github.com/gogf/gf/v2/util/gconv"
|
||||
@@ -37,7 +36,35 @@ type documentService struct{}
|
||||
|
||||
// Create 创建文件
|
||||
func (s *documentService) Create(ctx context.Context, req *dto.CreateDocumentReq) (res *dto.CreateDocumentRes, err error) {
|
||||
err = gfdb.DB(ctx).Transaction(ctx, func(ctx context.Context, tx gdb.TX) (err error) {
|
||||
err = gfdb.DB(ctx, public.DbNameKnowledge).Transaction(ctx, func(ctx context.Context, tx gdb.TX) (err error) {
|
||||
doc, err := dao.Document.Get(ctx, &dto.GetDocumentReq{
|
||||
DatasetId: req.DatasetId,
|
||||
Title: req.Title,
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if !g.IsEmpty(doc) && doc.Id > 0 {
|
||||
_, err = dao.Keyword.Delete(ctx, &dto.DeleteKeywordReq{
|
||||
DocumentId: doc.Id,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = dao.DocumentVector.Delete(ctx, &dto.DeleteDocumentVectorReq{
|
||||
DocumentId: doc.Id,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = dao.Document.Delete(ctx, &dto.DeleteDocumentReq{
|
||||
Id: doc.Id,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
var id int64
|
||||
id, err = dao.Document.Insert(ctx, req)
|
||||
if err != nil {
|
||||
@@ -74,11 +101,11 @@ func (s *documentService) Update(ctx context.Context, req *dto.UpdateDocumentReq
|
||||
|
||||
// Delete 删除文件
|
||||
func (s *documentService) Delete(ctx context.Context, req *dto.DeleteDocumentReq) (err error) {
|
||||
docs, err := dao.Document.GetByID(ctx, &dto.GetDocumentReq{Id: req.Id})
|
||||
docs, err := dao.Document.Get(ctx, &dto.GetDocumentReq{Id: req.Id})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = gfdb.DB(ctx).Transaction(ctx, func(ctx context.Context, tx gdb.TX) (err error) {
|
||||
err = gfdb.DB(ctx, public.DbNameKnowledge).Transaction(ctx, func(ctx context.Context, tx gdb.TX) (err error) {
|
||||
datasetReq := &dto.UpdateDatasetReq{
|
||||
Id: docs.DatasetId,
|
||||
DocumentCount: -1,
|
||||
@@ -92,6 +119,18 @@ func (s *documentService) Delete(ctx context.Context, req *dto.DeleteDocumentReq
|
||||
return
|
||||
}
|
||||
|
||||
if _, err = dao.Keyword.Delete(ctx, &dto.DeleteKeywordReq{
|
||||
DocumentId: docs.Id,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err = dao.DocumentVector.Delete(ctx, &dto.DeleteDocumentVectorReq{
|
||||
DocumentId: docs.Id,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err = dao.Task.DeleteByTaskId(ctx, &dto.DeleteTaskByTaskIdReq{
|
||||
TaskId: docs.Id,
|
||||
}); err != nil {
|
||||
@@ -106,7 +145,7 @@ func (s *documentService) Delete(ctx context.Context, req *dto.DeleteDocumentReq
|
||||
|
||||
// Get 获取文件详情
|
||||
func (s *documentService) Get(ctx context.Context, req *dto.GetDocumentReq) (res *dto.GetDocumentRes, err error) {
|
||||
r, err := dao.Document.GetByID(ctx, req)
|
||||
r, err := dao.Document.Get(ctx, req)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -136,7 +175,7 @@ func (s *documentService) List(ctx context.Context, req *dto.ListDocumentReq) (r
|
||||
func (s *documentService) Vector(ctx context.Context, req *dto.DocumentVectorReq) (err error) {
|
||||
// 1. 查询文件信息
|
||||
documentReq := dto.GetDocumentReq{Id: req.Id}
|
||||
doc, err := dao.Document.GetByID(ctx, &documentReq)
|
||||
doc, err := dao.Document.Get(ctx, &documentReq)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -172,16 +211,13 @@ func (s *documentService) Vector(ctx context.Context, req *dto.DocumentVectorReq
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// ======================
|
||||
// 核心：grpool + g.Try 最佳实践
|
||||
// ======================
|
||||
// 使用带超时的background context，避免HTTP请求完成后context被取消
|
||||
taskCtx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||
taskCtx = context.WithValue(taskCtx, "user", user)
|
||||
// 任务1: SQL 切分文档
|
||||
// 任务1: 语义 切分文档
|
||||
grpool.Add(taskCtx, func(ctx context.Context) {
|
||||
g.TryCatch(ctx, func(ctx context.Context) {
|
||||
if innerErr := s.sqlSplitDocument(ctx, doc); innerErr != nil {
|
||||
if innerErr := s.semanticSplitDocument(ctx, doc); innerErr != nil {
|
||||
cancel()
|
||||
}
|
||||
}, func(ctx context.Context, err error) {
|
||||
@@ -189,10 +225,10 @@ func (s *documentService) Vector(ctx context.Context, req *dto.DocumentVectorReq
|
||||
})
|
||||
})
|
||||
|
||||
// 任务2: ES 切分文档
|
||||
// 任务2: 递归 切分文档
|
||||
grpool.Add(taskCtx, func(ctx context.Context) {
|
||||
g.TryCatch(ctx, func(ctx context.Context) {
|
||||
if innerErr := s.esSplitDocument(ctx, doc); innerErr != nil {
|
||||
if innerErr := s.recursiveSplitDocument(ctx, doc); innerErr != nil {
|
||||
cancel()
|
||||
}
|
||||
}, func(ctx context.Context, err error) {
|
||||
@@ -327,8 +363,8 @@ func (s *documentService) extractDocument(ctx context.Context, doc *entity.Docum
|
||||
return
|
||||
}
|
||||
|
||||
// sqlSplitDocument SQL切分（支持取消）
|
||||
func (s *documentService) sqlSplitDocument(ctx context.Context, doc *entity.Document) (err error) {
|
||||
// semanticSplitDocument 语义切分
|
||||
func (s *documentService) semanticSplitDocument(ctx context.Context, doc *entity.Document) (err error) {
|
||||
// ========== 取消检查 1：方法入口 ==========
|
||||
if ctx.Err() != nil {
|
||||
err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
|
||||
@@ -354,7 +390,7 @@ func (s *documentService) sqlSplitDocument(ctx context.Context, doc *entity.Docu
|
||||
}
|
||||
|
||||
// 2. 语义切分文件
|
||||
docsSplit, err := eino.SemanticSplitDocument(ctx, docs)
|
||||
docsSplit, err := eino.SemanticSplitDocument(ctx, docs, model.ModelConfigTypeVectorDashScope.Code()) //TODO 后续替换成本地模型
|
||||
if err != nil {
|
||||
// 写入任务进度失败 任务类型为sql存储
|
||||
err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
|
||||
@@ -394,8 +430,8 @@ func (s *documentService) sqlSplitDocument(ctx context.Context, doc *entity.Docu
|
||||
}
|
||||
|
||||
contentHash := gmd5.MustEncryptString(t.Content)
|
||||
var success bool
|
||||
success, err = s.checkRepeat(ctx, public.KnowledgeContentHashSqlKey, contentHash)
|
||||
var isNew, needCopy bool
|
||||
isNew, needCopy, err = s.checkRepeatWithDocId(ctx, public.KnowledgeContentHashSqlKey, contentHash, doc.Id)
|
||||
if err != nil {
|
||||
// 写入任务进度失败 任务类型为sql存储
|
||||
err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
|
||||
@@ -406,7 +442,7 @@ func (s *documentService) sqlSplitDocument(ctx context.Context, doc *entity.Docu
|
||||
})
|
||||
return
|
||||
}
|
||||
if !success {
|
||||
if !isNew && !needCopy {
|
||||
continue
|
||||
}
|
||||
var metaData = make(map[string]any)
|
||||
@@ -415,7 +451,13 @@ func (s *documentService) sqlSplitDocument(ctx context.Context, doc *entity.Docu
|
||||
metaData[entity.DocumentCol.DatasetId] = doc.DatasetId
|
||||
metaData[entity.DocumentVectorCol.DocumentId] = doc.Id
|
||||
metaData[entity.DocumentVectorCol.ContentHash] = contentHash
|
||||
metaData[entity.DocumentVectorCol.ChunkIndex] = gconv.Int64(i)
|
||||
metaData[entity.DocumentVectorCol.ChunkIndex] = gconv.Int64(i + 1)
|
||||
if isNew {
|
||||
metaData["isNew"] = true
|
||||
}
|
||||
if needCopy {
|
||||
metaData["isNew"] = false
|
||||
}
|
||||
t.MetaData = metaData
|
||||
docsChunk = append(docsChunk, t)
|
||||
}
|
||||
@@ -468,8 +510,8 @@ func (s *documentService) sqlSplitDocument(ctx context.Context, doc *entity.Docu
|
||||
return
|
||||
}
|
||||
|
||||
// esSplitDocument ES切分（支持取消）
|
||||
func (s *documentService) esSplitDocument(ctx context.Context, doc *entity.Document) (err error) {
|
||||
// recursiveSplitDocument 递归切分
|
||||
func (s *documentService) recursiveSplitDocument(ctx context.Context, doc *entity.Document) (err error) {
|
||||
// ========== 取消检查 1：方法入口 ==========
|
||||
if ctx.Err() != nil {
|
||||
err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
|
||||
@@ -535,8 +577,8 @@ func (s *documentService) esSplitDocument(ctx context.Context, doc *entity.Docum
|
||||
}
|
||||
|
||||
contentHash := gmd5.MustEncryptString(t.Content)
|
||||
var success bool
|
||||
success, err = s.checkRepeat(ctx, public.KnowledgeContentHashEsKey, contentHash)
|
||||
var isNew, needCopy bool
|
||||
isNew, needCopy, err = s.checkRepeatWithDocId(ctx, public.KnowledgeContentHashEsKey, contentHash, doc.Id)
|
||||
if err != nil {
|
||||
// 写入任务进度失败 任务类型为es存储
|
||||
err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
|
||||
@@ -547,7 +589,7 @@ func (s *documentService) esSplitDocument(ctx context.Context, doc *entity.Docum
|
||||
})
|
||||
return
|
||||
}
|
||||
if !success {
|
||||
if !isNew && !needCopy {
|
||||
continue
|
||||
}
|
||||
meiliDocs = append(meiliDocs, map[string]interface{}{
|
||||
@@ -556,7 +598,7 @@ func (s *documentService) esSplitDocument(ctx context.Context, doc *entity.Docum
|
||||
entity.DocumentVectorCol.DocumentId: doc.Id,
|
||||
entity.DocumentVectorCol.Content: t.Content,
|
||||
entity.DocumentVectorCol.ContentHash: contentHash,
|
||||
entity.DocumentVectorCol.ChunkIndex: i,
|
||||
entity.DocumentVectorCol.ChunkIndex: i + 1,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -632,6 +674,7 @@ func (s *documentService) getHistoryData(ctx context.Context, doc *entity.Docume
|
||||
|
||||
// 3. Redis 无数据：根据 contentKey 类型选择查询方式
|
||||
var dictData = make([]*dto.DocumentVectorRPC, 0)
|
||||
|
||||
if public.KnowledgeContentHashSqlKey == contentKey {
|
||||
// SQL 方式：调用 HTTP 接口查询
|
||||
dictData, err = s.getHistoryDataFromHttp(ctx, doc)
|
||||
@@ -643,20 +686,16 @@ func (s *documentService) getHistoryData(ctx context.Context, doc *entity.Docume
|
||||
return err
|
||||
}
|
||||
|
||||
// 4. 把查询到的数据写入 Redis（600s过期）
|
||||
for _, item := range dictData {
|
||||
// 去除可能的 JSON 引号
|
||||
contentHash := strings.Trim(item.ContentHash, `"`)
|
||||
key := fmt.Sprintf(contentKey, contentHash)
|
||||
_, err = g.Redis().Set(ctx, key, true, gredis.SetOption{
|
||||
TTLOption: gredis.TTLOption{
|
||||
EX: gconv.PtrInt64(600),
|
||||
},
|
||||
NX: true,
|
||||
})
|
||||
// SAdd：把文档ID加入集合（自动去重，可存多个）
|
||||
_, err = g.Redis().SAdd(ctx, key, item.DocumentId)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// 设置过期时间
|
||||
_, _ = g.Redis().Expire(ctx, key, 600)
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -672,8 +711,10 @@ func (s *documentService) getHistoryDataFromHttp(ctx context.Context, doc *entit
|
||||
// 调用接口获取数据
|
||||
res, _, err := dao.DocumentVector.List(ctx, &dto.ListDocumentVectorReq{
|
||||
DatasetId: doc.DatasetId,
|
||||
Status: gconv.PtrInt8(1),
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = gconv.Struct(res, &dictData)
|
||||
return
|
||||
}
|
||||
@@ -705,17 +746,39 @@ func (s *documentService) getHistoryDataFromMeilisearch(ctx context.Context, doc
|
||||
return
|
||||
}
|
||||
|
||||
// checkRepeat 检查是否重复
|
||||
func (s *documentService) checkRepeat(ctx context.Context, contentKey, contentHash string) (success bool, err error) {
|
||||
var val *gvar.Var
|
||||
if val, err = g.Redis().Set(ctx, fmt.Sprintf(contentKey, contentHash), true, gredis.SetOption{
|
||||
TTLOption: gredis.TTLOption{
|
||||
EX: gconv.PtrInt64(600),
|
||||
},
|
||||
NX: true,
|
||||
}); err != nil {
|
||||
return
|
||||
// checkRepeatWithDocId 正确版：检查当前文档是否已存在该分片
|
||||
// 返回：isNew(是否需要生成向量)、isCrossDoc(是否跨文档需拷贝)、err
|
||||
func (s *documentService) checkRepeatWithDocId(ctx context.Context, contentKey string, contentHash string, currentDocId int64) (isNew bool, needCopy bool, err error) {
|
||||
key := fmt.Sprintf(contentKey, contentHash)
|
||||
|
||||
// 1. 检查当前文档ID是否在集合中
|
||||
exists, err := g.Redis().SIsMember(ctx, key, currentDocId)
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
success = val.Bool()
|
||||
return
|
||||
|
||||
// 情况1：当前文档已存在 → 完全跳过，不生成、不拷贝
|
||||
if !g.IsEmpty(exists) {
|
||||
return false, false, nil
|
||||
}
|
||||
|
||||
// 2. 检查 key 是否存在（是否有任何文档拥有该分片）
|
||||
keyExists, err := g.Redis().Exists(ctx, key)
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
|
||||
// 情况2：key 不存在 = 全新数据 → 需要生成向量
|
||||
if g.IsEmpty(keyExists) {
|
||||
// 把当前文档ID加入集合
|
||||
_, err = g.Redis().SAdd(ctx, key, currentDocId)
|
||||
_, _ = g.Redis().Expire(ctx, key, 600)
|
||||
return true, false, err
|
||||
}
|
||||
|
||||
// 情况3：key 存在，但当前文档不在集合中 = 跨文档重复 → 不生成，需拷贝
|
||||
// 把当前文档ID加入集合（记录归属关系）
|
||||
_, err = g.Redis().SAdd(ctx, key, currentDocId)
|
||||
_, _ = g.Redis().Expire(ctx, key, 600)
|
||||
return false, true, err
|
||||
}
|
||||
|
||||
@@ -4,17 +4,18 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"rag/common/eino"
|
||||
"rag/consts/model"
|
||||
"rag/consts/task"
|
||||
"rag/dao"
|
||||
"rag/model/dto"
|
||||
"rag/model/entity"
|
||||
|
||||
"gitea.com/red-future/common/beans"
|
||||
"github.com/cloudwego/eino/components/indexer"
|
||||
"github.com/cloudwego/eino/components/retriever"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/gogf/gf/v2/util/gconv"
|
||||
"github.com/pgvector/pgvector-go"
|
||||
)
|
||||
|
||||
var DocumentVector = new(documentVectorService)
|
||||
@@ -23,23 +24,32 @@ type documentVectorService struct{}
|
||||
|
||||
// Query 执行RAG查询
|
||||
func (s *documentVectorService) Query(ctx context.Context, req *dto.RAGQueryReq) (*dto.RAGQueryRes, error) {
|
||||
if req.TopK <= 0 {
|
||||
req.TopK = 5
|
||||
|
||||
modelInfo, err := dao.Model.Get(ctx, &dto.GetModelReq{
|
||||
ModelType: model.ModelTypeChat.Code(),
|
||||
})
|
||||
if err != nil {
|
||||
g.Log().Errorf(ctx, "获取模型失败: %v", err)
|
||||
return nil, fmt.Errorf("获取模型失败: %w", err)
|
||||
}
|
||||
if modelInfo == nil {
|
||||
g.Log().Errorf(ctx, "模型不存在: %v", model.ModelTypeChat.Code())
|
||||
return nil, fmt.Errorf("模型不存在: %w", err)
|
||||
}
|
||||
|
||||
// 4. 使用向量检索器进行查询
|
||||
r, err := eino.NewPGVectorRetriever(&eino.PGVectorRetrieverConfig{
|
||||
Embedder: eino.EmbedderDashscope,
|
||||
r, err := eino.NewPGVectorRetriever(ctx, &eino.PGVectorRetrieverConfig{
|
||||
DefaultTopK: req.TopK,
|
||||
})
|
||||
}, model.ModelConfigTypeVectorDashScope.Code()) //TODO 后续替换成本地模型
|
||||
if err != nil {
|
||||
g.Log().Errorf(ctx, "初始化向量检索器失败: %v", err)
|
||||
return nil, fmt.Errorf("初始化向量检索器失败: %w", err)
|
||||
}
|
||||
|
||||
// 5. 执行向量检索
|
||||
docs, err := r.Retrieve(ctx, req.Content, retriever.WithEmbedding(eino.EmbedderDashscope), retriever.WithDSLInfo(map[string]any{
|
||||
"dataset_ids": req.DatasetIds,
|
||||
docs, err := r.Retrieve(ctx, req.Content, retriever.WithDSLInfo(map[string]any{
|
||||
"dataset_ids": req.DatasetIds,
|
||||
"document_ids": req.DocumentIds,
|
||||
}))
|
||||
if err != nil {
|
||||
g.Log().Errorf(ctx, "向量检索失败: %v", err)
|
||||
@@ -53,7 +63,7 @@ func (s *documentVectorService) Query(ctx context.Context, req *dto.RAGQueryReq)
|
||||
return nil, fmt.Errorf("转换历史消息失败: %w", err)
|
||||
}
|
||||
|
||||
replyMsg, err := eino.NewChatModel(ctx, req.Content, docs, messages)
|
||||
replyMsg, err := eino.NewChatModel(ctx, req.Content, docs, messages, modelInfo.ConfigType)
|
||||
if err != nil {
|
||||
g.Log().Errorf(ctx, "向量检索失败: %v", err)
|
||||
return nil, fmt.Errorf("向量检索失败: %w", err)
|
||||
@@ -98,26 +108,108 @@ func (s *documentVectorService) DocsChunkMsg(ctx context.Context, msg any) (err
|
||||
TenantId: gconv.Uint64(docs[0].MetaData[entity.DocumentVectorCol.TenantId]),
|
||||
UserName: gconv.String(docs[0].MetaData[entity.DocumentVectorCol.Creator]),
|
||||
})
|
||||
idx := eino.NewPGVectorIndexer(&eino.PGVectorIndexerOptions{
|
||||
BatchSize: 10,
|
||||
})
|
||||
|
||||
documentId := gconv.Int64(docs[0].MetaData[entity.DocumentVectorCol.DocumentId])
|
||||
rows, err := idx.Store(ctx, docs, indexer.WithEmbedding(eino.EmbedderDashscope))
|
||||
if err != nil || rows == 0 {
|
||||
g.Log().Error(ctx, "DocsChunkMsg rows: , err:", rows, err)
|
||||
// 写入任务进度失败 任务类型为sql存储
|
||||
remark := " 向量存储数量: " + gconv.String(rows)
|
||||
if err != nil {
|
||||
remark = "向量存储失败: " + err.Error()
|
||||
|
||||
var docsStore = make([]*schema.Document, 0)
|
||||
var docsInsert = make([]*dto.VectorDocumentVectorMsg, 0)
|
||||
for _, doc := range docs {
|
||||
if gconv.Bool(doc.MetaData["isNew"]) {
|
||||
docsStore = append(docsStore, doc)
|
||||
} else {
|
||||
ck := new(dto.VectorDocumentVectorMsg)
|
||||
err = gconv.Struct(doc.MetaData, ck)
|
||||
ck.Content = doc.Content
|
||||
ck.VectorStatus = gconv.PtrInt8(1)
|
||||
ck.Status = gconv.PtrInt8(1)
|
||||
docsInsert = append(docsInsert, ck)
|
||||
}
|
||||
err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
|
||||
TaskId: documentId,
|
||||
TaskType: task.TaskTypeGenerateVector,
|
||||
Status: task.TaskStatusFailed,
|
||||
Remark: remark,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
if !g.IsEmpty(docsStore) {
|
||||
idx := eino.NewPGVectorIndexer(&eino.PGVectorIndexerOptions{
|
||||
BatchSize: 10,
|
||||
})
|
||||
var rows int64
|
||||
rows, err = idx.Store(ctx, docsStore, model.ModelConfigTypeVectorDashScope.Code()) //TODO 后续替换成本地模型
|
||||
|
||||
if err != nil || rows == 0 {
|
||||
g.Log().Error(ctx, "DocsChunkMsg rows: , err:", rows, err)
|
||||
// 写入任务进度失败 任务类型为sql存储
|
||||
remark := " 向量存储数量: " + gconv.String(rows)
|
||||
if err != nil {
|
||||
remark = "向量存储失败: " + err.Error()
|
||||
}
|
||||
err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
|
||||
TaskId: documentId,
|
||||
TaskType: task.TaskTypeGenerateVector,
|
||||
Status: task.TaskStatusFailed,
|
||||
Remark: remark,
|
||||
})
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if !g.IsEmpty(docsInsert) {
|
||||
// 1. 提取所有 contentHash
|
||||
contentHashs := make([]string, 0, len(docsInsert))
|
||||
for _, d := range docsInsert {
|
||||
contentHashs = append(contentHashs, d.ContentHash)
|
||||
}
|
||||
|
||||
// 2. 分页查询已存在的向量（一页1000，避免大查询）
|
||||
var existVectors []*entity.DocumentVector
|
||||
for page := 1; ; page++ {
|
||||
res, total, err := dao.DocumentVector.List(ctx, &dto.ListDocumentVectorReq{
|
||||
Page: &beans.Page{PageSize: 1000, PageNum: int64(page)},
|
||||
ContentHashs: contentHashs,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(res) == 0 {
|
||||
break
|
||||
}
|
||||
existVectors = append(existVectors, res...)
|
||||
if len(existVectors) >= total {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// 3. 构建哈希 -> 向量 的映射表（O(1) 查找，性能提升巨大）
|
||||
vectorMap := make(map[string]pgvector.Vector, len(existVectors))
|
||||
for _, v := range existVectors {
|
||||
vectorMap[v.ContentHash] = v.Vector
|
||||
}
|
||||
|
||||
// 4. 回填向量 + 过滤掉数据库已存在的数据（避免重复插入）
|
||||
for _, d := range docsInsert {
|
||||
// 回填已有向量
|
||||
if vec, ok := vectorMap[d.ContentHash]; ok {
|
||||
d.Vector = vec
|
||||
}
|
||||
}
|
||||
|
||||
var rows int64
|
||||
rows, err = dao.DocumentVector.BatchInsert(ctx, docsInsert)
|
||||
|
||||
if err != nil || rows == 0 {
|
||||
g.Log().Error(ctx, "DocsChunkMsg rows: , err:", rows, err)
|
||||
// 写入任务进度失败 任务类型为sql存储
|
||||
remark := " 向量存储数量: " + gconv.String(rows)
|
||||
if err != nil {
|
||||
remark = "向量存储失败: " + err.Error()
|
||||
}
|
||||
err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
|
||||
TaskId: documentId,
|
||||
TaskType: task.TaskTypeGenerateVector,
|
||||
Status: task.TaskStatusFailed,
|
||||
Remark: remark,
|
||||
})
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// 写入任务进度成功 任务类型为sql存储
|
||||
err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
|
||||
TaskId: documentId,
|
||||
|
||||
299
service/model.go
Normal file
299
service/model.go
Normal file
@@ -0,0 +1,299 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"rag/common/eino"
|
||||
"rag/consts/model"
|
||||
"rag/consts/task"
|
||||
"rag/dao"
|
||||
"rag/model/dto"
|
||||
"rag/model/entity"
|
||||
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/gogf/gf/v2/util/gconv"
|
||||
)
|
||||
|
||||
var ModelService = new(modelService)
|
||||
|
||||
type modelService struct{}
|
||||
|
||||
// GetModelAllEnums 获取模型全量枚举（模型类型 + 配置类型 合并）
|
||||
func (s *modelService) GetModelAllEnums(ctx context.Context, req *dto.GetModelAllEnumsReq) (res *dto.GetModelEnumRes, err error) {
|
||||
_, _ = ctx, req
|
||||
res = new(dto.GetModelEnumRes)
|
||||
|
||||
// 获取所有模型类型
|
||||
modelTypeRes := model.GetAllModelTypeEnums()
|
||||
|
||||
var options []dto.ModelEnumOption
|
||||
for _, mt := range modelTypeRes.Options {
|
||||
// 构造 modelType
|
||||
modelTypeStr := gconv.String(mt.Key)
|
||||
modelType := model.ModelType(gconv.PtrString(modelTypeStr))
|
||||
|
||||
// 获取对应配置类型
|
||||
configRes := model.GetAllModelConfigTypeEnums(modelType)
|
||||
|
||||
// 把 configRes.Options 转成目标类型
|
||||
var configList []dto.ModelKeyValue
|
||||
err = gconv.Structs(configRes.Options, &configList)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
options = append(options, dto.ModelEnumOption{
|
||||
Key: mt.Key,
|
||||
Value: mt.Value,
|
||||
ConfigTypes: configList,
|
||||
})
|
||||
}
|
||||
|
||||
res.Options = options
|
||||
return
|
||||
}
|
||||
|
||||
func (s *modelService) GetModelConfigFormFields(ctx context.Context, req *dto.GetModelConfigFormFieldsReq) (*dto.GetModelConfigFormFieldsRes, error) {
|
||||
_ = ctx
|
||||
|
||||
fields := make([]map[string]interface{}, 0)
|
||||
|
||||
// ===================== 固定基础字段（CreateModelReq 前4个）=====================
|
||||
// 1. 模型类型：固定只读字段
|
||||
fields = append(fields, map[string]interface{}{
|
||||
"name": "modelType",
|
||||
"label": "模型类型",
|
||||
"type": "text",
|
||||
"disabled": true,
|
||||
"required": true,
|
||||
"value": model.GetModelTypeDescByCode(req.ModelType),
|
||||
})
|
||||
|
||||
var configTypeValue = "未知类型"
|
||||
if *req.ModelType == *model.ModelTypeVector.Code() {
|
||||
configTypeValue = model.GetVectorDescByCode(req.ConfigType)
|
||||
} else if *req.ModelType == *model.ModelTypeChat.Code() {
|
||||
configTypeValue = model.GetChatDescByCode(req.ConfigType)
|
||||
}
|
||||
|
||||
// 2. 配置类型：固定只读字段
|
||||
fields = append(fields, map[string]interface{}{
|
||||
"name": "configType",
|
||||
"label": "配置类型",
|
||||
"type": "text",
|
||||
"disabled": true,
|
||||
"required": true,
|
||||
"value": configTypeValue,
|
||||
})
|
||||
|
||||
// 3. 基础信息
|
||||
fields = append(fields, []map[string]interface{}{
|
||||
{
|
||||
"name": "modelName",
|
||||
"label": "模型名称",
|
||||
"type": "input",
|
||||
"required": true,
|
||||
"placeholder": "例如：DeepSeek 对话模型",
|
||||
},
|
||||
{
|
||||
"name": "modelDesc",
|
||||
"label": "模型描述",
|
||||
"type": "textarea",
|
||||
"required": false,
|
||||
},
|
||||
}...)
|
||||
|
||||
// 4. 通用模型名称字段
|
||||
fields = append(fields, map[string]interface{}{
|
||||
"name": "model",
|
||||
"label": "模型类型",
|
||||
"type": "input",
|
||||
"required": true,
|
||||
"placeholder": "例如：deepseek-chat / text-embedding-3-small",
|
||||
})
|
||||
|
||||
// ===================== 动态配置内容 ConfigContent =====================
|
||||
|
||||
// 根据模型类型 + 配置类型生成动态字段
|
||||
switch *req.ModelType {
|
||||
case *model.ModelTypeChat.Code():
|
||||
switch *req.ConfigType {
|
||||
case *model.ModelConfigTypeChatArk.Code():
|
||||
fields = append(fields, map[string]interface{}{"name": "api_key", "label": "API Key", "type": "input", "required": true})
|
||||
|
||||
case *model.ModelConfigTypeChatArkBot.Code():
|
||||
fields = append(fields, map[string]interface{}{"name": "api_key", "label": "API Key", "type": "input", "required": true})
|
||||
|
||||
case *model.ModelConfigTypeChatClaude.Code():
|
||||
fields = append(fields, []map[string]interface{}{
|
||||
{"name": "by_bedrock", "label": "使用 AWS Bedrock", "type": "switch", "default": true},
|
||||
{"name": "access_key", "label": "Access Key", "type": "input"},
|
||||
{"name": "secret_access_key", "label": "Secret Access Key", "type": "input"},
|
||||
{"name": "region", "label": "Region", "type": "input"},
|
||||
{"name": "api_key", "label": "API Key", "type": "input"},
|
||||
{"name": "base_url", "label": "Base URL", "type": "input"},
|
||||
}...)
|
||||
|
||||
case *model.ModelConfigTypeChatDeepSeek.Code():
|
||||
fields = append(fields, []map[string]interface{}{
|
||||
{"name": "api_key", "label": "API Key", "type": "input", "required": true},
|
||||
{"name": "base_url", "label": "Base URL", "type": "input", "default": "https://api.deepseek.com"},
|
||||
}...)
|
||||
|
||||
case *model.ModelConfigTypeChatOllama.Code():
|
||||
fields = append(fields, map[string]interface{}{"name": "base_url", "label": "Base URL", "type": "input", "required": true, "default": "http://127.0.0.1:11434"})
|
||||
|
||||
case *model.ModelConfigTypeChatOpenAI.Code():
|
||||
fields = append(fields, []map[string]interface{}{
|
||||
{"name": "api_key", "label": "API Key", "type": "input", "required": true},
|
||||
{"name": "by_azure", "label": "使用 Azure", "type": "switch", "default": true},
|
||||
{"name": "base_url", "label": "Base URL", "type": "input"},
|
||||
{"name": "api_version", "label": "API Version", "type": "input"},
|
||||
}...)
|
||||
|
||||
case *model.ModelConfigTypeChatQianfan.Code():
|
||||
fields = append(fields, []map[string]interface{}{
|
||||
{"name": "access_key", "label": "Access Key", "type": "input", "required": true},
|
||||
{"name": "secret_key", "label": "Secret Key", "type": "input", "required": true},
|
||||
}...)
|
||||
|
||||
case *model.ModelConfigTypeChatQwen.Code():
|
||||
fields = append(fields, []map[string]interface{}{
|
||||
{"name": "api_key", "label": "API Key", "type": "input", "required": true},
|
||||
{"name": "base_url", "label": "Base URL", "type": "input"},
|
||||
}...)
|
||||
}
|
||||
|
||||
case *model.ModelTypeVector.Code():
|
||||
switch *req.ConfigType {
|
||||
case *model.ModelConfigTypeVectorArk.Code():
|
||||
fields = append(fields, []map[string]interface{}{
|
||||
{"name": "api_key", "label": "API Key", "type": "input", "required": true},
|
||||
{"name": "api_type", "label": "API Type", "type": "input"},
|
||||
}...)
|
||||
|
||||
case *model.ModelConfigTypeVectorOllama.Code():
|
||||
fields = append(fields, map[string]interface{}{"name": "base_url", "label": "Base URL", "type": "input", "required": true, "default": "http://127.0.0.1:11434"})
|
||||
|
||||
case *model.ModelConfigTypeVectorOpenAI.Code():
|
||||
fields = append(fields, []map[string]interface{}{
|
||||
{"name": "api_key", "label": "API Key", "type": "input", "required": true},
|
||||
{"name": "by_azure", "label": "使用 Azure", "type": "switch", "default": true},
|
||||
{"name": "base_url", "label": "Base URL", "type": "input"},
|
||||
{"name": "api_version", "label": "API Version", "type": "input"},
|
||||
}...)
|
||||
|
||||
case *model.ModelConfigTypeVectorQianfan.Code():
|
||||
fields = append(fields, []map[string]interface{}{
|
||||
{"name": "access_key", "label": "Access Key", "type": "input", "required": true},
|
||||
{"name": "secret_key", "label": "Secret Key", "type": "input", "required": true},
|
||||
}...)
|
||||
|
||||
case *model.ModelConfigTypeVectorTencentCloud.Code():
|
||||
fields = append(fields, []map[string]interface{}{
|
||||
{"name": "secret_id", "label": "Secret ID", "type": "input", "required": true},
|
||||
{"name": "secret_key", "label": "Secret Key", "type": "input", "required": true},
|
||||
{"name": "region", "label": "Region", "type": "input", "required": true, "default": "ap-beijing"},
|
||||
}...)
|
||||
case *model.ModelConfigTypeVectorDashScope.Code():
|
||||
fields = append(fields, map[string]interface{}{"name": "api_key", "label": "API Key", "type": "input", "required": true})
|
||||
}
|
||||
}
|
||||
|
||||
return &dto.GetModelConfigFormFieldsRes{
|
||||
ModelType: req.ModelType,
|
||||
ConfigType: req.ConfigType,
|
||||
Fields: fields,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *modelService) Create(ctx context.Context, req *dto.CreateModelReq) (res *dto.CreateModelRes, err error) {
|
||||
count, err := dao.Model.Count(ctx, &dto.GetModelReq{
|
||||
ModelType: req.ModelType,
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if count > 0 {
|
||||
err = gerror.New("模型配置已存在")
|
||||
return
|
||||
}
|
||||
var id int64
|
||||
id, err = dao.Model.Insert(ctx, req)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
res = &dto.CreateModelRes{Id: id}
|
||||
err = s.refresh(ctx, id)
|
||||
return
|
||||
}
|
||||
|
||||
func (s *modelService) Update(ctx context.Context, req *dto.UpdateModelReq) (err error) {
|
||||
count, err := dao.Task.Count(ctx, &dto.GetTaskReq{
|
||||
TaskStatus: task.TaskStatusRunning,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !g.IsEmpty(count) {
|
||||
err = gerror.New("任务正在执行中，模型配置暂时不可修改，请稍后再试")
|
||||
return
|
||||
}
|
||||
var updateCount int64
|
||||
updateCount, err = dao.Model.Update(ctx, req)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if !g.IsEmpty(updateCount) {
|
||||
err = s.refresh(ctx, req.Id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (s *modelService) refresh(ctx context.Context, id int64) (err error) {
|
||||
var modelDO *entity.Model
|
||||
modelDO, err = dao.Model.Get(ctx, &dto.GetModelReq{
|
||||
Id: id,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *modelDO.ModelType == *model.ModelTypeChat.Code() {
|
||||
if err = eino.RefreshTenantChatModel(ctx, modelDO); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if *modelDO.ModelType == *model.ModelTypeVector.Code() {
|
||||
if err = eino.RefreshTenantEmbedder(ctx, modelDO); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (s *modelService) Delete(ctx context.Context, req *dto.DeleteModelReq) (err error) {
|
||||
_, err = dao.Model.Delete(ctx, req)
|
||||
return
|
||||
}
|
||||
|
||||
func (s *modelService) Get(ctx context.Context, req *dto.GetModelReq) (res *dto.ModelVO, err error) {
|
||||
r, err := dao.Model.Get(ctx, req)
|
||||
err = gconv.Struct(r, &res)
|
||||
return
|
||||
}
|
||||
|
||||
func (s *modelService) List(ctx context.Context, req *dto.ListModelReq) (res *dto.ListModelRes, err error) {
|
||||
list, total, err := dao.Model.List(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res = &dto.ListModelRes{
|
||||
Total: total,
|
||||
}
|
||||
err = gconv.Struct(list, &res.List)
|
||||
return
|
||||
}
|
||||
@@ -2,6 +2,8 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"rag/consts/document"
|
||||
"rag/consts/public"
|
||||
"rag/consts/task"
|
||||
"rag/dao"
|
||||
"rag/model/dto"
|
||||
@@ -37,7 +39,7 @@ func (s *taskService) WriteTaskProgress(ctx context.Context, req *dto.WriteTaskP
|
||||
TaskType: req.TaskType,
|
||||
Status: req.Status,
|
||||
})
|
||||
completed = IsAllSubTasksCompleted(taskVO)
|
||||
completed = IsAllSubTasks(taskVO, task.TaskStatusCompleted)
|
||||
}
|
||||
|
||||
// 1. 查询是否已存在该文档的该类型任务
|
||||
@@ -49,7 +51,7 @@ func (s *taskService) WriteTaskProgress(ctx context.Context, req *dto.WriteTaskP
|
||||
g.Log().Errorf(ctx, "查询任务失败: %v", err)
|
||||
return err
|
||||
}
|
||||
err = gfdb.DB(ctx).Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
|
||||
err = gfdb.DB(ctx, public.DbNameKnowledge).Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
|
||||
// 2. 如果不存在，则创建新任务
|
||||
if g.IsEmpty(existTask) {
|
||||
createReq := &dto.CreateTaskReq{
|
||||
@@ -80,17 +82,36 @@ func (s *taskService) WriteTaskProgress(ctx context.Context, req *dto.WriteTaskP
|
||||
Status: task.TaskStatusCompleted,
|
||||
Remark: "文档解析完成",
|
||||
})
|
||||
if err != nil {
|
||||
g.Log().Errorf(ctx, "更新任务失败: %v", err)
|
||||
return err
|
||||
}
|
||||
_, err = dao.Document.Update(ctx, &dto.UpdateDocumentReq{
|
||||
Id: req.TaskId,
|
||||
VectorStatus: document.VectorStatusCompleted.Code(),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if task.TaskStatusFailed == req.Status {
|
||||
_, err = dao.Document.Update(ctx, &dto.UpdateDocumentReq{
|
||||
Id: req.TaskId,
|
||||
VectorStatus: document.VectorStatusFailed.Code(),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// IsAllSubTasksCompleted 判断三个子任务是否全部完成
|
||||
// 参数：传入当前文档的所有子任务列表
|
||||
func IsAllSubTasksCompleted(subTasks []*dto.TaskVO) bool {
|
||||
// IsAllSubTasks 判断三个子任务
|
||||
func IsAllSubTasks(subTasks []*dto.TaskVO, taskStatus task.TaskStatus) bool {
|
||||
// 必须包含 3 种任务类型
|
||||
hasKeywords := false
|
||||
hasVector := false
|
||||
@@ -98,7 +119,7 @@ func IsAllSubTasksCompleted(subTasks []*dto.TaskVO) bool {
|
||||
|
||||
for _, t := range subTasks {
|
||||
// 子任务必须是【已完成】状态才计数
|
||||
if t.Status == task.TaskStatusCompleted {
|
||||
if t.Status == taskStatus {
|
||||
switch t.TaskType {
|
||||
case task.TaskTypeExtractKeywords:
|
||||
hasKeywords = true
|
||||
@@ -113,3 +134,15 @@ func IsAllSubTasksCompleted(subTasks []*dto.TaskVO) bool {
|
||||
// 三个任务全部完成 → 返回true
|
||||
return hasKeywords && hasVector && hasFullText
|
||||
}
|
||||
|
||||
func (s *taskService) Get(ctx context.Context, req *dto.GetTaskReq) (res *dto.ListTaskRes, err error) {
|
||||
list, total, err := dao.Task.Get(ctx, req)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
res = &dto.ListTaskRes{
|
||||
Total: total,
|
||||
}
|
||||
err = gconv.Struct(list, &res.List)
|
||||
return
|
||||
}
|
||||
|
||||
43
update.sql
43
update.sql
@@ -206,6 +206,49 @@ COMMENT ON COLUMN rag_knowledge_task.remark IS '备注';
|
||||
|
||||
--------------------pgsql创建rag_knowledge_task表语句---------------------------
|
||||
|
||||
--------------------pgsql创建rag_knowledge_model表语句---------------------------
|
||||
-- 知识库模型配置表
|
||||
CREATE TABLE IF NOT EXISTS rag_knowledge_model (
|
||||
-- 基础字段（完全对齐项目规范）
|
||||
id BIGINT PRIMARY KEY, -- 主键ID（非自增）
|
||||
tenant_id BIGINT NOT NULL DEFAULT 0, -- 租户ID int8
|
||||
creator VARCHAR(64) NOT NULL,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updater VARCHAR(64) NOT NULL,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
deleted_at timestamp(6),
|
||||
|
||||
-- 业务字段
|
||||
dataset_id BIGINT NOT NULL, -- 数据集ID
|
||||
model_type VARCHAR(32) NOT NULL, -- 模型类型
|
||||
model_name VARCHAR(128) NOT NULL, -- 模型名称
|
||||
model_desc TEXT DEFAULT '', -- 模型描述
|
||||
model_config JSONB DEFAULT '{}'::JSONB -- 模型配置(JSONB)
|
||||
);
|
||||
|
||||
-- 索引（高频查询）
|
||||
CREATE INDEX idx_rkm_tenant_id ON rag_knowledge_model(tenant_id);
|
||||
CREATE INDEX idx_rkm_dataset_id ON rag_knowledge_model(dataset_id);
|
||||
CREATE INDEX idx_rkm_model_type ON rag_knowledge_model(model_type);
|
||||
CREATE INDEX idx_rkm_deleted_at ON rag_knowledge_model(deleted_at);
|
||||
|
||||
-- 表和字段注释
|
||||
COMMENT ON TABLE rag_knowledge_model IS '知识库模型配置表';
|
||||
COMMENT ON COLUMN rag_knowledge_model.id IS '主键ID（非自增）';
|
||||
COMMENT ON COLUMN rag_knowledge_model.tenant_id IS '租户ID';
|
||||
COMMENT ON COLUMN rag_knowledge_model.creator IS '创建人';
|
||||
COMMENT ON COLUMN rag_knowledge_model.created_at IS '创建时间';
|
||||
COMMENT ON COLUMN rag_knowledge_model.updater IS '更新人';
|
||||
COMMENT ON COLUMN rag_knowledge_model.updated_at IS '更新时间';
|
||||
COMMENT ON COLUMN rag_knowledge_model.deleted_at IS '删除时间（软删）';
|
||||
COMMENT ON COLUMN rag_knowledge_model.dataset_id IS '数据集ID';
|
||||
COMMENT ON COLUMN rag_knowledge_model.model_type IS '模型类型';
|
||||
COMMENT ON COLUMN rag_knowledge_model.model_name IS '模型名称';
|
||||
COMMENT ON COLUMN rag_knowledge_model.model_desc IS '模型描述';
|
||||
COMMENT ON COLUMN rag_knowledge_model.model_config IS '模型配置(JSONB)';
|
||||
|
||||
--------------------pgsql创建rag_knowledge_model表语句---------------------------
|
||||
|
||||
|
||||
--------------------pgsql创建rag_vector_dataset_index表语句---------------------------
|
||||
-- 向量数据集索引表
|
||||
|
||||
Reference in New Issue
Block a user