refactor: 重构文档处理流程和任务管理

main.go

@@ -11,6 +11,7 @@ import (
 
 	"gitea.com/red-future/common/http"
 	"gitea.com/red-future/common/jaeger"
+	"gitea.com/red-future/common/utils"
 	gmq "github.com/bjang03/gmq/core/gmq"
 	"github.com/bjang03/gmq/mq"
 	"github.com/bjang03/gmq/types"
@@ -27,22 +28,17 @@ func main() {
 		controller.Dataset,
 		controller.Document,
 		controller.DocumentChunk,
+		controller.Keyword,
+		controller.RAGQuery,
 	})
 
-	gmq.Init("config.yml")
-
-	if err := gmq.GetGmq("primary").GmqSubscribe(ctx, &mq.RedisSubMessage{
-		SubMessage: types.SubMessage{
-			Topic:        public.KnowledgeDocumentVectorStatusTopic,
-			ConsumerName: public.KnowledgeDocumentVectorStatusConsumer,
-			AutoAck:      public.KnowledgeDocumentVectorStatusAutoAck,
-			FetchCount:   public.KnowledgeDocumentVectorStatusBatchSize,
-			HandleFunc:   service.Document.DocsVectorStatusMsg,
-		},
-	}); err != nil {
-		return
+	err := utils.InitGseTool(ctx)
+	if err != nil {
+		g.Log().Error(ctx, "gse 分词工具初始化失败:", err)
 	}
 
+	gmq.Init("config.yml")
+
 	if err := gmq.GetGmq("primary").GmqSubscribe(ctx, &mq.RedisSubMessage{
 		SubMessage: types.SubMessage{
 			Topic:        public.KnowledgeDocumentChunkTopic,