red-future/rag
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: rag初始版

Browse Source
This commit is contained in:
曲皓东
2026-04-03 09:16:53 +08:00
commit 6f5c80da16
38 changed files with 3840 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
model/dto/dataset.go Normal file
View File

@@ -0,0 +1,69 @@
package dto
import (
"gitea.com/red-future/common/beans"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/os/gtime"
)
// CreateDatasetReq 创建数据集请求
type CreateDatasetReq struct {
g.Meta `path:"/createDataset" method:"post" tags:"知识库（数据集）管理" summary:"创建知识库（数据集）" dc:"创建知识库（数据集）"`
Name string `json:"name" v:"required#名称不能为空"`
Description string `json:"description"`
}
// CreateDatasetRes 创建数据集响应
type CreateDatasetRes struct {
Id int64 `json:"id,string"`
}
// UpdateDatasetReq 更新数据集请求
type UpdateDatasetReq struct {
g.Meta `path:"/updateDataset" method:"put" tags:"知识库（数据集）管理" summary:"更新知识库（数据集）" dc:"更新知识库（数据集）"`
Id int64 `json:"id" v:"required#ID不能为空"`
Name string `json:"name"`
Description string `json:"description"`
DocumentCount int64 `json:"documentCount"`
DocumentSize int64 `json:"documentSize"`
}
// DeleteDatasetReq 删除数据集请求
type DeleteDatasetReq struct {
g.Meta `path:"/deleteDataset" method:"delete" tags:"知识库（数据集）管理" summary:"删除知识库（数据集）" dc:"删除知识库（数据集）"`
Id int64 `json:"id" v:"required#ID不能为空"`
}
// GetDatasetReq 获取数据集请求
type GetDatasetReq struct {
g.Meta `path:"/getDataset" method:"get" tags:"知识库（数据集）管理" summary:"获取知识库（数据集）详情" dc:"获取知识库（数据集）详情"`
Id int64 `json:"id" v:"required#ID不能为空"`
}
// ListDatasetReq 数据集列表请求
type ListDatasetReq struct {
g.Meta `path:"/listDataset" method:"get" tags:"知识库（数据集）管理" summary:"获取知识库（数据集）列表" dc:"分页查询知识库（数据集）列表，支持多条件筛选"`
Page *beans.Page `json:"page"`
Keyword string `json:"keyword" dc:"关键词搜索"`
}
// ListDatasetRes 数据集列表响应
type ListDatasetRes struct {
List []*DatasetVO `json:"list"`
Total int `json:"total"`
}
type DatasetVO struct {
Id int64 `json:"id,string" dc:"id"`
Name string `json:"name" dc:"数据集名称"`
Description string `json:"description" dc:"数据集描述"`
DocumentCount int64 `json:"documentCount" dc:"文件数量"`
DocumentSize int64 `json:"documentSize" dc:"文件大小(字节)"`
CreatedAt *gtime.Time `json:"createdAt" dc:"创建时间"`
UpdatedAt *gtime.Time `json:"updatedAt" dc:"更新时间"`
}

1
model/dto/dataset_index.go Normal file
View File

@@ -0,0 +1 @@
package dto

108
model/dto/document.go Normal file
View File

@@ -0,0 +1,108 @@
package dto
import (
"rag/consts/document"
"gitea.com/red-future/common/beans"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/os/gtime"
)
// CreateDocumentReq 创建文件请求
type CreateDocumentReq struct {
g.Meta `path:"/createDocument" method:"post" tags:"文件管理" summary:"创建文件" dc:"创建文件"`
DatasetId int64 `json:"datasetId" v:"required#数据集ID不能为空"`
Title string `json:"title" v:"required#标题不能为空"`
Format string `json:"format" v:"required#格式不能为空"`
FileSize int64 `json:"fileSize" v:"required#大小不能为空"`
FilePath string `json:"filePath" v:"required#路径不能为空"`
}
// CreateDocumentRes 创建文件响应
type CreateDocumentRes struct {
Id int64 `json:"id,string"`
}
// UpdateDocumentReq 更新文件请求
type UpdateDocumentReq struct {
g.Meta `path:"/updateDocument" method:"put" tags:"文件管理" summary:"更新文件" dc:"更新文件"`
Id int64 `json:"id" v:"required#ID不能为空"`
Status document.Status `json:"status"`
VectorStatus document.VectorStatus `json:"vectorStatus"`
ChunkCount int64 `json:"chunkCount"`
}
// DeleteDocumentReq 删除文件请求
type DeleteDocumentReq struct {
g.Meta `path:"/deleteDocument" method:"delete" tags:"文件管理" summary:"删除文件" dc:"删除文件"`
Id int64 `json:"id" v:"required#ID不能为空"`
}
// GetDocumentReq 获取文件请求
type GetDocumentReq struct {
g.Meta `path:"/getDocument" method:"get" tags:"文件管理" summary:"获取文件详情" dc:"获取文件详情"`
Id int64 `json:"id" v:"required#ID不能为空"`
}
// ListDocumentReq 文件列表请求
type ListDocumentReq struct {
g.Meta `path:"/listDocument" method:"get" tags:"文件管理" summary:"获取文件列表" dc:"分页查询文件列表，支持多条件筛选"`
Page *beans.Page `json:"page"`
DatasetId int64 `json:"datasetId"`
Keyword string `json:"keyword" dc:"关键词搜索"`
Status document.Status `json:"status"`
}
// ListDocumentRes 文件列表响应
type ListDocumentRes struct {
List []*DocumentVO `json:"list"`
Total int `json:"total"`
}
type DocumentVO struct {
Id int64 `json:"id,string" dc:"id"`
DatasetId int64 `json:"datasetId,string"`
Title string `json:"title" dc:"文件标题"`
Status document.Status `json:"status" dc:"状态1启用/0停用"`
VectorStatus document.VectorStatus `json:"vectorStatus" dc:"向量化状态 状态: 1 待定, 2 处理, 3 完成, 4 失败"`
ChunkCount int64 `json:"chunkCount" dc:"分块数"`
FileSize int64 `json:"fileSize" dc:"文件大小"`
CreatedAt *gtime.Time `json:"createdAt" dc:"创建时间"`
UpdatedAt *gtime.Time `json:"updatedAt" dc:"更新时间"`
}
// ProcessDocumentReq 处理文件请求(向量化)
type ProcessDocumentReq struct {
g.Meta `path:"/getProcess" method:"get" tags:"文件管理" summary:"文件向量化处理" dc:"文件向量化处理"`
Id int64 `json:"id" v:"required#ID不能为空"`
DatasetId int64 `json:"datasetId" v:"required#数据集ID不能为空"`
}
// ProcessDocumentRes 处理文件响应
type ProcessDocumentRes struct {
ChunkCount int64 `json:"chunkCount"`
CostTime int64 `json:"costTime"`
}
type ListDocumentChunkRPC struct {
List []*DocumentChunkRPC `json:"list"`
}
type DocumentChunkRPC struct {
Id int64 `json:"id" dc:"id"`
DatasetId int64 `json:"datasetId" dc:"所属数据集ID"`
ContentHash string `json:"contentHash" dc:"内容hash"`
}
type KnowledgeDocumentMsg struct {
TenantId uint64 `json:"tenantId"`
Creator string `json:"creator"`
Id int64 `json:"id"`
VectorStatus document.VectorStatus `json:"vectorStatus"`
}

64
model/dto/document_chunk.go Normal file
View File

@@ -0,0 +1,64 @@
package dto
import (
"rag/consts/document"
"gitea.com/red-future/common/beans"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/os/gtime"
"github.com/pgvector/pgvector-go"
)
// UpdateDocumentChunkReq 更新文件块向量请求
type UpdateDocumentChunkReq struct {
g.Meta `path:"/updateDocumentChunk" method:"put" tags:"文件块向量管理" summary:"更新文件块" dc:"更新文件块"`
Id int64 `json:"id" v:"required#ID不能为空"`
Status document.Status `json:"status"`
}
// ListDocumentChunkReq 文件块向量列表请求
type ListDocumentChunkReq struct {
g.Meta `path:"/listDocumentChunk" method:"get" tags:"文件块向量管理" summary:"获取文件块向量列表" dc:"分页查询文件块向量列表，支持多条件筛选"`
Page *beans.Page `json:"page"`
DatasetId int64 `json:"datasetId"`
DocumentId int64 `json:"documentId"`
Status document.Status `json:"status"`
VectorStatus document.VectorStatus `json:"vectorStatus"`
}
// ListDocumentChunkRes 文件块向量列表响应
type ListDocumentChunkRes struct {
List []*DocumentChunkItem `json:"list"`
Total int `json:"total"`
}
type DocumentChunkItem struct {
Id int64 `json:"id,string" dc:"id"`
Status document.Status `json:"status" dc:"状态"`
VectorStatus document.VectorStatus `json:"vectorStatus" dc:"向量状态"`
DatasetId int64 `json:"datasetId,string" dc:"所属数据集ID"`
DocumentId int64 `json:"documentId,string" dc:"所属文档ID"`
Content string `json:"content" dc:"内容"`
ContentHash string `json:"contentHash" dc:"内容hash"`
ChunkIndex int64 `json:"chunkIndex" dc:"块索引"`
Vector []float64 `json:"vector" dc:"向量"`
Metadata map[string]interface{} `json:"metadata" dc:"元信息"`
CreatedAt *gtime.Time `json:"createdAt" dc:"创建时间"`
UpdatedAt *gtime.Time `json:"updatedAt" dc:"更新时间"`
}
type VectorDocumentChunkMsg struct {
TenantId uint64 `json:"tenantId"`
Creator string `json:"creator"`
DatasetId int64 `json:"datasetId"` // 数据集ID
DocumentId int64 `json:"documentId"` // 所属文档ID
Content string `json:"content"` // 原始内容
ContentHash string `json:"contentHash"` // 原始内容hash
ChunkIndex int64 `json:"chunkIndex"` // 第几块
Status document.Status `json:"status"`
VectorStatus document.VectorStatus `json:"vectorStatus"`
Vector pgvector.Vector `json:"vector"`
Metadata map[string]interface{} `json:"metadata"`
}

70
model/dto/keyword.go Normal file
View File

@@ -0,0 +1,70 @@
package dto
import (
"gitea.com/red-future/common/beans"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/os/gtime"
)
// CreateKeywordReq 创建关键词请求
type CreateKeywordReq struct {
g.Meta `path:"/createKeyword" method:"post" tags:"关键词管理" summary:"创建关键词" dc:"创建关键词"`
DatasetId int64 `json:"datasetId" v:"required#数据集ID不能为空"`
DocumentId int64 `json:"documentId" v:"required#文档ID不能为空"`
Word string `json:"word" v:"required#名称不能为空"`
Weight int16 `json:"weight" v:"required#权重不能为空"`
}
// CreateKeywordRes 创建关键词响应
type CreateKeywordRes struct {
Id int64 `json:"id,string"`
}
// UpdateKeywordReq 更新关键词请求
type UpdateKeywordReq struct {
g.Meta `path:"/updateKeyword" method:"put" tags:"关键词管理" summary:"更新关键词" dc:"更新关键词"`
Id int64 `json:"id" v:"required#ID不能为空"`
Word string `json:"word"`
Weight int16 `json:"weight"`
}
// DeleteKeywordReq 删除关键词请求
type DeleteKeywordReq struct {
g.Meta `path:"/deleteKeyword" method:"delete" tags:"关键词管理" summary:"删除关键词" dc:"删除关键词"`
Id int64 `json:"id" v:"required#ID不能为空"`
}
// GetKeywordReq 获取关键词请求
type GetKeywordReq struct {
g.Meta `path:"/getKeyword" method:"get" tags:"关键词管理" summary:"获取关键词详情" dc:"获取关键词详情"`
Id int64 `json:"id" v:"required#ID不能为空"`
}
// ListKeywordReq 关键词列表请求
type ListKeywordReq struct {
g.Meta `path:"/listKeyword" method:"get" tags:"关键词管理" summary:"获取关键词列表" dc:"分页查询关键词列表，支持多条件筛选"`
Page *beans.Page `json:"page"`
DatasetId int64 `json:"datasetId"`
DocumentId int64 `json:"documentId"`
Word string `json:"word"`
Keyword string `json:"keyword" dc:"关键词搜索"`
}
// ListKeywordRes 关键词列表响应
type ListKeywordRes struct {
List []*KeywordVO `json:"list"`
Total int `json:"total"`
}
type KeywordVO struct {
Id int64 `json:"id,string" dc:"id"`
Word string `json:"word" dc:"关键词名称"`
Weight int16 `json:"weight" dc:"权重"`
CreatedAt *gtime.Time `json:"createdAt" dc:"创建时间"`
UpdatedAt *gtime.Time `json:"updatedAt" dc:"更新时间"`
}

37
model/entity/dataset.go Normal file
View File

@@ -0,0 +1,37 @@
package entity
import (
"gitea.com/red-future/common/beans"
)
type datasetCol struct {
beans.SQLBaseCol
Name string
Description string
Embedding string
Dimension string
DocumentCount string
DocumentSize string
}
var DatasetCol = datasetCol{
SQLBaseCol: beans.DefSQLBaseCol,
Name: "name",
Description: "description",
Embedding: "embedding",
Dimension: "dimension",
DocumentCount: "document_count",
DocumentSize: "document_size",
}
// Dataset 数据集表
type Dataset struct {
beans.SQLBaseDO `orm:",inline"`
Name string `orm:"name" json:"name" dc:"数据集名称"`
Description string `orm:"description" json:"description" dc:"数据集描述"`
Embedding string `orm:"embedding" json:"embedding" dc:"向量模型"`
Dimension int `orm:"dimension" json:"dimension" dc:"向量维度"`
DocumentCount int64 `orm:"document_count" json:"documentCount" dc:"文档数量"`
DocumentSize int64 `orm:"document_size" json:"documentSize" dc:"文档大小"`
}

46
model/entity/dataset_index.go Normal file
View File

@@ -0,0 +1,46 @@
package entity
import "gitea.com/red-future/common/beans"
type datasetIndexCol struct {
beans.SQLBaseCol
Status string
VectorStatus string
DatasetId string
Name string
Collection string
Dimension string
FieldType string
MetricType string
VectorCount string
Description string
}
var DatasetIndexCol = datasetIndexCol{
SQLBaseCol: beans.DefSQLBaseCol,
Status: "status",
VectorStatus: "vector_status",
DatasetId: "dataset_id",
Name: "name",
Collection: "collection",
Dimension: "dimension",
FieldType: "field_type",
MetricType: "metric_type",
VectorCount: "vector_count",
Description: "description",
}
// DatasetIndex 数据集索引实体
type DatasetIndex struct {
beans.SQLBaseDO `orm:",inline"`
DatasetId int64 `orm:"dataset_id" json:"datasetId" dc:"数据集ID"`
Name string `orm:"name" json:"name" dc:"索引名称"`
Collection string `orm:"collection" json:"collection" dc:"向量集合名称"`
Dimension int `orm:"dimension" json:"dimension" dc:"向量维度"`
FieldType string `orm:"field_type" json:"fieldType" dc:"字段类型: float, binary"`
MetricType string `orm:"metric_type" json:"metricType" dc:"度量类型: L2, IP, COSINE"`
Status *int8 `orm:"status" json:"status" dc:"状态: creating, ready, error"`
VectorCount int64 `orm:"vector_count" json:"vectorCount" dc:"向量数量"`
Description string `orm:"description" json:"description" dc:"描述"`
}

64
model/entity/document.go Normal file
View File

@@ -0,0 +1,64 @@
package entity
import (
"gitea.com/red-future/common/beans"
"rag/consts/document"
)
type documentCol struct {
beans.SQLBaseCol
DatasetId string
Title string
Content string
Format string
Source string
SourceId string
Status string
VectorStatus string
ChunkCount string
FileSize string
FilePath string
Metadata string
}
var DocumentCol = documentCol{
SQLBaseCol: beans.DefSQLBaseCol,
DatasetId: "dataset_id",
Title: "title",
Content: "content",
Format: "format",
Source: "source",
SourceId: "source_id",
Status: "status",
VectorStatus: "vector_status",
ChunkCount: "chunk_count",
FileSize: "file_size",
FilePath: "file_path",
Metadata: "metadata",
}
// Document 文件实体
type Document struct {
beans.SQLBaseDO `orm:",inline"`
DatasetId int64 `orm:"dataset_id" json:"datasetId" dc:"数据集ID"`
Title string `orm:"title" json:"title" dc:"文件标题"`
Content string `orm:"content" json:"content" dc:"文件内容"`
Format string `orm:"format" json:"format" dc:"文件格式"`
Source string `orm:"source" json:"source" dc:"来源"`
SourceId string `orm:"source_id" json:"sourceId" dc:"来源ID"`
Status document.Status `orm:"status" json:"status" dc:"状态"`
VectorStatus document.VectorStatus `orm:"vector_status" json:"vectorStatus" dc:"向量状态"`
ChunkCount int64 `orm:"chunk_count" json:"chunkCount" dc:"切分块数量"`
FileSize int64 `orm:"file_size" json:"fileSize" dc:"文件大小"`
FilePath string `orm:"file_path" json:"filePath" dc:"文件存储路径"`
Metadata *Metadata `orm:"metadata" json:"metadata" dc:"文件元信息"`
}
// Metadata 文件元数据
type Metadata struct {
Author string `orm:"author" json:"author" dc:"作者"`
Tags []string `orm:"tags" json:"tags" dc:"标签"`
Custom map[string]string `orm:"custom" json:"custom" dc:"自定义字段"`
}

49
model/entity/document_chunk.go Normal file
View File

@@ -0,0 +1,49 @@
package entity
import (
"rag/consts/document"
"gitea.com/red-future/common/beans"
"github.com/pgvector/pgvector-go"
)
type documentChunkCol struct {
beans.SQLBaseCol
Status string
VectorStatus string
DatasetId string
DocumentId string
Content string
ContentHash string
ChunkIndex string
Vector string
Metadata string
}
var DocumentChunkCol = documentChunkCol{
SQLBaseCol: beans.DefSQLBaseCol,
Status: "status",
VectorStatus: "vector_status",
DatasetId: "dataset_id",
DocumentId: "document_id",
Content: "content",
ContentHash: "content_hash",
ChunkIndex: "chunk_index",
Vector: "vector",
Metadata: "metadata",
}
// DocumentChunk 文档切分块实体
type DocumentChunk struct {
beans.SQLBaseDO `orm:",inline"`
Status document.Status `orm:"status" json:"status" dc:"状态"`
VectorStatus document.VectorStatus `orm:"vector_status" json:"vectorStatus" dc:"向量状态"`
DatasetId int64 `orm:"dataset_id" json:"datasetId" dc:"数据集ID"`
DocumentId int64 `orm:"document_id" json:"documentId" dc:"文件ID"`
Content string `orm:"content" json:"content" dc:"切分块内容"`
ContentHash string `orm:"content_hash" json:"contentHash" dc:"切分块内容哈希"`
ChunkIndex int64 `orm:"chunk_index" json:"chunkIndex" dc:"切分块索引"`
Vector pgvector.Vector `orm:"vector" json:"vector" dc:"向量"`
Metadata map[string]interface{} `orm:"metadata" json:"metadata" dc:"元信息"`
}

27
model/entity/keyword.go Normal file
View File

@@ -0,0 +1,27 @@
package entity
import "gitea.com/red-future/common/beans"
type keywordCol struct {
beans.SQLBaseCol
DatasetId string
DocumentId string
Word string
Weight string
}
var KeywordCol = keywordCol{
SQLBaseCol: beans.DefSQLBaseCol,
DatasetId: "dataset_id",
DocumentId: "document_id",
Word: "word",
Weight: "weight",
}
type Keyword struct {
beans.SQLBaseDO `orm:",inline"`
DatasetId int64 `orm:"dataset_id" json:"datasetId" dc:"数据集ID"`
DocumentId int64 `orm:"document_id" json:"documentId" dc:"文件ID"`
Word string `orm:"word" json:"word" dc:"关键词"`
Weight int16 `orm:"weight" json:"weight" dc:"权重"`
}