重构数据引擎和报表引擎
This commit is contained in:
477
common/report/api.go
Normal file
477
common/report/api.go
Normal file
@@ -0,0 +1,477 @@
|
||||
package report
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"dataengine/common/report/config"
|
||||
"dataengine/common/report/ddlsync"
|
||||
"dataengine/common/report/executor"
|
||||
"dataengine/common/report/extract"
|
||||
"dataengine/common/report/model"
|
||||
|
||||
"gitea.redpowerfuture.com/red-future/common/db/gfdb"
|
||||
)
|
||||
|
||||
// ReportService 报表公共服务
|
||||
// 对外暴露的统一接口
|
||||
type ReportService struct {
|
||||
configLoader *config.ConfigLoader
|
||||
tableCreator *ddlsync.StatTableCreator
|
||||
queryExecutor *executor.QueryExecutor
|
||||
dailyExtractor *extract.DailyExtractor
|
||||
}
|
||||
|
||||
var defaultService *ReportService
|
||||
|
||||
// GetService 获取报表服务单例
|
||||
func GetService() *ReportService {
|
||||
if defaultService == nil {
|
||||
defaultService = &ReportService{
|
||||
configLoader: config.GetLoader(),
|
||||
tableCreator: ddlsync.NewStatTableCreator(),
|
||||
queryExecutor: executor.NewQueryExecutor(),
|
||||
dailyExtractor: extract.NewDailyExtractor(),
|
||||
}
|
||||
}
|
||||
return defaultService
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// 核心接口 1: 自动创建统计宽表
|
||||
// 首次抽取前调用
|
||||
// ============================================================
|
||||
|
||||
// AutoCreateStatTable 根据配置自动创建统计宽表
|
||||
// businessCode: 业务编码
|
||||
// reportCode: 报表编码
|
||||
func (s *ReportService) AutoCreateStatTable(ctx context.Context, businessCode, reportCode string) (*model.AutoCreateStatTableResp, error) {
|
||||
// 初始化系统表
|
||||
if err := initTables(ctx); err != nil {
|
||||
return nil, fmt.Errorf("初始化系统表失败: %w", err)
|
||||
}
|
||||
|
||||
resp, err := s.tableCreator.AutoCreateStatTable(ctx, businessCode, reportCode)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("AutoCreateStatTable 失败: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// 核心接口 2: 按天抽取数据
|
||||
// 业务层定时任务调用
|
||||
// ============================================================
|
||||
|
||||
// ExtractDailyData 按天抽取数据
|
||||
// businessCode: 业务编码
|
||||
// reportCode: 报表编码
|
||||
// statDate: 统计日期 yyyy-MM-dd
|
||||
// executor: 执行人
|
||||
func (s *ReportService) ExtractDailyData(ctx context.Context, businessCode, reportCode, statDate, executor string) (*model.ExtractDailyDataResp, error) {
|
||||
// 1. 先确保统计宽表存在
|
||||
report, err := s.configLoader.GetReport(ctx, businessCode, reportCode)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("获取报表配置失败: %w", err)
|
||||
}
|
||||
|
||||
// 检查表是否存在
|
||||
result, err := gfdb.DB(ctx).GetAll(ctx,
|
||||
"SELECT EXISTS (SELECT 1 FROM pg_tables WHERE tablename = $1) AS exists",
|
||||
strings.ToLower(report.StatTableName))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("检查统计宽表失败: %w", err)
|
||||
}
|
||||
tableExists := false
|
||||
if len(result) > 0 {
|
||||
tableExists = result[0]["exists"].Bool()
|
||||
}
|
||||
if !tableExists {
|
||||
// 表不存在，先创建
|
||||
if _, createErr := s.AutoCreateStatTable(ctx, businessCode, reportCode); createErr != nil {
|
||||
return nil, fmt.Errorf("创建统计宽表失败: %w", createErr)
|
||||
}
|
||||
}
|
||||
|
||||
resp, err := s.dailyExtractor.ExtractDailyData(ctx, businessCode, reportCode, statDate, executor)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ExtractDailyData 失败: %w", err)
|
||||
}
|
||||
|
||||
// 清除缓存
|
||||
s.configLoader.InvalidateCache(businessCode, reportCode)
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// 核心接口 3: 用户选择查询（最核心）
|
||||
// 前端用户选择条件 → 实时构建SQL → 返回报表数据
|
||||
// ============================================================
|
||||
|
||||
// QueryReportByUserSelect 根据用户选择实时查询报表数据
|
||||
// 不是自动生成报表，是用户在前端选择维度/指标/筛选/时间后实时查询展示
|
||||
func (s *ReportService) QueryReportByUserSelect(ctx context.Context, req *model.UserSelectQueryReq) (*model.UserSelectQueryResp, error) {
|
||||
// 参数校验
|
||||
if req.BusinessCode == "" {
|
||||
return nil, fmt.Errorf("businessCode 不能为空")
|
||||
}
|
||||
if req.ReportCode == "" {
|
||||
return nil, fmt.Errorf("reportCode 不能为空")
|
||||
}
|
||||
|
||||
resp, err := s.queryExecutor.QueryReportByUserSelect(ctx, req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("QueryReportByUserSelect 失败: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// 辅助接口
|
||||
// ============================================================
|
||||
|
||||
// GetReportFields 获取报表可用字段（按维度/指标/筛选分类）
|
||||
func (s *ReportService) GetReportFields(ctx context.Context, businessCode, reportCode string) (*model.GetReportFieldsResp, error) {
|
||||
resp, err := s.configLoader.GetReportFields(ctx, businessCode, reportCode)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("GetReportFields 失败: %w", err)
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// GetAllBusinesses 获取所有启用业务列表
|
||||
func (s *ReportService) GetAllBusinesses(ctx context.Context) ([]model.BusinessConfig, error) {
|
||||
return s.configLoader.GetAllBusinesses(ctx)
|
||||
}
|
||||
|
||||
// GetAllReports 获取业务下所有报表列表
|
||||
func (s *ReportService) GetAllReports(ctx context.Context, businessCode string) ([]model.ReportConfig, error) {
|
||||
return s.configLoader.GetAllReports(ctx, businessCode)
|
||||
}
|
||||
|
||||
// InvalidateCache 失效指定业务报表缓存
|
||||
func (s *ReportService) InvalidateCache(businessCode, reportCode string) {
|
||||
s.configLoader.InvalidateCache(businessCode, reportCode)
|
||||
}
|
||||
|
||||
// InitSystemTables 初始化系统表
|
||||
func (s *ReportService) InitSystemTables(ctx context.Context) error {
|
||||
return initTables(ctx)
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// 配置 CRUD: 业务
|
||||
// ============================================================
|
||||
|
||||
// SaveBusiness 保存业务配置（新增/修改合一）
|
||||
func (s *ReportService) SaveBusiness(ctx context.Context, req *model.SaveBusinessReq) (*model.SaveResult, error) {
|
||||
if err := initTables(ctx); err != nil {
|
||||
return nil, fmt.Errorf("初始化系统表失败: %w", err)
|
||||
}
|
||||
|
||||
biz := &model.BusinessConfig{
|
||||
BusinessCode: req.BusinessCode,
|
||||
BusinessName: req.BusinessName,
|
||||
Description: req.Description,
|
||||
Status: req.Status,
|
||||
Config: req.Config,
|
||||
Creator: req.Operator,
|
||||
Updater: req.Operator,
|
||||
}
|
||||
|
||||
if req.Status == "" {
|
||||
biz.Status = model.StatusActive
|
||||
}
|
||||
if biz.Config == nil {
|
||||
biz.Config = make(map[string]interface{})
|
||||
}
|
||||
|
||||
if req.ID != nil && *req.ID > 0 {
|
||||
// 更新
|
||||
biz.ID = *req.ID
|
||||
if err := s.configLoader.UpdateBusiness(ctx, biz); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &model.SaveResult{Success: true, ID: *req.ID, Message: "更新成功"}, nil
|
||||
}
|
||||
|
||||
// 新增
|
||||
id, err := s.configLoader.CreateBusiness(ctx, biz)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &model.SaveResult{Success: true, ID: id, Message: "创建成功"}, nil
|
||||
}
|
||||
|
||||
// DeleteBusiness 删除业务配置
|
||||
func (s *ReportService) DeleteBusiness(ctx context.Context, id int64) (*model.DeleteResult, error) {
|
||||
biz, err := s.configLoader.GetBusinessByID(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.configLoader.DeleteBusiness(ctx, id, biz.BusinessCode); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &model.DeleteResult{Success: true, Message: "删除成功"}, nil
|
||||
}
|
||||
|
||||
// GetBusiness 获取单个业务配置
|
||||
func (s *ReportService) GetBusiness(ctx context.Context, id int64) (*model.BusinessConfig, error) {
|
||||
return s.configLoader.GetBusinessByID(ctx, id)
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// 配置 CRUD: 报表
|
||||
// ============================================================
|
||||
|
||||
// SaveReport 保存报表配置（新增/修改合一）
|
||||
func (s *ReportService) SaveReport(ctx context.Context, req *model.SaveReportReq) (*model.SaveResult, error) {
|
||||
if err := initTables(ctx); err != nil {
|
||||
return nil, fmt.Errorf("初始化系统表失败: %w", err)
|
||||
}
|
||||
|
||||
rpt := &model.ReportConfig{
|
||||
BusinessCode: req.BusinessCode,
|
||||
ReportCode: req.ReportCode,
|
||||
ReportName: req.ReportName,
|
||||
Description: req.Description,
|
||||
Status: req.Status,
|
||||
StatTableName: req.StatTableName,
|
||||
StatTableComment: req.StatTableComment,
|
||||
DateField: req.DateField,
|
||||
PrimaryKeys: req.PrimaryKeys,
|
||||
ConflictKeys: req.ConflictKeys,
|
||||
Config: req.Config,
|
||||
Creator: req.Operator,
|
||||
Updater: req.Operator,
|
||||
}
|
||||
|
||||
if req.Status == "" {
|
||||
rpt.Status = model.StatusActive
|
||||
}
|
||||
if rpt.DateField == "" {
|
||||
rpt.DateField = "stat_date"
|
||||
}
|
||||
if rpt.PrimaryKeys == nil {
|
||||
rpt.PrimaryKeys = []string{"id"}
|
||||
}
|
||||
if rpt.ConflictKeys == nil {
|
||||
rpt.ConflictKeys = []string{rpt.DateField}
|
||||
}
|
||||
if rpt.Config == nil {
|
||||
rpt.Config = make(map[string]interface{})
|
||||
}
|
||||
|
||||
if req.ID != nil && *req.ID > 0 {
|
||||
rpt.ID = *req.ID
|
||||
if err := s.configLoader.UpdateReport(ctx, rpt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &model.SaveResult{Success: true, ID: *req.ID, Message: "更新成功"}, nil
|
||||
}
|
||||
|
||||
id, err := s.configLoader.CreateReport(ctx, rpt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &model.SaveResult{Success: true, ID: id, Message: "创建成功"}, nil
|
||||
}
|
||||
|
||||
// DeleteReport 删除报表配置
|
||||
func (s *ReportService) DeleteReport(ctx context.Context, id int64) (*model.DeleteResult, error) {
|
||||
rpt, err := s.configLoader.GetReportByID(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.configLoader.DeleteReport(ctx, id, rpt.BusinessCode, rpt.ReportCode); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &model.DeleteResult{Success: true, Message: "删除成功"}, nil
|
||||
}
|
||||
|
||||
// GetReport 获取单个报表配置
|
||||
func (s *ReportService) GetReport(ctx context.Context, id int64) (*model.ReportConfig, error) {
|
||||
return s.configLoader.GetReportByID(ctx, id)
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// 配置 CRUD: 字段
|
||||
// ============================================================
|
||||
|
||||
// SaveField 保存字段配置（新增/修改合一）
|
||||
func (s *ReportService) SaveField(ctx context.Context, req *model.SaveFieldReq) (*model.SaveResult, error) {
|
||||
if err := initTables(ctx); err != nil {
|
||||
return nil, fmt.Errorf("初始化系统表失败: %w", err)
|
||||
}
|
||||
|
||||
field := &model.FieldConfig{
|
||||
BusinessCode: req.BusinessCode,
|
||||
ReportCode: req.ReportCode,
|
||||
FieldCode: req.FieldCode,
|
||||
FieldName: req.FieldName,
|
||||
FieldType: req.FieldType,
|
||||
DataType: req.DataType,
|
||||
FieldRole: req.FieldRole,
|
||||
IsAggregatable: req.IsAggregatable,
|
||||
IsFilterable: req.IsFilterable,
|
||||
IsQueryable: req.IsQueryable,
|
||||
IsSortable: req.IsSortable,
|
||||
DefaultAggregate: req.DefaultAggregate,
|
||||
ValidAggregates: req.ValidAggregates,
|
||||
FilterOperators: req.FilterOperators,
|
||||
Expression: req.Expression,
|
||||
ExpressionType: req.ExpressionType,
|
||||
FormatPattern: req.FormatPattern,
|
||||
Unit: req.Unit,
|
||||
DictCode: req.DictCode,
|
||||
SortOrder: req.SortOrder,
|
||||
GroupName: req.GroupName,
|
||||
Status: req.Status,
|
||||
Creator: req.Operator,
|
||||
Updater: req.Operator,
|
||||
}
|
||||
|
||||
if req.Status == "" {
|
||||
field.Status = model.StatusActive
|
||||
}
|
||||
if field.DataType == "" {
|
||||
field.DataType = model.FieldTypeString
|
||||
}
|
||||
if field.ValidAggregates == nil {
|
||||
field.ValidAggregates = []string{}
|
||||
}
|
||||
if field.FilterOperators == nil {
|
||||
field.FilterOperators = []string{"=", "!=", ">", "<", ">=", "<=", "IN", "LIKE", "BETWEEN"}
|
||||
}
|
||||
|
||||
if req.ID != nil && *req.ID > 0 {
|
||||
field.ID = *req.ID
|
||||
if err := s.configLoader.UpdateField(ctx, field); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &model.SaveResult{Success: true, ID: *req.ID, Message: "更新成功"}, nil
|
||||
}
|
||||
|
||||
id, err := s.configLoader.CreateField(ctx, field)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &model.SaveResult{Success: true, ID: id, Message: "创建成功"}, nil
|
||||
}
|
||||
|
||||
// DeleteField 删除字段配置
|
||||
func (s *ReportService) DeleteField(ctx context.Context, id int64) (*model.DeleteResult, error) {
|
||||
field, err := s.configLoader.GetFieldByID(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.configLoader.DeleteField(ctx, id, field.BusinessCode, field.ReportCode); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &model.DeleteResult{Success: true, Message: "删除成功"}, nil
|
||||
}
|
||||
|
||||
// GetField 获取单个字段配置
|
||||
func (s *ReportService) GetField(ctx context.Context, id int64) (*model.FieldConfig, error) {
|
||||
return s.configLoader.GetFieldByID(ctx, id)
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// 配置 CRUD: 抽取配置
|
||||
// ============================================================
|
||||
|
||||
// SaveExtractConfig 保存抽取配置（新增/修改合一）
|
||||
func (s *ReportService) SaveExtractConfig(ctx context.Context, req *model.SaveExtractConfigReq) (*model.SaveResult, error) {
|
||||
if err := initTables(ctx); err != nil {
|
||||
return nil, fmt.Errorf("初始化系统表失败: %w", err)
|
||||
}
|
||||
|
||||
ec := &model.ExtractConfig{
|
||||
BusinessCode: req.BusinessCode,
|
||||
ReportCode: req.ReportCode,
|
||||
ExtractCode: req.ExtractCode,
|
||||
ExtractName: req.ExtractName,
|
||||
SourceTableName: req.SourceTableName,
|
||||
SourceTableAlias: req.SourceTableAlias,
|
||||
TargetTableName: req.TargetTableName,
|
||||
IsEnabled: req.IsEnabled,
|
||||
ExtractType: req.ExtractType,
|
||||
ExtractMode: req.ExtractMode,
|
||||
ExtractKeyField: req.ExtractKeyField,
|
||||
ExtractKeyFormat: req.ExtractKeyFormat,
|
||||
GroupByFields: req.GroupByFields,
|
||||
FilterExpression: req.FilterExpression,
|
||||
JoinConfigs: req.JoinConfigs,
|
||||
FieldMappings: req.FieldMappings,
|
||||
TransformRules: req.TransformRules,
|
||||
BatchSize: req.BatchSize,
|
||||
Status: req.Status,
|
||||
Creator: req.Operator,
|
||||
Updater: req.Operator,
|
||||
}
|
||||
|
||||
if req.Status == "" {
|
||||
ec.Status = model.StatusActive
|
||||
}
|
||||
if ec.ExtractType == "" {
|
||||
ec.ExtractType = model.ExtractTypeIncremental
|
||||
}
|
||||
if ec.ExtractMode == "" {
|
||||
ec.ExtractMode = model.ExtractModeDirect
|
||||
}
|
||||
if ec.BatchSize == 0 {
|
||||
ec.BatchSize = 1000
|
||||
}
|
||||
if ec.JoinConfigs == nil {
|
||||
ec.JoinConfigs = []model.JoinConfig{}
|
||||
}
|
||||
if ec.FieldMappings == nil {
|
||||
ec.FieldMappings = []model.FieldMapping{}
|
||||
}
|
||||
if ec.TransformRules == nil {
|
||||
ec.TransformRules = []model.TransformRule{}
|
||||
}
|
||||
if ec.GroupByFields == nil {
|
||||
ec.GroupByFields = []string{}
|
||||
}
|
||||
|
||||
if req.ID != nil && *req.ID > 0 {
|
||||
ec.ID = *req.ID
|
||||
if err := s.configLoader.UpdateExtractConfig(ctx, ec); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &model.SaveResult{Success: true, ID: *req.ID, Message: "更新成功"}, nil
|
||||
}
|
||||
|
||||
id, err := s.configLoader.CreateExtractConfig(ctx, ec)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &model.SaveResult{Success: true, ID: id, Message: "创建成功"}, nil
|
||||
}
|
||||
|
||||
// DeleteExtractConfig 删除抽取配置
|
||||
func (s *ReportService) DeleteExtractConfig(ctx context.Context, id int64) (*model.DeleteResult, error) {
|
||||
ec, err := s.configLoader.GetExtractConfigByID(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.configLoader.DeleteExtractConfig(ctx, id, ec.BusinessCode, ec.ReportCode); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &model.DeleteResult{Success: true, Message: "删除成功"}, nil
|
||||
}
|
||||
|
||||
// GetExtractConfig 获取单个抽取配置
|
||||
func (s *ReportService) GetExtractConfig(ctx context.Context, id int64) (*model.ExtractConfig, error) {
|
||||
return s.configLoader.GetExtractConfigByID(ctx, id)
|
||||
}
|
||||
|
||||
// GetExtractConfigs 获取业务报表下所有抽取配置
|
||||
func (s *ReportService) GetExtractConfigs(ctx context.Context, businessCode, reportCode string) ([]model.ExtractConfig, error) {
|
||||
return s.configLoader.GetExtractConfigs(ctx, businessCode, reportCode)
|
||||
}
|
||||
518
common/report/builder/sql_builder.go
Normal file
518
common/report/builder/sql_builder.go
Normal file
@@ -0,0 +1,518 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"dataengine/common/report/config"
|
||||
"dataengine/common/report/model"
|
||||
)
|
||||
|
||||
// SQLBuilder 动态SQL构建器
|
||||
type SQLBuilder struct {
|
||||
loader *config.ConfigLoader
|
||||
}
|
||||
|
||||
// NewSQLBuilder 创建SQL构建器
|
||||
func NewSQLBuilder() *SQLBuilder {
|
||||
return &SQLBuilder{
|
||||
loader: config.GetLoader(),
|
||||
}
|
||||
}
|
||||
|
||||
// BuildQuerySQL 根据用户选择构建查询SQL
|
||||
func (b *SQLBuilder) BuildQuerySQL(ctx context.Context, req *model.UserSelectQueryReq) (string, []interface{}, map[string]interface{}, error) {
|
||||
// 1. 校验配置
|
||||
report, err := b.loader.GetReport(ctx, req.BusinessCode, req.ReportCode)
|
||||
if err != nil {
|
||||
return "", nil, nil, fmt.Errorf("获取报表配置失败: %w", err)
|
||||
}
|
||||
|
||||
fieldMap, err := b.loader.GetFieldMap(ctx, req.BusinessCode, req.ReportCode)
|
||||
if err != nil {
|
||||
return "", nil, nil, fmt.Errorf("获取字段配置失败: %w", err)
|
||||
}
|
||||
|
||||
tableName := report.StatTableName
|
||||
|
||||
// 2. 构建 SELECT 部分
|
||||
selectClause, err := b.buildSelectClause(req, fieldMap, report)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
|
||||
// 3. 构建 FROM 部分
|
||||
fromClause := tableName
|
||||
|
||||
// 4. 构建 WHERE 部分
|
||||
whereClause, whereArgs, err := b.buildWhereClause(ctx, req, fieldMap, report)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
|
||||
// 5. 构建 GROUP BY 部分
|
||||
groupByClause, err := b.buildGroupByClause(req, fieldMap)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
|
||||
// 6. 构建 ORDER BY 部分
|
||||
orderByClause, err := b.buildOrderByClause(req, fieldMap)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
|
||||
// 7. 组合完整SQL
|
||||
var sql strings.Builder
|
||||
sql.WriteString("SELECT ")
|
||||
sql.WriteString(selectClause)
|
||||
sql.WriteString(" FROM ")
|
||||
sql.WriteString(fromClause)
|
||||
|
||||
if whereClause != "" {
|
||||
sql.WriteString(" WHERE ")
|
||||
sql.WriteString(whereClause)
|
||||
}
|
||||
|
||||
if groupByClause != "" {
|
||||
sql.WriteString(" GROUP BY ")
|
||||
sql.WriteString(groupByClause)
|
||||
}
|
||||
|
||||
if orderByClause != "" {
|
||||
sql.WriteString(" ORDER BY ")
|
||||
sql.WriteString(orderByClause)
|
||||
}
|
||||
|
||||
// 8. 统计总数SQL
|
||||
countSql := "SELECT COUNT(*) FROM " + fromClause
|
||||
if whereClause != "" {
|
||||
countSql += " WHERE " + whereClause
|
||||
}
|
||||
if groupByClause != "" {
|
||||
countSql = fmt.Sprintf("SELECT COUNT(*) FROM (SELECT 1 FROM %s WHERE %s GROUP BY %s) AS t",
|
||||
fromClause, whereClause, groupByClause)
|
||||
}
|
||||
|
||||
metadata := map[string]interface{}{
|
||||
"countSql": countSql,
|
||||
"tableName": tableName,
|
||||
"reportConfig": report,
|
||||
}
|
||||
|
||||
return sql.String(), whereArgs, metadata, nil
|
||||
}
|
||||
|
||||
// buildSelectClause 构建SELECT子句
|
||||
func (b *SQLBuilder) buildSelectClause(req *model.UserSelectQueryReq, fieldMap map[string]*model.FieldConfig, report *model.ReportConfig) (string, error) {
|
||||
var selectParts []string
|
||||
|
||||
// 1. 添加维度字段
|
||||
for _, dim := range req.Dimensions {
|
||||
dim = strings.TrimSpace(dim)
|
||||
if dim == "" {
|
||||
continue
|
||||
}
|
||||
fc, ok := fieldMap[dim]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("维度字段不存在: %s", dim)
|
||||
}
|
||||
if fc.FieldRole != model.RoleDimension && fc.FieldRole != model.RoleFilter {
|
||||
return "", fmt.Errorf("字段 %s 不可作为维度", dim)
|
||||
}
|
||||
selectParts = append(selectParts, dim)
|
||||
}
|
||||
|
||||
// 2. 添加指标字段(含聚合)
|
||||
if len(req.Indicators) == 0 {
|
||||
return "", fmt.Errorf("必须选择至少一个指标")
|
||||
}
|
||||
|
||||
for _, ind := range req.Indicators {
|
||||
fc, ok := fieldMap[ind.FieldCode]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("指标字段不存在: %s", ind.FieldCode)
|
||||
}
|
||||
|
||||
alias := ind.Alias
|
||||
if alias == "" {
|
||||
alias = ind.FieldCode
|
||||
}
|
||||
|
||||
agg := strings.ToUpper(ind.Aggregate)
|
||||
if agg == "" {
|
||||
agg = fc.DefaultAggregate
|
||||
if agg == "" {
|
||||
agg = model.AggregateSum
|
||||
}
|
||||
}
|
||||
|
||||
// 校验聚合方式
|
||||
if len(fc.ValidAggregates) > 0 {
|
||||
valid := false
|
||||
for _, v := range fc.ValidAggregates {
|
||||
if strings.ToUpper(v) == agg {
|
||||
valid = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !valid {
|
||||
return "", fmt.Errorf("字段 %s 不支持聚合方式 %s", ind.FieldCode, agg)
|
||||
}
|
||||
}
|
||||
|
||||
// 处理衍生指标(表达式)
|
||||
if fc.ExpressionType == "CALCULATED" && fc.Expression != "" {
|
||||
expr := b.parseExpression(fc.Expression, req.Indicators)
|
||||
selectParts = append(selectParts, fmt.Sprintf("%s AS %s", expr, alias))
|
||||
} else {
|
||||
selectParts = append(selectParts, fmt.Sprintf("%s(%s) AS %s", agg, ind.FieldCode, alias))
|
||||
}
|
||||
}
|
||||
|
||||
// 3. 添加时间分组字段
|
||||
if req.TimeGroup != "" && req.TimeGroup != "day" {
|
||||
dateField := report.DateField
|
||||
if dateField == "" {
|
||||
dateField = "stat_date"
|
||||
}
|
||||
timeGroupExpr := b.buildTimeGroupExpr(dateField, req.TimeGroup)
|
||||
selectParts = append(selectParts, timeGroupExpr)
|
||||
}
|
||||
|
||||
return strings.Join(selectParts, ", "), nil
|
||||
}
|
||||
|
||||
// buildWhereClause 构建WHERE子句
|
||||
func (b *SQLBuilder) buildWhereClause(ctx context.Context, req *model.UserSelectQueryReq, fieldMap map[string]*model.FieldConfig, report *model.ReportConfig) (string, []interface{}, error) {
|
||||
var conditions []string
|
||||
var args []interface{}
|
||||
|
||||
// 1. 租户过滤
|
||||
conditions = append(conditions, "tenant_id = 1")
|
||||
|
||||
// 2. 时间范围过滤
|
||||
if req.TimeRange != nil {
|
||||
dateField := report.DateField
|
||||
if dateField == "" {
|
||||
dateField = "stat_date"
|
||||
}
|
||||
|
||||
if req.TimeRange.StartDate != "" {
|
||||
conditions = append(conditions, fmt.Sprintf("%s >= ?", dateField))
|
||||
args = append(args, req.TimeRange.StartDate)
|
||||
}
|
||||
if req.TimeRange.EndDate != "" {
|
||||
conditions = append(conditions, fmt.Sprintf("%s <= ?", dateField))
|
||||
args = append(args, req.TimeRange.EndDate)
|
||||
}
|
||||
}
|
||||
|
||||
// 3. 业务过滤
|
||||
conditions = append(conditions, "business_code = ?")
|
||||
args = append(args, req.BusinessCode)
|
||||
|
||||
// 4. 用户筛选条件
|
||||
for _, filter := range req.Filters {
|
||||
fc, ok := fieldMap[filter.FieldCode]
|
||||
if !ok {
|
||||
return "", nil, fmt.Errorf("筛选字段不存在: %s", filter.FieldCode)
|
||||
}
|
||||
|
||||
if !fc.IsFilterable {
|
||||
return "", nil, fmt.Errorf("字段 %s 不可用于筛选", filter.FieldCode)
|
||||
}
|
||||
|
||||
op := strings.ToUpper(filter.Operator)
|
||||
if op == "" {
|
||||
op = "="
|
||||
}
|
||||
|
||||
// 校验操作符
|
||||
if len(fc.FilterOperators) > 0 {
|
||||
valid := false
|
||||
for _, v := range fc.FilterOperators {
|
||||
if strings.ToUpper(v) == op {
|
||||
valid = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !valid {
|
||||
return "", nil, fmt.Errorf("字段 %s 不支持操作符 %s", filter.FieldCode, op)
|
||||
}
|
||||
}
|
||||
|
||||
cond, vals, err := b.buildFilterCondition(filter, op, fc.FieldType)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
conditions = append(conditions, cond)
|
||||
args = append(args, vals...)
|
||||
}
|
||||
|
||||
return strings.Join(conditions, " AND "), args, nil
|
||||
}
|
||||
|
||||
// buildFilterCondition 构建单个筛选条件
|
||||
func (b *SQLBuilder) buildFilterCondition(filter model.FilterCondition, op string, fieldType string) (string, []interface{}, error) {
|
||||
field := filter.FieldCode
|
||||
var args []interface{}
|
||||
|
||||
switch op {
|
||||
case "=":
|
||||
return fmt.Sprintf("%s = ?", field), []interface{}{filter.Value}, nil
|
||||
case "!=":
|
||||
return fmt.Sprintf("%s != ?", field), []interface{}{filter.Value}, nil
|
||||
case ">":
|
||||
return fmt.Sprintf("%s > ?", field), []interface{}{filter.Value}, nil
|
||||
case "<":
|
||||
return fmt.Sprintf("%s < ?", field), []interface{}{filter.Value}, nil
|
||||
case ">=":
|
||||
return fmt.Sprintf("%s >= ?", field), []interface{}{filter.Value}, nil
|
||||
case "<=":
|
||||
return fmt.Sprintf("%s <= ?", field), []interface{}{filter.Value}, nil
|
||||
case "IN":
|
||||
values, err := b.convertToSlice(filter.Value)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
placeholders := make([]string, len(values))
|
||||
for i := range values {
|
||||
placeholders[i] = "?"
|
||||
args = append(args, values[i])
|
||||
}
|
||||
return fmt.Sprintf("%s IN (%s)", field, strings.Join(placeholders, ",")), args, nil
|
||||
case "LIKE":
|
||||
return fmt.Sprintf("%s LIKE ?", field), []interface{}{"%" + fmt.Sprintf("%v", filter.Value) + "%"}, nil
|
||||
case "BETWEEN":
|
||||
return fmt.Sprintf("%s BETWEEN ? AND ?", field), []interface{}{filter.Value, filter.Value2}, nil
|
||||
default:
|
||||
return fmt.Sprintf("%s = ?", field), []interface{}{filter.Value}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// buildGroupByClause 构建GROUP BY子句
|
||||
func (b *SQLBuilder) buildGroupByClause(req *model.UserSelectQueryReq, fieldMap map[string]*model.FieldConfig) (string, error) {
|
||||
var groupFields []string
|
||||
for _, dim := range req.Dimensions {
|
||||
fc, ok := fieldMap[dim]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if fc.FieldRole == model.RoleDimension || fc.FieldRole == model.RoleFilter {
|
||||
groupFields = append(groupFields, dim)
|
||||
}
|
||||
}
|
||||
|
||||
if len(groupFields) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
return strings.Join(groupFields, ", "), nil
|
||||
}
|
||||
|
||||
// buildOrderByClause 构建ORDER BY子句
|
||||
func (b *SQLBuilder) buildOrderByClause(req *model.UserSelectQueryReq, fieldMap map[string]*model.FieldConfig) (string, error) {
|
||||
if len(req.OrderBy) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
var orderParts []string
|
||||
for _, order := range req.OrderBy {
|
||||
field := order.FieldCode
|
||||
dir := strings.ToUpper(order.Direction)
|
||||
if dir == "" {
|
||||
dir = "ASC"
|
||||
}
|
||||
if dir != "ASC" && dir != "DESC" {
|
||||
return "", fmt.Errorf("排序方向必须是 ASC 或 DESC")
|
||||
}
|
||||
|
||||
fc, ok := fieldMap[field]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("排序字段不存在: %s", field)
|
||||
}
|
||||
|
||||
if !fc.IsSortable {
|
||||
return "", fmt.Errorf("字段 %s 不可排序", field)
|
||||
}
|
||||
|
||||
orderParts = append(orderParts, fmt.Sprintf("%s %s", field, dir))
|
||||
}
|
||||
|
||||
return strings.Join(orderParts, ", "), nil
|
||||
}
|
||||
|
||||
// buildTimeGroupExpr 构建时间分组表达式
|
||||
func (b *SQLBuilder) buildTimeGroupExpr(dateField, timeGroup string) string {
|
||||
switch timeGroup {
|
||||
case "week":
|
||||
return fmt.Sprintf("DATE_TRUNC('week', %s::date)::text AS time_group", dateField)
|
||||
case "month":
|
||||
return fmt.Sprintf("TO_CHAR(%s::date, 'YYYY-MM') AS time_group", dateField)
|
||||
case "quarter":
|
||||
return "TO_CHAR(" + dateField + "::date, 'YYYY-\"Q\"Q') AS time_group"
|
||||
default:
|
||||
return dateField + " AS time_group"
|
||||
}
|
||||
}
|
||||
|
||||
// parseExpression 解析衍生指标表达式
|
||||
func (b *SQLBuilder) parseExpression(expr string, indicators []model.IndicatorSelect) string {
|
||||
re := regexp.MustCompile(`\{([^}]+)\}`)
|
||||
return re.ReplaceAllStringFunc(expr, func(match string) string {
|
||||
fieldCode := match[1 : len(match)-1]
|
||||
for _, ind := range indicators {
|
||||
if ind.FieldCode == fieldCode {
|
||||
return fieldCode
|
||||
}
|
||||
}
|
||||
return match
|
||||
})
|
||||
}
|
||||
|
||||
// convertToSlice 转换为切片
|
||||
func (b *SQLBuilder) convertToSlice(v interface{}) ([]interface{}, error) {
|
||||
switch val := v.(type) {
|
||||
case []interface{}:
|
||||
return val, nil
|
||||
case []string:
|
||||
result := make([]interface{}, len(val))
|
||||
for i, s := range val {
|
||||
result[i] = s
|
||||
}
|
||||
return result, nil
|
||||
case string:
|
||||
parts := strings.Split(val, ",")
|
||||
result := make([]interface{}, len(parts))
|
||||
for i, p := range parts {
|
||||
result[i] = strings.TrimSpace(p)
|
||||
}
|
||||
return result, nil
|
||||
default:
|
||||
return []interface{}{v}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// BuildCountSQL 构建统计总数SQL
|
||||
func (b *SQLBuilder) BuildCountSQL(sql string) string {
|
||||
sql = regexp.MustCompile(`(?i)SELECT\s+.*?\s+FROM`).ReplaceAllString(sql, "SELECT COUNT(*) FROM")
|
||||
return sql
|
||||
}
|
||||
|
||||
// AddLimit 添加分页
|
||||
func (b *SQLBuilder) AddLimit(sql string, page, pageSize int) string {
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
if pageSize < 1 {
|
||||
pageSize = 20
|
||||
}
|
||||
if pageSize > 1000 {
|
||||
pageSize = 1000
|
||||
}
|
||||
offset := (page - 1) * pageSize
|
||||
return fmt.Sprintf("%s LIMIT %d OFFSET %d", sql, pageSize, offset)
|
||||
}
|
||||
|
||||
// GenerateInsertSQL 生成upsert SQL
|
||||
func (b *SQLBuilder) GenerateInsertSQL(tableName string, columns []string, conflictKeys []string) string {
|
||||
cols := strings.Join(columns, ", ")
|
||||
placeholders := make([]string, len(columns))
|
||||
for i := range columns {
|
||||
placeholders[i] = fmt.Sprintf("$%d", i+1)
|
||||
}
|
||||
placeholdersStr := strings.Join(placeholders, ", ")
|
||||
|
||||
var updateParts []string
|
||||
for _, col := range columns {
|
||||
if col == "id" || col == "created_at" {
|
||||
continue
|
||||
}
|
||||
updateParts = append(updateParts, fmt.Sprintf("%s = EXCLUDED.%s", col, col))
|
||||
}
|
||||
|
||||
sql := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", tableName, cols, placeholdersStr)
|
||||
|
||||
if len(conflictKeys) > 0 {
|
||||
sql += " ON CONFLICT (" + strings.Join(conflictKeys, ", ") + ")"
|
||||
sql += " DO UPDATE SET " + strings.Join(updateParts, ", ")
|
||||
}
|
||||
|
||||
return sql
|
||||
}
|
||||
|
||||
// BuildExtractSQL 构建数据抽取SQL
|
||||
func (b *SQLBuilder) BuildExtractSQL(ctx context.Context, extractConfig *model.ExtractConfig, statDate string) (string, []interface{}, error) {
|
||||
var selectParts []string
|
||||
var args []interface{}
|
||||
|
||||
// 基础字段
|
||||
selectParts = append(selectParts, "tenant_id")
|
||||
selectParts = append(selectParts, fmt.Sprintf("'%s' AS business_code", extractConfig.BusinessCode))
|
||||
selectParts = append(selectParts, fmt.Sprintf("'%s' AS stat_date", statDate))
|
||||
|
||||
// 字段映射
|
||||
sourceTable := extractConfig.SourceTableName
|
||||
if extractConfig.SourceTableAlias != "" {
|
||||
sourceTable = extractConfig.SourceTableAlias
|
||||
}
|
||||
|
||||
for _, mapping := range extractConfig.FieldMappings {
|
||||
targetField := mapping.TargetField
|
||||
sourceField := mapping.SourceField
|
||||
|
||||
if mapping.TransformRule != nil {
|
||||
expr := b.applyTransformRule(mapping.TransformRule, sourceField)
|
||||
selectParts = append(selectParts, fmt.Sprintf("%s AS %s", expr, targetField))
|
||||
} else {
|
||||
selectParts = append(selectParts, fmt.Sprintf("%s.%s AS %s", sourceTable, sourceField, targetField))
|
||||
}
|
||||
}
|
||||
|
||||
// 构建 FROM 和 JOIN
|
||||
fromClause := extractConfig.SourceTableName
|
||||
if extractConfig.SourceTableAlias != "" {
|
||||
fromClause += " " + extractConfig.SourceTableAlias
|
||||
}
|
||||
|
||||
for _, join := range extractConfig.JoinConfigs {
|
||||
joinType := "LEFT JOIN"
|
||||
if strings.ToUpper(join.JoinType) == "INNER" {
|
||||
joinType = "INNER JOIN"
|
||||
} else if strings.ToUpper(join.JoinType) == "RIGHT" {
|
||||
joinType = "RIGHT JOIN"
|
||||
}
|
||||
fromClause += fmt.Sprintf(" %s %s %s ON %s", joinType, join.JoinTable, join.JoinAlias, join.JoinCondition)
|
||||
}
|
||||
|
||||
// WHERE 条件
|
||||
whereClause := ""
|
||||
if extractConfig.FilterExpression != "" {
|
||||
whereClause = " WHERE " + extractConfig.FilterExpression
|
||||
}
|
||||
|
||||
sql := fmt.Sprintf("SELECT %s FROM %s%s", strings.Join(selectParts, ", "), fromClause, whereClause)
|
||||
|
||||
return sql, args, nil
|
||||
}
|
||||
|
||||
// applyTransformRule 应用转换规则
|
||||
func (b *SQLBuilder) applyTransformRule(rule *model.TransformRule, sourceField string) string {
|
||||
switch rule.RuleType {
|
||||
case "CALCULATE":
|
||||
if rule.Expression != "" {
|
||||
return strings.ReplaceAll(rule.Expression, "{source}", sourceField)
|
||||
}
|
||||
case "FORMAT":
|
||||
if rule.Format != "" {
|
||||
return fmt.Sprintf("TO_CHAR(%s, '%s')", sourceField, rule.Format)
|
||||
}
|
||||
case "MAPPING":
|
||||
// 运行时映射，需要在代码中处理
|
||||
return sourceField
|
||||
}
|
||||
return sourceField
|
||||
}
|
||||
664
common/report/config/loader.go
Normal file
664
common/report/config/loader.go
Normal file
@@ -0,0 +1,664 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"dataengine/common/report/model"
|
||||
|
||||
"gitea.redpowerfuture.com/red-future/common/db/gfdb"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
)
|
||||
|
||||
// ConfigLoader 配置加载器
|
||||
type ConfigLoader struct {
|
||||
mu sync.RWMutex
|
||||
// 缓存
|
||||
businessCache map[string]*model.BusinessConfig
|
||||
reportCache map[string]*model.ReportConfig
|
||||
fieldCache map[string][]model.FieldConfig
|
||||
extractCache map[string][]model.ExtractConfig
|
||||
}
|
||||
|
||||
var (
|
||||
defaultLoader *ConfigLoader
|
||||
once sync.Once
|
||||
)
|
||||
|
||||
// GetLoader 获取配置加载器单例
|
||||
func GetLoader() *ConfigLoader {
|
||||
once.Do(func() {
|
||||
defaultLoader = &ConfigLoader{
|
||||
businessCache: make(map[string]*model.BusinessConfig),
|
||||
reportCache: make(map[string]*model.ReportConfig),
|
||||
fieldCache: make(map[string][]model.FieldConfig),
|
||||
extractCache: make(map[string][]model.ExtractConfig),
|
||||
}
|
||||
})
|
||||
return defaultLoader
|
||||
}
|
||||
|
||||
// GetBusiness 获取业务配置
|
||||
func (l *ConfigLoader) GetBusiness(ctx context.Context, businessCode string) (*model.BusinessConfig, error) {
|
||||
l.mu.RLock()
|
||||
if biz, ok := l.businessCache[businessCode]; ok {
|
||||
l.mu.RUnlock()
|
||||
return biz, nil
|
||||
}
|
||||
l.mu.RUnlock()
|
||||
|
||||
var biz model.BusinessConfig
|
||||
r, err := gfdb.DB(ctx).Model(ctx, "report_business_config").
|
||||
Where("business_code", businessCode).
|
||||
Where("status", model.StatusActive).
|
||||
One()
|
||||
if err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, fmt.Errorf("业务配置不存在: %s", businessCode)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if r.IsEmpty() {
|
||||
return nil, fmt.Errorf("业务配置不存在: %s", businessCode)
|
||||
}
|
||||
if err = r.Struct(&biz); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if biz.Config == nil {
|
||||
biz.Config = make(map[string]interface{})
|
||||
}
|
||||
|
||||
l.mu.Lock()
|
||||
l.businessCache[businessCode] = &biz
|
||||
l.mu.Unlock()
|
||||
|
||||
return &biz, nil
|
||||
}
|
||||
|
||||
// GetReport 获取报表配置
|
||||
func (l *ConfigLoader) GetReport(ctx context.Context, businessCode, reportCode string) (*model.ReportConfig, error) {
|
||||
key := businessCode + ":" + reportCode
|
||||
l.mu.RLock()
|
||||
if rpt, ok := l.reportCache[key]; ok {
|
||||
l.mu.RUnlock()
|
||||
return rpt, nil
|
||||
}
|
||||
l.mu.RUnlock()
|
||||
|
||||
var rpt model.ReportConfig
|
||||
r, err := gfdb.DB(ctx).Model(ctx, "report_report_config").
|
||||
Where("business_code", businessCode).
|
||||
Where("report_code", reportCode).
|
||||
Where("status", model.StatusActive).
|
||||
One()
|
||||
if err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, fmt.Errorf("报表配置不存在: %s/%s", businessCode, reportCode)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if r.IsEmpty() {
|
||||
return nil, fmt.Errorf("报表配置不存在: %s/%s", businessCode, reportCode)
|
||||
}
|
||||
if err = r.Struct(&rpt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if rpt.PrimaryKeys == nil {
|
||||
rpt.PrimaryKeys = []string{"id"}
|
||||
}
|
||||
if rpt.ConflictKeys == nil {
|
||||
rpt.ConflictKeys = []string{"stat_date"}
|
||||
}
|
||||
if rpt.Config == nil {
|
||||
rpt.Config = make(map[string]interface{})
|
||||
}
|
||||
|
||||
l.mu.Lock()
|
||||
l.reportCache[key] = &rpt
|
||||
l.mu.Unlock()
|
||||
|
||||
return &rpt, nil
|
||||
}
|
||||
|
||||
// GetFields 获取报表字段配置
|
||||
func (l *ConfigLoader) GetFields(ctx context.Context, businessCode, reportCode string) ([]model.FieldConfig, error) {
|
||||
key := businessCode + ":" + reportCode
|
||||
l.mu.RLock()
|
||||
if fields, ok := l.fieldCache[key]; ok {
|
||||
l.mu.RUnlock()
|
||||
return fields, nil
|
||||
}
|
||||
l.mu.RUnlock()
|
||||
|
||||
var fields []model.FieldConfig
|
||||
err := gfdb.DB(ctx).Model(ctx, "report_field_config").
|
||||
Where("business_code", businessCode).
|
||||
Where("report_code", reportCode).
|
||||
Where("status", model.StatusActive).
|
||||
Order("sort_order ASC").
|
||||
Scan(&fields)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for i := range fields {
|
||||
if fields[i].ValidAggregates == nil {
|
||||
fields[i].ValidAggregates = []string{}
|
||||
}
|
||||
if fields[i].FilterOperators == nil {
|
||||
fields[i].FilterOperators = []string{"=", "!=", ">", "<", ">=", "<=", "IN", "LIKE", "BETWEEN"}
|
||||
}
|
||||
}
|
||||
|
||||
l.mu.Lock()
|
||||
l.fieldCache[key] = fields
|
||||
l.mu.Unlock()
|
||||
|
||||
return fields, nil
|
||||
}
|
||||
|
||||
// GetFieldMap 获取字段配置Map
|
||||
func (l *ConfigLoader) GetFieldMap(ctx context.Context, businessCode, reportCode string) (map[string]*model.FieldConfig, error) {
|
||||
fields, err := l.GetFields(ctx, businessCode, reportCode)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fieldMap := make(map[string]*model.FieldConfig)
|
||||
for i := range fields {
|
||||
fieldMap[fields[i].FieldCode] = &fields[i]
|
||||
}
|
||||
return fieldMap, nil
|
||||
}
|
||||
|
||||
// GetExtractConfigs 获取抽取配置
|
||||
func (l *ConfigLoader) GetExtractConfigs(ctx context.Context, businessCode, reportCode string) ([]model.ExtractConfig, error) {
|
||||
key := businessCode + ":" + reportCode
|
||||
l.mu.RLock()
|
||||
if configs, ok := l.extractCache[key]; ok {
|
||||
l.mu.RUnlock()
|
||||
return configs, nil
|
||||
}
|
||||
l.mu.RUnlock()
|
||||
|
||||
var configs []model.ExtractConfig
|
||||
err := gfdb.DB(ctx).Model(ctx, "report_extract_config").
|
||||
Where("business_code", businessCode).
|
||||
Where("report_code", reportCode).
|
||||
Where("status", model.StatusActive).
|
||||
Where("is_enabled", true).
|
||||
Scan(&configs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for i := range configs {
|
||||
if configs[i].JoinConfigs == nil {
|
||||
configs[i].JoinConfigs = []model.JoinConfig{}
|
||||
}
|
||||
if configs[i].FieldMappings == nil {
|
||||
configs[i].FieldMappings = []model.FieldMapping{}
|
||||
}
|
||||
if configs[i].TransformRules == nil {
|
||||
configs[i].TransformRules = []model.TransformRule{}
|
||||
}
|
||||
if configs[i].GroupByFields == nil {
|
||||
configs[i].GroupByFields = []string{}
|
||||
}
|
||||
if configs[i].ExtractMode == "" {
|
||||
configs[i].ExtractMode = model.ExtractModeDirect
|
||||
}
|
||||
}
|
||||
|
||||
l.mu.Lock()
|
||||
l.extractCache[key] = configs
|
||||
l.mu.Unlock()
|
||||
|
||||
return configs, nil
|
||||
}
|
||||
|
||||
// GetExtractLog 获取抽取记录
|
||||
func (l *ConfigLoader) GetExtractLog(ctx context.Context, businessCode, reportCode, extractCode, statDate string) (*model.ExtractLog, error) {
|
||||
var log model.ExtractLog
|
||||
r, err := gfdb.DB(ctx).Model(ctx, "report_extract_log").
|
||||
Where("business_code", businessCode).
|
||||
Where("report_code", reportCode).
|
||||
Where("extract_code", extractCode).
|
||||
Where("stat_date", statDate).
|
||||
One()
|
||||
if err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if r.IsEmpty() {
|
||||
return nil, nil
|
||||
}
|
||||
if err = r.Struct(&log); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &log, nil
|
||||
}
|
||||
|
||||
// CreateExtractLog 创建抽取记录
|
||||
func (l *ConfigLoader) CreateExtractLog(ctx context.Context, log *model.ExtractLog) error {
|
||||
data, _ := json.Marshal(log)
|
||||
var m map[string]interface{}
|
||||
json.Unmarshal(data, &m)
|
||||
delete(m, "id")
|
||||
_, err := gfdb.DB(ctx).Model(ctx, "report_extract_log").Data(m).Save()
|
||||
return err
|
||||
}
|
||||
|
||||
// UpdateExtractLog 更新抽取记录
|
||||
func (l *ConfigLoader) UpdateExtractLog(ctx context.Context, log *model.ExtractLog) error {
|
||||
data, _ := json.Marshal(log)
|
||||
var m map[string]interface{}
|
||||
json.Unmarshal(data, &m)
|
||||
delete(m, "id")
|
||||
delete(m, "created_at")
|
||||
_, err := gfdb.DB(ctx).Model(ctx, "report_extract_log").
|
||||
Where("business_code", log.BusinessCode).
|
||||
Where("report_code", log.ReportCode).
|
||||
Where("extract_code", log.ExtractCode).
|
||||
Where("stat_date", log.StatDate).
|
||||
Data(m).
|
||||
Update()
|
||||
return err
|
||||
}
|
||||
|
||||
// InvalidateCache 失效缓存
|
||||
func (l *ConfigLoader) InvalidateCache(businessCode, reportCode string) {
|
||||
l.mu.Lock()
|
||||
delete(l.businessCache, businessCode)
|
||||
delete(l.reportCache, businessCode+":"+reportCode)
|
||||
delete(l.fieldCache, businessCode+":"+reportCode)
|
||||
delete(l.extractCache, businessCode+":"+reportCode)
|
||||
l.mu.Unlock()
|
||||
}
|
||||
|
||||
// InvalidateBusinessCache 只失效业务缓存（不影响报表/字段）
|
||||
func (l *ConfigLoader) InvalidateBusinessCache(businessCode string) {
|
||||
l.mu.Lock()
|
||||
delete(l.businessCache, businessCode)
|
||||
l.mu.Unlock()
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// CRUD: BusinessConfig
|
||||
// ============================================================
|
||||
|
||||
// CreateBusiness 创建业务配置
|
||||
func (l *ConfigLoader) CreateBusiness(ctx context.Context, biz *model.BusinessConfig) (int64, error) {
|
||||
data, _ := json.Marshal(biz)
|
||||
var m map[string]interface{}
|
||||
json.Unmarshal(data, &m)
|
||||
delete(m, "id")
|
||||
delete(m, "created_at")
|
||||
delete(m, "updated_at")
|
||||
delete(m, "deleted_at")
|
||||
|
||||
result, err := gfdb.DB(ctx).Model(ctx, "report_business_config").Data(m).Insert()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("创建业务配置失败: %w", err)
|
||||
}
|
||||
id, _ := result.LastInsertId()
|
||||
l.InvalidateBusinessCache(biz.BusinessCode)
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// UpdateBusiness 更新业务配置
|
||||
func (l *ConfigLoader) UpdateBusiness(ctx context.Context, biz *model.BusinessConfig) error {
|
||||
data, _ := json.Marshal(biz)
|
||||
var m map[string]interface{}
|
||||
json.Unmarshal(data, &m)
|
||||
delete(m, "id")
|
||||
delete(m, "created_at")
|
||||
delete(m, "deleted_at")
|
||||
|
||||
_, err := gfdb.DB(ctx).Model(ctx, "report_business_config").
|
||||
Where("id", biz.ID).
|
||||
Data(m).
|
||||
Update()
|
||||
if err != nil {
|
||||
return fmt.Errorf("更新业务配置失败: %w", err)
|
||||
}
|
||||
l.InvalidateBusinessCache(biz.BusinessCode)
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteBusiness 删除业务配置（软删除）
|
||||
func (l *ConfigLoader) DeleteBusiness(ctx context.Context, id int64, businessCode string) error {
|
||||
_, err := gfdb.DB(ctx).Model(ctx, "report_business_config").
|
||||
Where("id", id).
|
||||
Data(map[string]interface{}{
|
||||
"status": model.StatusInactive,
|
||||
"deleted_at": "NOW()",
|
||||
}).
|
||||
Update()
|
||||
if err != nil {
|
||||
return fmt.Errorf("删除业务配置失败: %w", err)
|
||||
}
|
||||
l.InvalidateBusinessCache(businessCode)
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetBusinessByID 根据ID获取业务配置
|
||||
func (l *ConfigLoader) GetBusinessByID(ctx context.Context, id int64) (*model.BusinessConfig, error) {
|
||||
var biz model.BusinessConfig
|
||||
r, err := gfdb.DB(ctx).Model(ctx, "report_business_config").
|
||||
Where("id", id).
|
||||
One()
|
||||
if err != nil {
|
||||
g.Log().Infof(ctx, "[GetBusinessByID] id=%d, err=%v", id, err)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, fmt.Errorf("业务配置不存在: id=%d", id)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if r.IsEmpty() {
|
||||
return nil, fmt.Errorf("业务配置不存在: id=%d", id)
|
||||
}
|
||||
if err = r.Struct(&biz); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
g.Log().Infof(ctx, "[GetBusinessByID] id=%d, biz.ID=%d, biz.BusinessCode=%s",
|
||||
id, biz.ID, biz.BusinessCode)
|
||||
return &biz, nil
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// CRUD: ReportConfig
|
||||
// ============================================================
|
||||
|
||||
// CreateReport 创建报表配置
|
||||
func (l *ConfigLoader) CreateReport(ctx context.Context, rpt *model.ReportConfig) (int64, error) {
|
||||
data, _ := json.Marshal(rpt)
|
||||
var m map[string]interface{}
|
||||
json.Unmarshal(data, &m)
|
||||
delete(m, "id")
|
||||
delete(m, "created_at")
|
||||
delete(m, "updated_at")
|
||||
delete(m, "deleted_at")
|
||||
|
||||
result, err := gfdb.DB(ctx).Model(ctx, "report_report_config").Data(m).Insert()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("创建报表配置失败: %w", err)
|
||||
}
|
||||
id, _ := result.LastInsertId()
|
||||
l.InvalidateCache(rpt.BusinessCode, rpt.ReportCode)
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// UpdateReport 更新报表配置
|
||||
func (l *ConfigLoader) UpdateReport(ctx context.Context, rpt *model.ReportConfig) error {
|
||||
data, _ := json.Marshal(rpt)
|
||||
var m map[string]interface{}
|
||||
json.Unmarshal(data, &m)
|
||||
delete(m, "id")
|
||||
delete(m, "created_at")
|
||||
delete(m, "deleted_at")
|
||||
|
||||
_, err := gfdb.DB(ctx).Model(ctx, "report_report_config").
|
||||
Where("id", rpt.ID).
|
||||
Data(m).
|
||||
Update()
|
||||
if err != nil {
|
||||
return fmt.Errorf("更新报表配置失败: %w", err)
|
||||
}
|
||||
l.InvalidateCache(rpt.BusinessCode, rpt.ReportCode)
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteReport 删除报表配置（软删除）
|
||||
func (l *ConfigLoader) DeleteReport(ctx context.Context, id int64, businessCode, reportCode string) error {
|
||||
_, err := gfdb.DB(ctx).Model(ctx, "report_report_config").
|
||||
Where("id", id).
|
||||
Data(map[string]interface{}{
|
||||
"status": model.StatusInactive,
|
||||
"deleted_at": "NOW()",
|
||||
}).
|
||||
Update()
|
||||
if err != nil {
|
||||
return fmt.Errorf("删除报表配置失败: %w", err)
|
||||
}
|
||||
l.InvalidateCache(businessCode, reportCode)
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetReportByID 根据ID获取报表配置
|
||||
func (l *ConfigLoader) GetReportByID(ctx context.Context, id int64) (*model.ReportConfig, error) {
|
||||
var rpt model.ReportConfig
|
||||
r, err := gfdb.DB(ctx).Model(ctx, "report_report_config").
|
||||
Where("id", id).
|
||||
One()
|
||||
if err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, fmt.Errorf("报表配置不存在: id=%d", id)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if r.IsEmpty() {
|
||||
return nil, fmt.Errorf("报表配置不存在: id=%d", id)
|
||||
}
|
||||
if err = r.Struct(&rpt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &rpt, nil
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// CRUD: FieldConfig
|
||||
// ============================================================
|
||||
|
||||
// CreateField 创建字段配置
|
||||
func (l *ConfigLoader) CreateField(ctx context.Context, field *model.FieldConfig) (int64, error) {
|
||||
data, _ := json.Marshal(field)
|
||||
var m map[string]interface{}
|
||||
json.Unmarshal(data, &m)
|
||||
delete(m, "id")
|
||||
delete(m, "created_at")
|
||||
delete(m, "updated_at")
|
||||
delete(m, "deleted_at")
|
||||
|
||||
result, err := gfdb.DB(ctx).Model(ctx, "report_field_config").Data(m).Insert()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("创建字段配置失败: %w", err)
|
||||
}
|
||||
id, _ := result.LastInsertId()
|
||||
l.InvalidateCache(field.BusinessCode, field.ReportCode)
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// UpdateField 更新字段配置
|
||||
func (l *ConfigLoader) UpdateField(ctx context.Context, field *model.FieldConfig) error {
|
||||
data, _ := json.Marshal(field)
|
||||
var m map[string]interface{}
|
||||
json.Unmarshal(data, &m)
|
||||
delete(m, "id")
|
||||
delete(m, "created_at")
|
||||
delete(m, "deleted_at")
|
||||
|
||||
_, err := gfdb.DB(ctx).Model(ctx, "report_field_config").
|
||||
Where("id", field.ID).
|
||||
Data(m).
|
||||
Update()
|
||||
if err != nil {
|
||||
return fmt.Errorf("更新字段配置失败: %w", err)
|
||||
}
|
||||
l.InvalidateCache(field.BusinessCode, field.ReportCode)
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteField 删除字段配置（软删除）
|
||||
func (l *ConfigLoader) DeleteField(ctx context.Context, id int64, businessCode, reportCode string) error {
|
||||
_, err := gfdb.DB(ctx).Model(ctx, "report_field_config").
|
||||
Where("id", id).
|
||||
Data(map[string]interface{}{
|
||||
"status": model.StatusInactive,
|
||||
"deleted_at": "NOW()",
|
||||
}).
|
||||
Update()
|
||||
if err != nil {
|
||||
return fmt.Errorf("删除字段配置失败: %w", err)
|
||||
}
|
||||
l.InvalidateCache(businessCode, reportCode)
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetFieldByID 根据ID获取字段配置
|
||||
func (l *ConfigLoader) GetFieldByID(ctx context.Context, id int64) (*model.FieldConfig, error) {
|
||||
var field model.FieldConfig
|
||||
r, err := gfdb.DB(ctx).Model(ctx, "report_field_config").
|
||||
Where("id", id).
|
||||
One()
|
||||
if err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, fmt.Errorf("字段配置不存在: id=%d", id)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if r.IsEmpty() {
|
||||
return nil, fmt.Errorf("字段配置不存在: id=%d", id)
|
||||
}
|
||||
if err = r.Struct(&field); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &field, nil
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// CRUD: ExtractConfig
|
||||
// ============================================================
|
||||
|
||||
// CreateExtractConfig 创建抽取配置
|
||||
func (l *ConfigLoader) CreateExtractConfig(ctx context.Context, ec *model.ExtractConfig) (int64, error) {
|
||||
data, _ := json.Marshal(ec)
|
||||
var m map[string]interface{}
|
||||
json.Unmarshal(data, &m)
|
||||
delete(m, "id")
|
||||
delete(m, "created_at")
|
||||
delete(m, "updated_at")
|
||||
delete(m, "deleted_at")
|
||||
|
||||
result, err := gfdb.DB(ctx).Model(ctx, "report_extract_config").Data(m).Insert()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("创建抽取配置失败: %w", err)
|
||||
}
|
||||
id, _ := result.LastInsertId()
|
||||
l.InvalidateCache(ec.BusinessCode, ec.ReportCode)
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// UpdateExtractConfig 更新抽取配置
|
||||
func (l *ConfigLoader) UpdateExtractConfig(ctx context.Context, ec *model.ExtractConfig) error {
|
||||
data, _ := json.Marshal(ec)
|
||||
var m map[string]interface{}
|
||||
json.Unmarshal(data, &m)
|
||||
delete(m, "id")
|
||||
delete(m, "created_at")
|
||||
delete(m, "deleted_at")
|
||||
|
||||
_, err := gfdb.DB(ctx).Model(ctx, "report_extract_config").
|
||||
Where("id", ec.ID).
|
||||
Data(m).
|
||||
Update()
|
||||
if err != nil {
|
||||
return fmt.Errorf("更新抽取配置失败: %w", err)
|
||||
}
|
||||
l.InvalidateCache(ec.BusinessCode, ec.ReportCode)
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteExtractConfig 删除抽取配置（软删除）
|
||||
func (l *ConfigLoader) DeleteExtractConfig(ctx context.Context, id int64, businessCode, reportCode string) error {
|
||||
_, err := gfdb.DB(ctx).Model(ctx, "report_extract_config").
|
||||
Where("id", id).
|
||||
Data(map[string]interface{}{
|
||||
"status": model.StatusInactive,
|
||||
"deleted_at": "NOW()",
|
||||
}).
|
||||
Update()
|
||||
if err != nil {
|
||||
return fmt.Errorf("删除抽取配置失败: %w", err)
|
||||
}
|
||||
l.InvalidateCache(businessCode, reportCode)
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetExtractConfigByID 根据ID获取抽取配置
|
||||
func (l *ConfigLoader) GetExtractConfigByID(ctx context.Context, id int64) (*model.ExtractConfig, error) {
|
||||
var ec model.ExtractConfig
|
||||
r, err := gfdb.DB(ctx).Model(ctx, "report_extract_config").
|
||||
Where("id", id).
|
||||
One()
|
||||
if err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, fmt.Errorf("抽取配置不存在: id=%d", id)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if r.IsEmpty() {
|
||||
return nil, fmt.Errorf("抽取配置不存在: id=%d", id)
|
||||
}
|
||||
if err = r.Struct(&ec); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ec, nil
|
||||
}
|
||||
|
||||
// GetAllBusinesses 获取所有业务配置
|
||||
func (l *ConfigLoader) GetAllBusinesses(ctx context.Context) ([]model.BusinessConfig, error) {
|
||||
var businesses []model.BusinessConfig
|
||||
err := gfdb.DB(ctx).Model(ctx, "report_business_config").
|
||||
Where("status", model.StatusActive).
|
||||
Order("id ASC").
|
||||
Scan(&businesses)
|
||||
return businesses, err
|
||||
}
|
||||
|
||||
// GetAllReports 获取所有报表配置
|
||||
func (l *ConfigLoader) GetAllReports(ctx context.Context, businessCode string) ([]model.ReportConfig, error) {
|
||||
var reports []model.ReportConfig
|
||||
err := gfdb.DB(ctx).Model(ctx, "report_report_config").
|
||||
Where("business_code", businessCode).
|
||||
Where("status", model.StatusActive).
|
||||
Order("id ASC").
|
||||
Scan(&reports)
|
||||
return reports, err
|
||||
}
|
||||
|
||||
// GetReportFields 获取报表可用字段(按角色分类)
|
||||
func (l *ConfigLoader) GetReportFields(ctx context.Context, businessCode, reportCode string) (*model.GetReportFieldsResp, error) {
|
||||
fields, err := l.GetFields(ctx, businessCode, reportCode)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp := &model.GetReportFieldsResp{
|
||||
BusinessCode: businessCode,
|
||||
ReportCode: reportCode,
|
||||
Dimensions: []model.FieldConfig{},
|
||||
Indicators: []model.FieldConfig{},
|
||||
Filters: []model.FieldConfig{},
|
||||
}
|
||||
|
||||
for _, f := range fields {
|
||||
switch f.FieldRole {
|
||||
case model.RoleDimension:
|
||||
resp.Dimensions = append(resp.Dimensions, f)
|
||||
case model.RoleIndicator:
|
||||
resp.Indicators = append(resp.Indicators, f)
|
||||
case model.RoleFilter, model.RoleFilterOnly:
|
||||
resp.Filters = append(resp.Filters, f)
|
||||
}
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
210
common/report/ddlsync/creator.go
Normal file
210
common/report/ddlsync/creator.go
Normal file
@@ -0,0 +1,210 @@
|
||||
package ddlsync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"dataengine/common/report/config"
|
||||
"dataengine/common/report/model"
|
||||
|
||||
"gitea.redpowerfuture.com/red-future/common/db/gfdb"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// StatTableCreator 统计宽表创建器
|
||||
type StatTableCreator struct {
|
||||
loader *config.ConfigLoader
|
||||
}
|
||||
|
||||
// NewStatTableCreator 创建统计宽表创建器
|
||||
func NewStatTableCreator() *StatTableCreator {
|
||||
return &StatTableCreator{
|
||||
loader: config.GetLoader(),
|
||||
}
|
||||
}
|
||||
|
||||
// AutoCreateStatTable 根据配置自动创建统计宽表
|
||||
func (c *StatTableCreator) AutoCreateStatTable(ctx context.Context, businessCode, reportCode string) (*model.AutoCreateStatTableResp, error) {
|
||||
start := time.Now()
|
||||
logger := logrus.WithFields(logrus.Fields{
|
||||
"businessCode": businessCode,
|
||||
"reportCode": reportCode,
|
||||
})
|
||||
|
||||
// 1. 获取报表配置
|
||||
report, err := c.loader.GetReport(ctx, businessCode, reportCode)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("获取报表配置失败: %w", err)
|
||||
}
|
||||
|
||||
// 2. 获取字段配置
|
||||
fields, err := c.loader.GetFields(ctx, businessCode, reportCode)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("获取字段配置失败: %w", err)
|
||||
}
|
||||
|
||||
if len(fields) == 0 {
|
||||
return nil, fmt.Errorf("报表字段配置为空，请先配置字段")
|
||||
}
|
||||
|
||||
// 3. 构建建表SQL
|
||||
tableName := report.StatTableName
|
||||
sql := c.buildCreateTableSQL(tableName, report, fields)
|
||||
|
||||
logger.Infof("创建统计宽表: %s", tableName)
|
||||
|
||||
// 4. 执行建表
|
||||
_, err = gfdb.DB(ctx).Exec(ctx, sql)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("建表失败: %w", err)
|
||||
}
|
||||
|
||||
// 5. 创建冲突唯一索引
|
||||
if len(report.ConflictKeys) > 0 {
|
||||
indexName := fmt.Sprintf("uq_%s_conflict", tableName)
|
||||
indexCols := strings.Join(report.ConflictKeys, ", ")
|
||||
indexSQL := fmt.Sprintf("CREATE UNIQUE INDEX IF NOT EXISTS %s ON %s (%s)", indexName, tableName, indexCols)
|
||||
if _, err := gfdb.DB(ctx).Exec(ctx, indexSQL); err != nil {
|
||||
logger.Warnf("创建冲突索引失败: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 6. 添加字段注释
|
||||
for _, f := range fields {
|
||||
if f.FieldName != "" {
|
||||
commentSQL := fmt.Sprintf("COMMENT ON COLUMN %s.%s IS '%s'", tableName, f.FieldCode, strings.ReplaceAll(f.FieldName, "'", "''"))
|
||||
if _, err := gfdb.DB(ctx).Exec(ctx, commentSQL); err != nil {
|
||||
logger.Warnf("添加字段注释失败 [%s]: %v", f.FieldCode, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
execTime := time.Since(start).Milliseconds()
|
||||
logger.Infof("统计宽表 %s 创建完成，字段数: %d，耗时: %dms", tableName, len(fields), execTime)
|
||||
|
||||
return &model.AutoCreateStatTableResp{
|
||||
Success: true,
|
||||
TableName: tableName,
|
||||
ColumnCount: len(fields),
|
||||
ExecTimeMs: execTime,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// buildCreateTableSQL 构建建表SQL
|
||||
func (c *StatTableCreator) buildCreateTableSQL(tableName string, report *model.ReportConfig, fields []model.FieldConfig) string {
|
||||
var cols []string
|
||||
|
||||
// 基础字段
|
||||
cols = append(cols, "id BIGSERIAL PRIMARY KEY")
|
||||
cols = append(cols, "tenant_id BIGINT NOT NULL DEFAULT 0")
|
||||
cols = append(cols, "business_code VARCHAR(64) NOT NULL DEFAULT ''")
|
||||
cols = append(cols, "creator VARCHAR(64) DEFAULT ''")
|
||||
cols = append(cols, "created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()")
|
||||
cols = append(cols, "updater VARCHAR(64) DEFAULT ''")
|
||||
cols = append(cols, "updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()")
|
||||
cols = append(cols, "deleted_at TIMESTAMP WITH TIME ZONE")
|
||||
|
||||
// 日期维度字段
|
||||
dateField := report.DateField
|
||||
if dateField == "" {
|
||||
dateField = "stat_date"
|
||||
}
|
||||
cols = append(cols, fmt.Sprintf("%s VARCHAR(16) NOT NULL DEFAULT ''", dateField))
|
||||
|
||||
// 业务字段
|
||||
for _, f := range fields {
|
||||
colDef := c.fieldTypeToColumn(f.FieldCode, f.FieldType)
|
||||
cols = append(cols, colDef)
|
||||
}
|
||||
|
||||
// 原始数据
|
||||
cols = append(cols, "raw_data JSONB DEFAULT '{}'")
|
||||
|
||||
sql := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (\n %s\n)", tableName, strings.Join(cols, ",\n "))
|
||||
|
||||
return sql
|
||||
}
|
||||
|
||||
// fieldTypeToColumn 字段类型转PG列类型
|
||||
func (c *StatTableCreator) fieldTypeToColumn(fieldCode, fieldType string) string {
|
||||
switch fieldType {
|
||||
case model.FieldTypeInt, model.FieldTypeFloat:
|
||||
return fmt.Sprintf("%s NUMERIC(20,4) DEFAULT 0", fieldCode)
|
||||
case model.FieldTypeDate:
|
||||
return fmt.Sprintf("%s VARCHAR(16) DEFAULT ''", fieldCode)
|
||||
case model.FieldTypeDatetime:
|
||||
return fmt.Sprintf("%s TIMESTAMP WITH TIME ZONE", fieldCode)
|
||||
case model.FieldTypeJsonb:
|
||||
return fmt.Sprintf("%s JSONB DEFAULT '{}'", fieldCode)
|
||||
default: // STRING
|
||||
return fmt.Sprintf("%s VARCHAR(256) DEFAULT ''", fieldCode)
|
||||
}
|
||||
}
|
||||
|
||||
// DropStatTable 删除统计宽表
|
||||
func (c *StatTableCreator) DropStatTable(ctx context.Context, businessCode, reportCode string) error {
|
||||
report, err := c.loader.GetReport(ctx, businessCode, reportCode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sql := fmt.Sprintf("DROP TABLE IF EXISTS %s CASCADE", report.StatTableName)
|
||||
_, err = gfdb.DB(ctx).Exec(ctx, sql)
|
||||
return err
|
||||
}
|
||||
|
||||
// TableExists 检查表是否存在
|
||||
func (c *StatTableCreator) TableExists(ctx context.Context, tableName string) (bool, error) {
|
||||
result, err := gfdb.DB(ctx).GetAll(ctx, "SELECT COUNT(*) FROM pg_tables WHERE tablename = $1", strings.ToLower(tableName))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if len(result) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
count := result[0]["count"].Int()
|
||||
return count > 0, nil
|
||||
}
|
||||
|
||||
// GetTableColumns 获取表字段列表
|
||||
func (c *StatTableCreator) GetTableColumns(ctx context.Context, tableName string) ([]string, error) {
|
||||
var columns []string
|
||||
sql := `SELECT column_name FROM information_schema.columns WHERE table_name = $1 AND table_schema = 'public' ORDER BY ordinal_position`
|
||||
rows, err := gfdb.DB(ctx).GetAll(ctx, sql, strings.ToLower(tableName))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, row := range rows.List() {
|
||||
if col, ok := row["column_name"].(string); ok {
|
||||
columns = append(columns, col)
|
||||
}
|
||||
}
|
||||
return columns, nil
|
||||
}
|
||||
|
||||
// AlterTableAddColumns 为已存在的表添加新字段
|
||||
func (c *StatTableCreator) AlterTableAddColumns(ctx context.Context, tableName string, newFields []model.FieldConfig) error {
|
||||
existingCols, err := c.GetTableColumns(ctx, tableName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
existingMap := make(map[string]bool)
|
||||
for _, col := range existingCols {
|
||||
existingMap[col] = true
|
||||
}
|
||||
|
||||
for _, f := range newFields {
|
||||
if existingMap[f.FieldCode] {
|
||||
continue
|
||||
}
|
||||
colDef := c.fieldTypeToColumn(f.FieldCode, f.FieldType)
|
||||
sql := fmt.Sprintf("ALTER TABLE %s ADD COLUMN IF NOT EXISTS %s", tableName, colDef)
|
||||
if _, err := gfdb.DB(ctx).Exec(ctx, sql); err != nil {
|
||||
return fmt.Errorf("添加字段 %s 失败: %w", f.FieldCode, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
666
common/report/example_usage.go
Normal file
666
common/report/example_usage.go
Normal file
@@ -0,0 +1,666 @@
|
||||
package report
|
||||
|
||||
// ============================================================
|
||||
// 通用报表引擎 - 完整调用示例
|
||||
// ============================================================
|
||||
//
|
||||
// 包名: dataengine/common/report
|
||||
// 入口: report.GetService() → *ReportService
|
||||
//
|
||||
// 接口一览:
|
||||
// ┌──────────────┬─────────────────────────────────────────────────┐
|
||||
// │ 分类 │ 接口 │
|
||||
// ├──────────────┼─────────────────────────────────────────────────┤
|
||||
// │ 配置 CRUD │ SaveBusiness / DelBusiness / GetBusiness │
|
||||
// │ │ SaveReport / DelReport / GetReport │
|
||||
// │ │ SaveField / DelField / GetField │
|
||||
// │ │ SaveExtractConfig / DelExtractConfig / Get.. │
|
||||
// ├──────────────┼─────────────────────────────────────────────────┤
|
||||
// │ 数据抽取 │ ExtractDailyData / AutoCreateStatTable │
|
||||
// ├──────────────┼─────────────────────────────────────────────────┤
|
||||
// │ 报表查询 │ QueryReportByUserSelect │
|
||||
// ├──────────────┼─────────────────────────────────────────────────┤
|
||||
// │ 辅助查询 │ GetAllBusinesses / GetAllReports │
|
||||
// │ │ GetReportFields / GetExtractConfigs │
|
||||
// └──────────────┴─────────────────────────────────────────────────┘
|
||||
//
|
||||
// ============================================================
|
||||
|
||||
// ============================================================================
|
||||
// 场景一：新平台零代码接入（快手电商为例）
|
||||
// ============================================================================
|
||||
//
|
||||
// 背景：快手订单数据已通过数据引擎同步到 kuaishou_order_list 表（每条订单一行）。
|
||||
// 需求：按店铺+天聚合订单数据 → 自动建统计宽表 → 抽取 → 前端自由查询报表。
|
||||
//
|
||||
// 全程只调 API 不写 SQL，前端管理后台可直接操作。
|
||||
|
||||
/*
|
||||
package example
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"dataengine/common/report"
|
||||
"dataengine/common/report/model"
|
||||
)
|
||||
|
||||
func KuaishouExample() {
|
||||
ctx := context.Background()
|
||||
svc := report.GetService()
|
||||
|
||||
// ─── Step 1: 注册业务 ───────────────────────────────────────────
|
||||
// 一个业务就是一个数据源平台（快手/抖音/淘宝/...）
|
||||
_, _ = svc.SaveBusiness(ctx, &model.SaveBusinessReq{
|
||||
BusinessCode: "KUAISHOU", // 唯一标识，后续所有接口都用它
|
||||
BusinessName: "快手电商",
|
||||
Description: "快手平台电商业务线",
|
||||
Status: model.StatusActive,
|
||||
Operator: "admin",
|
||||
})
|
||||
|
||||
// ─── Step 2: 注册报表 ───────────────────────────────────────────
|
||||
// 一个业务可以有多个报表（店铺日报、商品日报、主播日报...）
|
||||
_, _ = svc.SaveReport(ctx, &model.SaveReportReq{
|
||||
BusinessCode: "KUAISHOU",
|
||||
ReportCode: "shop_daily_report", // 报表唯一编码
|
||||
ReportName: "快手店铺日报",
|
||||
StatTableName: "stat_kuaishou_shop_daily", // 统计宽表名（自动创建）
|
||||
DateField: "stat_date", // 日期字段
|
||||
ConflictKeys: []string{"shop_id", "stat_date"}, // 唯一约束（upsert 依据）
|
||||
Operator: "admin",
|
||||
})
|
||||
|
||||
// ─── Step 3: 配置字段（前端可选择的所有维度/指标/筛选） ─────────
|
||||
// 这是前端"自定义报表"的数据源 —— 前端建好字段后，
|
||||
// 用户选择哪些维度、哪些指标、怎么筛选，实时查。
|
||||
|
||||
// 3a. 维度字段：分组依据，不可聚合
|
||||
dimensions := []model.SaveFieldReq{
|
||||
{BusinessCode: "KUAISHOU", ReportCode: "shop_daily_report",
|
||||
FieldCode: "shop_id", FieldName: "店铺ID", FieldType: "STRING",
|
||||
FieldRole: "DIMENSION", IsSortable: true, SortOrder: 1, GroupName: "店铺", Operator: "admin"},
|
||||
{BusinessCode: "KUAISHOU", ReportCode: "shop_daily_report",
|
||||
FieldCode: "shop_name", FieldName: "店铺名称", FieldType: "STRING",
|
||||
FieldRole: "DIMENSION", IsSortable: true, SortOrder: 2, GroupName: "店铺", Operator: "admin"},
|
||||
{BusinessCode: "KUAISHOU", ReportCode: "shop_daily_report",
|
||||
FieldCode: "stat_date", FieldName: "统计日期", FieldType: "DATE",
|
||||
FieldRole: "DIMENSION", IsSortable: true, SortOrder: 3, GroupName: "时间", Operator: "admin"},
|
||||
}
|
||||
|
||||
// 3b. 指标字段：聚合度量，前端可选 SUM/COUNT/AVG/MAX/MIN
|
||||
indicators := []model.SaveFieldReq{
|
||||
{BusinessCode: "KUAISHOU", ReportCode: "shop_daily_report",
|
||||
FieldCode: "order_count", FieldName: "订单数", FieldType: "INT",
|
||||
FieldRole: "INDICATOR", IsAggregatable: true, DefaultAggregate: "SUM",
|
||||
ValidAggregates: []string{"SUM", "COUNT", "AVG", "MAX", "MIN"},
|
||||
SortOrder: 10, GroupName: "订单", Operator: "admin"},
|
||||
{BusinessCode: "KUAISHOU", ReportCode: "shop_daily_report",
|
||||
FieldCode: "order_amount", FieldName: "订单金额(元)", FieldType: "FLOAT",
|
||||
FieldRole: "INDICATOR", IsAggregatable: true, DefaultAggregate: "SUM",
|
||||
ValidAggregates: []string{"SUM", "AVG", "MAX", "MIN"},
|
||||
SortOrder: 11, GroupName: "金额", Unit: "元", Operator: "admin"},
|
||||
{BusinessCode: "KUAISHOU", ReportCode: "shop_daily_report",
|
||||
FieldCode: "paid_amount", FieldName: "实付金额(元)", FieldType: "FLOAT",
|
||||
FieldRole: "INDICATOR", IsAggregatable: true, DefaultAggregate: "SUM",
|
||||
ValidAggregates: []string{"SUM", "AVG"},
|
||||
SortOrder: 12, GroupName: "金额", Unit: "元", Operator: "admin"},
|
||||
{BusinessCode: "KUAISHOU", ReportCode: "shop_daily_report",
|
||||
FieldCode: "refund_amount", FieldName: "退款金额(元)", FieldType: "FLOAT",
|
||||
FieldRole: "INDICATOR", IsAggregatable: true, DefaultAggregate: "SUM",
|
||||
ValidAggregates: []string{"SUM", "AVG"},
|
||||
SortOrder: 13, GroupName: "退款", Unit: "元", Operator: "admin"},
|
||||
{BusinessCode: "KUAISHOU", ReportCode: "shop_daily_report",
|
||||
FieldCode: "buyer_count", FieldName: "下单买家数", FieldType: "INT",
|
||||
FieldRole: "INDICATOR", IsAggregatable: true, DefaultAggregate: "COUNT",
|
||||
ValidAggregates: []string{"COUNT"},
|
||||
SortOrder: 14, GroupName: "用户", Operator: "admin"},
|
||||
// 衍生指标：退款率 = 退款金额 / 订单金额 * 100
|
||||
{BusinessCode: "KUAISHOU", ReportCode: "shop_daily_report",
|
||||
FieldCode: "refund_rate", FieldName: "退款率", FieldType: "FLOAT",
|
||||
FieldRole: "INDICATOR", IsAggregatable: true, DefaultAggregate: "AVG",
|
||||
Expression: "{refund_amount} / NULLIF({order_amount}, 0) * 100",
|
||||
ExpressionType: "CALCULATED", FormatPattern: "#,##0.00",
|
||||
Unit: "%", SortOrder: 20, GroupName: "退款", Operator: "admin"},
|
||||
}
|
||||
|
||||
// 3c. 筛选字段：纯筛选，不出现在 SELECT 中
|
||||
filters := []model.SaveFieldReq{
|
||||
{BusinessCode: "KUAISHOU", ReportCode: "shop_daily_report",
|
||||
FieldCode: "order_status", FieldName: "订单状态", FieldType: "STRING",
|
||||
FieldRole: "FILTER", IsFilterable: true,
|
||||
FilterOperators: []string{"=", "IN"},
|
||||
SortOrder: 30, GroupName: "筛选", Operator: "admin"},
|
||||
}
|
||||
|
||||
for _, f := range dimensions {
|
||||
_, _ = svc.SaveField(ctx, &f)
|
||||
}
|
||||
for _, f := range indicators {
|
||||
_, _ = svc.SaveField(ctx, &f)
|
||||
}
|
||||
for _, f := range filters {
|
||||
_, _ = svc.SaveField(ctx, &f)
|
||||
}
|
||||
|
||||
// ─── Step 4: 配置数据抽取规则 ────────────────────────────────────
|
||||
// 关键：AGGREGATE 模式，从源表聚合到统计宽表
|
||||
_, _ = svc.SaveExtractConfig(ctx, &model.SaveExtractConfigReq{
|
||||
BusinessCode: "KUAISHOU",
|
||||
ReportCode: "shop_daily_report",
|
||||
ExtractCode: "extract_shop_daily",
|
||||
ExtractName: "快手店铺订单按天聚合",
|
||||
SourceTableName: "kuaishou_order_list", // ← 源表（订单明细）
|
||||
SourceTableAlias: "o",
|
||||
TargetTableName: "stat_kuaishou_shop_daily", // ← 目标（统计宽表）
|
||||
ExtractType: model.ExtractTypeIncremental, // 增量抽取
|
||||
ExtractMode: model.ExtractModeAggregate, // ← 聚合模式
|
||||
ExtractKeyField: "created_at", // 增量依据字段
|
||||
GroupByFields: []string{"shop_id"}, // ← GROUP BY
|
||||
FilterExpression: "o.order_status != 'CANCELLED'", // 过滤取消订单
|
||||
// 字段映射：源表字段 → 目标表字段 + 聚合函数
|
||||
FieldMappings: []model.FieldMapping{
|
||||
{SourceField: "shop_id", TargetField: "shop_id", FieldType: "STRING"},
|
||||
{SourceField: "shop_name", TargetField: "shop_name", FieldType: "STRING"},
|
||||
{SourceField: "id", TargetField: "order_count", FieldType: "INT", AggregateFunction: "COUNT"},
|
||||
{SourceField: "order_amount", TargetField: "order_amount", FieldType: "FLOAT", AggregateFunction: "SUM"},
|
||||
{SourceField: "paid_amount", TargetField: "paid_amount", FieldType: "FLOAT", AggregateFunction: "SUM"},
|
||||
{SourceField: "refund_amount",TargetField: "refund_amount",FieldType: "FLOAT", AggregateFunction: "SUM"},
|
||||
{SourceField: "buyer_id", TargetField: "buyer_count", FieldType: "INT", AggregateFunction: "COUNT"},
|
||||
},
|
||||
BatchSize: 1000,
|
||||
Operator: "admin",
|
||||
})
|
||||
|
||||
// ─── Step 5: 每天定时抽取（cron/k8s CronJob 中调用） ─────────
|
||||
// ExtractDailyData 内部：
|
||||
// 1. 检测 stat_kuaishou_shop_daily 表是否存在
|
||||
// 2. 不存在 → 根据 FieldConfig 自动 CREATE TABLE
|
||||
// 3. 从 kuaishou_order_list 按 AGGREGATE 模式抽取当天数据
|
||||
// 4. UPSERT 到统计宽表
|
||||
today := time.Now().Format("2006-01-02")
|
||||
resp, _ := svc.ExtractDailyData(ctx, "KUAISHOU", "shop_daily_report", today, "cron")
|
||||
fmt.Printf("[%s] 抽取完成: 总%d 成功%d 失败%d 耗时%dms\n",
|
||||
today, resp.TotalCount, resp.SuccessCount, resp.FailCount, resp.ExecTimeMs)
|
||||
|
||||
// 实际抽取生成的 SQL（AGGREGATE 模式）:
|
||||
//
|
||||
// INSERT INTO stat_kuaishou_shop_daily (...)
|
||||
// SELECT ROW_NUMBER() OVER () AS id,
|
||||
// '2026-06-10' AS stat_date,
|
||||
// o.shop_id, o.shop_name,
|
||||
// COUNT(o.id) AS order_count,
|
||||
// SUM(o.order_amount) AS order_amount,
|
||||
// SUM(o.paid_amount) AS paid_amount,
|
||||
// SUM(o.refund_amount) AS refund_amount,
|
||||
// COUNT(o.buyer_id) AS buyer_count
|
||||
// FROM kuaishou_order_list o
|
||||
// WHERE o.created_at::date = '2026-06-10'
|
||||
// GROUP BY o.shop_id
|
||||
|
||||
// ─── Step 6: 前端查询 ──────────────────────────────────────────
|
||||
|
||||
// 6a. 前端先拉取可用字段列表
|
||||
fields, _ := svc.GetReportFields(ctx, "KUAISHOU", "shop_daily_report")
|
||||
// 返回:
|
||||
// dimensions: [shop_id, shop_name, stat_date]
|
||||
// indicators: [order_count, order_amount, paid_amount, refund_amount, buyer_count, refund_rate]
|
||||
// filters: [order_status]
|
||||
// 前端据此渲染选择器面板
|
||||
|
||||
// 6b. 用户选择: 维度=店铺, 指标=金额+订单数, 时间=近7天, 排名=TOP10
|
||||
top10Req := &model.UserSelectQueryReq{
|
||||
BusinessCode: "KUAISHOU",
|
||||
ReportCode: "shop_daily_report",
|
||||
Dimensions: []string{"shop_id", "shop_name"},
|
||||
Indicators: []model.IndicatorSelect{
|
||||
{FieldCode: "order_amount", Aggregate: "SUM", Alias: "total_amount"},
|
||||
{FieldCode: "order_count", Aggregate: "SUM", Alias: "total_orders"},
|
||||
{FieldCode: "paid_amount", Aggregate: "SUM", Alias: "total_paid"},
|
||||
{FieldCode: "refund_rate", Aggregate: "AVG", Alias: "avg_refund_rate"},
|
||||
},
|
||||
TimeRange: &model.TimeRange{
|
||||
StartDate: time.Now().AddDate(0, 0, -7).Format("2006-01-02"),
|
||||
EndDate: today,
|
||||
},
|
||||
OrderBy: []model.OrderCondition{{FieldCode: "total_amount", Direction: "DESC"}},
|
||||
Page: 1,
|
||||
PageSize: 10,
|
||||
}
|
||||
top10Resp, _ := svc.QueryReportByUserSelect(ctx, top10Req)
|
||||
fmt.Printf("TOP10 销售额排行 (总数=%d):\n", top10Resp.Total)
|
||||
for i, row := range top10Resp.List {
|
||||
b, _ := json.Marshal(row)
|
||||
fmt.Printf(" %d. %s\n", i+1, string(b))
|
||||
}
|
||||
// 实际生成 SQL:
|
||||
// SELECT o.shop_id, o.shop_name,
|
||||
// SUM(o.order_amount) AS total_amount,
|
||||
// SUM(o.order_count) AS total_orders,
|
||||
// SUM(o.paid_amount) AS total_paid,
|
||||
// AVG(refund_amount / NULLIF(order_amount,0) * 100) AS avg_refund_rate
|
||||
// FROM stat_kuaishou_shop_daily o
|
||||
// WHERE stat_date BETWEEN '2026-06-03' AND '2026-06-10'
|
||||
// GROUP BY o.shop_id, o.shop_name
|
||||
// ORDER BY total_amount DESC
|
||||
// LIMIT 10 OFFSET 0
|
||||
|
||||
// 6c. 用户切换维度: 每日趋势（聚合所有店铺）
|
||||
trendReq := &model.UserSelectQueryReq{
|
||||
BusinessCode: "KUAISHOU",
|
||||
ReportCode: "shop_daily_report",
|
||||
Dimensions: []string{"stat_date"},
|
||||
Indicators: []model.IndicatorSelect{
|
||||
{FieldCode: "order_amount", Aggregate: "SUM", Alias: "daily_amount"},
|
||||
{FieldCode: "order_count", Aggregate: "SUM", Alias: "daily_orders"},
|
||||
{FieldCode: "refund_amount",Aggregate: "SUM", Alias: "daily_refund"},
|
||||
},
|
||||
TimeRange: &model.TimeRange{
|
||||
StartDate: time.Now().AddDate(0, 0, -30).Format("2006-01-02"),
|
||||
EndDate: today,
|
||||
},
|
||||
TimeGroup: "day",
|
||||
OrderBy: []model.OrderCondition{{FieldCode: "stat_date", Direction: "ASC"}},
|
||||
PageSize: 100,
|
||||
}
|
||||
trendResp, _ := svc.QueryReportByUserSelect(ctx, trendReq)
|
||||
fmt.Printf("30天趋势 (共%d行):\n", trendResp.Total)
|
||||
|
||||
// 6d. 加筛选条件: 只看活跃订单，金额>10000
|
||||
filteredReq := &model.UserSelectQueryReq{
|
||||
BusinessCode: "KUAISHOU",
|
||||
ReportCode: "shop_daily_report",
|
||||
Dimensions: []string{"shop_id", "shop_name"},
|
||||
Indicators: []model.IndicatorSelect{
|
||||
{FieldCode: "order_amount", Aggregate: "SUM", Alias: "total_amount"},
|
||||
},
|
||||
Filters: []model.FilterCondition{
|
||||
{FieldCode: "order_status", Operator: "=", Value: "ACTIVE"},
|
||||
{FieldCode: "total_amount", Operator: ">=", Value: 10000}, // 指标别名也可筛选
|
||||
},
|
||||
OrderBy: []model.OrderCondition{{FieldCode: "total_amount", Direction: "DESC"}},
|
||||
PageSize: 20,
|
||||
}
|
||||
_ = filteredReq
|
||||
|
||||
// 6e. 按周汇总趋势
|
||||
weeklyReq := &model.UserSelectQueryReq{
|
||||
BusinessCode: "KUAISHOU",
|
||||
ReportCode: "shop_daily_report",
|
||||
Dimensions: []string{"shop_id"},
|
||||
Indicators: []model.IndicatorSelect{
|
||||
{FieldCode: "order_amount", Aggregate: "SUM", Alias: "weekly_amount"},
|
||||
},
|
||||
TimeGroup: "week",
|
||||
OrderBy: []model.OrderCondition{{FieldCode: "weekly_amount", Direction: "DESC"}},
|
||||
PageSize: 50,
|
||||
}
|
||||
_ = weeklyReq
|
||||
}
|
||||
|
||||
|
||||
// ============================================================================
|
||||
// 场景二：已有配置管理（CRUD 二次开发参考）
|
||||
// ============================================================================
|
||||
|
||||
func CRUDExample() {
|
||||
ctx := context.Background()
|
||||
svc := report.GetService()
|
||||
|
||||
// ── 业务 CRUD ────────────────────────────────────────────
|
||||
|
||||
// 新增
|
||||
result, _ := svc.SaveBusiness(ctx, &model.SaveBusinessReq{
|
||||
BusinessCode: "DOUYIN", BusinessName: "抖音电商",
|
||||
Operator: "admin",
|
||||
})
|
||||
businessId := result.ID
|
||||
|
||||
// 修改（传 ID 即修改）
|
||||
result, _ = svc.SaveBusiness(ctx, &model.SaveBusinessReq{
|
||||
ID: &businessId, BusinessCode: "DOUYIN",
|
||||
BusinessName: "抖音电商（新版）", Operator: "admin",
|
||||
})
|
||||
|
||||
// 查询
|
||||
biz, _ := svc.GetBusiness(ctx, businessId)
|
||||
fmt.Printf("%s: %s\n", biz.BusinessCode, biz.BusinessName)
|
||||
|
||||
// 删除
|
||||
svc.DeleteBusiness(ctx, businessId)
|
||||
|
||||
// 全部列表
|
||||
allBiz, _ := svc.GetAllBusinesses(ctx)
|
||||
for _, b := range allBiz {
|
||||
fmt.Printf("- %s (%s)\n", b.BusinessCode, b.BusinessName)
|
||||
}
|
||||
|
||||
// ── 报表 CRUD ────────────────────────────────────────────
|
||||
|
||||
reportResult, _ := svc.SaveReport(ctx, &model.SaveReportReq{
|
||||
BusinessCode: "DOUYIN",
|
||||
ReportCode: "shop_daily_report",
|
||||
ReportName: "抖音店铺日报",
|
||||
StatTableName: "stat_douyin_shop_daily",
|
||||
ConflictKeys: []string{"shop_id", "stat_date"},
|
||||
Operator: "admin",
|
||||
})
|
||||
reportId := reportResult.ID
|
||||
|
||||
rpt, _ := svc.GetReport(ctx, reportId)
|
||||
svc.DeleteReport(ctx, reportId)
|
||||
_ = rpt
|
||||
|
||||
// ── 字段 CRUD ────────────────────────────────────────────
|
||||
|
||||
// 新增字段（id 不传 = 新增）
|
||||
fieldResult, _ := svc.SaveField(ctx, &model.SaveFieldReq{
|
||||
BusinessCode: "DOUYIN", ReportCode: "shop_daily_report",
|
||||
FieldCode: "order_amount", FieldName: "订单金额",
|
||||
FieldType: "FLOAT", FieldRole: "INDICATOR",
|
||||
IsAggregatable: true, DefaultAggregate: "SUM",
|
||||
ValidAggregates: []string{"SUM", "AVG", "MAX", "MIN"},
|
||||
SortOrder: 10, GroupName: "金额", Operator: "admin",
|
||||
})
|
||||
fieldId := fieldResult.ID
|
||||
|
||||
// 修改字段（传 id = 更新）
|
||||
svc.SaveField(ctx, &model.SaveFieldReq{
|
||||
ID: &fieldId,
|
||||
FieldName: "订单金额（元）",
|
||||
Operator: "admin",
|
||||
// ... 只传要修改的字段，未传的保持原值
|
||||
})
|
||||
|
||||
f, _ := svc.GetField(ctx, fieldId)
|
||||
svc.DeleteField(ctx, fieldId)
|
||||
_ = f
|
||||
|
||||
// ── 抽取配置 CRUD ────────────────────────────────────────
|
||||
|
||||
ecResult, _ := svc.SaveExtractConfig(ctx, &model.SaveExtractConfigReq{
|
||||
BusinessCode: "DOUYIN", ReportCode: "shop_daily_report",
|
||||
ExtractCode: "extract_daily", ExtractName: "按天聚合抽取",
|
||||
SourceTableName: "douyin_order_list", SourceTableAlias: "o",
|
||||
TargetTableName: "stat_douyin_shop_daily",
|
||||
ExtractMode: "AGGREGATE",
|
||||
ExtractKeyField: "created_at",
|
||||
GroupByFields: []string{"shop_id"},
|
||||
FieldMappings: []model.FieldMapping{
|
||||
{SourceField: "id", TargetField: "order_count", FieldType: "INT", AggregateFunction: "COUNT"},
|
||||
{SourceField: "order_amount", TargetField: "order_amount", FieldType: "FLOAT", AggregateFunction: "SUM"},
|
||||
},
|
||||
Operator: "admin",
|
||||
})
|
||||
ecId := ecResult.ID
|
||||
|
||||
ec, _ := svc.GetExtractConfig(ctx, ecId)
|
||||
allEc, _ := svc.GetExtractConfigs(ctx, "DOUYIN", "shop_daily_report")
|
||||
svc.DeleteExtractConfig(ctx, ecId)
|
||||
_, _ = ec, allEc
|
||||
}
|
||||
|
||||
|
||||
// ============================================================================
|
||||
// 场景三：任意平台接入的通用模式（零硬编码）
|
||||
// ============================================================================
|
||||
//
|
||||
// 假设要接入淘宝平台，taobao_order_list 表已有数据。
|
||||
// 全程不写任何代码，只需调 CRUD API。
|
||||
|
||||
func GenericPlatformExample() {
|
||||
ctx := context.Background()
|
||||
svc := report.GetService()
|
||||
|
||||
// 1. 前端注册业务
|
||||
svc.SaveBusiness(ctx, &model.SaveBusinessReq{
|
||||
BusinessCode: "TAOBAO", BusinessName: "淘宝电商",
|
||||
Operator: "admin",
|
||||
})
|
||||
|
||||
// 2. 注册报表 + 指定统计宽表名（以后表名不再手工管理）
|
||||
svc.SaveReport(ctx, &model.SaveReportReq{
|
||||
BusinessCode: "TAOBAO",
|
||||
ReportCode: "shop_daily_report",
|
||||
ReportName: "淘宝店铺日报",
|
||||
StatTableName: "stat_taobao_shop_daily",
|
||||
ConflictKeys: []string{"shop_id", "stat_date"},
|
||||
Operator: "admin",
|
||||
})
|
||||
|
||||
// 3. 前端选择统计维度（自由组合，随时增删改）
|
||||
svc.SaveField(ctx, &model.SaveFieldReq{
|
||||
BusinessCode: "TAOBAO", ReportCode: "shop_daily_report",
|
||||
FieldCode: "shop_id", FieldName: "店铺ID",
|
||||
FieldType: "STRING", FieldRole: "DIMENSION",
|
||||
SortOrder: 1, GroupName: "店铺", Operator: "admin",
|
||||
})
|
||||
svc.SaveField(ctx, &model.SaveFieldReq{
|
||||
BusinessCode: "TAOBAO", ReportCode: "shop_daily_report",
|
||||
FieldCode: "shop_name", FieldName: "店铺名称",
|
||||
FieldType: "STRING", FieldRole: "DIMENSION",
|
||||
SortOrder: 2, GroupName: "店铺", Operator: "admin",
|
||||
})
|
||||
svc.SaveField(ctx, &model.SaveFieldReq{
|
||||
BusinessCode: "TAOBAO", ReportCode: "shop_daily_report",
|
||||
FieldCode: "stat_date", FieldName: "统计日期",
|
||||
FieldType: "DATE", FieldRole: "DIMENSION",
|
||||
SortOrder: 3, GroupName: "时间", Operator: "admin",
|
||||
})
|
||||
|
||||
// 4. 前端选择统计指标（自由组合，随时增删改）
|
||||
svc.SaveField(ctx, &model.SaveFieldReq{
|
||||
BusinessCode: "TAOBAO", ReportCode: "shop_daily_report",
|
||||
FieldCode: "order_count", FieldName: "订单数",
|
||||
FieldType: "INT", FieldRole: "INDICATOR",
|
||||
IsAggregatable: true, DefaultAggregate: "SUM",
|
||||
ValidAggregates: []string{"SUM", "COUNT", "AVG"},
|
||||
SortOrder: 10, GroupName: "订单", Operator: "admin",
|
||||
})
|
||||
svc.SaveField(ctx, &model.SaveFieldReq{
|
||||
BusinessCode: "TAOBAO", ReportCode: "shop_daily_report",
|
||||
FieldCode: "order_amount", FieldName: "订单金额",
|
||||
FieldType: "FLOAT", FieldRole: "INDICATOR",
|
||||
IsAggregatable: true, DefaultAggregate: "SUM",
|
||||
ValidAggregates: []string{"SUM", "AVG", "MAX", "MIN"},
|
||||
SortOrder: 11, GroupName: "金额", Unit: "元", Operator: "admin",
|
||||
})
|
||||
svc.SaveField(ctx, &model.SaveFieldReq{
|
||||
BusinessCode: "TAOBAO", ReportCode: "shop_daily_report",
|
||||
FieldCode: "buyer_count", FieldName: "下单买家数",
|
||||
FieldType: "INT", FieldRole: "INDICATOR",
|
||||
IsAggregatable: true, DefaultAggregate: "COUNT",
|
||||
ValidAggregates: []string{"COUNT"},
|
||||
SortOrder: 12, GroupName: "用户", Operator: "admin",
|
||||
})
|
||||
|
||||
// 5. 配置抽取规则
|
||||
svc.SaveExtractConfig(ctx, &model.SaveExtractConfigReq{
|
||||
BusinessCode: "TAOBAO", ReportCode: "shop_daily_report",
|
||||
ExtractCode: "extract_daily", ExtractName: "淘宝按天聚合",
|
||||
SourceTableName: "taobao_order_list", SourceTableAlias: "o",
|
||||
TargetTableName: "stat_taobao_shop_daily",
|
||||
ExtractMode: "AGGREGATE",
|
||||
ExtractKeyField: "created_at",
|
||||
GroupByFields: []string{"shop_id"},
|
||||
FieldMappings: []model.FieldMapping{
|
||||
{SourceField: "shop_id", TargetField: "shop_id", FieldType: "STRING"},
|
||||
{SourceField: "shop_name", TargetField: "shop_name", FieldType: "STRING"},
|
||||
{SourceField: "id", TargetField: "order_count", FieldType: "INT", AggregateFunction: "COUNT"},
|
||||
{SourceField: "order_amount", TargetField: "order_amount", FieldType: "FLOAT", AggregateFunction: "SUM"},
|
||||
{SourceField: "buyer_id", TargetField: "buyer_count", FieldType: "INT", AggregateFunction: "COUNT"},
|
||||
},
|
||||
BatchSize: 1000,
|
||||
Operator: "admin",
|
||||
})
|
||||
|
||||
// 6. 定时任务每天执行
|
||||
svc.ExtractDailyData(ctx, "TAOBAO", "shop_daily_report", "2026-06-10", "cron")
|
||||
|
||||
// 7. 前端实时查询
|
||||
req := &model.UserSelectQueryReq{
|
||||
BusinessCode: "TAOBAO", ReportCode: "shop_daily_report",
|
||||
Dimensions: []string{"shop_id", "shop_name"},
|
||||
Indicators: []model.IndicatorSelect{
|
||||
{FieldCode: "order_amount", Aggregate: "SUM", Alias: "total"},
|
||||
{FieldCode: "order_count", Aggregate: "SUM", Alias: "orders"},
|
||||
},
|
||||
TimeRange: &model.TimeRange{StartDate: "2026-06-01", EndDate: "2026-06-10"},
|
||||
Page: 1, PageSize: 20,
|
||||
}
|
||||
resp, _ := svc.QueryReportByUserSelect(ctx, req)
|
||||
fmt.Printf("查询结果: 共%d条, 耗时%dms\n", resp.Total, resp.ExecTimeMs)
|
||||
|
||||
// 平台接入完毕。全程零代码改动。
|
||||
}
|
||||
|
||||
|
||||
// ============================================================================
|
||||
// 场景四：在外部业务服务（如 goview-report）中调用
|
||||
// ============================================================================
|
||||
|
||||
func ExternalServiceExample() {
|
||||
ctx := context.Background()
|
||||
svc := report.GetService()
|
||||
|
||||
// 直接用，不需要初始化，内部自动懒加载和建表。
|
||||
today := time.Now().Format("2006-01-02")
|
||||
|
||||
// 按天抽取
|
||||
svc.ExtractDailyData(ctx, "KUAISHOU", "shop_daily_report", today, "cron")
|
||||
|
||||
// 实时查询
|
||||
svc.QueryReportByUserSelect(ctx, &model.UserSelectQueryReq{
|
||||
BusinessCode: "KUAISHOU", ReportCode: "shop_daily_report",
|
||||
Dimensions: []string{"shop_id", "shop_name"},
|
||||
Indicators: []model.IndicatorSelect{
|
||||
{FieldCode: "order_amount", Aggregate: "SUM", Alias: "total_amount"},
|
||||
},
|
||||
TimeRange: &model.TimeRange{StartDate: "2026-06-01", EndDate: today},
|
||||
Page: 1, PageSize: 20,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
// ============================================================================
|
||||
// 场景五：Direct 模式（逐行抽取，不做聚合）
|
||||
// ============================================================================
|
||||
//
|
||||
// 适用于源表已经是一行一条统计记录的场景（如已预聚合的报表源表）。
|
||||
|
||||
func DirectModeExample() {
|
||||
ctx := context.Background()
|
||||
svc := report.GetService()
|
||||
|
||||
svc.SaveBusiness(ctx, &model.SaveBusinessReq{
|
||||
BusinessCode: "HDWL", BusinessName: "HDWL业务", Operator: "admin",
|
||||
})
|
||||
svc.SaveReport(ctx, &model.SaveReportReq{
|
||||
BusinessCode: "HDWL", ReportCode: "shop_daily_stat",
|
||||
ReportName: "HDWL店铺日报", StatTableName: "stat_hdwl_shop_daily",
|
||||
ConflictKeys: []string{"report_date"},
|
||||
Operator: "admin",
|
||||
})
|
||||
|
||||
// Direct 模式：不聚合，逐行映射
|
||||
svc.SaveExtractConfig(ctx, &model.SaveExtractConfigReq{
|
||||
BusinessCode: "HDWL", ReportCode: "shop_daily_stat",
|
||||
ExtractCode: "extract_direct", ExtractName: "直接逐行抽取",
|
||||
SourceTableName: "hdwl_daily_summary", SourceTableAlias: "s",
|
||||
TargetTableName: "stat_hdwl_shop_daily",
|
||||
ExtractMode: "DIRECT", // ← DIRECT 模式
|
||||
ExtractKeyField: "report_date",
|
||||
FieldMappings: []model.FieldMapping{
|
||||
{SourceField: "shop_id", TargetField: "shop_id", FieldType: "STRING"},
|
||||
{SourceField: "shop_name", TargetField: "shop_name", FieldType: "STRING"},
|
||||
{SourceField: "total_sale", TargetField: "sale_amount", FieldType: "FLOAT"},
|
||||
{SourceField: "total_orders", TargetField: "order_count", FieldType: "INT"},
|
||||
},
|
||||
Operator: "admin",
|
||||
})
|
||||
|
||||
svc.ExtractDailyData(ctx, "HDWL", "shop_daily_stat", "2026-06-10", "cron")
|
||||
}
|
||||
|
||||
|
||||
// ============================================================================
|
||||
// 场景六：前端交互流程参考
|
||||
// ============================================================================
|
||||
//
|
||||
// 前端代码调用示例（伪代码，展示 API 调用顺序）：
|
||||
//
|
||||
// // 1. 页面加载 → 获取业务列表 → 渲染下拉框
|
||||
// GET /api/report/businesses
|
||||
// 返回: [{ businessCode: "KUAISHOU", businessName: "快手电商" }, ...]
|
||||
//
|
||||
// // 2. 用户选择业务 → 获取报表列表
|
||||
// GET /api/report/reports?businessCode=KUAISHOU
|
||||
// 返回: [{ reportCode: "shop_daily_report", reportName: "快手店铺日报" }, ...]
|
||||
//
|
||||
// // 3. 用户选择报表 → 获取可用字段 → 渲染维度/指标/筛选选择器
|
||||
// GET /api/report/fields?businessCode=KUAISHOU&reportCode=shop_daily_report
|
||||
// 返回: { dimensions: [...], indicators: [...], filters: [...] }
|
||||
//
|
||||
// // 4. 用户选好条件 → 查询
|
||||
// POST /api/report/query
|
||||
// Body: {
|
||||
// "businessCode": "KUAISHOU", "reportCode": "shop_daily_report",
|
||||
// "dimensions": ["shop_id", "shop_name"],
|
||||
// "indicators": [
|
||||
// {"fieldCode": "order_amount", "aggregate": "SUM", "alias": "total"},
|
||||
// {"fieldCode": "order_count", "aggregate": "SUM", "alias": "count"}
|
||||
// ],
|
||||
// "timeRange": {"startDate": "2026-06-01", "endDate": "2026-06-10"},
|
||||
// "orderBy": [{"fieldCode": "total", "direction": "DESC"}],
|
||||
// "page": 1, "pageSize": 20
|
||||
// }
|
||||
// 返回: { list: [...], total: 152, page: 1, totalPages: 8, execTimeMs: 45 }
|
||||
//
|
||||
// // 5. 翻页/换维度/换指标 → 重新调 Step 4
|
||||
//
|
||||
// 管理后台（用户可自行维护配置）：
|
||||
//
|
||||
// // 新增业务
|
||||
// POST /api/report/business/save
|
||||
// Body: { "businessCode": "TAOBAO", "businessName": "淘宝电商" }
|
||||
//
|
||||
// // 新增/修改字段（用户自定义统计维度）
|
||||
// POST /api/report/field/save
|
||||
// Body: { "businessCode": "TAOBAO", "reportCode": "shop_daily_report",
|
||||
// "fieldCode": "category", "fieldName": "商品类目",
|
||||
// "fieldType": "STRING", "fieldRole": "DIMENSION" }
|
||||
//
|
||||
// // 配置抽取规则
|
||||
// POST /api/report/extractConfig/save
|
||||
// Body: { "businessCode": "TAOBAO", "reportCode": "shop_daily_report",
|
||||
// "extractCode": "extract_daily", "sourceTableName": "taobao_order_list",
|
||||
// "extractMode": "AGGREGATE", "groupByFields": ["shop_id"],
|
||||
// "fieldMappings": [...] }
|
||||
|
||||
|
||||
// ============================================================================
|
||||
// Direct vs AGGREGATE 模式对比
|
||||
// ============================================================================
|
||||
//
|
||||
// ┌──────────┬──────────────────────────────────────────────┐
|
||||
// │ 模式 │ 行为 │
|
||||
// ├──────────┼──────────────────────────────────────────────┤
|
||||
// │ DIRECT │ 逐行映射，1:1 从源表复制到目标表 │
|
||||
// │ │ 适用：源表已是一行一统计 │
|
||||
// │ │ SQL: SELECT ... FROM source │
|
||||
// ├──────────┼──────────────────────────────────────────────┤
|
||||
// │ AGGREGATE│ GROUP BY + 聚合函数，N:1 聚合 │
|
||||
// │ │ 适用：源表是明细表（如订单行） │
|
||||
// │ │ SQL: SELECT ... SUM() COUNT() ... │
|
||||
// │ │ FROM source GROUP BY groupByFields │
|
||||
// └──────────┴──────────────────────────────────────────────┘
|
||||
//
|
||||
// 可聚合函数: SUM / COUNT / AVG / MAX / MIN
|
||||
// 字段角色: DIMENSION(维度) / INDICATOR(指标) / FILTER(筛选) / FILTER_ONLY
|
||||
// 字段类型: STRING / INT / FLOAT / DATE / DATETIME / JSONB
|
||||
// 操作符: = / != / > / < / >= / <= / IN / LIKE / BETWEEN
|
||||
// 时间分组: day / week / month / quarter
|
||||
*/
|
||||
192
common/report/executor/executor.go
Normal file
192
common/report/executor/executor.go
Normal file
@@ -0,0 +1,192 @@
|
||||
package executor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"dataengine/common/report/builder"
|
||||
"dataengine/common/report/model"
|
||||
|
||||
"gitea.redpowerfuture.com/red-future/common/db/gfdb"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// QueryExecutor SQL执行器
|
||||
type QueryExecutor struct {
|
||||
sqlBuilder *builder.SQLBuilder
|
||||
}
|
||||
|
||||
// NewQueryExecutor 创建查询执行器
|
||||
func NewQueryExecutor() *QueryExecutor {
|
||||
return &QueryExecutor{
|
||||
sqlBuilder: builder.NewSQLBuilder(),
|
||||
}
|
||||
}
|
||||
|
||||
// QueryReportByUserSelect 根据用户选择实时查询报表数据（核心接口）
|
||||
func (e *QueryExecutor) QueryReportByUserSelect(ctx context.Context, req *model.UserSelectQueryReq) (*model.UserSelectQueryResp, error) {
|
||||
start := time.Now()
|
||||
logger := logrus.WithFields(logrus.Fields{
|
||||
"businessCode": req.BusinessCode,
|
||||
"reportCode": req.ReportCode,
|
||||
})
|
||||
|
||||
// 1. 参数校验
|
||||
if err := e.validateReq(req); err != nil {
|
||||
return nil, fmt.Errorf("参数校验失败: %w", err)
|
||||
}
|
||||
|
||||
// 2. 构建查询SQL
|
||||
querySQL, queryArgs, metadata, err := e.sqlBuilder.BuildQuerySQL(ctx, req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("构建查询SQL失败: %w", err)
|
||||
}
|
||||
|
||||
logger.Debugf("查询SQL: %s args: %v", querySQL, queryArgs)
|
||||
|
||||
// 3. 获取记录总数
|
||||
total, err := e.executeCount(ctx, metadata, queryArgs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("查询总数失败: %w", err)
|
||||
}
|
||||
|
||||
// 4. 分页查询
|
||||
if req.Page < 1 {
|
||||
req.Page = 1
|
||||
}
|
||||
if req.PageSize < 1 {
|
||||
req.PageSize = 20
|
||||
}
|
||||
if req.PageSize > 1000 {
|
||||
req.PageSize = 1000
|
||||
}
|
||||
|
||||
pagedSQL := e.sqlBuilder.AddLimit(querySQL, req.Page, req.PageSize)
|
||||
logger.Debugf("分页SQL: %s", pagedSQL)
|
||||
|
||||
dataResult, err := gfdb.DB(ctx).GetAll(ctx, pagedSQL, queryArgs...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("查询数据失败: %w", err)
|
||||
}
|
||||
|
||||
var list []map[string]interface{}
|
||||
if dataResult.Len() > 0 {
|
||||
list = dataResult.List()
|
||||
}
|
||||
|
||||
// 计算总页数
|
||||
totalPages := int(math.Ceil(float64(total) / float64(req.PageSize)))
|
||||
|
||||
execTime := time.Since(start).Milliseconds()
|
||||
|
||||
resp := &model.UserSelectQueryResp{
|
||||
List: list,
|
||||
Total: total,
|
||||
Page: req.Page,
|
||||
PageSize: req.PageSize,
|
||||
TotalPages: totalPages,
|
||||
Sql: querySQL,
|
||||
ExecTimeMs: execTime,
|
||||
}
|
||||
|
||||
logger.Infof("报表查询完成, 总数:%d 耗时:%dms", total, execTime)
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// executeCount 执行总数统计
|
||||
func (e *QueryExecutor) executeCount(ctx context.Context, metadata map[string]interface{}, args []interface{}) (int64, error) {
|
||||
countSql, ok := metadata["countSql"].(string)
|
||||
if !ok || countSql == "" {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
result, err := gfdb.DB(ctx).GetAll(ctx, countSql, args...)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("统计总数失败: %w", err)
|
||||
}
|
||||
|
||||
var total int64
|
||||
if result.Len() > 0 {
|
||||
for k, v := range result.List()[0] {
|
||||
_ = k
|
||||
if n, ok := v.(int64); ok {
|
||||
total = n
|
||||
} else {
|
||||
total = result[0]["count"].Int64()
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
// validateReq 校验请求参数
|
||||
func (e *QueryExecutor) validateReq(req *model.UserSelectQueryReq) error {
|
||||
if req.BusinessCode == "" {
|
||||
return fmt.Errorf("业务编码不能为空")
|
||||
}
|
||||
if req.ReportCode == "" {
|
||||
return fmt.Errorf("报表编码不能为空")
|
||||
}
|
||||
if len(req.Indicators) == 0 {
|
||||
return fmt.Errorf("必须选择至少一个指标")
|
||||
}
|
||||
|
||||
// 校验指标
|
||||
for i, ind := range req.Indicators {
|
||||
if ind.FieldCode == "" {
|
||||
return fmt.Errorf("第%d个指标字段编码不能为空", i+1)
|
||||
}
|
||||
if ind.Aggregate != "" {
|
||||
agg := strings.ToUpper(ind.Aggregate)
|
||||
validAggs := map[string]bool{
|
||||
"SUM": true,
|
||||
"COUNT": true,
|
||||
"AVG": true,
|
||||
"MAX": true,
|
||||
"MIN": true,
|
||||
}
|
||||
if !validAggs[agg] {
|
||||
return fmt.Errorf("不支持的聚合方式: %s", ind.Aggregate)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 校验时间分组
|
||||
if req.TimeGroup != "" {
|
||||
validGroups := map[string]bool{
|
||||
"day": true,
|
||||
"week": true,
|
||||
"month": true,
|
||||
"quarter": true,
|
||||
}
|
||||
if !validGroups[req.TimeGroup] {
|
||||
return fmt.Errorf("不支持的时间分组: %s", req.TimeGroup)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// RawQuery 执行原始SQL查询（用于特殊场景）
|
||||
func (e *QueryExecutor) RawQuery(ctx context.Context, sql string, args ...interface{}) ([]map[string]interface{}, error) {
|
||||
result, err := gfdb.DB(ctx).GetAll(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("原始查询失败: %w", err)
|
||||
}
|
||||
|
||||
if result.Len() > 0 {
|
||||
return result.List(), nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// ExecRaw 执行原始SQL（INSERT/UPDATE/DELETE）
|
||||
func (e *QueryExecutor) ExecRaw(ctx context.Context, sql string, args ...interface{}) error {
|
||||
_, err := gfdb.DB(ctx).Exec(ctx, sql, args...)
|
||||
return err
|
||||
}
|
||||
644
common/report/extract/extract.go
Normal file
644
common/report/extract/extract.go
Normal file
@@ -0,0 +1,644 @@
|
||||
package extract
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"dataengine/common/report/config"
|
||||
"dataengine/common/report/model"
|
||||
|
||||
"gitea.redpowerfuture.com/red-future/common/db/gfdb"
|
||||
"github.com/gogf/gf/v2/util/gconv"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// DailyExtractor 天级数据抽取器
|
||||
type DailyExtractor struct {
|
||||
loader *config.ConfigLoader
|
||||
}
|
||||
|
||||
// NewDailyExtractor 创建抽取器
|
||||
func NewDailyExtractor() *DailyExtractor {
|
||||
return &DailyExtractor{
|
||||
loader: config.GetLoader(),
|
||||
}
|
||||
}
|
||||
|
||||
// ExtractDailyData 按天抽取数据（业务层定时任务调用）
|
||||
func (e *DailyExtractor) ExtractDailyData(ctx context.Context, businessCode, reportCode, statDate, executor string) (*model.ExtractDailyDataResp, error) {
|
||||
start := time.Now()
|
||||
logger := logrus.WithFields(logrus.Fields{
|
||||
"businessCode": businessCode,
|
||||
"reportCode": reportCode,
|
||||
"statDate": statDate,
|
||||
})
|
||||
|
||||
// 1. 获取报表配置
|
||||
report, err := e.loader.GetReport(ctx, businessCode, reportCode)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("获取报表配置失败: %w", err)
|
||||
}
|
||||
|
||||
// 2. 获取抽取配置
|
||||
extractConfigs, err := e.loader.GetExtractConfigs(ctx, businessCode, reportCode)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("获取抽取配置失败: %w", err)
|
||||
}
|
||||
|
||||
if len(extractConfigs) == 0 {
|
||||
return nil, fmt.Errorf("没有可用的抽取配置")
|
||||
}
|
||||
|
||||
// 3. 获取字段配置
|
||||
fieldMap, err := e.loader.GetFieldMap(ctx, businessCode, reportCode)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("获取字段配置失败: %w", err)
|
||||
}
|
||||
|
||||
// 4. 确保统计宽表存在
|
||||
if err := e.ensureStatTableExists(ctx, report, fieldMap); err != nil {
|
||||
return nil, fmt.Errorf("确保统计宽表存在失败: %w", err)
|
||||
}
|
||||
|
||||
totalCount := 0
|
||||
successCount := 0
|
||||
failCount := 0
|
||||
var lastErr error
|
||||
|
||||
// 5. 遍历每个抽取配置
|
||||
for _, ec := range extractConfigs {
|
||||
// 检查幂等性
|
||||
exLog, err := e.loader.GetExtractLog(ctx, businessCode, reportCode, ec.ExtractCode, statDate)
|
||||
if err != nil {
|
||||
logger.Errorf("获取抽取记录失败: %v", err)
|
||||
}
|
||||
|
||||
if exLog != nil && exLog.Status == model.ExtractStatusSuccess {
|
||||
logger.Infof("抽取配置 %s 日期 %s 已完成，跳过", ec.ExtractCode, statDate)
|
||||
continue
|
||||
}
|
||||
|
||||
// 创建抽取记录
|
||||
extractLog := &model.ExtractLog{
|
||||
BusinessCode: businessCode,
|
||||
ReportCode: reportCode,
|
||||
ExtractCode: ec.ExtractCode,
|
||||
StatDate: statDate,
|
||||
ExtractType: ec.ExtractType,
|
||||
Status: model.ExtractStatusRunning,
|
||||
Executor: executor,
|
||||
StartTime: &start,
|
||||
}
|
||||
_ = e.loader.CreateExtractLog(ctx, extractLog)
|
||||
|
||||
// 执行抽取
|
||||
c, s, f, err := e.executeExtract(ctx, &ec, report, fieldMap, statDate)
|
||||
totalCount += c
|
||||
successCount += s
|
||||
failCount += f
|
||||
|
||||
// 更新抽取记录
|
||||
now := time.Now()
|
||||
extractLog.EndTime = &now
|
||||
extractLog.TotalCount = c
|
||||
extractLog.SuccessCount = s
|
||||
extractLog.FailCount = f
|
||||
|
||||
if err != nil {
|
||||
extractLog.Status = model.ExtractStatusFailed
|
||||
extractLog.ErrorMessage = err.Error()
|
||||
lastErr = err
|
||||
logger.Errorf("抽取配置 %s 执行失败: %v", ec.ExtractCode, err)
|
||||
} else {
|
||||
extractLog.Status = model.ExtractStatusSuccess
|
||||
logger.Infof("抽取配置 %s 完成, 总数:%d 成功:%d 失败:%d", ec.ExtractCode, c, s, f)
|
||||
}
|
||||
|
||||
if updateErr := e.loader.UpdateExtractLog(ctx, extractLog); updateErr != nil {
|
||||
logger.Errorf("更新抽取记录失败: %v", updateErr)
|
||||
}
|
||||
}
|
||||
|
||||
execTime := time.Since(start).Milliseconds()
|
||||
logger.Infof("按天抽取完成, 总数:%d 成功:%d 失败:%d 耗时:%dms", totalCount, successCount, failCount, execTime)
|
||||
|
||||
resp := &model.ExtractDailyDataResp{
|
||||
Success: lastErr == nil,
|
||||
TotalCount: totalCount,
|
||||
SuccessCount: successCount,
|
||||
FailCount: failCount,
|
||||
ExecTimeMs: execTime,
|
||||
}
|
||||
|
||||
if lastErr != nil {
|
||||
resp.ErrorMsg = lastErr.Error()
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// executeExtract 执行单个抽取配置
|
||||
func (e *DailyExtractor) executeExtract(ctx context.Context, ec *model.ExtractConfig, report *model.ReportConfig, fieldMap map[string]*model.FieldConfig, statDate string) (total, success, fail int, err error) {
|
||||
logger := logrus.WithField("extractCode", ec.ExtractCode)
|
||||
|
||||
// 1. 构建抽取SQL
|
||||
extractSQL, whereArgs, err := e.buildExtractSQL(ctx, ec, report, statDate)
|
||||
if err != nil {
|
||||
return 0, 0, 0, fmt.Errorf("构建抽取SQL失败: %w", err)
|
||||
}
|
||||
|
||||
logger.Debugf("抽取SQL: %s", extractSQL)
|
||||
|
||||
// 2. 分批抽取
|
||||
batchSize := ec.BatchSize
|
||||
if batchSize <= 0 {
|
||||
batchSize = 1000
|
||||
}
|
||||
|
||||
offset := 0
|
||||
for {
|
||||
// 添加分页
|
||||
pagedSQL := fmt.Sprintf("%s LIMIT %d OFFSET %d", extractSQL, batchSize, offset)
|
||||
args := append(whereArgs)
|
||||
|
||||
rows, queryErr := gfdb.DB(ctx).GetAll(ctx, pagedSQL, args...)
|
||||
if queryErr != nil {
|
||||
return total, success, fail, fmt.Errorf("抽取查询失败: %w", queryErr)
|
||||
}
|
||||
|
||||
batchCount := rows.Len()
|
||||
if batchCount == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
// 3. 应用转换规则（仅 DIRECT 模式需注入审计字段，AGGREGATE 模式已由SQL处理）
|
||||
dataList := rows.List()
|
||||
if ec.ExtractMode != model.ExtractModeAggregate {
|
||||
for i := range dataList {
|
||||
e.applyTransformRules(ec, dataList[i])
|
||||
dataList[i]["tenant_id"] = 1
|
||||
dataList[i]["business_code"] = ec.BusinessCode
|
||||
}
|
||||
}
|
||||
|
||||
// 4. 写入统计宽表
|
||||
c, _, writeErr := e.batchUpsert(ctx, report.StatTableName, report.ConflictKeys, dataList)
|
||||
if writeErr != nil {
|
||||
logger.Errorf("批量写入失败 (offset=%d): %v", offset, writeErr)
|
||||
fail += batchCount
|
||||
} else {
|
||||
success += c
|
||||
}
|
||||
|
||||
total += batchCount
|
||||
offset += batchSize
|
||||
|
||||
if batchCount < batchSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return total, success, fail, nil
|
||||
}
|
||||
|
||||
// buildExtractSQL 构建抽取SQL
|
||||
func (e *DailyExtractor) buildExtractSQL(ctx context.Context, ec *model.ExtractConfig, report *model.ReportConfig, statDate string) (string, []interface{}, error) {
|
||||
var args []interface{}
|
||||
|
||||
sourceTable := ec.SourceTableName
|
||||
if ec.SourceTableAlias != "" {
|
||||
sourceTable = ec.SourceTableAlias
|
||||
} else {
|
||||
sourceTable = "s"
|
||||
}
|
||||
|
||||
// 日期字段
|
||||
dateField := report.DateField
|
||||
if dateField == "" {
|
||||
dateField = "stat_date"
|
||||
}
|
||||
|
||||
// 判断抽取模式
|
||||
mode := ec.ExtractMode
|
||||
if mode == "" {
|
||||
mode = model.ExtractModeDirect
|
||||
}
|
||||
|
||||
if mode == model.ExtractModeAggregate {
|
||||
return e.buildAggregateExtractSQL(ec, report, sourceTable, dateField, statDate)
|
||||
}
|
||||
|
||||
// === 默认 DIRECT 模式：逐行抽取 ===
|
||||
return e.buildDirectExtractSQL(ec, report, sourceTable, dateField, statDate), args, nil
|
||||
}
|
||||
|
||||
// buildDirectExtractSQL 逐行抽取模式SQL（直接映射，不做聚合）
|
||||
func (e *DailyExtractor) buildDirectExtractSQL(ec *model.ExtractConfig, report *model.ReportConfig, sourceTable, dateField, statDate string) string {
|
||||
var selectParts []string
|
||||
|
||||
// 基础审计字段（常量注入）
|
||||
selectParts = append(selectParts, "0 AS id")
|
||||
selectParts = append(selectParts, "1 AS tenant_id")
|
||||
selectParts = append(selectParts, fmt.Sprintf("'%s' AS business_code", ec.BusinessCode))
|
||||
selectParts = append(selectParts, "'system' AS creator")
|
||||
selectParts = append(selectParts, "NOW() AS created_at")
|
||||
selectParts = append(selectParts, "'system' AS updater")
|
||||
selectParts = append(selectParts, "NOW() AS updated_at")
|
||||
selectParts = append(selectParts, "NULL::TIMESTAMP AS deleted_at")
|
||||
|
||||
// 日期字段
|
||||
selectParts = append(selectParts, fmt.Sprintf("'%s' AS %s", statDate, dateField))
|
||||
|
||||
// 原始数据
|
||||
selectParts = append(selectParts, "'{}'::JSONB AS raw_data")
|
||||
|
||||
// 字段映射
|
||||
for _, mapping := range ec.FieldMappings {
|
||||
targetField := mapping.TargetField
|
||||
sourceField := mapping.SourceField
|
||||
|
||||
var expr string
|
||||
if mapping.TransformRule != nil {
|
||||
expr = e.applyTransformExpr(mapping.TransformRule, fmt.Sprintf("%s.%s", sourceTable, sourceField))
|
||||
} else {
|
||||
expr = fmt.Sprintf("%s.%s", sourceTable, sourceField)
|
||||
}
|
||||
|
||||
if mapping.DefaultValue != nil {
|
||||
expr = fmt.Sprintf("COALESCE(%s, '%v')", expr, mapping.DefaultValue)
|
||||
}
|
||||
|
||||
selectParts = append(selectParts, fmt.Sprintf("%s AS %s", expr, targetField))
|
||||
}
|
||||
|
||||
// FROM + JOIN
|
||||
fromClause := e.buildFromClause(ec, sourceTable)
|
||||
|
||||
// JOIN 字段映射
|
||||
selectParts = append(selectParts, e.buildJoinFieldSelects(ec)...)
|
||||
|
||||
// WHERE
|
||||
whereClause := e.buildWhereClause(ec, sourceTable, statDate)
|
||||
|
||||
return fmt.Sprintf("SELECT %s FROM %s %s", strings.Join(selectParts, ", "), fromClause, whereClause)
|
||||
}
|
||||
|
||||
// buildAggregateExtractSQL 聚合抽取模式SQL（GROUP BY + SUM/COUNT/AVG）
|
||||
func (e *DailyExtractor) buildAggregateExtractSQL(ec *model.ExtractConfig, report *model.ReportConfig, sourceTable, dateField, statDate string) (string, []interface{}, error) {
|
||||
var selectParts []string
|
||||
var groupByParts []string
|
||||
var args []interface{}
|
||||
|
||||
// 基础审计字段（聚合模式下用常量）
|
||||
selectParts = append(selectParts, "ROW_NUMBER() OVER () AS id") // 伪自增ID
|
||||
selectParts = append(selectParts, "1 AS tenant_id")
|
||||
selectParts = append(selectParts, fmt.Sprintf("'%s' AS business_code", ec.BusinessCode))
|
||||
selectParts = append(selectParts, "'system' AS creator")
|
||||
selectParts = append(selectParts, "NOW() AS created_at")
|
||||
selectParts = append(selectParts, "'system' AS updater")
|
||||
selectParts = append(selectParts, "NOW() AS updated_at")
|
||||
selectParts = append(selectParts, "NULL::TIMESTAMP AS deleted_at")
|
||||
|
||||
// 日期字段（常量）
|
||||
selectParts = append(selectParts, fmt.Sprintf("'%s' AS %s", statDate, dateField))
|
||||
|
||||
// 原始数据
|
||||
selectParts = append(selectParts, "'{}'::JSONB AS raw_data")
|
||||
|
||||
// GroupByFields 集合（快速查找）
|
||||
gbySet := make(map[string]bool)
|
||||
for _, gbf := range ec.GroupByFields {
|
||||
gbySet[gbf] = true
|
||||
}
|
||||
|
||||
// 添加 GroupBy 字段到 SELECT 和 GROUP BY
|
||||
for _, gbf := range ec.GroupByFields {
|
||||
selectParts = append(selectParts, fmt.Sprintf("%s.%s", sourceTable, gbf))
|
||||
groupByParts = append(groupByParts, fmt.Sprintf("%s.%s", sourceTable, gbf))
|
||||
}
|
||||
|
||||
// 字段映射：根据 AggregateFunction 决定聚合方式
|
||||
for _, mapping := range ec.FieldMappings {
|
||||
targetField := mapping.TargetField
|
||||
sourceField := mapping.SourceField
|
||||
|
||||
// 构建源表达式
|
||||
var sourceExpr string
|
||||
if mapping.TransformRule != nil {
|
||||
sourceExpr = e.applyTransformExpr(mapping.TransformRule, fmt.Sprintf("%s.%s", sourceTable, sourceField))
|
||||
} else {
|
||||
sourceExpr = fmt.Sprintf("%s.%s", sourceTable, sourceField)
|
||||
}
|
||||
|
||||
// 判断是否需要聚合
|
||||
aggFunc := strings.ToUpper(mapping.AggregateFunction)
|
||||
if aggFunc != "" && !gbySet[sourceField] {
|
||||
// 聚合字段：SUM(s.xxx) / COUNT(s.xxx) / AVG(s.xxx)
|
||||
expr := fmt.Sprintf("%s(%s)", aggFunc, sourceExpr)
|
||||
if mapping.DefaultValue != nil {
|
||||
expr = fmt.Sprintf("COALESCE(%s, %v)", expr, mapping.DefaultValue)
|
||||
}
|
||||
selectParts = append(selectParts, fmt.Sprintf("%s AS %s", expr, targetField))
|
||||
} else if gbySet[sourceField] {
|
||||
// GroupBy 字段不需要重复加入 SELECT（已通过 groupByFields 处理）
|
||||
continue
|
||||
} else {
|
||||
// 非聚合字段，也未在 GroupBy 中 → 用 MAX/MIN 取值（兼容 PG only_full_group_by）
|
||||
expr := fmt.Sprintf("MAX(%s)", sourceExpr)
|
||||
if mapping.DefaultValue != nil {
|
||||
expr = fmt.Sprintf("COALESCE(%s, %v)", expr, mapping.DefaultValue)
|
||||
}
|
||||
selectParts = append(selectParts, fmt.Sprintf("%s AS %s", expr, targetField))
|
||||
}
|
||||
}
|
||||
|
||||
// FROM + JOIN
|
||||
fromClause := e.buildFromClause(ec, sourceTable)
|
||||
|
||||
// WHERE
|
||||
whereClause := e.buildWhereClause(ec, sourceTable, statDate)
|
||||
|
||||
// 组合 SQL
|
||||
sql := fmt.Sprintf("SELECT %s FROM %s %s",
|
||||
strings.Join(selectParts, ", "),
|
||||
fromClause,
|
||||
whereClause)
|
||||
|
||||
// GROUP BY
|
||||
if len(groupByParts) > 0 {
|
||||
sql += " GROUP BY " + strings.Join(groupByParts, ", ")
|
||||
}
|
||||
|
||||
return sql, args, nil
|
||||
}
|
||||
|
||||
// buildFromClause 构建FROM + JOIN子句
|
||||
func (e *DailyExtractor) buildFromClause(ec *model.ExtractConfig, sourceTable string) string {
|
||||
fromClause := fmt.Sprintf("%s %s", ec.SourceTableName, sourceTable)
|
||||
|
||||
for _, join := range ec.JoinConfigs {
|
||||
joinType := "LEFT JOIN"
|
||||
jType := strings.ToUpper(join.JoinType)
|
||||
if jType == "INNER" {
|
||||
joinType = "INNER JOIN"
|
||||
} else if jType == "RIGHT" {
|
||||
joinType = "RIGHT JOIN"
|
||||
}
|
||||
|
||||
joinAlias := join.JoinAlias
|
||||
if joinAlias == "" {
|
||||
joinAlias = join.JoinTable
|
||||
}
|
||||
|
||||
fromClause += fmt.Sprintf(" %s %s %s ON %s", joinType, join.JoinTable, joinAlias, join.JoinCondition)
|
||||
}
|
||||
|
||||
return fromClause
|
||||
}
|
||||
|
||||
// buildJoinFieldSelects 构建JOIN表的字段映射SELECT部分
|
||||
func (e *DailyExtractor) buildJoinFieldSelects(ec *model.ExtractConfig) []string {
|
||||
var parts []string
|
||||
for _, join := range ec.JoinConfigs {
|
||||
joinAlias := join.JoinAlias
|
||||
if joinAlias == "" {
|
||||
joinAlias = join.JoinTable
|
||||
}
|
||||
for _, jm := range join.FieldMappings {
|
||||
targetField := jm.TargetField
|
||||
sourceExpr := fmt.Sprintf("%s.%s", joinAlias, jm.SourceField)
|
||||
if jm.TransformRule != nil {
|
||||
sourceExpr = e.applyTransformExpr(jm.TransformRule, sourceExpr)
|
||||
}
|
||||
parts = append(parts, fmt.Sprintf("%s AS %s", sourceExpr, targetField))
|
||||
}
|
||||
}
|
||||
return parts
|
||||
}
|
||||
|
||||
// buildWhereClause 构建WHERE子句
|
||||
func (e *DailyExtractor) buildWhereClause(ec *model.ExtractConfig, sourceTable, statDate string) string {
|
||||
var whereConditions []string
|
||||
|
||||
// 日期范围（增量抽取）
|
||||
if ec.ExtractType == model.ExtractTypeIncremental && ec.ExtractKeyField != "" {
|
||||
dateCondition := fmt.Sprintf("%s.%s::date = '%s'", sourceTable, ec.ExtractKeyField, statDate)
|
||||
whereConditions = append(whereConditions, dateCondition)
|
||||
}
|
||||
|
||||
// 自定义过滤条件
|
||||
if ec.FilterExpression != "" {
|
||||
whereConditions = append(whereConditions, ec.FilterExpression)
|
||||
}
|
||||
|
||||
if len(whereConditions) == 0 {
|
||||
return ""
|
||||
}
|
||||
return "WHERE " + strings.Join(whereConditions, " AND ")
|
||||
}
|
||||
|
||||
// applyTransformExpr 应用转换表达式
|
||||
func (e *DailyExtractor) applyTransformExpr(rule *model.TransformRule, sourceExpr string) string {
|
||||
switch rule.RuleType {
|
||||
case "CALCULATE":
|
||||
if rule.Expression != "" {
|
||||
return strings.ReplaceAll(rule.Expression, "{source}", sourceExpr)
|
||||
}
|
||||
case "FORMAT":
|
||||
if rule.Format != "" {
|
||||
return fmt.Sprintf("TO_CHAR(%s, '%s')", sourceExpr, rule.Format)
|
||||
}
|
||||
case "MAPPING":
|
||||
// 在代码中运行时做映射
|
||||
return sourceExpr
|
||||
}
|
||||
return sourceExpr
|
||||
}
|
||||
|
||||
// applyTransformRules 应用运行时转换规则（映射等代码转换）
|
||||
func (e *DailyExtractor) applyTransformRules(ec *model.ExtractConfig, row map[string]interface{}) {
|
||||
for _, rule := range ec.TransformRules {
|
||||
if rule.RuleType != "MAPPING" {
|
||||
continue
|
||||
}
|
||||
|
||||
sourceField := rule.Expression // 存储源字段名
|
||||
targetField := rule.RuleCode // 存储目标字段名
|
||||
|
||||
if sourceVal, ok := row[sourceField]; ok {
|
||||
strVal := gconv.String(sourceVal)
|
||||
if mapped, exists := rule.Mapping[strVal]; exists {
|
||||
row[targetField] = mapped
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ensureStatTableExists 确保统计宽表存在
|
||||
func (e *DailyExtractor) ensureStatTableExists(ctx context.Context, report *model.ReportConfig, fieldMap map[string]*model.FieldConfig) error {
|
||||
tableName := report.StatTableName
|
||||
|
||||
// 检查表是否存在
|
||||
result, err := gfdb.DB(ctx).GetAll(ctx, "SELECT COUNT(*) FROM pg_tables WHERE tablename = $1", strings.ToLower(tableName))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
count := 0
|
||||
if len(result) > 0 {
|
||||
count = result[0]["count"].Int()
|
||||
}
|
||||
|
||||
if count == 0 {
|
||||
// 需要建表
|
||||
return e.createStatTable(ctx, report, fieldMap)
|
||||
}
|
||||
|
||||
logrus.Infof("统计宽表 %s 已存在", tableName)
|
||||
return nil
|
||||
}
|
||||
|
||||
// createStatTable 创建统计宽表
|
||||
func (e *DailyExtractor) createStatTable(ctx context.Context, report *model.ReportConfig, fieldMap map[string]*model.FieldConfig) error {
|
||||
var cols []string
|
||||
|
||||
// 标准审计字段
|
||||
cols = append(cols, "id BIGSERIAL PRIMARY KEY")
|
||||
cols = append(cols, "tenant_id BIGINT NOT NULL DEFAULT 0")
|
||||
cols = append(cols, "business_code VARCHAR(64) NOT NULL DEFAULT ''")
|
||||
cols = append(cols, "creator VARCHAR(64) DEFAULT ''")
|
||||
cols = append(cols, "created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()")
|
||||
cols = append(cols, "updater VARCHAR(64) DEFAULT ''")
|
||||
cols = append(cols, "updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()")
|
||||
cols = append(cols, "deleted_at TIMESTAMP WITH TIME ZONE")
|
||||
|
||||
// 日期字段
|
||||
dateField := report.DateField
|
||||
if dateField == "" {
|
||||
dateField = "stat_date"
|
||||
}
|
||||
cols = append(cols, fmt.Sprintf("%s VARCHAR(16) NOT NULL DEFAULT ''", dateField))
|
||||
|
||||
// 业务字段
|
||||
for _, fc := range fieldMap {
|
||||
fc := fc
|
||||
colType := fieldTypeToPG(fc.FieldType)
|
||||
cols = append(cols, fmt.Sprintf("%s %s", fc.FieldCode, colType))
|
||||
}
|
||||
|
||||
// 原始数据
|
||||
cols = append(cols, "raw_data JSONB DEFAULT '{}'")
|
||||
|
||||
tableName := report.StatTableName
|
||||
sql := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (\n %s\n)", tableName, strings.Join(cols, ",\n "))
|
||||
|
||||
logrus.Infof("创建统计宽表: %s", tableName)
|
||||
|
||||
if _, err := gfdb.DB(ctx).Exec(ctx, sql); err != nil {
|
||||
return fmt.Errorf("建表失败: %w", err)
|
||||
}
|
||||
|
||||
// 冲突唯一索引
|
||||
if len(report.ConflictKeys) > 0 {
|
||||
indexName := fmt.Sprintf("uq_%s_conflict", tableName)
|
||||
indexCols := strings.Join(report.ConflictKeys, ", ")
|
||||
indexSQL := fmt.Sprintf("CREATE UNIQUE INDEX IF NOT EXISTS %s ON %s (%s)", indexName, tableName, indexCols)
|
||||
if _, err := gfdb.DB(ctx).Exec(ctx, indexSQL); err != nil {
|
||||
logrus.Warnf("创建冲突索引失败: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 字段注释
|
||||
for _, fc := range fieldMap {
|
||||
fc := fc
|
||||
if fc.FieldName != "" {
|
||||
escaped := strings.ReplaceAll(fc.FieldName, "'", "''")
|
||||
commentSQL := fmt.Sprintf("COMMENT ON COLUMN %s.%s IS '%s'", tableName, fc.FieldCode, escaped)
|
||||
if _, err := gfdb.DB(ctx).Exec(ctx, commentSQL); err != nil {
|
||||
logrus.Warnf("添加字段注释失败 [%s.%s]: %v", tableName, fc.FieldCode, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// batchUpsert 批量upsert写入
|
||||
func (e *DailyExtractor) batchUpsert(ctx context.Context, tableName string, conflictKeys []string, rows []map[string]interface{}) (int, []string, error) {
|
||||
if len(rows) == 0 {
|
||||
return 0, nil, nil
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
for i := range rows {
|
||||
if rows[i] == nil {
|
||||
rows[i] = make(map[string]interface{})
|
||||
}
|
||||
rows[i]["updated_at"] = now
|
||||
}
|
||||
|
||||
batchSize := 100
|
||||
total := 0
|
||||
var allColumns []string
|
||||
|
||||
for i := 0; i < len(rows); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(rows) {
|
||||
end = len(rows)
|
||||
}
|
||||
batch := rows[i:end]
|
||||
|
||||
m := gfdb.DB(ctx).Model(ctx, tableName).Data(batch)
|
||||
if len(conflictKeys) > 0 {
|
||||
keys := make([]interface{}, len(conflictKeys))
|
||||
for j, k := range conflictKeys {
|
||||
keys[j] = k
|
||||
}
|
||||
m = m.OnConflict(keys...)
|
||||
}
|
||||
|
||||
_, err := m.Save()
|
||||
if err != nil {
|
||||
logrus.Errorf("批量写入 %s 失败: %v", tableName, err)
|
||||
// 逐条重试
|
||||
for _, row := range batch {
|
||||
mm := gfdb.DB(ctx).Model(ctx, tableName).Data(row)
|
||||
if len(conflictKeys) > 0 {
|
||||
keys := make([]interface{}, len(conflictKeys))
|
||||
for j, k := range conflictKeys {
|
||||
keys[j] = k
|
||||
}
|
||||
mm = mm.OnConflict(keys...)
|
||||
}
|
||||
if _, e := mm.Save(); e != nil {
|
||||
logrus.Errorf("逐条写入失败: %v", e)
|
||||
} else {
|
||||
total++
|
||||
}
|
||||
}
|
||||
} else {
|
||||
total += len(batch)
|
||||
}
|
||||
}
|
||||
|
||||
return total, allColumns, nil
|
||||
}
|
||||
|
||||
// fieldTypeToPG 字段类型转PG类型
|
||||
func fieldTypeToPG(fieldType string) string {
|
||||
switch fieldType {
|
||||
case model.FieldTypeInt:
|
||||
return "NUMERIC(20,0) DEFAULT 0"
|
||||
case model.FieldTypeFloat:
|
||||
return "NUMERIC(20,4) DEFAULT 0"
|
||||
case model.FieldTypeDate:
|
||||
return "VARCHAR(16) DEFAULT ''"
|
||||
case model.FieldTypeDatetime:
|
||||
return "TIMESTAMP WITH TIME ZONE"
|
||||
case model.FieldTypeJsonb:
|
||||
return "JSONB DEFAULT '{}'"
|
||||
default:
|
||||
return "VARCHAR(256) DEFAULT ''"
|
||||
}
|
||||
}
|
||||
424
common/report/model/model.go
Normal file
424
common/report/model/model.go
Normal file
@@ -0,0 +1,424 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// ============================================================
|
||||
// 实体定义
|
||||
// ============================================================
|
||||
|
||||
// BusinessConfig 业务配置
|
||||
type BusinessConfig struct {
|
||||
ID int64 `orm:"id" json:"id"`
|
||||
TenantId uint64 `orm:"tenant_id" json:"tenant_id"`
|
||||
BusinessCode string `orm:"business_code" json:"businessCode"`
|
||||
BusinessName string `orm:"business_name" json:"businessName"`
|
||||
Description string `orm:"description" json:"description"`
|
||||
Status string `orm:"status" json:"status"`
|
||||
Config map[string]interface{} `orm:"config" json:"config"`
|
||||
Creator string `orm:"creator" json:"creator"`
|
||||
CreatedAt *time.Time `orm:"created_at" json:"createdAt"`
|
||||
Updater string `orm:"updater" json:"updater"`
|
||||
UpdatedAt *time.Time `orm:"updated_at" json:"updatedAt"`
|
||||
DeletedAt *time.Time `orm:"deleted_at" json:"deletedAt"`
|
||||
}
|
||||
|
||||
// ReportConfig 报表配置
|
||||
type ReportConfig struct {
|
||||
ID int64 `orm:"id" json:"id"`
|
||||
TenantId uint64 `orm:"tenant_id" json:"tenant_id"`
|
||||
BusinessCode string `orm:"business_code" json:"businessCode"`
|
||||
ReportCode string `orm:"report_code" json:"reportCode"`
|
||||
ReportName string `orm:"report_name" json:"reportName"`
|
||||
Description string `orm:"description" json:"description"`
|
||||
Status string `orm:"status" json:"status"`
|
||||
StatTableName string `orm:"stat_table_name" json:"statTableName"`
|
||||
StatTableComment string `orm:"stat_table_comment" json:"statTableComment"`
|
||||
DateField string `orm:"date_field" json:"dateField"`
|
||||
PrimaryKeys []string `orm:"primary_keys" json:"primaryKeys"`
|
||||
ConflictKeys []string `orm:"conflict_keys" json:"conflictKeys"`
|
||||
Config map[string]interface{} `orm:"config" json:"config"`
|
||||
Creator string `orm:"creator" json:"creator"`
|
||||
CreatedAt *time.Time `orm:"created_at" json:"createdAt"`
|
||||
Updater string `orm:"updater" json:"updater"`
|
||||
UpdatedAt *time.Time `orm:"updated_at" json:"updatedAt"`
|
||||
DeletedAt *time.Time `orm:"deleted_at" json:"deletedAt"`
|
||||
}
|
||||
|
||||
// FieldConfig 字段配置
|
||||
type FieldConfig struct {
|
||||
ID int64 `orm:"id" json:"id"`
|
||||
TenantId uint64 `orm:"tenant_id" json:"tenant_id"`
|
||||
BusinessCode string `orm:"business_code" json:"businessCode"`
|
||||
ReportCode string `orm:"report_code" json:"reportCode"`
|
||||
FieldCode string `orm:"field_code" json:"fieldCode"`
|
||||
FieldName string `orm:"field_name" json:"fieldName"`
|
||||
FieldType string `orm:"field_type" json:"fieldType"`
|
||||
DataType string `orm:"data_type" json:"dataType"`
|
||||
FieldRole string `orm:"field_role" json:"fieldRole"`
|
||||
IsAggregatable bool `orm:"is_aggregatable" json:"isAggregatable"`
|
||||
IsFilterable bool `orm:"is_filterable" json:"isFilterable"`
|
||||
IsQueryable bool `orm:"is_queryable" json:"isQueryable"`
|
||||
IsSortable bool `orm:"is_sortable" json:"isSortable"`
|
||||
DefaultAggregate string `orm:"default_aggregate" json:"defaultAggregate"`
|
||||
ValidAggregates []string `orm:"valid_aggregates" json:"validAggregates"`
|
||||
FilterOperators []string `orm:"filter_operators" json:"filterOperators"`
|
||||
Expression string `orm:"expression" json:"expression"`
|
||||
ExpressionType string `orm:"expression_type" json:"expressionType"`
|
||||
FormatPattern string `orm:"format_pattern" json:"formatPattern"`
|
||||
Unit string `orm:"unit" json:"unit"`
|
||||
DictCode string `orm:"dict_code" json:"dictCode"`
|
||||
SortOrder int `orm:"sort_order" json:"sortOrder"`
|
||||
GroupName string `orm:"group_name" json:"groupName"`
|
||||
Status string `orm:"status" json:"status"`
|
||||
Creator string `orm:"creator" json:"creator"`
|
||||
CreatedAt *time.Time `orm:"created_at" json:"createdAt"`
|
||||
Updater string `orm:"updater" json:"updater"`
|
||||
UpdatedAt *time.Time `orm:"updated_at" json:"updatedAt"`
|
||||
DeletedAt *time.Time `orm:"deleted_at" json:"deletedAt"`
|
||||
}
|
||||
|
||||
// ExtractConfig 抽取配置
|
||||
type ExtractConfig struct {
|
||||
ID int64 `orm:"id" json:"id"`
|
||||
TenantId uint64 `orm:"tenant_id" json:"tenant_id"`
|
||||
BusinessCode string `orm:"business_code" json:"businessCode"`
|
||||
ReportCode string `orm:"report_code" json:"reportCode"`
|
||||
ExtractCode string `orm:"extract_code" json:"extractCode"`
|
||||
ExtractName string `orm:"extract_name" json:"extractName"`
|
||||
SourceTableName string `orm:"source_table_name" json:"sourceTableName"`
|
||||
SourceTableAlias string `orm:"source_table_alias" json:"sourceTableAlias"`
|
||||
TargetTableName string `orm:"target_table_name" json:"targetTableName"`
|
||||
IsEnabled bool `orm:"is_enabled" json:"isEnabled"`
|
||||
ExtractType string `orm:"extract_type" json:"extractType"`
|
||||
ExtractMode string `orm:"extract_mode" json:"extractMode"`
|
||||
ExtractKeyField string `orm:"extract_key_field" json:"extractKeyField"`
|
||||
ExtractKeyFormat string `orm:"extract_key_format" json:"extractKeyFormat"`
|
||||
GroupByFields []string `orm:"group_by_fields" json:"groupByFields"`
|
||||
FilterExpression string `orm:"filter_expression" json:"filterExpression"`
|
||||
JoinConfigs []JoinConfig `orm:"join_configs" json:"joinConfigs"`
|
||||
FieldMappings []FieldMapping `orm:"field_mappings" json:"fieldMappings"`
|
||||
TransformRules []TransformRule `orm:"transform_rules" json:"transformRules"`
|
||||
BatchSize int `orm:"batch_size" json:"batchSize"`
|
||||
Status string `orm:"status" json:"status"`
|
||||
Creator string `orm:"creator" json:"creator"`
|
||||
CreatedAt *time.Time `orm:"created_at" json:"createdAt"`
|
||||
Updater string `orm:"updater" json:"updater"`
|
||||
UpdatedAt *time.Time `orm:"updated_at" json:"updatedAt"`
|
||||
DeletedAt *time.Time `orm:"deleted_at" json:"deletedAt"`
|
||||
}
|
||||
|
||||
// ExtractLog 抽取记录
|
||||
type ExtractLog struct {
|
||||
ID int64 `orm:"id" json:"id"`
|
||||
BusinessCode string `orm:"business_code" json:"businessCode"`
|
||||
ReportCode string `orm:"report_code" json:"reportCode"`
|
||||
ExtractCode string `orm:"extract_code" json:"extractCode"`
|
||||
StatDate string `orm:"stat_date" json:"statDate"`
|
||||
ExtractType string `orm:"extract_type" json:"extractType"`
|
||||
Status string `orm:"status" json:"status"`
|
||||
TotalCount int `orm:"total_count" json:"totalCount"`
|
||||
SuccessCount int `orm:"success_count" json:"successCount"`
|
||||
FailCount int `orm:"fail_count" json:"failCount"`
|
||||
StartTime *time.Time `orm:"start_time" json:"startTime"`
|
||||
EndTime *time.Time `orm:"end_time" json:"endTime"`
|
||||
ErrorMessage string `orm:"error_message" json:"errorMessage"`
|
||||
Executor string `orm:"executor" json:"executor"`
|
||||
CreatedAt *time.Time `orm:"created_at" json:"createdAt"`
|
||||
UpdatedAt *time.Time `orm:"updated_at" json:"updatedAt"`
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// 辅助结构
|
||||
// ============================================================
|
||||
|
||||
// JoinConfig 关联配置
|
||||
type JoinConfig struct {
|
||||
JoinTable string `json:"joinTable"`
|
||||
JoinAlias string `json:"joinAlias"`
|
||||
JoinType string `json:"joinType"` // LEFT/RIGHT/INNER
|
||||
JoinCondition string `json:"joinCondition"`
|
||||
FieldMappings []FieldMapping `json:"fieldMappings"`
|
||||
}
|
||||
|
||||
// FieldMapping 字段映射
|
||||
type FieldMapping struct {
|
||||
SourceField string `json:"sourceField"`
|
||||
TargetField string `json:"targetField"`
|
||||
FieldType string `json:"fieldType"`
|
||||
AggregateFunction string `json:"aggregateFunction"`
|
||||
DefaultValue interface{} `json:"defaultValue"`
|
||||
TransformRule *TransformRule `json:"transformRule,omitempty"`
|
||||
}
|
||||
|
||||
// TransformRule 转换规则
|
||||
type TransformRule struct {
|
||||
RuleCode string `json:"ruleCode"`
|
||||
RuleType string `json:"ruleType"` // DIRECT/MAPPING/FORMAT/CALCULATE
|
||||
Expression string `json:"expression"`
|
||||
Format string `json:"format"`
|
||||
Mapping map[string]interface{} `json:"mapping"`
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// 前端请求/响应结构体
|
||||
// ============================================================
|
||||
|
||||
// UserSelectQueryReq 用户选择查询请求
|
||||
type UserSelectQueryReq struct {
|
||||
BusinessCode string `json:"businessCode" v:"required" dc:"业务编码"`
|
||||
ReportCode string `json:"reportCode" v:"required" dc:"报表编码"`
|
||||
Dimensions []string `json:"dimensions" dc:"统计维度列表，如 shop_id/anchor_id/date"`
|
||||
Indicators []IndicatorSelect `json:"indicators" dc:"统计指标列表(含聚合方式)"`
|
||||
Filters []FilterCondition `json:"filters" dc:"筛选条件列表"`
|
||||
TimeRange *TimeRange `json:"timeRange" dc:"时间范围"`
|
||||
TimeGroup string `json:"timeGroup" dc:"时间分组: day/week/month/quarter"`
|
||||
OrderBy []OrderCondition `json:"orderBy" dc:"排序条件"`
|
||||
Page int `json:"page" dc:"页码" d:"1"`
|
||||
PageSize int `json:"pageSize" dc:"每页条数" d:"20"`
|
||||
}
|
||||
|
||||
// IndicatorSelect 指标选择
|
||||
type IndicatorSelect struct {
|
||||
FieldCode string `json:"fieldCode" dc:"字段编码"`
|
||||
Aggregate string `json:"aggregate" dc:"聚合方式: SUM/COUNT/AVG/MAX/MIN"`
|
||||
Alias string `json:"alias" dc:"别名"`
|
||||
}
|
||||
|
||||
// FilterCondition 筛选条件
|
||||
type FilterCondition struct {
|
||||
FieldCode string `json:"fieldCode" dc:"字段编码"`
|
||||
Operator string `json:"operator" dc:"操作符: =/!=/>/</>=/<=/IN/LIKE/BETWEEN"`
|
||||
Value interface{} `json:"value" dc:"值"`
|
||||
Value2 interface{} `json:"value2" dc:"第二个值(BETWEEN时使用)"`
|
||||
}
|
||||
|
||||
// TimeRange 时间范围
|
||||
type TimeRange struct {
|
||||
StartDate string `json:"startDate" dc:"开始日期 yyyy-MM-dd"`
|
||||
EndDate string `json:"endDate" dc:"结束日期 yyyy-MM-dd"`
|
||||
}
|
||||
|
||||
// OrderCondition 排序条件
|
||||
type OrderCondition struct {
|
||||
FieldCode string `json:"fieldCode" dc:"字段编码"`
|
||||
Direction string `json:"direction" dc:"排序方向: ASC/DESC"`
|
||||
}
|
||||
|
||||
// UserSelectQueryResp 用户选择查询响应
|
||||
type UserSelectQueryResp struct {
|
||||
List []map[string]interface{} `json:"list" dc:"数据列表"`
|
||||
Total int64 `json:"total" dc:"总数"`
|
||||
Page int `json:"page" dc:"当前页"`
|
||||
PageSize int `json:"pageSize" dc:"每页条数"`
|
||||
TotalPages int `json:"totalPages" dc:"总页数"`
|
||||
Sql string `json:"sql,omitempty" dc:"执行的SQL(调试用)"`
|
||||
ExecTimeMs int64 `json:"execTimeMs" dc:"执行耗时(毫秒)"`
|
||||
}
|
||||
|
||||
// ExtractDailyDataReq 按天抽取数据请求
|
||||
type ExtractDailyDataReq struct {
|
||||
BusinessCode string `json:"businessCode" v:"required" dc:"业务编码"`
|
||||
ReportCode string `json:"reportCode" v:"required" dc:"报表编码"`
|
||||
StatDate string `json:"statDate" v:"required" dc:"统计日期 yyyy-MM-dd"`
|
||||
Executor string `json:"executor" dc:"执行人"`
|
||||
}
|
||||
|
||||
// ExtractDailyDataResp 按天抽取数据响应
|
||||
type ExtractDailyDataResp struct {
|
||||
Success bool `json:"success" dc:"是否成功"`
|
||||
TotalCount int `json:"totalCount" dc:"总记录数"`
|
||||
SuccessCount int `json:"successCount" dc:"成功记录数"`
|
||||
FailCount int `json:"failCount" dc:"失败记录数"`
|
||||
ExecTimeMs int64 `json:"execTimeMs" dc:"执行耗时(毫秒)"`
|
||||
ErrorMsg string `json:"errorMsg" dc:"错误信息"`
|
||||
}
|
||||
|
||||
// AutoCreateStatTableReq 自动创建统计宽表请求
|
||||
type AutoCreateStatTableReq struct {
|
||||
BusinessCode string `json:"businessCode" v:"required" dc:"业务编码"`
|
||||
ReportCode string `json:"reportCode" v:"required" dc:"报表编码"`
|
||||
Creator string `json:"creator" dc:"创建人"`
|
||||
}
|
||||
|
||||
// AutoCreateStatTableResp 自动创建统计宽表响应
|
||||
type AutoCreateStatTableResp struct {
|
||||
Success bool `json:"success" dc:"是否成功"`
|
||||
TableName string `json:"tableName" dc:"创建的表名"`
|
||||
ColumnCount int `json:"columnCount" dc:"字段数量"`
|
||||
ExecTimeMs int64 `json:"execTimeMs" dc:"执行耗时(毫秒)"`
|
||||
}
|
||||
|
||||
// GetReportFieldsResp 获取报表可用字段响应
|
||||
type GetReportFieldsResp struct {
|
||||
BusinessCode string `json:"businessCode" dc:"业务编码"`
|
||||
ReportCode string `json:"reportCode" dc:"报表编码"`
|
||||
Dimensions []FieldConfig `json:"dimensions" dc:"维度字段列表"`
|
||||
Indicators []FieldConfig `json:"indicators" dc:"指标字段列表"`
|
||||
Filters []FieldConfig `json:"filters" dc:"筛选字段列表"`
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// 配置 CRUD 请求/响应
|
||||
// ============================================================
|
||||
|
||||
// SaveBusinessReq 保存业务配置请求（新增/修改合一）
|
||||
type SaveBusinessReq struct {
|
||||
ID *int64 `json:"id"` // 有值为更新，无值为新增
|
||||
BusinessCode string `json:"businessCode" v:"required" dc:"业务编码"`
|
||||
BusinessName string `json:"businessName" v:"required" dc:"业务名称"`
|
||||
Description string `json:"description" dc:"描述"`
|
||||
Status string `json:"status" dc:"状态 ACTIVE/INACTIVE" d:"ACTIVE"`
|
||||
Config map[string]interface{} `json:"config" dc:"扩展配置"`
|
||||
Operator string `json:"operator" dc:"操作人"`
|
||||
}
|
||||
|
||||
// SaveReportReq 保存报表配置请求
|
||||
type SaveReportReq struct {
|
||||
ID *int64 `json:"id"`
|
||||
BusinessCode string `json:"businessCode" v:"required" dc:"业务编码"`
|
||||
ReportCode string `json:"reportCode" v:"required" dc:"报表编码"`
|
||||
ReportName string `json:"reportName" v:"required" dc:"报表名称"`
|
||||
Description string `json:"description" dc:"描述"`
|
||||
Status string `json:"status" dc:"状态" d:"ACTIVE"`
|
||||
StatTableName string `json:"statTableName" v:"required" dc:"统计宽表名"`
|
||||
StatTableComment string `json:"statTableComment" dc:"统计宽表注释"`
|
||||
DateField string `json:"dateField" dc:"日期字段" d:"stat_date"`
|
||||
PrimaryKeys []string `json:"primaryKeys" dc:"主键字段"`
|
||||
ConflictKeys []string `json:"conflictKeys" dc:"冲突键（唯一索引）"`
|
||||
Config map[string]interface{} `json:"config" dc:"扩展配置"`
|
||||
Operator string `json:"operator" dc:"操作人"`
|
||||
}
|
||||
|
||||
// SaveFieldReq 保存字段配置请求
|
||||
type SaveFieldReq struct {
|
||||
ID *int64 `json:"id"`
|
||||
BusinessCode string `json:"businessCode" v:"required" dc:"业务编码"`
|
||||
ReportCode string `json:"reportCode" v:"required" dc:"报表编码"`
|
||||
FieldCode string `json:"fieldCode" v:"required" dc:"字段编码"`
|
||||
FieldName string `json:"fieldName" v:"required" dc:"字段名称"`
|
||||
FieldType string `json:"fieldType" v:"required" dc:"字段类型 STRING/INT/FLOAT/DATE/DATETIME/JSONB"`
|
||||
DataType string `json:"dataType" dc:"数据存储类型" d:"STRING"`
|
||||
FieldRole string `json:"fieldRole" v:"required" dc:"字段角色 DIMENSION/INDICATOR/FILTER/FILTER_ONLY"`
|
||||
IsAggregatable bool `json:"isAggregatable" dc:"是否可聚合"`
|
||||
IsFilterable bool `json:"isFilterable" dc:"是否可筛选" d:"true"`
|
||||
IsQueryable bool `json:"isQueryable" dc:"是否可查询" d:"true"`
|
||||
IsSortable bool `json:"isSortable" dc:"是否可排序" d:"true"`
|
||||
DefaultAggregate string `json:"defaultAggregate" dc:"默认聚合方式"`
|
||||
ValidAggregates []string `json:"validAggregates" dc:"可选聚合列表"`
|
||||
FilterOperators []string `json:"filterOperators" dc:"可选操作符列表"`
|
||||
Expression string `json:"expression" dc:"表达式（衍生字段）"`
|
||||
ExpressionType string `json:"expressionType" dc:"表达式类型 DIRECT/CALCULATED"`
|
||||
FormatPattern string `json:"formatPattern" dc:"格式化模板"`
|
||||
Unit string `json:"unit" dc:"单位"`
|
||||
DictCode string `json:"dictCode" dc:"字典编码"`
|
||||
SortOrder int `json:"sortOrder" dc:"排序"`
|
||||
GroupName string `json:"groupName" dc:"分组名称"`
|
||||
Status string `json:"status" dc:"状态" d:"ACTIVE"`
|
||||
Operator string `json:"operator" dc:"操作人"`
|
||||
}
|
||||
|
||||
// SaveExtractConfigReq 保存抽取配置请求
|
||||
type SaveExtractConfigReq struct {
|
||||
ID *int64 `json:"id"`
|
||||
BusinessCode string `json:"businessCode" v:"required" dc:"业务编码"`
|
||||
ReportCode string `json:"reportCode" v:"required" dc:"报表编码"`
|
||||
ExtractCode string `json:"extractCode" v:"required" dc:"抽取编码"`
|
||||
ExtractName string `json:"extractName" v:"required" dc:"抽取名称"`
|
||||
SourceTableName string `json:"sourceTableName" v:"required" dc:"源表名"`
|
||||
SourceTableAlias string `json:"sourceTableAlias" dc:"源表别名"`
|
||||
TargetTableName string `json:"targetTableName" v:"required" dc:"目标表名"`
|
||||
IsEnabled bool `json:"isEnabled" dc:"是否启用" d:"true"`
|
||||
ExtractType string `json:"extractType" dc:"抽取类型 FULL/INCREMENTAL" d:"INCREMENTAL"`
|
||||
ExtractMode string `json:"extractMode" dc:"抽取模式 DIRECT/AGGREGATE" d:"DIRECT"`
|
||||
ExtractKeyField string `json:"extractKeyField" dc:"抽取关键字段（增量依据）"`
|
||||
ExtractKeyFormat string `json:"extractKeyFormat" dc:"关键字段格式"`
|
||||
GroupByFields []string `json:"groupByFields" dc:"GROUP BY 字段列表"`
|
||||
FilterExpression string `json:"filterExpression" dc:"过滤表达式"`
|
||||
JoinConfigs []JoinConfig `json:"joinConfigs" dc:"JOIN配置"`
|
||||
FieldMappings []FieldMapping `json:"fieldMappings" dc:"字段映射列表"`
|
||||
TransformRules []TransformRule `json:"transformRules" dc:"转换规则列表"`
|
||||
BatchSize int `json:"batchSize" dc:"批处理大小" d:"1000"`
|
||||
Status string `json:"status" dc:"状态" d:"ACTIVE"`
|
||||
Operator string `json:"operator" dc:"操作人"`
|
||||
}
|
||||
|
||||
// IdReq 通用 ID 请求
|
||||
type IdReq struct {
|
||||
ID int64 `json:"id" v:"required" dc:"主键ID"`
|
||||
}
|
||||
|
||||
// SaveResult 写操作通用返回
|
||||
type SaveResult struct {
|
||||
Success bool `json:"success"`
|
||||
ID int64 `json:"id"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// DeleteResult 删除结果
|
||||
type DeleteResult struct {
|
||||
Success bool `json:"success"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// GetExtractConfigsReq 获取抽取配置列表请求
|
||||
type GetExtractConfigsReq struct {
|
||||
BusinessCode string `json:"businessCode" v:"required" dc:"业务编码"`
|
||||
ReportCode string `json:"reportCode" v:"required" dc:"报表编码"`
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// 常量定义
|
||||
// ============================================================
|
||||
|
||||
const (
|
||||
// 状态
|
||||
StatusActive = "ACTIVE"
|
||||
StatusInactive = "INACTIVE"
|
||||
|
||||
// 字段角色
|
||||
RoleDimension = "DIMENSION"
|
||||
RoleIndicator = "INDICATOR"
|
||||
RoleFilter = "FILTER"
|
||||
RoleFilterOnly = "FILTER_ONLY"
|
||||
|
||||
// 字段类型
|
||||
FieldTypeString = "STRING"
|
||||
FieldTypeInt = "INT"
|
||||
FieldTypeFloat = "FLOAT"
|
||||
FieldTypeDate = "DATE"
|
||||
FieldTypeDatetime = "DATETIME"
|
||||
FieldTypeJsonb = "JSONB"
|
||||
|
||||
// 聚合方式
|
||||
AggregateSum = "SUM"
|
||||
AggregateCount = "COUNT"
|
||||
AggregateAvg = "AVG"
|
||||
AggregateMax = "MAX"
|
||||
AggregateMin = "MIN"
|
||||
|
||||
// 操作符
|
||||
OperatorEq = "="
|
||||
OperatorNe = "!="
|
||||
OperatorGt = ">"
|
||||
OperatorLt = "<"
|
||||
OperatorGe = ">="
|
||||
OperatorLe = "<="
|
||||
OperatorIn = "IN"
|
||||
OperatorLike = "LIKE"
|
||||
OperatorBetween = "BETWEEN"
|
||||
|
||||
// 抽取类型
|
||||
ExtractTypeFull = "FULL"
|
||||
ExtractTypeIncremental = "INCREMENTAL"
|
||||
|
||||
// 抽取模式
|
||||
ExtractModeDirect = "DIRECT" // 逐行抽取（默认，源表每行 → 宽表一行）
|
||||
ExtractModeAggregate = "AGGREGATE" // 聚合抽取（按 GROUP BY 聚合，SUM/COUNT/AVG）
|
||||
|
||||
// 抽取状态
|
||||
ExtractStatusRunning = "RUNNING"
|
||||
ExtractStatusSuccess = "SUCCESS"
|
||||
ExtractStatusFailed = "FAILED"
|
||||
)
|
||||
146
common/report/report.go
Normal file
146
common/report/report.go
Normal file
@@ -0,0 +1,146 @@
|
||||
package report
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"gitea.redpowerfuture.com/red-future/common/db/gfdb"
|
||||
)
|
||||
|
||||
// initTables 初始化系统表
|
||||
func initTables(ctx context.Context) error {
|
||||
ddls := []string{
|
||||
// 业务配置表
|
||||
`CREATE TABLE IF NOT EXISTS report_business_config (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL DEFAULT 0,
|
||||
business_code VARCHAR(64) NOT NULL,
|
||||
business_name VARCHAR(128) NOT NULL,
|
||||
description TEXT DEFAULT '',
|
||||
status VARCHAR(16) NOT NULL DEFAULT 'ACTIVE',
|
||||
config JSONB DEFAULT '{}',
|
||||
creator VARCHAR(64) DEFAULT '',
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
||||
updater VARCHAR(64) DEFAULT '',
|
||||
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
||||
deleted_at TIMESTAMP WITH TIME ZONE
|
||||
)`,
|
||||
|
||||
// 报表配置表
|
||||
`CREATE TABLE IF NOT EXISTS report_report_config (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL DEFAULT 0,
|
||||
business_code VARCHAR(64) NOT NULL,
|
||||
report_code VARCHAR(64) NOT NULL,
|
||||
report_name VARCHAR(128) NOT NULL,
|
||||
description TEXT DEFAULT '',
|
||||
status VARCHAR(16) NOT NULL DEFAULT 'ACTIVE',
|
||||
stat_table_name VARCHAR(128) NOT NULL,
|
||||
stat_table_comment VARCHAR(256) DEFAULT '',
|
||||
date_field VARCHAR(64) DEFAULT 'stat_date',
|
||||
primary_keys JSONB DEFAULT '["id"]'::jsonb,
|
||||
conflict_keys JSONB DEFAULT '["stat_date"]'::jsonb,
|
||||
config JSONB DEFAULT '{}',
|
||||
creator VARCHAR(64) DEFAULT '',
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
||||
updater VARCHAR(64) DEFAULT '',
|
||||
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
||||
deleted_at TIMESTAMP WITH TIME ZONE,
|
||||
CONSTRAINT uk_business_report_code UNIQUE (tenant_id, business_code, report_code)
|
||||
)`,
|
||||
|
||||
// 字段配置表
|
||||
`CREATE TABLE IF NOT EXISTS report_field_config (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL DEFAULT 0,
|
||||
business_code VARCHAR(64) NOT NULL,
|
||||
report_code VARCHAR(64) NOT NULL,
|
||||
field_code VARCHAR(64) NOT NULL,
|
||||
field_name VARCHAR(128) NOT NULL,
|
||||
field_type VARCHAR(32) NOT NULL,
|
||||
data_type VARCHAR(32) NOT NULL DEFAULT 'STRING',
|
||||
field_role VARCHAR(32) NOT NULL,
|
||||
is_aggregatable BOOLEAN DEFAULT FALSE,
|
||||
is_filterable BOOLEAN DEFAULT TRUE,
|
||||
is_queryable BOOLEAN DEFAULT TRUE,
|
||||
is_sortable BOOLEAN DEFAULT TRUE,
|
||||
default_aggregate VARCHAR(32) DEFAULT '',
|
||||
valid_aggregates JSONB DEFAULT '[]'::jsonb,
|
||||
filter_operators JSONB DEFAULT '["=","!=",">","<",">=","<=","IN","LIKE","BETWEEN"]'::jsonb,
|
||||
expression VARCHAR(512) DEFAULT '',
|
||||
expression_type VARCHAR(32) DEFAULT '',
|
||||
format_pattern VARCHAR(64) DEFAULT '',
|
||||
unit VARCHAR(32) DEFAULT '',
|
||||
dict_code VARCHAR(64) DEFAULT '',
|
||||
sort_order INT DEFAULT 0,
|
||||
group_name VARCHAR(64) DEFAULT '',
|
||||
status VARCHAR(16) NOT NULL DEFAULT 'ACTIVE',
|
||||
creator VARCHAR(64) DEFAULT '',
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
||||
updater VARCHAR(64) DEFAULT '',
|
||||
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
||||
deleted_at TIMESTAMP WITH TIME ZONE,
|
||||
CONSTRAINT uk_business_report_field_code UNIQUE (tenant_id, business_code, report_code, field_code)
|
||||
)`,
|
||||
|
||||
// 抽取配置表
|
||||
`CREATE TABLE IF NOT EXISTS report_extract_config (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL DEFAULT 0,
|
||||
business_code VARCHAR(64) NOT NULL,
|
||||
report_code VARCHAR(64) NOT NULL,
|
||||
extract_code VARCHAR(64) NOT NULL,
|
||||
extract_name VARCHAR(128) NOT NULL,
|
||||
source_table_name VARCHAR(128) NOT NULL,
|
||||
source_table_alias VARCHAR(64) DEFAULT '',
|
||||
target_table_name VARCHAR(128) NOT NULL,
|
||||
is_enabled BOOLEAN DEFAULT TRUE,
|
||||
extract_type VARCHAR(32) NOT NULL DEFAULT 'FULL',
|
||||
extract_mode VARCHAR(32) NOT NULL DEFAULT 'DIRECT',
|
||||
extract_key_field VARCHAR(64) DEFAULT '',
|
||||
extract_key_format VARCHAR(64) DEFAULT '',
|
||||
group_by_fields JSONB DEFAULT '[]'::jsonb,
|
||||
filter_expression TEXT DEFAULT '',
|
||||
join_configs JSONB DEFAULT '[]'::jsonb,
|
||||
field_mappings JSONB DEFAULT '[]'::jsonb,
|
||||
transform_rules JSONB DEFAULT '[]'::jsonb,
|
||||
batch_size INT DEFAULT 1000,
|
||||
status VARCHAR(16) NOT NULL DEFAULT 'ACTIVE',
|
||||
creator VARCHAR(64) DEFAULT '',
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
||||
updater VARCHAR(64) DEFAULT '',
|
||||
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
||||
deleted_at TIMESTAMP WITH TIME ZONE,
|
||||
CONSTRAINT uk_business_report_extract_code UNIQUE (tenant_id, business_code, report_code, extract_code)
|
||||
)`,
|
||||
|
||||
// 抽取记录表
|
||||
`CREATE TABLE IF NOT EXISTS report_extract_log (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL DEFAULT 0,
|
||||
business_code VARCHAR(64) NOT NULL,
|
||||
report_code VARCHAR(64) NOT NULL,
|
||||
extract_code VARCHAR(64) NOT NULL,
|
||||
stat_date VARCHAR(16) NOT NULL,
|
||||
extract_type VARCHAR(32) NOT NULL,
|
||||
status VARCHAR(16) NOT NULL DEFAULT 'RUNNING',
|
||||
total_count INT DEFAULT 0,
|
||||
success_count INT DEFAULT 0,
|
||||
fail_count INT DEFAULT 0,
|
||||
start_time TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
||||
end_time TIMESTAMP WITH TIME ZONE,
|
||||
error_message TEXT DEFAULT '',
|
||||
executor VARCHAR(64) DEFAULT '',
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
||||
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
||||
CONSTRAINT uk_extract_keys UNIQUE (tenant_id, business_code, report_code, extract_code, stat_date)
|
||||
)`,
|
||||
}
|
||||
|
||||
for _, ddl := range ddls {
|
||||
if _, err := gfdb.DB(ctx).Exec(ctx, ddl); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user