645 lines
19 KiB
Go
645 lines
19 KiB
Go
package extract
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"strings"
|
||
"time"
|
||
|
||
"dataengine/common/report/config"
|
||
"dataengine/common/report/model"
|
||
|
||
"gitea.redpowerfuture.com/red-future/common/db/gfdb"
|
||
"github.com/gogf/gf/v2/util/gconv"
|
||
"github.com/sirupsen/logrus"
|
||
)
|
||
|
||
// DailyExtractor 天级数据抽取器
|
||
type DailyExtractor struct {
|
||
loader *config.ConfigLoader
|
||
}
|
||
|
||
// NewDailyExtractor 创建抽取器
|
||
func NewDailyExtractor() *DailyExtractor {
|
||
return &DailyExtractor{
|
||
loader: config.GetLoader(),
|
||
}
|
||
}
|
||
|
||
// ExtractDailyData 按天抽取数据（业务层定时任务调用）
|
||
func (e *DailyExtractor) ExtractDailyData(ctx context.Context, businessCode, reportCode, statDate, executor string) (*model.ExtractDailyDataResp, error) {
|
||
start := time.Now()
|
||
logger := logrus.WithFields(logrus.Fields{
|
||
"businessCode": businessCode,
|
||
"reportCode": reportCode,
|
||
"statDate": statDate,
|
||
})
|
||
|
||
// 1. 获取报表配置
|
||
report, err := e.loader.GetReport(ctx, businessCode, reportCode)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("获取报表配置失败: %w", err)
|
||
}
|
||
|
||
// 2. 获取抽取配置
|
||
extractConfigs, err := e.loader.GetExtractConfigs(ctx, businessCode, reportCode)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("获取抽取配置失败: %w", err)
|
||
}
|
||
|
||
if len(extractConfigs) == 0 {
|
||
return nil, fmt.Errorf("没有可用的抽取配置")
|
||
}
|
||
|
||
// 3. 获取字段配置
|
||
fieldMap, err := e.loader.GetFieldMap(ctx, businessCode, reportCode)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("获取字段配置失败: %w", err)
|
||
}
|
||
|
||
// 4. 确保统计宽表存在
|
||
if err := e.ensureStatTableExists(ctx, report, fieldMap); err != nil {
|
||
return nil, fmt.Errorf("确保统计宽表存在失败: %w", err)
|
||
}
|
||
|
||
totalCount := 0
|
||
successCount := 0
|
||
failCount := 0
|
||
var lastErr error
|
||
|
||
// 5. 遍历每个抽取配置
|
||
for _, ec := range extractConfigs {
|
||
// 检查幂等性
|
||
exLog, err := e.loader.GetExtractLog(ctx, businessCode, reportCode, ec.ExtractCode, statDate)
|
||
if err != nil {
|
||
logger.Errorf("获取抽取记录失败: %v", err)
|
||
}
|
||
|
||
if exLog != nil && exLog.Status == model.ExtractStatusSuccess {
|
||
logger.Infof("抽取配置 %s 日期 %s 已完成，跳过", ec.ExtractCode, statDate)
|
||
continue
|
||
}
|
||
|
||
// 创建抽取记录
|
||
extractLog := &model.ExtractLog{
|
||
BusinessCode: businessCode,
|
||
ReportCode: reportCode,
|
||
ExtractCode: ec.ExtractCode,
|
||
StatDate: statDate,
|
||
ExtractType: ec.ExtractType,
|
||
Status: model.ExtractStatusRunning,
|
||
Executor: executor,
|
||
StartTime: &start,
|
||
}
|
||
_ = e.loader.CreateExtractLog(ctx, extractLog)
|
||
|
||
// 执行抽取
|
||
c, s, f, err := e.executeExtract(ctx, &ec, report, fieldMap, statDate)
|
||
totalCount += c
|
||
successCount += s
|
||
failCount += f
|
||
|
||
// 更新抽取记录
|
||
now := time.Now()
|
||
extractLog.EndTime = &now
|
||
extractLog.TotalCount = c
|
||
extractLog.SuccessCount = s
|
||
extractLog.FailCount = f
|
||
|
||
if err != nil {
|
||
extractLog.Status = model.ExtractStatusFailed
|
||
extractLog.ErrorMessage = err.Error()
|
||
lastErr = err
|
||
logger.Errorf("抽取配置 %s 执行失败: %v", ec.ExtractCode, err)
|
||
} else {
|
||
extractLog.Status = model.ExtractStatusSuccess
|
||
logger.Infof("抽取配置 %s 完成, 总数:%d 成功:%d 失败:%d", ec.ExtractCode, c, s, f)
|
||
}
|
||
|
||
if updateErr := e.loader.UpdateExtractLog(ctx, extractLog); updateErr != nil {
|
||
logger.Errorf("更新抽取记录失败: %v", updateErr)
|
||
}
|
||
}
|
||
|
||
execTime := time.Since(start).Milliseconds()
|
||
logger.Infof("按天抽取完成, 总数:%d 成功:%d 失败:%d 耗时:%dms", totalCount, successCount, failCount, execTime)
|
||
|
||
resp := &model.ExtractDailyDataResp{
|
||
Success: lastErr == nil,
|
||
TotalCount: totalCount,
|
||
SuccessCount: successCount,
|
||
FailCount: failCount,
|
||
ExecTimeMs: execTime,
|
||
}
|
||
|
||
if lastErr != nil {
|
||
resp.ErrorMsg = lastErr.Error()
|
||
}
|
||
|
||
return resp, nil
|
||
}
|
||
|
||
// executeExtract 执行单个抽取配置
|
||
func (e *DailyExtractor) executeExtract(ctx context.Context, ec *model.ExtractConfig, report *model.ReportConfig, fieldMap map[string]*model.FieldConfig, statDate string) (total, success, fail int, err error) {
|
||
logger := logrus.WithField("extractCode", ec.ExtractCode)
|
||
|
||
// 1. 构建抽取SQL
|
||
extractSQL, whereArgs, err := e.buildExtractSQL(ctx, ec, report, statDate)
|
||
if err != nil {
|
||
return 0, 0, 0, fmt.Errorf("构建抽取SQL失败: %w", err)
|
||
}
|
||
|
||
logger.Debugf("抽取SQL: %s", extractSQL)
|
||
|
||
// 2. 分批抽取
|
||
batchSize := ec.BatchSize
|
||
if batchSize <= 0 {
|
||
batchSize = 1000
|
||
}
|
||
|
||
offset := 0
|
||
for {
|
||
// 添加分页
|
||
pagedSQL := fmt.Sprintf("%s LIMIT %d OFFSET %d", extractSQL, batchSize, offset)
|
||
args := append(whereArgs)
|
||
|
||
rows, queryErr := gfdb.DB(ctx).GetAll(ctx, pagedSQL, args...)
|
||
if queryErr != nil {
|
||
return total, success, fail, fmt.Errorf("抽取查询失败: %w", queryErr)
|
||
}
|
||
|
||
batchCount := rows.Len()
|
||
if batchCount == 0 {
|
||
break
|
||
}
|
||
|
||
// 3. 应用转换规则（仅 DIRECT 模式需注入审计字段，AGGREGATE 模式已由SQL处理）
|
||
dataList := rows.List()
|
||
if ec.ExtractMode != model.ExtractModeAggregate {
|
||
for i := range dataList {
|
||
e.applyTransformRules(ec, dataList[i])
|
||
dataList[i]["tenant_id"] = 1
|
||
dataList[i]["business_code"] = ec.BusinessCode
|
||
}
|
||
}
|
||
|
||
// 4. 写入统计宽表
|
||
c, _, writeErr := e.batchUpsert(ctx, report.StatTableName, report.ConflictKeys, dataList)
|
||
if writeErr != nil {
|
||
logger.Errorf("批量写入失败 (offset=%d): %v", offset, writeErr)
|
||
fail += batchCount
|
||
} else {
|
||
success += c
|
||
}
|
||
|
||
total += batchCount
|
||
offset += batchSize
|
||
|
||
if batchCount < batchSize {
|
||
break
|
||
}
|
||
}
|
||
|
||
return total, success, fail, nil
|
||
}
|
||
|
||
// buildExtractSQL 构建抽取SQL
|
||
func (e *DailyExtractor) buildExtractSQL(ctx context.Context, ec *model.ExtractConfig, report *model.ReportConfig, statDate string) (string, []interface{}, error) {
|
||
var args []interface{}
|
||
|
||
sourceTable := ec.SourceTableName
|
||
if ec.SourceTableAlias != "" {
|
||
sourceTable = ec.SourceTableAlias
|
||
} else {
|
||
sourceTable = "s"
|
||
}
|
||
|
||
// 日期字段
|
||
dateField := report.DateField
|
||
if dateField == "" {
|
||
dateField = "stat_date"
|
||
}
|
||
|
||
// 判断抽取模式
|
||
mode := ec.ExtractMode
|
||
if mode == "" {
|
||
mode = model.ExtractModeDirect
|
||
}
|
||
|
||
if mode == model.ExtractModeAggregate {
|
||
return e.buildAggregateExtractSQL(ec, report, sourceTable, dateField, statDate)
|
||
}
|
||
|
||
// === 默认 DIRECT 模式：逐行抽取 ===
|
||
return e.buildDirectExtractSQL(ec, report, sourceTable, dateField, statDate), args, nil
|
||
}
|
||
|
||
// buildDirectExtractSQL 逐行抽取模式SQL（直接映射，不做聚合）
|
||
func (e *DailyExtractor) buildDirectExtractSQL(ec *model.ExtractConfig, report *model.ReportConfig, sourceTable, dateField, statDate string) string {
|
||
var selectParts []string
|
||
|
||
// 基础审计字段（常量注入）
|
||
selectParts = append(selectParts, "0 AS id")
|
||
selectParts = append(selectParts, "1 AS tenant_id")
|
||
selectParts = append(selectParts, fmt.Sprintf("'%s' AS business_code", ec.BusinessCode))
|
||
selectParts = append(selectParts, "'system' AS creator")
|
||
selectParts = append(selectParts, "NOW() AS created_at")
|
||
selectParts = append(selectParts, "'system' AS updater")
|
||
selectParts = append(selectParts, "NOW() AS updated_at")
|
||
selectParts = append(selectParts, "NULL::TIMESTAMP AS deleted_at")
|
||
|
||
// 日期字段
|
||
selectParts = append(selectParts, fmt.Sprintf("'%s' AS %s", statDate, dateField))
|
||
|
||
// 原始数据
|
||
selectParts = append(selectParts, "'{}'::JSONB AS raw_data")
|
||
|
||
// 字段映射
|
||
for _, mapping := range ec.FieldMappings {
|
||
targetField := mapping.TargetField
|
||
sourceField := mapping.SourceField
|
||
|
||
var expr string
|
||
if mapping.TransformRule != nil {
|
||
expr = e.applyTransformExpr(mapping.TransformRule, fmt.Sprintf("%s.%s", sourceTable, sourceField))
|
||
} else {
|
||
expr = fmt.Sprintf("%s.%s", sourceTable, sourceField)
|
||
}
|
||
|
||
if mapping.DefaultValue != nil {
|
||
expr = fmt.Sprintf("COALESCE(%s, '%v')", expr, mapping.DefaultValue)
|
||
}
|
||
|
||
selectParts = append(selectParts, fmt.Sprintf("%s AS %s", expr, targetField))
|
||
}
|
||
|
||
// FROM + JOIN
|
||
fromClause := e.buildFromClause(ec, sourceTable)
|
||
|
||
// JOIN 字段映射
|
||
selectParts = append(selectParts, e.buildJoinFieldSelects(ec)...)
|
||
|
||
// WHERE
|
||
whereClause := e.buildWhereClause(ec, sourceTable, statDate)
|
||
|
||
return fmt.Sprintf("SELECT %s FROM %s %s", strings.Join(selectParts, ", "), fromClause, whereClause)
|
||
}
|
||
|
||
// buildAggregateExtractSQL 聚合抽取模式SQL（GROUP BY + SUM/COUNT/AVG）
|
||
func (e *DailyExtractor) buildAggregateExtractSQL(ec *model.ExtractConfig, report *model.ReportConfig, sourceTable, dateField, statDate string) (string, []interface{}, error) {
|
||
var selectParts []string
|
||
var groupByParts []string
|
||
var args []interface{}
|
||
|
||
// 基础审计字段（聚合模式下用常量）
|
||
selectParts = append(selectParts, "ROW_NUMBER() OVER () AS id") // 伪自增ID
|
||
selectParts = append(selectParts, "1 AS tenant_id")
|
||
selectParts = append(selectParts, fmt.Sprintf("'%s' AS business_code", ec.BusinessCode))
|
||
selectParts = append(selectParts, "'system' AS creator")
|
||
selectParts = append(selectParts, "NOW() AS created_at")
|
||
selectParts = append(selectParts, "'system' AS updater")
|
||
selectParts = append(selectParts, "NOW() AS updated_at")
|
||
selectParts = append(selectParts, "NULL::TIMESTAMP AS deleted_at")
|
||
|
||
// 日期字段（常量）
|
||
selectParts = append(selectParts, fmt.Sprintf("'%s' AS %s", statDate, dateField))
|
||
|
||
// 原始数据
|
||
selectParts = append(selectParts, "'{}'::JSONB AS raw_data")
|
||
|
||
// GroupByFields 集合（快速查找）
|
||
gbySet := make(map[string]bool)
|
||
for _, gbf := range ec.GroupByFields {
|
||
gbySet[gbf] = true
|
||
}
|
||
|
||
// 添加 GroupBy 字段到 SELECT 和 GROUP BY
|
||
for _, gbf := range ec.GroupByFields {
|
||
selectParts = append(selectParts, fmt.Sprintf("%s.%s", sourceTable, gbf))
|
||
groupByParts = append(groupByParts, fmt.Sprintf("%s.%s", sourceTable, gbf))
|
||
}
|
||
|
||
// 字段映射：根据 AggregateFunction 决定聚合方式
|
||
for _, mapping := range ec.FieldMappings {
|
||
targetField := mapping.TargetField
|
||
sourceField := mapping.SourceField
|
||
|
||
// 构建源表达式
|
||
var sourceExpr string
|
||
if mapping.TransformRule != nil {
|
||
sourceExpr = e.applyTransformExpr(mapping.TransformRule, fmt.Sprintf("%s.%s", sourceTable, sourceField))
|
||
} else {
|
||
sourceExpr = fmt.Sprintf("%s.%s", sourceTable, sourceField)
|
||
}
|
||
|
||
// 判断是否需要聚合
|
||
aggFunc := strings.ToUpper(mapping.AggregateFunction)
|
||
if aggFunc != "" && !gbySet[sourceField] {
|
||
// 聚合字段：SUM(s.xxx) / COUNT(s.xxx) / AVG(s.xxx)
|
||
expr := fmt.Sprintf("%s(%s)", aggFunc, sourceExpr)
|
||
if mapping.DefaultValue != nil {
|
||
expr = fmt.Sprintf("COALESCE(%s, %v)", expr, mapping.DefaultValue)
|
||
}
|
||
selectParts = append(selectParts, fmt.Sprintf("%s AS %s", expr, targetField))
|
||
} else if gbySet[sourceField] {
|
||
// GroupBy 字段不需要重复加入 SELECT（已通过 groupByFields 处理）
|
||
continue
|
||
} else {
|
||
// 非聚合字段，也未在 GroupBy 中 → 用 MAX/MIN 取值（兼容 PG only_full_group_by）
|
||
expr := fmt.Sprintf("MAX(%s)", sourceExpr)
|
||
if mapping.DefaultValue != nil {
|
||
expr = fmt.Sprintf("COALESCE(%s, %v)", expr, mapping.DefaultValue)
|
||
}
|
||
selectParts = append(selectParts, fmt.Sprintf("%s AS %s", expr, targetField))
|
||
}
|
||
}
|
||
|
||
// FROM + JOIN
|
||
fromClause := e.buildFromClause(ec, sourceTable)
|
||
|
||
// WHERE
|
||
whereClause := e.buildWhereClause(ec, sourceTable, statDate)
|
||
|
||
// 组合 SQL
|
||
sql := fmt.Sprintf("SELECT %s FROM %s %s",
|
||
strings.Join(selectParts, ", "),
|
||
fromClause,
|
||
whereClause)
|
||
|
||
// GROUP BY
|
||
if len(groupByParts) > 0 {
|
||
sql += " GROUP BY " + strings.Join(groupByParts, ", ")
|
||
}
|
||
|
||
return sql, args, nil
|
||
}
|
||
|
||
// buildFromClause 构建FROM + JOIN子句
|
||
func (e *DailyExtractor) buildFromClause(ec *model.ExtractConfig, sourceTable string) string {
|
||
fromClause := fmt.Sprintf("%s %s", ec.SourceTableName, sourceTable)
|
||
|
||
for _, join := range ec.JoinConfigs {
|
||
joinType := "LEFT JOIN"
|
||
jType := strings.ToUpper(join.JoinType)
|
||
if jType == "INNER" {
|
||
joinType = "INNER JOIN"
|
||
} else if jType == "RIGHT" {
|
||
joinType = "RIGHT JOIN"
|
||
}
|
||
|
||
joinAlias := join.JoinAlias
|
||
if joinAlias == "" {
|
||
joinAlias = join.JoinTable
|
||
}
|
||
|
||
fromClause += fmt.Sprintf(" %s %s %s ON %s", joinType, join.JoinTable, joinAlias, join.JoinCondition)
|
||
}
|
||
|
||
return fromClause
|
||
}
|
||
|
||
// buildJoinFieldSelects 构建JOIN表的字段映射SELECT部分
|
||
func (e *DailyExtractor) buildJoinFieldSelects(ec *model.ExtractConfig) []string {
|
||
var parts []string
|
||
for _, join := range ec.JoinConfigs {
|
||
joinAlias := join.JoinAlias
|
||
if joinAlias == "" {
|
||
joinAlias = join.JoinTable
|
||
}
|
||
for _, jm := range join.FieldMappings {
|
||
targetField := jm.TargetField
|
||
sourceExpr := fmt.Sprintf("%s.%s", joinAlias, jm.SourceField)
|
||
if jm.TransformRule != nil {
|
||
sourceExpr = e.applyTransformExpr(jm.TransformRule, sourceExpr)
|
||
}
|
||
parts = append(parts, fmt.Sprintf("%s AS %s", sourceExpr, targetField))
|
||
}
|
||
}
|
||
return parts
|
||
}
|
||
|
||
// buildWhereClause 构建WHERE子句
|
||
func (e *DailyExtractor) buildWhereClause(ec *model.ExtractConfig, sourceTable, statDate string) string {
|
||
var whereConditions []string
|
||
|
||
// 日期范围（增量抽取）
|
||
if ec.ExtractType == model.ExtractTypeIncremental && ec.ExtractKeyField != "" {
|
||
dateCondition := fmt.Sprintf("%s.%s::date = '%s'", sourceTable, ec.ExtractKeyField, statDate)
|
||
whereConditions = append(whereConditions, dateCondition)
|
||
}
|
||
|
||
// 自定义过滤条件
|
||
if ec.FilterExpression != "" {
|
||
whereConditions = append(whereConditions, ec.FilterExpression)
|
||
}
|
||
|
||
if len(whereConditions) == 0 {
|
||
return ""
|
||
}
|
||
return "WHERE " + strings.Join(whereConditions, " AND ")
|
||
}
|
||
|
||
// applyTransformExpr 应用转换表达式
|
||
func (e *DailyExtractor) applyTransformExpr(rule *model.TransformRule, sourceExpr string) string {
|
||
switch rule.RuleType {
|
||
case "CALCULATE":
|
||
if rule.Expression != "" {
|
||
return strings.ReplaceAll(rule.Expression, "{source}", sourceExpr)
|
||
}
|
||
case "FORMAT":
|
||
if rule.Format != "" {
|
||
return fmt.Sprintf("TO_CHAR(%s, '%s')", sourceExpr, rule.Format)
|
||
}
|
||
case "MAPPING":
|
||
// 在代码中运行时做映射
|
||
return sourceExpr
|
||
}
|
||
return sourceExpr
|
||
}
|
||
|
||
// applyTransformRules 应用运行时转换规则（映射等代码转换）
|
||
func (e *DailyExtractor) applyTransformRules(ec *model.ExtractConfig, row map[string]interface{}) {
|
||
for _, rule := range ec.TransformRules {
|
||
if rule.RuleType != "MAPPING" {
|
||
continue
|
||
}
|
||
|
||
sourceField := rule.Expression // 存储源字段名
|
||
targetField := rule.RuleCode // 存储目标字段名
|
||
|
||
if sourceVal, ok := row[sourceField]; ok {
|
||
strVal := gconv.String(sourceVal)
|
||
if mapped, exists := rule.Mapping[strVal]; exists {
|
||
row[targetField] = mapped
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// ensureStatTableExists 确保统计宽表存在
|
||
func (e *DailyExtractor) ensureStatTableExists(ctx context.Context, report *model.ReportConfig, fieldMap map[string]*model.FieldConfig) error {
|
||
tableName := report.StatTableName
|
||
|
||
// 检查表是否存在
|
||
result, err := gfdb.DB(ctx).GetAll(ctx, "SELECT COUNT(*) FROM pg_tables WHERE tablename = $1", strings.ToLower(tableName))
|
||
if err != nil {
|
||
return err
|
||
}
|
||
count := 0
|
||
if len(result) > 0 {
|
||
count = result[0]["count"].Int()
|
||
}
|
||
|
||
if count == 0 {
|
||
// 需要建表
|
||
return e.createStatTable(ctx, report, fieldMap)
|
||
}
|
||
|
||
logrus.Infof("统计宽表 %s 已存在", tableName)
|
||
return nil
|
||
}
|
||
|
||
// createStatTable 创建统计宽表
|
||
func (e *DailyExtractor) createStatTable(ctx context.Context, report *model.ReportConfig, fieldMap map[string]*model.FieldConfig) error {
|
||
var cols []string
|
||
|
||
// 标准审计字段
|
||
cols = append(cols, "id BIGSERIAL PRIMARY KEY")
|
||
cols = append(cols, "tenant_id BIGINT NOT NULL DEFAULT 0")
|
||
cols = append(cols, "business_code VARCHAR(64) NOT NULL DEFAULT ''")
|
||
cols = append(cols, "creator VARCHAR(64) DEFAULT ''")
|
||
cols = append(cols, "created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()")
|
||
cols = append(cols, "updater VARCHAR(64) DEFAULT ''")
|
||
cols = append(cols, "updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()")
|
||
cols = append(cols, "deleted_at TIMESTAMP WITH TIME ZONE")
|
||
|
||
// 日期字段
|
||
dateField := report.DateField
|
||
if dateField == "" {
|
||
dateField = "stat_date"
|
||
}
|
||
cols = append(cols, fmt.Sprintf("%s VARCHAR(16) NOT NULL DEFAULT ''", dateField))
|
||
|
||
// 业务字段
|
||
for _, fc := range fieldMap {
|
||
fc := fc
|
||
colType := fieldTypeToPG(fc.FieldType)
|
||
cols = append(cols, fmt.Sprintf("%s %s", fc.FieldCode, colType))
|
||
}
|
||
|
||
// 原始数据
|
||
cols = append(cols, "raw_data JSONB DEFAULT '{}'")
|
||
|
||
tableName := report.StatTableName
|
||
sql := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (\n %s\n)", tableName, strings.Join(cols, ",\n "))
|
||
|
||
logrus.Infof("创建统计宽表: %s", tableName)
|
||
|
||
if _, err := gfdb.DB(ctx).Exec(ctx, sql); err != nil {
|
||
return fmt.Errorf("建表失败: %w", err)
|
||
}
|
||
|
||
// 冲突唯一索引
|
||
if len(report.ConflictKeys) > 0 {
|
||
indexName := fmt.Sprintf("uq_%s_conflict", tableName)
|
||
indexCols := strings.Join(report.ConflictKeys, ", ")
|
||
indexSQL := fmt.Sprintf("CREATE UNIQUE INDEX IF NOT EXISTS %s ON %s (%s)", indexName, tableName, indexCols)
|
||
if _, err := gfdb.DB(ctx).Exec(ctx, indexSQL); err != nil {
|
||
logrus.Warnf("创建冲突索引失败: %v", err)
|
||
}
|
||
}
|
||
|
||
// 字段注释
|
||
for _, fc := range fieldMap {
|
||
fc := fc
|
||
if fc.FieldName != "" {
|
||
escaped := strings.ReplaceAll(fc.FieldName, "'", "''")
|
||
commentSQL := fmt.Sprintf("COMMENT ON COLUMN %s.%s IS '%s'", tableName, fc.FieldCode, escaped)
|
||
if _, err := gfdb.DB(ctx).Exec(ctx, commentSQL); err != nil {
|
||
logrus.Warnf("添加字段注释失败 [%s.%s]: %v", tableName, fc.FieldCode, err)
|
||
}
|
||
}
|
||
}
|
||
|
||
return nil
|
||
}
|
||
|
||
// batchUpsert 批量upsert写入
|
||
func (e *DailyExtractor) batchUpsert(ctx context.Context, tableName string, conflictKeys []string, rows []map[string]interface{}) (int, []string, error) {
|
||
if len(rows) == 0 {
|
||
return 0, nil, nil
|
||
}
|
||
|
||
now := time.Now()
|
||
for i := range rows {
|
||
if rows[i] == nil {
|
||
rows[i] = make(map[string]interface{})
|
||
}
|
||
rows[i]["updated_at"] = now
|
||
}
|
||
|
||
batchSize := 100
|
||
total := 0
|
||
var allColumns []string
|
||
|
||
for i := 0; i < len(rows); i += batchSize {
|
||
end := i + batchSize
|
||
if end > len(rows) {
|
||
end = len(rows)
|
||
}
|
||
batch := rows[i:end]
|
||
|
||
m := gfdb.DB(ctx).Model(ctx, tableName).Data(batch)
|
||
if len(conflictKeys) > 0 {
|
||
keys := make([]interface{}, len(conflictKeys))
|
||
for j, k := range conflictKeys {
|
||
keys[j] = k
|
||
}
|
||
m = m.OnConflict(keys...)
|
||
}
|
||
|
||
_, err := m.Save()
|
||
if err != nil {
|
||
logrus.Errorf("批量写入 %s 失败: %v", tableName, err)
|
||
// 逐条重试
|
||
for _, row := range batch {
|
||
mm := gfdb.DB(ctx).Model(ctx, tableName).Data(row)
|
||
if len(conflictKeys) > 0 {
|
||
keys := make([]interface{}, len(conflictKeys))
|
||
for j, k := range conflictKeys {
|
||
keys[j] = k
|
||
}
|
||
mm = mm.OnConflict(keys...)
|
||
}
|
||
if _, e := mm.Save(); e != nil {
|
||
logrus.Errorf("逐条写入失败: %v", e)
|
||
} else {
|
||
total++
|
||
}
|
||
}
|
||
} else {
|
||
total += len(batch)
|
||
}
|
||
}
|
||
|
||
return total, allColumns, nil
|
||
}
|
||
|
||
// fieldTypeToPG 字段类型转PG类型
|
||
func fieldTypeToPG(fieldType string) string {
|
||
switch fieldType {
|
||
case model.FieldTypeInt:
|
||
return "NUMERIC(20,0) DEFAULT 0"
|
||
case model.FieldTypeFloat:
|
||
return "NUMERIC(20,4) DEFAULT 0"
|
||
case model.FieldTypeDate:
|
||
return "VARCHAR(16) DEFAULT ''"
|
||
case model.FieldTypeDatetime:
|
||
return "TIMESTAMP WITH TIME ZONE"
|
||
case model.FieldTypeJsonb:
|
||
return "JSONB DEFAULT '{}'"
|
||
default:
|
||
return "VARCHAR(256) DEFAULT ''"
|
||
}
|
||
}
|