抽取数据添加补偿机制

dao/copydata/api_account_report_detail_dao.go

@@ -9,6 +9,7 @@ import (
 
 	"gitea.com/red-future/common/db/gfdb"
 	"github.com/gogf/gf/v2/util/gconv"
+	"github.com/sirupsen/logrus"
 )
 
 var CidAccountReportDetail = new(cidAccountReportDetailDao)
@@ -29,7 +30,7 @@ func (d *cidAccountReportDetailDao) Insert(ctx context.Context, req *dto.CidAcco
 	return r.LastInsertId()
 }
 
-// BatchInsert 批量插入广告数据报表详情
+// BatchInsert 批量插入广告数据报表详情（使用 OnConflict 实现幂等性）
 func (d *cidAccountReportDetailDao) BatchInsert(ctx context.Context, reqs []*dto.CidAccountReportDetailItem) (successCount int64, failCount int64, failedIndexes []int64, err error) {
 	if len(reqs) == 0 {
 		return 0, 0, nil, errors.New("批量插入数据不能为空")
@@ -64,9 +65,19 @@ func (d *cidAccountReportDetailDao) BatchInsert(ctx context.Context, reqs []*dto
 			continue
 		}
 
-		// 执行批量插入
-		_, err = gfdb.DB(ctx).Model(ctx, consts.CidAccountReportDetailTable).Data(entityList).Insert()
+		// 执行批量插入，使用 OnConflict 实现幂等性
+		_, err = gfdb.DB(ctx).Model(ctx, consts.CidAccountReportDetailTable).
+			Data(entityList).
+			OnConflict(
+				"report_date_str",
+				"page_number",
+				"campaign_id",
+				"creative_id",
+			).
+			Save()
+
 		if err != nil {
+			logrus.Warnf("批量插入失败，尝试逐条插入: %v", err)
 			// 批量插入失败，尝试逐条插入
 			for k := range batch {
 				_, singleErr := d.Insert(ctx, batch[k])
@@ -84,3 +95,66 @@ func (d *cidAccountReportDetailDao) BatchInsert(ctx context.Context, reqs []*dto
 
 	return successCount, failCount, failedIndexes, nil
 }
+
+// DeleteByDateRange 按日期范围删除数据（用于补偿前去重）
+func (d *cidAccountReportDetailDao) DeleteByDateRange(ctx context.Context, advertiserID int64, startDateStr, endDateStr string) (int64, error) {
+	cols := (&entity.CidAccountReportDetail{}).GetCols()
+
+	result, err := gfdb.DB(ctx).Model(ctx, consts.CidAccountReportDetailTable).
+		Where(cols.ReportDateStr+" >= ? AND "+cols.ReportDateStr+" <= ?", startDateStr, endDateStr).
+		Delete()
+
+	if err != nil {
+		return 0, err
+	}
+
+	affected, _ := result.RowsAffected()
+	return affected, nil
+}
+
+// BatchInsertInTx 在事务中批量插入
+func (d *cidAccountReportDetailDao) BatchInsertInTx(ctx context.Context, tx interface{}, reqs []*dto.CidAccountReportDetailItem) (successCount int64, failCount int64, err error) {
+	if len(reqs) == 0 {
+		return 0, 0, errors.New("批量插入数据不能为空")
+	}
+
+	batchSize := 100
+	successCount = 0
+	failCount = 0
+
+	for i := 0; i < len(reqs); i += batchSize {
+		end := i + batchSize
+		if end > len(reqs) {
+			end = len(reqs)
+		}
+
+		batch := reqs[i:end]
+		entityList := make([]*entity.CidAccountReportDetail, 0, len(batch))
+
+		for _, req := range batch {
+			var entityData entity.CidAccountReportDetail
+			if err = gconv.Struct(req, &entityData); err != nil {
+				failCount++
+				logrus.Errorf("数据转换失败: %v", err)
+				continue
+			}
+			entityList = append(entityList, &entityData)
+		}
+
+		if len(entityList) == 0 {
+			continue
+		}
+
+		_, txErr := gfdb.DB(ctx).Model(ctx, consts.CidAccountReportDetailTable).Data(entityList).Insert()
+		if txErr != nil {
+			logrus.Errorf("批量插入失败 batch[%d:%d]: %v", i, end, txErr)
+			failCount += int64(len(entityList))
+			err = txErr
+			continue
+		}
+
+		successCount += int64(len(entityList))
+	}
+
+	return successCount, failCount, err
+}