red-future/data-engine
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

抽取数据添加补偿机制

Browse Source
This commit is contained in:
李明康
2026-04-08 09:03:20 +08:00
parent 14db0dd2fe
commit 5f2c9c3855
19 changed files with 1058 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
sync/campaign_report_sync.go → sync/account_report_sync.go
View File

@@ -10,39 +10,39 @@ import (
"github.com/sirupsen/logrus"
)
type CampaignReportSync struct {
type AccountReportSync struct {
*BaseReportSync
converter *DataConverter
mockGen *MockDataGenerator
}
func NewCampaignReportSync() *CampaignReportSync {
return &CampaignReportSync{
func NewAccountReportSync() *AccountReportSync {
return &AccountReportSync{
BaseReportSync: NewBaseReportSync(),
converter: NewDataConverter(),
mockGen: NewMockDataGenerator(),
}
}
func (c *CampaignReportSync) FetchReport(ctx context.Context, params interface{}) (interface{}, error) {
req, ok := params.(*CampaignReportRequest)
func (c *AccountReportSync) FetchReport(ctx context.Context, params interface{}) (interface{}, error) {
req, ok := params.(*AccountReportRequest)
if !ok {
return nil, fmt.Errorf("参数类型错误，期望 CampaignReportRequest 类型")
return nil, fmt.Errorf("参数类型错误，期望 AccountReportRequest 类型")
}
useMock := false
if useMock {
logrus.Info("使用 Mock 数据")
return c.mockGen.GenerateCampaignReportResponse(), nil
return c.mockGen.GenerateAccountReportResponse(), nil
}
respBytes, err := NewHttpClient("https://ad.e.kuaishou.com", 0).Post(ctx, "/rest/openapi/gw/esp/report/campaignReport", req)
respBytes, err := NewHttpClient("https://ad.e.kuaishou.com", 0).Post(ctx, "/rest/openapi/gw/esp/report/accountReport", req)
if err != nil {
return nil, fmt.Errorf("调用 API 失败：%w", err)
}
var response CampaignReportResponse
var response AccountReportResponse
if err := json.Unmarshal(respBytes, &response); err != nil {
return nil, fmt.Errorf("解析响应失败：%w", err)
}
@@ -54,22 +54,22 @@ func (c *CampaignReportSync) FetchReport(ctx context.Context, params interface{}
return &response, nil
}
func (c *CampaignReportSync) ConvertToSum(apiData interface{}, dataType string) interface{} {
response, ok := apiData.(*CampaignReportResponse)
func (c *AccountReportSync) ConvertToSum(apiData interface{}, dataType string) interface{} {
response, ok := apiData.(*AccountReportResponse)
if !ok || response.Data == nil || response.Data.Sum == nil {
return nil
}
return c.converter.ConvertToSumItem(response.Data.Sum, dataType)
return c.converter.ConvertToSumItem(response.Data.Sum, dataType, 0)
}
func (c *CampaignReportSync) ConvertToDetails(apiData interface{}, dataType string) []interface{} {
response, ok := apiData.(*CampaignReportResponse)
func (c *AccountReportSync) ConvertToDetails(apiData interface{}, dataType string) []interface{} {
response, ok := apiData.(*AccountReportResponse)
if !ok || response.Data == nil || len(response.Data.Detail) == 0 {
return nil
}
detailItems := c.converter.ConvertToDetailItems(response.Data.Detail, dataType)
detailItems := c.converter.ConvertToDetailItems(response.Data.Detail, dataType, 0)
result := make([]interface{}, len(detailItems))
for i, item := range detailItems {
@@ -78,7 +78,7 @@ func (c *CampaignReportSync) ConvertToDetails(apiData interface{}, dataType stri
return result
}
func (c *CampaignReportSync) SaveSum(ctx context.Context, data interface{}) (int64, error) {
func (c *AccountReportSync) SaveSum(ctx context.Context, data interface{}) (int64, error) {
sumItem, ok := data.(*dto.CidAccountReportSumItem)
if !ok {
return 0, fmt.Errorf("数据类型错误，期望 CidAccountReportSumItem 类型")
@@ -92,7 +92,7 @@ func (c *CampaignReportSync) SaveSum(ctx context.Context, data interface{}) (int
return res.Id, nil
}
func (c *CampaignReportSync) SaveDetails(ctx context.Context, data []interface{}) (int64, int64, error) {
func (c *AccountReportSync) SaveDetails(ctx context.Context, data []interface{}) (int64, int64, error) {
detailItems := make([]*dto.CidAccountReportDetailItem, len(data))
for i, item := range data {
detailItem, ok := item.(*dto.CidAccountReportDetailItem)

40
sync/campaign_report_types.go → sync/account_report_types.go
View File

@@ -1,17 +1,17 @@
package sync
type CampaignReportRequest struct {
AdvertiserID int64 `json:"advertiser_id"`
StartTime int64 `json:"start_time"`
EndTime int64 `json:"end_time"`
SelectColumns []string `json:"select_columns"`
GroupType int `json:"group_type"`
QueryVersion int `json:"query_version"`
SelectParam *CampaignSelectParam `json:"select_param,omitempty"`
PageInfo *PageInfo `json:"page_info,omitempty"`
type AccountReportRequest struct {
AdvertiserID int64 `json:"advertiser_id"`
StartTime int64 `json:"start_time"`
EndTime int64 `json:"end_time"`
SelectColumns []string `json:"select_columns"`
GroupType int `json:"group_type"`
QueryVersion int `json:"query_version"`
SelectParam *AccountSelectParam `json:"select_param,omitempty"`
PageInfo *PageInfo `json:"page_info,omitempty"`
}
type CampaignSelectParam struct {
type AccountSelectParam struct {
CampaignIDs []int64 `json:"campaign_ids,omitempty"`
AuthorID int64 `json:"author_id,omitempty"`
AdTypeStr string `json:"ad_type_str,omitempty"`
@@ -33,19 +33,19 @@ type PageInfo struct {
TotalCount int `json:"total_count"`
}
type CampaignReportResponse struct {
Code int `json:"code"`
Message string `json:"message"`
Data *CampaignReportData `json:"data"`
type AccountReportResponse struct {
Code int `json:"code"`
Message string `json:"message"`
Data *AccountReportData `json:"data"`
}
type CampaignReportData struct {
Sum *CampaignReportSum `json:"sum"`
Detail []*CampaignReportItem `json:"detail"`
TotalCount int `json:"total_count"`
type AccountReportData struct {
Sum *AccountReportSum `json:"sum"`
Detail []*AccountReportItem `json:"detail"`
TotalCount int `json:"total_count"`
}
type CampaignReportSum struct {
type AccountReportSum struct {
T0OrderPaymentAmt string `json:"t0_order_payment_amt"`
CreativeMaterialType string `json:"creative_material_type"`
LiveName string `json:"live_name"`
@@ -232,4 +232,4 @@ type CampaignReportSum struct {
CidVoucherCost *float64 `json:"cid_voucher_cost"`
}
type CampaignReportItem CampaignReportSum
type AccountReportItem AccountReportSum

14
sync/data_converter.go
View File

@@ -10,7 +10,7 @@ func NewDataConverter() *DataConverter {
return &DataConverter{}
}
func (c *DataConverter) ConvertToSumItem(apiData *CampaignReportSum, dataType string) *copydata.CidAccountReportSumItem {
func (c *DataConverter) ConvertToSumItem(apiData *AccountReportSum, dataType string, pageNumber int) *copydata.CidAccountReportSumItem {
if apiData == nil {
return nil
}
@@ -191,6 +191,7 @@ func (c *DataConverter) ConvertToSumItem(apiData *CampaignReportSum, dataType st
ItemEntranceClkCnt: apiData.ItemEntranceClkCnt,
ShowCnt: apiData.ShowCnt,
ReportDateStr: apiData.ReportDateStr,
PageNumber: pageNumber,
CampaignId: apiData.CampaignId,
CampaignName: apiData.CampaignName,
UnitId: apiData.UnitId,
@@ -204,26 +205,26 @@ func (c *DataConverter) ConvertToSumItem(apiData *CampaignReportSum, dataType st
}
}
func (c *DataConverter) ConvertToDetailItems(apiItems []*CampaignReportItem, dataType string) []*copydata.CidAccountReportDetailItem {
func (c *DataConverter) ConvertToDetailItems(apiItems []*AccountReportItem, dataType string, pageNumber int) []*copydata.CidAccountReportDetailItem {
if len(apiItems) == 0 {
return nil
}
result := make([]*copydata.CidAccountReportDetailItem, 0, len(apiItems))
for _, item := range apiItems {
detailItem := c.convertItemToDetail(item, dataType)
detailItem := c.convertItemToDetail(item, dataType, pageNumber)
result = append(result, detailItem)
}
return result
}
func (c *DataConverter) convertItemToDetail(apiItem *CampaignReportItem, dataType string) *copydata.CidAccountReportDetailItem {
func (c *DataConverter) convertItemToDetail(apiItem *AccountReportItem, dataType string, pageNumber int) *copydata.CidAccountReportDetailItem {
if apiItem == nil {
return nil
}
item := (*CampaignReportSum)(apiItem)
sumItem := c.ConvertToSumItem(item, dataType)
item := (*AccountReportSum)(apiItem)
sumItem := c.ConvertToSumItem(item, dataType, pageNumber)
return &copydata.CidAccountReportDetailItem{
DataType: sumItem.DataType,
@@ -401,6 +402,7 @@ func (c *DataConverter) convertItemToDetail(apiItem *CampaignReportItem, dataTyp
ItemEntranceClkCnt: sumItem.ItemEntranceClkCnt,
ShowCnt: sumItem.ShowCnt,
ReportDateStr: sumItem.ReportDateStr,
PageNumber: pageNumber,
CampaignId: sumItem.CampaignId,
CampaignName: sumItem.CampaignName,
UnitId: sumItem.UnitId,

22
sync/mock_generator.go
View File

@@ -15,15 +15,15 @@ func NewMockDataGenerator() *MockDataGenerator {
}
}
func (m *MockDataGenerator) GenerateCampaignReportRequest() *CampaignReportRequest {
return &CampaignReportRequest{
func (m *MockDataGenerator) GenerateAccountReportRequest() *AccountReportRequest {
return &AccountReportRequest{
AdvertiserID: 10001,
StartTime: time.Now().AddDate(0, 0, -30).UnixNano() / 1e6,
EndTime: time.Now().UnixNano() / 1e6,
SelectColumns: []string{"impression", "click", "cost", "t0GMV"},
GroupType: 1,
QueryVersion: 1,
SelectParam: &CampaignSelectParam{
SelectParam: &AccountSelectParam{
CampaignIDs: []int64{1, 2, 3},
},
PageInfo: &PageInfo{
@@ -33,14 +33,14 @@ func (m *MockDataGenerator) GenerateCampaignReportRequest() *CampaignReportReque
}
}
func (m *MockDataGenerator) GenerateCampaignReportResponse() *CampaignReportResponse {
func (m *MockDataGenerator) GenerateAccountReportResponse() *AccountReportResponse {
sumData := m.generateSumData()
detailData := m.generateDetailData(5)
return &CampaignReportResponse{
return &AccountReportResponse{
Code: 0,
Message: "success",
Data: &CampaignReportData{
Data: &AccountReportData{
Sum: sumData,
Detail: detailData,
TotalCount: len(detailData),
@@ -48,12 +48,12 @@ func (m *MockDataGenerator) GenerateCampaignReportResponse() *CampaignReportResp
}
}
func (m *MockDataGenerator) generateSumData() *CampaignReportSum {
func (m *MockDataGenerator) generateSumData() *AccountReportSum {
cost := m.randomFloat(1000, 10000)
impression := m.randomInt64(10000, 100000)
click := m.randomInt64(100, 1000)
return &CampaignReportSum{
return &AccountReportSum{
T0OrderPaymentAmt: "888.99",
CreativeMaterialType: "视频素材类型",
LiveName: "测试直播间",
@@ -241,10 +241,10 @@ func (m *MockDataGenerator) generateSumData() *CampaignReportSum {
}
}
func (m *MockDataGenerator) generateDetailData(count int) []*CampaignReportItem {
items := make([]*CampaignReportItem, count)
func (m *MockDataGenerator) generateDetailData(count int) []*AccountReportItem {
items := make([]*AccountReportItem, count)
for i := 0; i < count; i++ {
items[i] = (*CampaignReportItem)(m.generateSumData())
items[i] = (*AccountReportItem)(m.generateSumData())
}
return items
}

12
sync/quick_sync.go
View File

@@ -7,17 +7,17 @@ import (
"github.com/sirupsen/logrus"
)
func SyncCampaignReportWithMock(ctx context.Context) error {
func SyncAccountReportWithMock(ctx context.Context) error {
syncService := NewSyncService()
req := &CampaignReportRequest{
req := &AccountReportRequest{
AdvertiserID: 10001,
StartTime: time.Now().AddDate(0, 0, -30).UnixNano() / 1e6,
EndTime: time.Now().UnixNano() / 1e6,
SelectColumns: []string{"impression", "click", "cost", "t0GMV"},
GroupType: 1,
QueryVersion: 1,
SelectParam: &CampaignSelectParam{
SelectParam: &AccountSelectParam{
CampaignIDs: []int64{1, 2, 3},
},
PageInfo: &PageInfo{
@@ -26,7 +26,7 @@ func SyncCampaignReportWithMock(ctx context.Context) error {
},
}
result, err := syncService.SyncCampaignReport(ctx, req, true)
result, err := syncService.SyncAccountReport(ctx, req, true)
if err != nil {
logrus.Errorf("同步失败：%v", err)
return err
@@ -36,10 +36,10 @@ func SyncCampaignReportWithMock(ctx context.Context) error {
return nil
}
func SyncCampaignReportWithRealAPI(ctx context.Context, req *CampaignReportRequest) error {
func SyncAccountReportWithRealAPI(ctx context.Context, req *AccountReportRequest) error {
syncService := NewSyncService()
result, err := syncService.SyncCampaignReport(ctx, req, false)
result, err := syncService.SyncAccountReport(ctx, req, false)
if err != nil {
logrus.Errorf("同步失败：%v", err)
return err

328
sync/sync_service.go
View File

@@ -1,14 +1,15 @@
package sync
import (
dao "cid/dao/copydata"
dto "cid/model/dto/copydata"
taskDto "cid/model/dto/copydata"
"cid/service/copydata"
"context"
"encoding/json"
"fmt"
"time"
"gitea.com/red-future/common/beans"
"github.com/sirupsen/logrus"
)
@@ -27,32 +28,44 @@ func NewSyncService() *SyncService {
}
type SyncResult struct {
SumSuccess bool `json:"sum_success"`
SumID int64 `json:"sum_id"`
DetailSuccess bool `json:"detail_success"`
DetailCount int `json:"detail_count"`
DetailSuccessCount int64 `json:"detail_success_count"`
DetailFailCount int64 `json:"detail_fail_count"`
Error error `json:"error"`
SumSuccess bool `json:"sum_success"`
SumID int64 `json:"sum_id"`
DetailSuccess bool `json:"detail_success"`
DetailCount int `json:"detail_count"`
DetailSuccessCount int64 `json:"detail_success_count"`
DetailFailCount int64 `json:"detail_fail_count"`
Error error `json:"error"`
TaskLogID int64 `json:"task_log_id"`
PageResults []*PageSyncResult `json:"page_results,omitempty"`
}
func (s *SyncService) SyncCampaignReport(ctx context.Context, req *CampaignReportRequest, useMock bool) (*SyncResult, error) {
type PageSyncResult struct {
PageNumber int `json:"page_number"`
PageTaskLogID int64 `json:"page_task_log_id"`
Success bool `json:"success"`
RecordCount int `json:"record_count"`
DurationMs int64 `json:"duration_ms"`
ErrorMessage string `json:"error_message,omitempty"`
RetryCount int `json:"retry_count"`
}
func (s *SyncService) SyncAccountReport(ctx context.Context, req *AccountReportRequest, useMock bool) (*SyncResult, error) {
result := &SyncResult{}
var responseData *CampaignReportResponse
var responseData *AccountReportResponse
if useMock {
logrus.Info("使用 Mock 数据同步快手广告计划报表")
responseData = s.mockGen.GenerateCampaignReportResponse()
logrus.Info("使用 Mock 数据同步快手广告账户报表")
responseData = s.mockGen.GenerateAccountReportResponse()
} else {
logrus.Info("从真实 API 同步快手广告计划报表")
respBytes, err := s.httpClient.Post(ctx, "/rest/openapi/gw/esp/report/campaignReport", req)
logrus.Info("从真实 API 同步快手广告账户报表")
respBytes, err := s.httpClient.Post(ctx, "/rest/openapi/gw/esp/report/accountReport", req)
if err != nil {
result.Error = fmt.Errorf("调用 API 失败：%w", err)
return result, result.Error
}
responseData = &CampaignReportResponse{}
responseData = &AccountReportResponse{}
if err := json.Unmarshal(respBytes, responseData); err != nil {
result.Error = fmt.Errorf("解析响应失败：%w", err)
return result, result.Error
@@ -65,9 +78,7 @@ func (s *SyncService) SyncCampaignReport(ctx context.Context, req *CampaignRepor
}
if responseData.Data.Sum != nil {
sumItem := s.converter.ConvertToSumItem(responseData.Data.Sum, "campaign_report")
ctx = context.WithValue(ctx, "user", &beans.User{UserName: "admin"})
sumItem := s.converter.ConvertToSumItem(responseData.Data.Sum, "account_report", req.PageInfo.CurrentPage)
sumResult, saveErr := s.saveSumData(ctx, sumItem)
if saveErr != nil {
logrus.Errorf("保存汇总数据失败：%v", saveErr)
@@ -80,7 +91,7 @@ func (s *SyncService) SyncCampaignReport(ctx context.Context, req *CampaignRepor
}
if len(responseData.Data.Detail) > 0 {
detailItems := s.converter.ConvertToDetailItems(responseData.Data.Detail, "campaign_report")
detailItems := s.converter.ConvertToDetailItems(responseData.Data.Detail, "account_report", req.PageInfo.CurrentPage)
detailResult, saveErr := s.saveDetailData(ctx, detailItems)
if saveErr != nil {
logrus.Errorf("保存明细数据失败：%v", saveErr)
@@ -90,39 +101,117 @@ func (s *SyncService) SyncCampaignReport(ctx context.Context, req *CampaignRepor
result.DetailCount = len(detailItems)
result.DetailSuccessCount = detailResult.SuccessCount
result.DetailFailCount = detailResult.FailCount
logrus.Infof("成功保存明细数据，成功=%d, 失败=%d", detailResult.SuccessCount, detailResult.FailCount)
logrus.Infof("成功保存 %d 条明细数据（成功=%d, 失败=%d）", len(detailItems), detailResult.SuccessCount, detailResult.FailCount)
}
}
return result, result.Error
}
// SyncCampaignReportWithPagination 带分页处理的同步方法（支持全量数据抽取）
func (s *SyncService) SyncCampaignReportWithPagination(ctx context.Context, req *CampaignReportRequest, useMock bool, maxRetries int) (*SyncResult, error) {
aggregatedResult := &SyncResult{
SumSuccess: false,
SumID: 0,
// SyncAccountReportWithPagination 带分页处理的同步方法（支持全量数据抽取）
func (s *SyncService) SyncAccountReportWithPagination(ctx context.Context, req *AccountReportRequest, useMock bool, maxRetries int) (*SyncResult, error) {
startTime := time.Now()
parentTaskID := fmt.Sprintf("%d_%d_account", req.AdvertiserID, req.StartTime)
logReq := &taskDto.CreateSyncTaskLogReq{
TaskID: parentTaskID,
TaskType: "account_report",
AdvertiserID: req.AdvertiserID,
StartTime: time.UnixMilli(req.StartTime),
EndTime: time.UnixMilli(req.EndTime),
Status: "pending",
MaxRetry: maxRetries,
RequestParams: req,
}
parentLogID, err := dao.SyncTaskLog.Create(ctx, logReq)
if err != nil {
logrus.Errorf("创建主任务日志失败：%v", err)
}
updateParentLog := func(status, errMsg, errorCode string, summary interface{}) {
if parentLogID == 0 {
return
}
duration := time.Since(startTime).Milliseconds()
updateReq := &taskDto.UpdateSyncTaskLogReq{
ID: parentLogID,
Status: status,
ErrorMessage: errMsg,
ErrorCode: errorCode,
DurationMs: &duration,
}
if status == "success" || status == "manual_review" {
completedAt := time.Now()
updateReq.CompletedAt = completedAt
}
if summary != nil {
updateReq.ResultSummary = summary
}
if err := dao.SyncTaskLog.Update(ctx, updateReq); err != nil {
logrus.Errorf("更新主任务日志失败：%v", err)
}
}
updateParentLog("running", "", "", nil)
aggregatedResult := &SyncResult{
SumSuccess: false,
SumID: 0,
TaskLogID: parentLogID,
PageResults: make([]*PageSyncResult, 0),
}
allDetailItems := make([]*dto.CidAccountReportDetailItem, 0)
totalCount := 0
currentPage := 1
pageSize := 100
successPages := 0
failedPages := 0
if req.PageInfo == nil {
req.PageInfo = &PageInfo{}
}
var totalPages int
for {
logrus.Infof(">>> 正在同步第 %d 页数据...", currentPage)
req.PageInfo.CurrentPage = currentPage
req.PageInfo.PageSize = pageSize
result, err := s.SyncWithRetry(ctx, req, useMock, maxRetries)
pageTaskID := fmt.Sprintf("%s_page_%d", parentTaskID, currentPage)
pageStartTime := time.Now()
pageResult := &PageSyncResult{
PageNumber: currentPage,
Success: false,
RecordCount: 0,
RetryCount: 0,
}
result, err := s.syncSinglePageWithTask(ctx, req, useMock, maxRetries, pageTaskID, currentPage)
pageDuration := time.Since(pageStartTime).Milliseconds()
pageResult.DurationMs = pageDuration
if err != nil {
logrus.Errorf("第 %d 页同步失败：%v", currentPage, err)
return aggregatedResult, err
pageResult.ErrorMessage = err.Error()
failedPages++
aggregatedResult.PageResults = append(aggregatedResult.PageResults, pageResult)
if failedPages > maxRetries {
logrus.Warnf("失败页数超过阈值 %d，终止同步", maxRetries)
break
}
currentPage++
time.Sleep(500 * time.Millisecond)
continue
}
if result.SumSuccess && aggregatedResult.SumID == 0 {
@@ -132,17 +221,18 @@ func (s *SyncService) SyncCampaignReportWithPagination(ctx context.Context, req
}
if result.DetailSuccess && result.DetailCount > 0 {
detailItems := s.extractDetailItems(req, useMock)
if len(detailItems) > 0 {
allDetailItems = append(allDetailItems, detailItems...)
totalCount += len(detailItems)
logrus.Infof("✓ 第 %d 页获取到 %d 条明细数据，累计 %d 条", currentPage, len(detailItems), totalCount)
}
totalCount += result.DetailCount
pageResult.Success = true
pageResult.RecordCount = result.DetailCount
successPages++
logrus.Infof("✓ 第 %d 页获取到 %d 条明细数据，累计 %d 条", currentPage, result.DetailCount, totalCount)
}
aggregatedResult.PageResults = append(aggregatedResult.PageResults, pageResult)
currentData := s.fetchCurrentData(req, useMock)
if currentData != nil && currentData.TotalCount > 0 {
totalPages := (currentData.TotalCount + pageSize - 1) / pageSize
totalPages = (currentData.TotalCount + pageSize - 1) / pageSize
logrus.Infof("总记录数：%d, 总页数：%d, 当前页：%d/%d",
currentData.TotalCount, totalPages, currentPage, totalPages)
@@ -161,70 +251,148 @@ func (s *SyncService) SyncCampaignReportWithPagination(ctx context.Context, req
time.Sleep(300 * time.Millisecond)
}
if len(allDetailItems) > 0 {
logrus.Infof("开始批量保存 %d 条明细数据...", len(allDetailItems))
detailResult, saveErr := s.saveDetailData(ctx, allDetailItems)
if saveErr != nil {
logrus.Errorf("批量保存明细数据失败：%v", saveErr)
aggregatedResult.Error = fmt.Errorf("批量保存明细数据失败：%w", saveErr)
logrus.Infof("分页同步完成 - 成功:%d页, 失败:%d页, 总明细:%d条",
successPages, failedPages, totalCount)
// 统计所有子任务的结果
totalDetailCount := 0
var totalSuccessCount int64
var totalFailCount int64
for _, pageResult := range aggregatedResult.PageResults {
if pageResult.Success {
totalDetailCount += pageResult.RecordCount
totalSuccessCount++
} else {
aggregatedResult.DetailSuccess = true
aggregatedResult.DetailCount = len(allDetailItems)
aggregatedResult.DetailSuccessCount = detailResult.SuccessCount
aggregatedResult.DetailFailCount = detailResult.FailCount
logrus.Infof("✓ 批量保存明细数据完成，成功=%d, 失败=%d",
detailResult.SuccessCount, detailResult.FailCount)
totalFailCount++
}
}
aggregatedResult.DetailCount = totalDetailCount
aggregatedResult.DetailSuccessCount = totalSuccessCount
aggregatedResult.DetailFailCount = totalFailCount
if failedPages > 0 {
logrus.Warnf("存在 %d 个失败的页面，主任务标记为部分失败", failedPages)
summary := map[string]interface{}{
"sum_id": aggregatedResult.SumID,
"detail_count": totalDetailCount,
"total_pages": totalPages,
"success_pages": successPages,
"failed_pages": failedPages,
"page_results": aggregatedResult.PageResults,
}
updateParentLog("partial_failed", fmt.Sprintf("%d 个页面同步失败", failedPages), "PAGE_SYNC_FAILED", summary)
} else {
logrus.Info("没有明细数据需要保存")
logrus.Info("✓ 所有页面同步成功")
summary := map[string]interface{}{
"sum_id": aggregatedResult.SumID,
"detail_count": totalDetailCount,
"total_pages": totalPages,
"success_pages": successPages,
"failed_pages": 0,
"page_results": aggregatedResult.PageResults,
}
updateParentLog("success", "", "", summary)
}
return aggregatedResult, aggregatedResult.Error
}
func (s *SyncService) extractDetailItems(req *CampaignReportRequest, useMock bool) []*dto.CidAccountReportDetailItem {
if useMock {
responseData := s.mockGen.GenerateCampaignReportResponse()
if responseData == nil || responseData.Data == nil || len(responseData.Data.Detail) == 0 {
return nil
}
return s.converter.ConvertToDetailItems(responseData.Data.Detail, "campaign_report")
}
respBytes, err := s.httpClient.Post(context.Background(), "/rest/openapi/gw/esp/report/campaignReport", req)
if err != nil {
logrus.Errorf("重新获取数据失败：%v", err)
return nil
}
responseData := &CampaignReportResponse{}
if err := json.Unmarshal(respBytes, responseData); err != nil {
logrus.Errorf("解析响应失败：%v", err)
return nil
}
if responseData.Code != 0 || responseData.Data == nil || len(responseData.Data.Detail) == 0 {
return nil
}
return s.converter.ConvertToDetailItems(responseData.Data.Detail, "campaign_report")
func (s *SyncService) SyncSinglePageWithTask(ctx context.Context, req *AccountReportRequest, useMock bool, maxRetries int, pageTaskID string, pageNumber int) (*SyncResult, error) {
return s.syncSinglePageWithTask(ctx, req, useMock, maxRetries, pageTaskID, pageNumber)
}
func (s *SyncService) fetchCurrentData(req *CampaignReportRequest, useMock bool) *CampaignReportData {
func (s *SyncService) syncSinglePageWithTask(ctx context.Context, req *AccountReportRequest, useMock bool, maxRetries int, pageTaskID string, pageNumber int) (*SyncResult, error) {
pageStartTime := time.Now()
pageLogReq := &taskDto.CreateSyncTaskLogReq{
TaskID: pageTaskID,
TaskType: "account_report_page",
AdvertiserID: req.AdvertiserID,
StartTime: time.UnixMilli(req.StartTime),
EndTime: time.UnixMilli(req.EndTime),
Status: "pending",
MaxRetry: maxRetries,
PageInfo: req.PageInfo,
RequestParams: map[string]interface{}{
"page_number": pageNumber,
"page_size": req.PageInfo.PageSize,
},
}
pageLogID, err := dao.SyncTaskLog.Create(ctx, pageLogReq)
if err != nil {
logrus.Errorf("创建分页任务日志失败：%v", err)
}
updatePageLog := func(status, errMsg, errorCode string, retryCount int) {
if pageLogID == 0 {
return
}
duration := time.Since(pageStartTime).Milliseconds()
updateReq := &taskDto.UpdateSyncTaskLogReq{
ID: pageLogID,
Status: status,
ErrorMessage: errMsg,
ErrorCode: errorCode,
DurationMs: &duration,
}
if retryCount > 0 {
updateReq.RetryCount = &retryCount
}
if status == "success" || status == "failed" {
completedAt := time.Now()
updateReq.CompletedAt = completedAt
}
if err := dao.SyncTaskLog.Update(ctx, updateReq); err != nil {
logrus.Errorf("更新分页任务日志失败：%v", err)
}
}
updatePageLog("running", "", "", 0)
logrus.Infof(">>> 开始同步第 %d 页数据...", pageNumber)
result, err := s.SyncWithRetry(ctx, req, useMock, maxRetries)
if err != nil {
updatePageLog("failed", err.Error(), "PAGE_SYNC_FAILED", 0)
return result, err
}
summary := map[string]interface{}{
"page_number": pageNumber,
"detail_count": result.DetailCount,
"sum_saved": result.SumSuccess,
}
updatePageLog("success", "", "", 0)
logrus.Debugf("分页任务 %s 完成: %v", pageTaskID, summary)
return result, nil
}
func (s *SyncService) fetchCurrentData(req *AccountReportRequest, useMock bool) *AccountReportData {
if useMock {
responseData := s.mockGen.GenerateCampaignReportResponse()
responseData := s.mockGen.GenerateAccountReportResponse()
if responseData != nil && responseData.Data != nil {
return responseData.Data
}
return nil
}
respBytes, err := s.httpClient.Post(context.Background(), "/rest/openapi/gw/esp/report/campaignReport", req)
respBytes, err := s.httpClient.Post(context.Background(), "/rest/openapi/gw/esp/report/accountReport", req)
if err != nil {
return nil
}
responseData := &CampaignReportResponse{}
responseData := &AccountReportResponse{}
if err := json.Unmarshal(respBytes, responseData); err != nil {
return nil
}
@@ -247,12 +415,12 @@ func (s *SyncService) saveDetailData(ctx context.Context, items []*dto.CidAccoun
return copydata.CidAccountReportDetail.BatchCreate(ctx, req)
}
func (s *SyncService) SyncWithRetry(ctx context.Context, req *CampaignReportRequest, useMock bool, maxRetries int) (*SyncResult, error) {
func (s *SyncService) SyncWithRetry(ctx context.Context, req *AccountReportRequest, useMock bool, maxRetries int) (*SyncResult, error) {
var lastResult *SyncResult
var lastErr error
for attempt := 0; attempt <= maxRetries; attempt++ {
result, err := s.SyncCampaignReport(ctx, req, useMock)
result, err := s.SyncAccountReport(ctx, req, useMock)
lastResult = result
lastErr = err

28
sync/sync_test.go
View File

@@ -32,7 +32,7 @@ func init() {
func TestMockDataGeneration(t *testing.T) {
mockGen := NewMockDataGenerator()
req := mockGen.GenerateCampaignReportRequest()
req := mockGen.GenerateAccountReportRequest()
if req == nil {
t.Error("请求数据生成失败")
return
@@ -45,23 +45,19 @@ func TestDataConverter(t *testing.T) {
converter := NewDataConverter()
mockGen := NewMockDataGenerator()
responseData := mockGen.GenerateCampaignReportResponse()
responseData := mockGen.GenerateAccountReportResponse()
if responseData == nil || responseData.Data.Sum == nil {
t.Fatal("Mock 数据生成失败")
}
sumItem := converter.ConvertToSumItem(responseData.Data.Sum, "campaign_report")
sumItem := converter.ConvertToSumItem(responseData.Data.Sum, "account_report", 1)
if sumItem == nil {
t.Fatal("转换为汇总数据失败")
}
if sumItem.CampaignName == "" {
t.Error("计划名称为空")
}
fmt.Printf("✓ 汇总数据转换成功：计划=%s\n", sumItem.CampaignName)
detailItems := converter.ConvertToDetailItems(responseData.Data.Detail, "campaign_report")
detailItems := converter.ConvertToDetailItems(responseData.Data.Detail, "account_report", 1)
if len(detailItems) == 0 {
t.Fatal("转换为明细数据失败")
}
@@ -69,11 +65,11 @@ func TestDataConverter(t *testing.T) {
fmt.Printf("✓ 明细数据转换成功：数量=%d\n", len(detailItems))
}
func TestSyncCampaignReportWithDB(t *testing.T) {
func TestSyncAccountReportWithDB(t *testing.T) {
ctx := gctx.New()
syncService := NewSyncService()
req := &CampaignReportRequest{
req := &AccountReportRequest{
AdvertiserID: 10001,
StartTime: time.Now().AddDate(0, 0, -30).UnixNano() / 1e6,
EndTime: time.Now().UnixNano() / 1e6,
@@ -82,7 +78,7 @@ func TestSyncCampaignReportWithDB(t *testing.T) {
QueryVersion: 1,
}
result, err := syncService.SyncCampaignReport(ctx, req, true)
result, err := syncService.SyncAccountReport(ctx, req, true)
if err != nil {
t.Logf("同步失败（可能是数据库问题）: %v", err)
return
@@ -98,7 +94,7 @@ func TestSyncCampaignReportWithDB(t *testing.T) {
// ctx := gctx.New()
// syncService := NewSyncService()
//
// req := &CampaignReportRequest{
// req := &AccountReportRequest{
// AdvertiserID: 10001,
// StartTime: time.Now().AddDate(0, 0, -30).UnixNano() / 1e6,
// EndTime: time.Now().UnixNano() / 1e6,
@@ -108,7 +104,7 @@ func TestSyncCampaignReportWithDB(t *testing.T) {
// }
//
// logrus.Info("=== 开始执行定时同步任务 ===")
// result, err := syncService.SyncCampaignReportWithPagination(ctx, req, true, 3)
// result, err := syncService.SyncAccountReportWithPagination(ctx, req, true, 3)
// if err != nil {
// t.Logf("定时同步任务失败：%v", err)
// return
@@ -120,10 +116,10 @@ func TestSyncCampaignReportWithDB(t *testing.T) {
// result.DetailCount, result.DetailSuccessCount, result.DetailFailCount)
//}
func BenchmarkSyncCampaignReport(b *testing.B) {
func BenchmarkSyncAccountReport(b *testing.B) {
ctx := gctx.New()
syncService := NewSyncService()
req := &CampaignReportRequest{
req := &AccountReportRequest{
AdvertiserID: 10001,
StartTime: time.Now().AddDate(0, 0, -30).UnixNano() / 1e6,
EndTime: time.Now().UnixNano() / 1e6,
@@ -134,6 +130,6 @@ func BenchmarkSyncCampaignReport(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _ = syncService.SyncCampaignReport(ctx, req, true)
_, _ = syncService.SyncAccountReport(ctx, req, true)
}
}