data-engine/sync/sync_service.go

package sync

import (
	dao "cid/dao/copydata"
	dto "cid/model/dto/copydata"
	taskDto "cid/model/dto/copydata"
	"cid/service/copydata"
	"context"
	"encoding/json"
	"fmt"
	"time"

	"github.com/sirupsen/logrus"
)

type SyncService struct {
	httpClient *HttpClient
	converter  *DataConverter
	mockGen    *MockDataGenerator
}

func NewSyncService() *SyncService {
	return &SyncService{
		httpClient: NewHttpClient("https://ad.e.kuaishou.com", 0),
		converter:  NewDataConverter(),
		mockGen:    NewMockDataGenerator(),
	}
}

type SyncResult struct {
	SumSuccess         bool              `json:"sum_success"`
	SumID              int64             `json:"sum_id"`
	DetailSuccess      bool              `json:"detail_success"`
	DetailCount        int               `json:"detail_count"`
	DetailSuccessCount int64             `json:"detail_success_count"`
	DetailFailCount    int64             `json:"detail_fail_count"`
	Error              error             `json:"error"`
	TaskLogID          int64             `json:"task_log_id"`
	PageResults        []*PageSyncResult `json:"page_results,omitempty"`
}

type PageSyncResult struct {
	PageNumber    int    `json:"page_number"`
	PageTaskLogID int64  `json:"page_task_log_id"`
	Success       bool   `json:"success"`
	RecordCount   int    `json:"record_count"`
	DurationMs    int64  `json:"duration_ms"`
	ErrorMessage  string `json:"error_message,omitempty"`
	RetryCount    int    `json:"retry_count"`
}

func (s *SyncService) SyncAccountReport(ctx context.Context, req *AccountReportRequest, useMock bool) (*SyncResult, error) {
	result := &SyncResult{}

	var responseData *AccountReportResponse

	if useMock {
		logrus.Info("使用 Mock 数据同步快手广告账户报表")
		responseData = s.mockGen.GenerateAccountReportResponse()
	} else {
		logrus.Info("从真实 API 同步快手广告账户报表")
		respBytes, err := s.httpClient.Post(ctx, "/rest/openapi/gw/esp/report/accountReport", req)
		if err != nil {
			result.Error = fmt.Errorf("调用 API 失败：%w", err)
			return result, result.Error
		}

		responseData = &AccountReportResponse{}
		if err := json.Unmarshal(respBytes, responseData); err != nil {
			result.Error = fmt.Errorf("解析响应失败：%w", err)
			return result, result.Error
		}

		if responseData.Code != 0 {
			result.Error = fmt.Errorf("API 返回错误：code=%d, message=%s", responseData.Code, responseData.Message)
			return result, result.Error
		}
	}

	if responseData.Data.Sum != nil {
		sumItem := s.converter.ConvertToSumItem(responseData.Data.Sum, "account_report", req.PageInfo.CurrentPage)
		sumResult, saveErr := s.saveSumData(ctx, sumItem)
		if saveErr != nil {
			logrus.Errorf("保存汇总数据失败：%v", saveErr)
			result.Error = fmt.Errorf("保存汇总数据失败：%w", saveErr)
		} else {
			result.SumSuccess = true
			result.SumID = sumResult.Id
			logrus.Infof("成功保存汇总数据，ID=%d", sumResult.Id)
		}
	}

	if len(responseData.Data.Detail) > 0 {
		detailItems := s.converter.ConvertToDetailItems(responseData.Data.Detail, "account_report", req.PageInfo.CurrentPage)
		detailResult, saveErr := s.saveDetailData(ctx, detailItems)
		if saveErr != nil {
			logrus.Errorf("保存明细数据失败：%v", saveErr)
			result.Error = fmt.Errorf("保存明细数据失败：%w", saveErr)
		} else {
			result.DetailSuccess = true
			result.DetailCount = len(detailItems)
			result.DetailSuccessCount = detailResult.SuccessCount
			result.DetailFailCount = detailResult.FailCount
			logrus.Infof("成功保存 %d 条明细数据（成功=%d, 失败=%d）", len(detailItems), detailResult.SuccessCount, detailResult.FailCount)
		}
	}

	return result, result.Error
}

// SyncAccountReportWithPagination 带分页处理的同步方法（支持全量数据抽取）
func (s *SyncService) SyncAccountReportWithPagination(ctx context.Context, req *AccountReportRequest, useMock bool, maxRetries int) (*SyncResult, error) {
	startTime := time.Now()
	parentTaskID := fmt.Sprintf("%d_%d_account", req.AdvertiserID, req.StartTime)

	logReq := &taskDto.CreateSyncTaskLogReq{
		TaskID:        parentTaskID,
		TaskType:      "account_report",
		AdvertiserID:  req.AdvertiserID,
		StartTime:     time.UnixMilli(req.StartTime),
		EndTime:       time.UnixMilli(req.EndTime),
		Status:        "pending",
		MaxRetry:      maxRetries,
		RequestParams: req,
	}

	parentLogID, err := dao.SyncTaskLog.Create(ctx, logReq)
	if err != nil {
		logrus.Errorf("创建主任务日志失败：%v", err)
	}

	updateParentLog := func(status, errMsg, errorCode string, summary interface{}) {
		if parentLogID == 0 {
			return
		}
		duration := time.Since(startTime).Milliseconds()
		updateReq := &taskDto.UpdateSyncTaskLogReq{
			ID:           parentLogID,
			Status:       status,
			ErrorMessage: errMsg,
			ErrorCode:    errorCode,
			DurationMs:   &duration,
		}

		if status == "success" || status == "manual_review" {
			completedAt := time.Now()
			updateReq.CompletedAt = completedAt
		}

		if summary != nil {
			updateReq.ResultSummary = summary
		}

		if err := dao.SyncTaskLog.Update(ctx, updateReq); err != nil {
			logrus.Errorf("更新主任务日志失败：%v", err)
		}
	}

	updateParentLog("running", "", "", nil)

	aggregatedResult := &SyncResult{
		SumSuccess:  false,
		SumID:       0,
		TaskLogID:   parentLogID,
		PageResults: make([]*PageSyncResult, 0),
	}

	totalCount := 0
	currentPage := 1
	pageSize := 100
	successPages := 0
	failedPages := 0

	if req.PageInfo == nil {
		req.PageInfo = &PageInfo{}
	}

	var totalPages int

	for {
		logrus.Infof(">>> 正在同步第 %d 页数据...", currentPage)

		req.PageInfo.CurrentPage = currentPage
		req.PageInfo.PageSize = pageSize

		pageTaskID := fmt.Sprintf("%s_page_%d", parentTaskID, currentPage)
		pageStartTime := time.Now()

		pageResult := &PageSyncResult{
			PageNumber:  currentPage,
			Success:     false,
			RecordCount: 0,
			RetryCount:  0,
		}

		result, err := s.syncSinglePageWithTask(ctx, req, useMock, maxRetries, pageTaskID, currentPage)
		pageDuration := time.Since(pageStartTime).Milliseconds()
		pageResult.DurationMs = pageDuration

		if err != nil {
			logrus.Errorf("第 %d 页同步失败：%v", currentPage, err)
			pageResult.ErrorMessage = err.Error()
			failedPages++

			aggregatedResult.PageResults = append(aggregatedResult.PageResults, pageResult)

			if failedPages > maxRetries {
				logrus.Warnf("失败页数超过阈值 %d，终止同步", maxRetries)
				break
			}

			currentPage++
			time.Sleep(500 * time.Millisecond)
			continue
		}

		if result.SumSuccess && aggregatedResult.SumID == 0 {
			aggregatedResult.SumSuccess = true
			aggregatedResult.SumID = result.SumID
			logrus.Infof("✓ 汇总数据已保存，ID=%d", result.SumID)
		}

		if result.DetailSuccess && result.DetailCount > 0 {
			totalCount += result.DetailCount
			pageResult.Success = true
			pageResult.RecordCount = result.DetailCount
			successPages++
			logrus.Infof("✓ 第 %d 页获取到 %d 条明细数据，累计 %d 条", currentPage, result.DetailCount, totalCount)
		}

		aggregatedResult.PageResults = append(aggregatedResult.PageResults, pageResult)

		currentData := s.fetchCurrentData(req, useMock)
		if currentData != nil && currentData.TotalCount > 0 {
			totalPages = (currentData.TotalCount + pageSize - 1) / pageSize
			logrus.Infof("总记录数：%d, 总页数：%d, 当前页：%d/%d",
				currentData.TotalCount, totalPages, currentPage, totalPages)

			if currentPage >= totalPages {
				logrus.Infof("✓ 已同步所有页面数据，共 %d 页，%d 条记录", totalPages, currentData.TotalCount)
				break
			}
		}

		if result.DetailCount < pageSize {
			logrus.Infof("✓ 当前页数据不足 %d 条，已到达最后一页", pageSize)
			break
		}

		currentPage++
		time.Sleep(300 * time.Millisecond)
	}

	logrus.Infof("分页同步完成 - 成功:%d页, 失败:%d页, 总明细:%d条",
		successPages, failedPages, totalCount)

	// 统计所有子任务的结果
	totalDetailCount := 0
	var totalSuccessCount int64
	var totalFailCount int64

	for _, pageResult := range aggregatedResult.PageResults {
		if pageResult.Success {
			totalDetailCount += pageResult.RecordCount
			totalSuccessCount++
		} else {
			totalFailCount++
		}
	}

	aggregatedResult.DetailCount = totalDetailCount
	aggregatedResult.DetailSuccessCount = totalSuccessCount
	aggregatedResult.DetailFailCount = totalFailCount

	if failedPages > 0 {
		logrus.Warnf("存在 %d 个失败的页面，主任务标记为部分失败", failedPages)

		summary := map[string]interface{}{
			"sum_id":        aggregatedResult.SumID,
			"detail_count":  totalDetailCount,
			"total_pages":   totalPages,
			"success_pages": successPages,
			"failed_pages":  failedPages,
			"page_results":  aggregatedResult.PageResults,
		}
		updateParentLog("partial_failed", fmt.Sprintf("%d 个页面同步失败", failedPages), "PAGE_SYNC_FAILED", summary)
	} else {
		logrus.Info("✓ 所有页面同步成功")

		summary := map[string]interface{}{
			"sum_id":        aggregatedResult.SumID,
			"detail_count":  totalDetailCount,
			"total_pages":   totalPages,
			"success_pages": successPages,
			"failed_pages":  0,
			"page_results":  aggregatedResult.PageResults,
		}
		updateParentLog("success", "", "", summary)
	}

	return aggregatedResult, aggregatedResult.Error
}

func (s *SyncService) SyncSinglePageWithTask(ctx context.Context, req *AccountReportRequest, useMock bool, maxRetries int, pageTaskID string, pageNumber int) (*SyncResult, error) {
	return s.syncSinglePageWithTask(ctx, req, useMock, maxRetries, pageTaskID, pageNumber)
}

func (s *SyncService) syncSinglePageWithTask(ctx context.Context, req *AccountReportRequest, useMock bool, maxRetries int, pageTaskID string, pageNumber int) (*SyncResult, error) {
	pageStartTime := time.Now()

	pageLogReq := &taskDto.CreateSyncTaskLogReq{
		TaskID:       pageTaskID,
		TaskType:     "account_report_page",
		AdvertiserID: req.AdvertiserID,
		StartTime:    time.UnixMilli(req.StartTime),
		EndTime:      time.UnixMilli(req.EndTime),
		Status:       "pending",
		MaxRetry:     maxRetries,
		PageInfo:     req.PageInfo,
		RequestParams: map[string]interface{}{
			"page_number": pageNumber,
			"page_size":   req.PageInfo.PageSize,
		},
	}

	pageLogID, err := dao.SyncTaskLog.Create(ctx, pageLogReq)
	if err != nil {
		logrus.Errorf("创建分页任务日志失败：%v", err)
	}

	updatePageLog := func(status, errMsg, errorCode string, retryCount int) {
		if pageLogID == 0 {
			return
		}
		duration := time.Since(pageStartTime).Milliseconds()
		updateReq := &taskDto.UpdateSyncTaskLogReq{
			ID:           pageLogID,
			Status:       status,
			ErrorMessage: errMsg,
			ErrorCode:    errorCode,
			DurationMs:   &duration,
		}

		if retryCount > 0 {
			updateReq.RetryCount = &retryCount
		}

		if status == "success" || status == "failed" {
			completedAt := time.Now()
			updateReq.CompletedAt = completedAt
		}

		if err := dao.SyncTaskLog.Update(ctx, updateReq); err != nil {
			logrus.Errorf("更新分页任务日志失败：%v", err)
		}
	}

	updatePageLog("running", "", "", 0)

	logrus.Infof(">>> 开始同步第 %d 页数据...", pageNumber)

	result, err := s.SyncWithRetry(ctx, req, useMock, maxRetries)

	if err != nil {
		updatePageLog("failed", err.Error(), "PAGE_SYNC_FAILED", 0)
		return result, err
	}

	summary := map[string]interface{}{
		"page_number":  pageNumber,
		"detail_count": result.DetailCount,
		"sum_saved":    result.SumSuccess,
	}
	updatePageLog("success", "", "", 0)

	logrus.Debugf("分页任务 %s 完成: %v", pageTaskID, summary)

	return result, nil
}

func (s *SyncService) fetchCurrentData(req *AccountReportRequest, useMock bool) *AccountReportData {
	if useMock {
		responseData := s.mockGen.GenerateAccountReportResponse()
		if responseData != nil && responseData.Data != nil {
			return responseData.Data
		}
		return nil
	}

	respBytes, err := s.httpClient.Post(context.Background(), "/rest/openapi/gw/esp/report/accountReport", req)
	if err != nil {
		return nil
	}

	responseData := &AccountReportResponse{}
	if err := json.Unmarshal(respBytes, responseData); err != nil {
		return nil
	}

	if responseData.Code == 0 && responseData.Data != nil {
		return responseData.Data
	}

	return nil
}

func (s *SyncService) saveSumData(ctx context.Context, item *dto.CidAccountReportSumItem) (*dto.CreateCidAccountReportSumRes, error) {
	return copydata.CidAccountReportDetail.CreateSum(ctx, item)
}

func (s *SyncService) saveDetailData(ctx context.Context, items []*dto.CidAccountReportDetailItem) (*dto.BatchCreateCidAccountReportDetailRes, error) {
	req := &dto.BatchCreateCidAccountReportDetailReq{
		Items: items,
	}
	return copydata.CidAccountReportDetail.BatchCreate(ctx, req)
}

func (s *SyncService) SyncWithRetry(ctx context.Context, req *AccountReportRequest, useMock bool, maxRetries int) (*SyncResult, error) {
	var lastResult *SyncResult
	var lastErr error

	for attempt := 0; attempt <= maxRetries; attempt++ {
		result, err := s.SyncAccountReport(ctx, req, useMock)
		lastResult = result
		lastErr = err

		if err == nil {
			logrus.Infof("同步成功，尝试次数：%d", attempt+1)
			return result, nil
		}

		logrus.Warnf("同步失败，第 %d 次重试，错误：%v", attempt+1, err)
	}

	return lastResult, lastErr
}