重构数据引擎
This commit is contained in:
@@ -1,396 +0,0 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
entities "dataengine/entities"
|
||||
)
|
||||
|
||||
// APIService API服务接口
|
||||
type APIService interface {
|
||||
GetCampaignReport(params *entities.RequestParams) (*entities.APIResponse, error)
|
||||
GetCampaignReportByTimeRange(advertiserID int64, startTime, endTime int64) (*entities.APIResponse, error)
|
||||
GetAllCampaignReport(params *entities.RequestParams) (*entities.APIResponse, error)
|
||||
GetAllCampaignReportByTimeRange(advertiserID int64, startTime, endTime int64) (*entities.APIResponse, error)
|
||||
GetCampaignReportWithCallback(params *entities.RequestParams, callback func(*entities.APIResponse, error) bool) error
|
||||
}
|
||||
|
||||
// KuaishouAPIService 快手API服务实现
|
||||
type KuaishouAPIService struct {
|
||||
BaseURL string
|
||||
AccessToken string
|
||||
HTTPClient *http.Client
|
||||
MaxRetries int
|
||||
RetryDelay time.Duration
|
||||
RateLimit time.Duration // 请求间隔，避免触发限流
|
||||
}
|
||||
|
||||
// NewKuaishouAPIService 创建API服务实例
|
||||
func NewKuaishouAPIService(accessToken string) *KuaishouAPIService {
|
||||
return &KuaishouAPIService{
|
||||
BaseURL: "https://ad.e.kuaishou.com/rest/openapi/gw/esp/report",
|
||||
AccessToken: accessToken,
|
||||
HTTPClient: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
MaxRetries: 3,
|
||||
RetryDelay: 2 * time.Second,
|
||||
RateLimit: 200 * time.Millisecond, // 默认200ms间隔
|
||||
}
|
||||
}
|
||||
|
||||
// SetRateLimit 设置请求间隔
|
||||
func (s *KuaishouAPIService) SetRateLimit(delay time.Duration) {
|
||||
s.RateLimit = delay
|
||||
}
|
||||
|
||||
// GetCampaignReport 获取广告计划报表（单页）
|
||||
func (s *KuaishouAPIService) GetCampaignReport(params *entities.RequestParams) (*entities.APIResponse, error) {
|
||||
for retry := 0; retry <= s.MaxRetries; retry++ {
|
||||
response, err := s.doRequest(params)
|
||||
if err == nil {
|
||||
return response, nil
|
||||
}
|
||||
|
||||
// 判断是否重试
|
||||
if retry < s.MaxRetries {
|
||||
time.Sleep(s.RetryDelay)
|
||||
continue
|
||||
}
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("请求失败，已达到最大重试次数")
|
||||
}
|
||||
|
||||
// doRequest 执行实际请求
|
||||
func (s *KuaishouAPIService) doRequest(params *entities.RequestParams) (*entities.APIResponse, error) {
|
||||
// 序列化请求参数
|
||||
jsonData, err := json.Marshal(params)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("序列化失败: %v", err)
|
||||
}
|
||||
|
||||
// 创建HTTP请求
|
||||
url := fmt.Sprintf("%s/campaignReport", s.BaseURL)
|
||||
req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("创建请求失败: %v", err)
|
||||
}
|
||||
|
||||
// 设置请求头
|
||||
s.setHeaders(req)
|
||||
|
||||
// 发送请求
|
||||
resp, err := s.HTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("请求失败: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// 检查HTTP状态码
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := ioutil.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("HTTP状态码错误: %d, 响应: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
// 读取响应
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("读取响应失败: %v", err)
|
||||
}
|
||||
|
||||
// 解析响应
|
||||
var apiResponse entities.APIResponse
|
||||
err = json.Unmarshal(body, &apiResponse)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("解析响应失败: %v", err)
|
||||
}
|
||||
|
||||
// 检查API返回码
|
||||
if apiResponse.Code != 0 {
|
||||
return nil, fmt.Errorf("API返回错误: %s (code: %d)", apiResponse.Message, apiResponse.Code)
|
||||
}
|
||||
|
||||
return &apiResponse, nil
|
||||
}
|
||||
|
||||
// GetAllCampaignReport 获取所有分页数据
|
||||
func (s *KuaishouAPIService) GetAllCampaignReport(params *entities.RequestParams) (*entities.APIResponse, error) {
|
||||
// 1. 先获取第一页数据，得到总数
|
||||
params.ResetPage()
|
||||
firstPage, err := s.GetCampaignReport(params)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("获取第一页数据失败: %v", err)
|
||||
}
|
||||
|
||||
if firstPage.Data == nil {
|
||||
return firstPage, nil // 没有数据
|
||||
}
|
||||
|
||||
totalCount := firstPage.Data.TotalCount
|
||||
pageSize := params.PageInfo.PageSize
|
||||
|
||||
// 2. 如果只有一页，直接返回
|
||||
if totalCount <= pageSize {
|
||||
return firstPage, nil
|
||||
}
|
||||
|
||||
// 3. 计算总页数
|
||||
totalPages := int(math.Ceil(float64(totalCount) / float64(pageSize)))
|
||||
|
||||
// 4. 创建结果集
|
||||
allResponses := make([]*entities.APIResponse, 0, totalPages)
|
||||
allResponses = append(allResponses, firstPage)
|
||||
|
||||
// 5. 并发获取剩余页数据
|
||||
pageChan := make(chan *pageResult, totalPages-1)
|
||||
|
||||
// 启动goroutine获取剩余页
|
||||
for page := 2; page <= totalPages; page++ {
|
||||
go func(currentPage int) {
|
||||
// 复制参数并设置页码
|
||||
pageParams := *params
|
||||
pageParams.PageInfo = &entities.PageInfo{
|
||||
CurrentPage: currentPage,
|
||||
PageSize: pageSize,
|
||||
}
|
||||
|
||||
response, err := s.GetCampaignReport(&pageParams)
|
||||
pageChan <- &pageResult{
|
||||
Page: currentPage,
|
||||
Response: response,
|
||||
Error: err,
|
||||
}
|
||||
|
||||
// 控制请求频率
|
||||
time.Sleep(s.RateLimit)
|
||||
}(page)
|
||||
}
|
||||
|
||||
// 6. 收集结果
|
||||
errors := make([]error, 0)
|
||||
for i := 0; i < totalPages-1; i++ {
|
||||
result := <-pageChan
|
||||
if result.Error != nil {
|
||||
errors = append(errors, fmt.Errorf("第%d页获取失败: %v", result.Page, result.Error))
|
||||
} else {
|
||||
allResponses = append(allResponses, result.Response)
|
||||
}
|
||||
}
|
||||
|
||||
// 7. 合并数据
|
||||
mergedResponse := entities.MergeData(allResponses)
|
||||
|
||||
// 8. 如果有错误，返回错误信息
|
||||
if len(errors) > 0 {
|
||||
// 可以记录错误日志，但继续返回已获取的数据
|
||||
fmt.Printf("部分页面获取失败: %v\n", errors)
|
||||
}
|
||||
|
||||
return mergedResponse, nil
|
||||
}
|
||||
|
||||
// GetAllCampaignReportByTimeRange 根据时间范围获取所有数据
|
||||
func (s *KuaishouAPIService) GetAllCampaignReportByTimeRange(advertiserID int64, startTime, endTime int64) (*entities.APIResponse, error) {
|
||||
params := entities.NewRequestParams()
|
||||
params.AdvertiserID = advertiserID
|
||||
params.StartTime = startTime
|
||||
params.EndTime = endTime
|
||||
|
||||
return s.GetAllCampaignReport(params)
|
||||
}
|
||||
|
||||
// GetCampaignReportWithCallback 使用回调函数获取所有数据（流式处理，避免内存占用过大）
|
||||
func (s *KuaishouAPIService) GetCampaignReportWithCallback(params *entities.RequestParams, callback func(*entities.APIResponse, error) bool) error {
|
||||
// 1. 先获取第一页数据，得到总数
|
||||
params.ResetPage()
|
||||
firstPage, err := s.GetCampaignReport(params)
|
||||
if err != nil {
|
||||
return fmt.Errorf("获取第一页数据失败: %v", err)
|
||||
}
|
||||
|
||||
// 调用回调处理第一页
|
||||
if !callback(firstPage, nil) {
|
||||
return nil // 回调要求停止
|
||||
}
|
||||
|
||||
if firstPage.Data == nil || firstPage.Data.TotalCount <= params.PageInfo.PageSize {
|
||||
return nil // 只有一页
|
||||
}
|
||||
|
||||
totalCount := firstPage.Data.TotalCount
|
||||
pageSize := params.PageInfo.PageSize
|
||||
totalPages := int(math.Ceil(float64(totalCount) / float64(pageSize)))
|
||||
|
||||
// 2. 顺序获取剩余页数据（避免并发导致API限流）
|
||||
for page := 2; page <= totalPages; page++ {
|
||||
// 复制参数并设置页码
|
||||
pageParams := *params
|
||||
pageParams.PageInfo = &entities.PageInfo{
|
||||
CurrentPage: page,
|
||||
PageSize: pageSize,
|
||||
}
|
||||
|
||||
response, err := s.GetCampaignReport(&pageParams)
|
||||
if !callback(response, err) {
|
||||
return nil // 回调要求停止
|
||||
}
|
||||
|
||||
// 控制请求频率
|
||||
time.Sleep(s.RateLimit)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetCampaignReportSequential 顺序获取所有数据（简单实现）
|
||||
func (s *KuaishouAPIService) GetCampaignReportSequential(params *entities.RequestParams) (*entities.APIResponse, error) {
|
||||
// 1. 先获取第一页数据，得到总数
|
||||
params.ResetPage()
|
||||
firstPage, err := s.GetCampaignReport(params)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("获取第一页数据失败: %v", err)
|
||||
}
|
||||
|
||||
if firstPage.Data == nil {
|
||||
return firstPage, nil
|
||||
}
|
||||
|
||||
totalCount := firstPage.Data.TotalCount
|
||||
pageSize := params.PageInfo.PageSize
|
||||
|
||||
// 2. 如果只有一页，直接返回
|
||||
if totalCount <= pageSize {
|
||||
return firstPage, nil
|
||||
}
|
||||
|
||||
// 3. 顺序获取所有数据
|
||||
allDetails := make([]*entities.ReportDetail, 0, totalCount)
|
||||
if firstPage.Data.Detail != nil {
|
||||
allDetails = append(allDetails, firstPage.Data.Detail...)
|
||||
}
|
||||
|
||||
totalPages := int(math.Ceil(float64(totalCount) / float64(pageSize)))
|
||||
|
||||
for page := 2; page <= totalPages; page++ {
|
||||
// 复制参数并设置页码
|
||||
pageParams := *params
|
||||
pageParams.PageInfo = &entities.PageInfo{
|
||||
CurrentPage: page,
|
||||
PageSize: pageSize,
|
||||
}
|
||||
|
||||
response, err := s.GetCampaignReport(&pageParams)
|
||||
if err != nil {
|
||||
// 记录错误但继续获取
|
||||
fmt.Printf("第%d页获取失败: %v\n", page, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if response.Data != nil && response.Data.Detail != nil {
|
||||
allDetails = append(allDetails, response.Data.Detail...)
|
||||
}
|
||||
|
||||
// 控制请求频率
|
||||
time.Sleep(s.RateLimit)
|
||||
}
|
||||
|
||||
// 4. 构建完整响应
|
||||
completeResponse := &entities.APIResponse{
|
||||
Code: firstPage.Code,
|
||||
Message: firstPage.Message,
|
||||
Data: &entities.ReportData{
|
||||
Sum: firstPage.Data.Sum,
|
||||
Detail: allDetails,
|
||||
TotalCount: totalCount,
|
||||
},
|
||||
}
|
||||
|
||||
return completeResponse, nil
|
||||
}
|
||||
|
||||
// setHeaders 设置请求头
|
||||
func (s *KuaishouAPIService) setHeaders(req *http.Request) {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Accept", "application/json")
|
||||
if s.AccessToken != "" {
|
||||
req.Header.Set("Access-Token", s.AccessToken)
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", s.AccessToken))
|
||||
}
|
||||
}
|
||||
|
||||
// pageResult 分页结果
|
||||
type pageResult struct {
|
||||
Page int
|
||||
Response *entities.APIResponse
|
||||
Error error
|
||||
}
|
||||
|
||||
// ReportStatistics 报表统计
|
||||
type ReportStatistics struct {
|
||||
TotalImpression int64
|
||||
TotalClick int64
|
||||
TotalCost float64
|
||||
TotalGMV float64
|
||||
TotalT0GMV float64
|
||||
TotalT0OrderCnt int64
|
||||
PageCount int
|
||||
RecordCount int
|
||||
}
|
||||
|
||||
// CalculateStatistics 计算统计信息
|
||||
func CalculateStatistics(response *entities.APIResponse) *ReportStatistics {
|
||||
stats := &ReportStatistics{}
|
||||
|
||||
if response.Data == nil {
|
||||
return stats
|
||||
}
|
||||
|
||||
stats.PageCount = 1 // 默认值
|
||||
stats.RecordCount = response.Data.TotalCount
|
||||
|
||||
if response.Data.Detail != nil {
|
||||
for _, detail := range response.Data.Detail {
|
||||
stats.TotalImpression += detail.Impression
|
||||
stats.TotalClick += detail.Click
|
||||
stats.TotalCost += detail.CostTotal
|
||||
stats.TotalGMV += detail.GMV
|
||||
stats.TotalT0GMV += detail.T0GMV
|
||||
stats.TotalT0OrderCnt += detail.T0OrderCnt
|
||||
}
|
||||
}
|
||||
|
||||
return stats
|
||||
}
|
||||
|
||||
// GetAverageMetrics 获取平均指标
|
||||
func (stats *ReportStatistics) GetAverageMetrics() map[string]float64 {
|
||||
metrics := make(map[string]float64)
|
||||
|
||||
if stats.TotalImpression > 0 {
|
||||
metrics["ctr"] = float64(stats.TotalClick) / float64(stats.TotalImpression) * 100
|
||||
}
|
||||
|
||||
if stats.TotalClick > 0 {
|
||||
metrics["cpc"] = stats.TotalCost / float64(stats.TotalClick)
|
||||
}
|
||||
|
||||
if stats.TotalCost > 0 {
|
||||
metrics["roi"] = stats.TotalGMV / stats.TotalCost
|
||||
metrics["t0_roi"] = stats.TotalT0GMV / stats.TotalCost
|
||||
}
|
||||
|
||||
if stats.TotalImpression > 0 {
|
||||
metrics["cpm"] = stats.TotalCost / float64(stats.TotalImpression) * 1000
|
||||
}
|
||||
|
||||
return metrics
|
||||
}
|
||||
@@ -1,113 +0,0 @@
|
||||
package copydata
|
||||
|
||||
import (
|
||||
"context"
|
||||
dao "dataengine/dao/copydata"
|
||||
dto "dataengine/model/dto/copydata"
|
||||
"errors"
|
||||
|
||||
"gitea.com/red-future/common/beans"
|
||||
)
|
||||
|
||||
type cidAccountReportDetailService struct{}
|
||||
|
||||
// CidAccountReportDetail 广告数据报表详情服务
|
||||
var CidAccountReportDetail = new(cidAccountReportDetailService)
|
||||
|
||||
// Create 创建广告数据报表详情
|
||||
func (s *cidAccountReportDetailService) Create(ctx context.Context, req *dto.CidAccountReportDetailItem) (res *dto.CreateCidAccountReportDetailRes, err error) {
|
||||
// 验证必要字段
|
||||
if req.DataType == "" {
|
||||
return nil, errors.New("数据类型不能为空")
|
||||
}
|
||||
if req.ReportDateStr == "" {
|
||||
return nil, errors.New("报告日期不能为空")
|
||||
}
|
||||
|
||||
// 插入数据库
|
||||
id, err := dao.CidAccountReportDetail.Insert(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res = &dto.CreateCidAccountReportDetailRes{
|
||||
Id: id,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// BatchCreate 批量创建广告数据报表详情
|
||||
func (s *cidAccountReportDetailService) BatchCreate(ctx context.Context, req *dto.BatchCreateCidAccountReportDetailReq) (res *dto.BatchCreateCidAccountReportDetailRes, err error) {
|
||||
ctx = context.WithValue(ctx, "user", &beans.User{UserName: "admin"})
|
||||
// 验证数据
|
||||
if len(req.Items) == 0 {
|
||||
return nil, errors.New("批量创建数据不能为空")
|
||||
}
|
||||
|
||||
//// 验证每条数据
|
||||
//for i, item := range req.Items {
|
||||
// if item.DataType == "" {
|
||||
// return nil, errors.New("第" + string(rune(i+1)) + "条数据的数据类型不能为空")
|
||||
// }
|
||||
// if item.ReportDateStr == "" {
|
||||
// return nil, errors.New("第" + string(rune(i+1)) + "条数据的报告日期不能为空")
|
||||
// }
|
||||
//}
|
||||
|
||||
// 批量插入数据库
|
||||
successCount, failCount, failedIndexes, err := dao.CidAccountReportDetail.BatchInsert(ctx, req.Items)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res = &dto.BatchCreateCidAccountReportDetailRes{
|
||||
SuccessCount: successCount,
|
||||
FailCount: failCount,
|
||||
FailedItems: failedIndexes,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// CreateSum Create 创建广告数据报表汇总
|
||||
func (s *cidAccountReportDetailService) CreateSum(ctx context.Context, req *dto.CidAccountReportSumItem) (res *dto.CreateCidAccountReportSumRes, err error) {
|
||||
// 验证必要字段
|
||||
if req.DataType == "" {
|
||||
return nil, errors.New("数据类型不能为空")
|
||||
}
|
||||
if req.ReportDateStr == "" {
|
||||
return nil, errors.New("报告日期不能为空")
|
||||
}
|
||||
|
||||
// 插入数据库
|
||||
id, err := dao.CidAccountReportSum.Insert(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res = &dto.CreateCidAccountReportSumRes{
|
||||
Id: id,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// BatchCreateSum 批量创建广告数据报表汇总
|
||||
func (s *cidAccountReportDetailService) BatchCreateSum(ctx context.Context, req *dto.BatchCreateCidAccountReportSumReq) (res *dto.BatchCreateCidAccountReportSumRes, err error) {
|
||||
ctx = context.WithValue(ctx, "user", &beans.User{UserName: "admin"})
|
||||
// 验证数据
|
||||
if len(req.Items) == 0 {
|
||||
return nil, errors.New("批量创建数据不能为空")
|
||||
}
|
||||
|
||||
// 批量插入数据库
|
||||
successCount, failCount, failedIndexes, err := dao.CidAccountReportSum.BatchInsert(ctx, req.Items)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res = &dto.BatchCreateCidAccountReportSumRes{
|
||||
SuccessCount: successCount,
|
||||
FailCount: failCount,
|
||||
FailedItems: failedIndexes,
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -1,97 +0,0 @@
|
||||
package copydata
|
||||
|
||||
import (
|
||||
"context"
|
||||
dao "dataengine/dao/copydata"
|
||||
dto "dataengine/model/dto/copydata"
|
||||
"errors"
|
||||
|
||||
"gitea.com/red-future/common/beans"
|
||||
)
|
||||
|
||||
type campaignReportSumService struct{}
|
||||
|
||||
// CampaignReportSum 广告计划效果指标表服务
|
||||
var CampaignReportSum = new(campaignReportSumService)
|
||||
|
||||
// Create 创建广告计划效果指标表
|
||||
func (s *campaignReportSumService) Create(ctx context.Context, req *dto.CampaignReportSumItem) (res *dto.CreateCampaignReportSumRes, err error) {
|
||||
// 验证必要字段
|
||||
if req.ReportDateStr == "" {
|
||||
return nil, errors.New("报告日期不能为空")
|
||||
}
|
||||
|
||||
// 插入数据库
|
||||
id, err := dao.CampaignReportSum.Insert(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res = &dto.CreateCampaignReportSumRes{
|
||||
Id: id,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// BatchCreate 批量创建广告计划效果指标表
|
||||
func (s *campaignReportSumService) BatchCreate(ctx context.Context, req *dto.BatchCreateCampaignReportSumReq) (res *dto.BatchCreateCampaignReportSumRes, err error) {
|
||||
ctx = context.WithValue(ctx, "user", &beans.User{UserName: "admin"})
|
||||
// 验证数据
|
||||
if len(req.Items) == 0 {
|
||||
return nil, errors.New("批量创建数据不能为空")
|
||||
}
|
||||
|
||||
// 批量插入数据库
|
||||
successCount, failCount, failedIndexes, err := dao.CampaignReportSum.BatchInsert(ctx, req.Items)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res = &dto.BatchCreateCampaignReportSumRes{
|
||||
SuccessCount: successCount,
|
||||
FailCount: failCount,
|
||||
FailedItems: failedIndexes,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// CreateDetail 创建广告效果指标表
|
||||
func (s *campaignReportSumService) CreateDetail(ctx context.Context, req *dto.CampaignReportDetailItem) (res *dto.CreateCampaignReportDetailRes, err error) {
|
||||
// 验证必要字段
|
||||
if req.ReportDateStr == "" {
|
||||
return nil, errors.New("报告日期不能为空")
|
||||
}
|
||||
|
||||
// 插入数据库
|
||||
id, err := dao.CampaignReportDetail.Insert(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res = &dto.CreateCampaignReportDetailRes{
|
||||
Id: id,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// BatchCreateDetail 批量创建广告效果指标表
|
||||
func (s *campaignReportSumService) BatchCreateDetail(ctx context.Context, req *dto.BatchCreateCampaignReportDetailReq) (res *dto.BatchCreateCampaignReportDetailRes, err error) {
|
||||
ctx = context.WithValue(ctx, "user", &beans.User{UserName: "admin"})
|
||||
// 验证数据
|
||||
if len(req.Items) == 0 {
|
||||
return nil, errors.New("批量创建数据不能为空")
|
||||
}
|
||||
|
||||
// 批量插入数据库
|
||||
successCount, failCount, failedIndexes, err := dao.CampaignReportDetail.BatchInsert(ctx, req.Items)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res = &dto.BatchCreateCampaignReportDetailRes{
|
||||
SuccessCount: successCount,
|
||||
FailCount: failCount,
|
||||
FailedItems: failedIndexes,
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -1,91 +0,0 @@
|
||||
package copydata
|
||||
|
||||
import (
|
||||
"context"
|
||||
dao "dataengine/dao/copydata"
|
||||
dto "dataengine/model/dto/copydata"
|
||||
"errors"
|
||||
|
||||
"gitea.com/red-future/common/beans"
|
||||
)
|
||||
|
||||
type creativeReportSumService struct{}
|
||||
|
||||
// CreativeReportSum 广告效果指标表服务
|
||||
var CreativeReportSum = new(creativeReportSumService)
|
||||
|
||||
// Create 创建广告效果指标表
|
||||
func (s *creativeReportSumService) Create(ctx context.Context, req *dto.CreativeReportSumItem) (res *dto.CreateCreativeReportSumRes, err error) {
|
||||
// 验证必要字段
|
||||
if req.ReportDateStr == "" {
|
||||
return nil, errors.New("报告日期不能为空")
|
||||
}
|
||||
|
||||
// 插入数据库
|
||||
id, err := dao.CreativeReportSum.Insert(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res = &dto.CreateCreativeReportSumRes{
|
||||
Id: id,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// BatchCreate 批量创建广告效果指标表
|
||||
func (s *creativeReportSumService) BatchCreate(ctx context.Context, req *dto.BatchCreateCreativeReportSumReq) (res *dto.BatchCreateCreativeReportSumRes, err error) {
|
||||
ctx = context.WithValue(ctx, "user", &beans.User{UserName: "admin"})
|
||||
// 验证数据
|
||||
if len(req.Items) == 0 {
|
||||
return nil, errors.New("批量创建数据不能为空")
|
||||
}
|
||||
|
||||
// 批量插入数据库
|
||||
successCount, failCount, failedIndexes, err := dao.CreativeReportSum.BatchInsert(ctx, req.Items)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res = &dto.BatchCreateCreativeReportSumRes{
|
||||
SuccessCount: successCount,
|
||||
FailCount: failCount,
|
||||
FailedItems: failedIndexes,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (s *creativeReportSumService) CreateDetail(ctx context.Context, req *dto.CreativeReportDetailItem) (res *dto.CreateCreativeReportDetailRes, err error) {
|
||||
if req.ReportDateStr == "" {
|
||||
return nil, errors.New("报告日期不能为空")
|
||||
}
|
||||
|
||||
id, err := dao.CreativeReportDetail.Insert(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res = &dto.CreateCreativeReportDetailRes{
|
||||
Id: id,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (s *creativeReportSumService) BatchCreateDetail(ctx context.Context, req *dto.BatchCreateCreativeReportDetailReq) (res *dto.BatchCreateCreativeReportDetailRes, err error) {
|
||||
ctx = context.WithValue(ctx, "user", &beans.User{UserName: "admin"})
|
||||
if len(req.Items) == 0 {
|
||||
return nil, errors.New("批量创建数据不能为空")
|
||||
}
|
||||
|
||||
successCount, failCount, failedIndexes, err := dao.CreativeReportDetail.BatchInsert(ctx, req.Items)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res = &dto.BatchCreateCreativeReportDetailRes{
|
||||
SuccessCount: successCount,
|
||||
FailCount: failCount,
|
||||
FailedItems: failedIndexes,
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -1,261 +0,0 @@
|
||||
package copydata
|
||||
|
||||
import (
|
||||
"context"
|
||||
dao "dataengine/dao/copydata"
|
||||
dto "dataengine/model/dto/copydata"
|
||||
entity "dataengine/model/entity/copydata"
|
||||
"errors"
|
||||
|
||||
"gitea.com/red-future/common/beans"
|
||||
)
|
||||
|
||||
type materialReportService struct{}
|
||||
|
||||
// MaterialReport 素材报表数据服务
|
||||
var MaterialReport = new(materialReportService)
|
||||
|
||||
// Create 创建素材报表数据
|
||||
func (s *materialReportService) Create(ctx context.Context, req *dto.MaterialReportItem) (res *dto.CreateMaterialReportRes, err error) {
|
||||
if req.ReportDateStr == "" {
|
||||
return nil, errors.New("报告日期不能为空")
|
||||
}
|
||||
|
||||
id, err := dao.MaterialReport.Insert(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res = &dto.CreateMaterialReportRes{
|
||||
Id: id,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// BatchCreate 批量创建素材报表数据
|
||||
func (s *materialReportService) BatchCreate(ctx context.Context, req *dto.BatchCreateMaterialReportReq) (res *dto.BatchCreateMaterialReportRes, err error) {
|
||||
ctx = context.WithValue(ctx, "user", &beans.User{UserName: "admin"})
|
||||
|
||||
if len(req.Items) == 0 {
|
||||
return nil, errors.New("批量创建数据不能为空")
|
||||
}
|
||||
|
||||
successCount, failCount, failedIndexes, err := dao.MaterialReport.BatchInsert(ctx, req.Items)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res = &dto.BatchCreateMaterialReportRes{
|
||||
SuccessCount: successCount,
|
||||
FailCount: failCount,
|
||||
FailedItems: failedIndexes,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// List 获取素材报表数据列表
|
||||
func (s *materialReportService) List(ctx context.Context, req *dto.ListMaterialReportReq) (res *dto.ListMaterialReportRes, err error) {
|
||||
list, total, err := dao.MaterialReport.List(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
items := make([]*dto.MaterialReportItem, len(list))
|
||||
for i, item := range list {
|
||||
items[i] = convertMaterialEntityToDTO(item)
|
||||
}
|
||||
|
||||
res = &dto.ListMaterialReportRes{
|
||||
List: items,
|
||||
Total: total,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// convertEntityToDTO 将实体转换为 DTO
|
||||
func convertMaterialEntityToDTO(entity *entity.MaterialReport) *dto.MaterialReportItem {
|
||||
return &dto.MaterialReportItem{
|
||||
CreatedBy: entity.CreatedBy,
|
||||
UpdatedBy: entity.UpdatedBy,
|
||||
T0OrderPaymentAmt: entity.T0OrderPaymentAmt,
|
||||
CreativeMaterialType: entity.CreativeMaterialType,
|
||||
LiveName: entity.LiveName,
|
||||
AuthorId: entity.AuthorId,
|
||||
PicUrl: entity.PicUrl,
|
||||
PicName: entity.PicName,
|
||||
PicId: entity.PicId,
|
||||
CoverUrl: entity.CoverUrl,
|
||||
CoverId: entity.CoverId,
|
||||
ItemOrderConversionRatio: entity.ItemOrderConversionRatio,
|
||||
ItemCardClickRatio: entity.ItemCardClickRatio,
|
||||
ItemCardClkCnt: entity.ItemCardClkCnt,
|
||||
LivePlayCntCost: entity.LivePlayCntCost,
|
||||
AdMerchantFollowCost: entity.AdMerchantFollowCost,
|
||||
AdMerchantFollow: entity.AdMerchantFollow,
|
||||
NetT0OrderCnt: entity.NetT0OrderCnt,
|
||||
NetT0Roi: entity.NetT0Roi,
|
||||
NetT0Gmv: entity.NetT0Gmv,
|
||||
PhotoName: entity.PhotoName,
|
||||
PhotoIdStr: entity.PhotoIdStr,
|
||||
PhotoId: entity.PhotoId,
|
||||
ModPriceSegment: entity.ModPriceSegment,
|
||||
AgeSegment: entity.AgeSegment,
|
||||
Province: entity.Province,
|
||||
Gender: entity.Gender,
|
||||
AdPhotoPlayedFiveRatio: entity.AdPhotoPlayedFiveRatio,
|
||||
AdPhotoPlayedThreeRatio: entity.AdPhotoPlayedThreeRatio,
|
||||
OrderSubmitRoi: entity.OrderSubmitRoi,
|
||||
OrderSubmitAmt: entity.OrderSubmitAmt,
|
||||
EventOrderSubmitCost: entity.EventOrderSubmitCost,
|
||||
EventOrderSubmit: entity.EventOrderSubmit,
|
||||
EventOrderPaidRoi: entity.EventOrderPaidRoi,
|
||||
EventAppInvoked: entity.EventAppInvoked,
|
||||
EventAddShoppingCart: entity.EventAddShoppingCart,
|
||||
ConversionNumCost: entity.ConversionNumCost,
|
||||
AdEffectivePlayNum: entity.AdEffectivePlayNum,
|
||||
AdItemClick: entity.AdItemClick,
|
||||
MerchantProductId: entity.MerchantProductId,
|
||||
CostTotal: entity.CostTotal,
|
||||
AdShow: entity.AdShow,
|
||||
AdShow1kCost: entity.AdShow1kCost,
|
||||
Impression: entity.Impression,
|
||||
PhotoClick: entity.PhotoClick,
|
||||
PhotoClickRatio: entity.PhotoClickRatio,
|
||||
Click: entity.Click,
|
||||
ActionbarClick: entity.ActionbarClick,
|
||||
ActionbarClickCost: entity.ActionbarClickCost,
|
||||
EspClickRatio: entity.EspClickRatio,
|
||||
ActionRatio: entity.ActionRatio,
|
||||
AdItemCount: entity.AdItemCount,
|
||||
EspLivePlayedSeconds: entity.EspLivePlayedSeconds,
|
||||
PlayedThreeSeconds: entity.PlayedThreeSeconds,
|
||||
Play3sRatio: entity.Play3sRatio,
|
||||
PlayedFiveSeconds: entity.PlayedFiveSeconds,
|
||||
Play5sRatio: entity.Play5sRatio,
|
||||
PlayedEnd: entity.PlayedEnd,
|
||||
PlayEndRatio: entity.PlayEndRatio,
|
||||
Share: entity.Share,
|
||||
Comment: entity.Comment,
|
||||
Likes: entity.Likes,
|
||||
Report: entity.Report,
|
||||
Block: entity.Block,
|
||||
ItemNegative: entity.ItemNegative,
|
||||
LiveShare: entity.LiveShare,
|
||||
LiveComment: entity.LiveComment,
|
||||
LiveReward: entity.LiveReward,
|
||||
EffectivePlayCount: entity.EffectivePlayCount,
|
||||
EffectivePlayRatio: entity.EffectivePlayRatio,
|
||||
ConversionNum: entity.ConversionNum,
|
||||
ConversionCostEsp: entity.ConversionCostEsp,
|
||||
Roi: entity.Roi,
|
||||
Gmv: entity.Gmv,
|
||||
T0Gmv: entity.T0Gmv,
|
||||
T1Gmv: entity.T1Gmv,
|
||||
T7Gmv: entity.T7Gmv,
|
||||
T15Gmv: entity.T15Gmv,
|
||||
T30Gmv: entity.T30Gmv,
|
||||
T0Roi: entity.T0Roi,
|
||||
T1Roi: entity.T1Roi,
|
||||
T7Roi: entity.T7Roi,
|
||||
T15Roi: entity.T15Roi,
|
||||
T30Roi: entity.T30Roi,
|
||||
PaiedOrder: entity.PaiedOrder,
|
||||
OrderRatio: entity.OrderRatio,
|
||||
T0OrderCnt: entity.T0OrderCnt,
|
||||
T0OrderCntCost: entity.T0OrderCntCost,
|
||||
T0OrderCntRatio: entity.T0OrderCntRatio,
|
||||
T1OrderCnt: entity.T1OrderCnt,
|
||||
T7OrderCnt: entity.T7OrderCnt,
|
||||
T15OrderCnt: entity.T15OrderCnt,
|
||||
T30OrderCnt: entity.T30OrderCnt,
|
||||
MerchantRecoFans: entity.MerchantRecoFans,
|
||||
T1Retention: entity.T1Retention,
|
||||
T7Retention: entity.T7Retention,
|
||||
T15Retention: entity.T15Retention,
|
||||
T30Retention: entity.T30Retention,
|
||||
T1RetentionRatio: entity.T1RetentionRatio,
|
||||
T7RetentionRatio: entity.T7RetentionRatio,
|
||||
T15RetentionRatio: entity.T15RetentionRatio,
|
||||
T30RetentionRatio: entity.T30RetentionRatio,
|
||||
ReservationSuccess: entity.ReservationSuccess,
|
||||
ReservationCost: entity.ReservationCost,
|
||||
StandardLivePlayedStarted: entity.StandardLivePlayedStarted,
|
||||
AdLivePlayCnt: entity.AdLivePlayCnt,
|
||||
AdLivePlayCntCost: entity.AdLivePlayCntCost,
|
||||
LiveAudienceCost: entity.LiveAudienceCost,
|
||||
LiveEventGoodsView: entity.LiveEventGoodsView,
|
||||
GoodsClickRatio: entity.GoodsClickRatio,
|
||||
DirectAttrPlatNewBuyerCnt: entity.DirectAttrPlatNewBuyerCnt,
|
||||
T30AttrPlatTotalBuyerCnt: entity.T30AttrPlatTotalBuyerCnt,
|
||||
DirectAttrSellerNewBuyerCnt: entity.DirectAttrSellerNewBuyerCnt,
|
||||
T30AttrSellerTotalBuyerCnt: entity.T30AttrSellerTotalBuyerCnt,
|
||||
T3Gmv: entity.T3Gmv,
|
||||
T3OrderCnt: entity.T3OrderCnt,
|
||||
T3Roi: entity.T3Roi,
|
||||
T7IndirectOrderAmt: entity.T7IndirectOrderAmt,
|
||||
T7IndirectOrderCnt: entity.T7IndirectOrderCnt,
|
||||
FansT0GmvPerFans: entity.FansT0GmvPerFans,
|
||||
FansT3GmvPerFans: entity.FansT3GmvPerFans,
|
||||
FansT7GmvPerFans: entity.FansT7GmvPerFans,
|
||||
FansT15GmvPerFans: entity.FansT15GmvPerFans,
|
||||
FansT30GmvPerFans: entity.FansT30GmvPerFans,
|
||||
RecoFansCost: entity.RecoFansCost,
|
||||
QcpxWhiteboxDirectOrderPaymentAmt: entity.QcpxWhiteboxDirectOrderPaymentAmt,
|
||||
QcpxWhiteboxDirectOrderCnt: entity.QcpxWhiteboxDirectOrderCnt,
|
||||
FansT0Gmv: entity.FansT0Gmv,
|
||||
FansT1Gmv: entity.FansT1Gmv,
|
||||
FansT7Gmv: entity.FansT7Gmv,
|
||||
FansT15Gmv: entity.FansT15Gmv,
|
||||
FansT30Gmv: entity.FansT30Gmv,
|
||||
FansT0Roi: entity.FansT0Roi,
|
||||
FansT1Roi: entity.FansT1Roi,
|
||||
FansT7Roi: entity.FansT7Roi,
|
||||
FansT15Roi: entity.FansT15Roi,
|
||||
FansT30Roi: entity.FansT30Roi,
|
||||
T0ShopNewBuyerOrderPaymentAmt: entity.T0ShopNewBuyerOrderPaymentAmt,
|
||||
T1ShopNewBuyerOrderPaymentAmt: entity.T1ShopNewBuyerOrderPaymentAmt,
|
||||
T3ShopNewBuyerOrderPaymentAmt: entity.T3ShopNewBuyerOrderPaymentAmt,
|
||||
T7ShopNewBuyerOrderPaymentAmt: entity.T7ShopNewBuyerOrderPaymentAmt,
|
||||
T15ShopNewBuyerOrderPaymentAmt: entity.T15ShopNewBuyerOrderPaymentAmt,
|
||||
T30ShopNewBuyerOrderPaymentAmt: entity.T30ShopNewBuyerOrderPaymentAmt,
|
||||
T0ShopNewBuyerOrderCnt: entity.T0ShopNewBuyerOrderCnt,
|
||||
T1ShopNewBuyerOrderCnt: entity.T1ShopNewBuyerOrderCnt,
|
||||
T3ShopNewBuyerOrderCnt: entity.T3ShopNewBuyerOrderCnt,
|
||||
T7ShopNewBuyerOrderCnt: entity.T7ShopNewBuyerOrderCnt,
|
||||
T15ShopNewBuyerOrderCnt: entity.T15ShopNewBuyerOrderCnt,
|
||||
T30ShopNewBuyerOrderCnt: entity.T30ShopNewBuyerOrderCnt,
|
||||
T1NewBuyerRepurchaseRatio: entity.T1NewBuyerRepurchaseRatio,
|
||||
T3NewBuyerRepurchaseRatio: entity.T3NewBuyerRepurchaseRatio,
|
||||
T7NewBuyerRepurchaseRatio: entity.T7NewBuyerRepurchaseRatio,
|
||||
T15NewBuyerRepurchaseRatio: entity.T15NewBuyerRepurchaseRatio,
|
||||
T30NewBuyerRepurchaseRatio: entity.T30NewBuyerRepurchaseRatio,
|
||||
T0ShopNewBuyerRoi: entity.T0ShopNewBuyerRoi,
|
||||
T1ShopNewBuyerRoi: entity.T1ShopNewBuyerRoi,
|
||||
T3ShopNewBuyerRoi: entity.T3ShopNewBuyerRoi,
|
||||
T7ShopNewBuyerRoi: entity.T7ShopNewBuyerRoi,
|
||||
T15ShopNewBuyerRoi: entity.T15ShopNewBuyerRoi,
|
||||
T30ShopNewBuyerRoi: entity.T30ShopNewBuyerRoi,
|
||||
CreateCardOrderCnt: entity.CreateCardOrderCnt,
|
||||
ForwardTsCreateCardOrderCnt: entity.ForwardTsCreateCardOrderCnt,
|
||||
CreateCardOrderCost: entity.CreateCardOrderCost,
|
||||
ForwardTsCreateCardOrderCost: entity.ForwardTsCreateCardOrderCost,
|
||||
ActivateCardOrderCnt: entity.ActivateCardOrderCnt,
|
||||
ForwardTsActivateCardOrderCnt: entity.ForwardTsActivateCardOrderCnt,
|
||||
ActivateCardOrderCost: entity.ActivateCardOrderCost,
|
||||
ForwardTsActivateCardOrderCost: entity.ForwardTsActivateCardOrderCost,
|
||||
CreateCardOrderRatio: entity.CreateCardOrderRatio,
|
||||
ForwardTsCreateCardOrderRatio: entity.ForwardTsCreateCardOrderRatio,
|
||||
ActivateCardOrderCntRatio: entity.ActivateCardOrderCntRatio,
|
||||
ForwardTsActivateCardOrderRatio: entity.ForwardTsActivateCardOrderRatio,
|
||||
LivePlayCnt: entity.LivePlayCnt,
|
||||
ItemEntranceClkCnt: entity.ItemEntranceClkCnt,
|
||||
ShowCnt: entity.ShowCnt,
|
||||
ReportDateStr: entity.ReportDateStr,
|
||||
CampaignId: entity.CampaignId,
|
||||
CampaignName: entity.CampaignName,
|
||||
UnitId: entity.UnitId,
|
||||
UnitName: entity.UnitName,
|
||||
CreativeId: entity.CreativeId,
|
||||
CreativeName: entity.CreativeName,
|
||||
}
|
||||
}
|
||||
@@ -1,237 +0,0 @@
|
||||
package copydata
|
||||
|
||||
import (
|
||||
"context"
|
||||
dao "dataengine/dao/copydata"
|
||||
dto "dataengine/model/dto/copydata"
|
||||
entity "dataengine/model/entity/copydata"
|
||||
"errors"
|
||||
)
|
||||
|
||||
var PopulationReportService = new(populationReportService)
|
||||
|
||||
type populationReportService struct{}
|
||||
|
||||
// Create 创建人群报表数据
|
||||
func (s *populationReportService) Create(ctx context.Context, req *dto.PopulationReportItem) (*dto.CreatePopulationReportRes, error) {
|
||||
if req.ReportDateStr == "" {
|
||||
return nil, errors.New("报告日期不能为空")
|
||||
}
|
||||
|
||||
id, err := dao.PopulationReport.Insert(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &dto.CreatePopulationReportRes{Id: id}, nil
|
||||
}
|
||||
|
||||
// BatchCreate 批量创建人群报表数据
|
||||
func (s *populationReportService) BatchCreate(ctx context.Context, req []*dto.PopulationReportItem) (*dto.BatchCreatePopulationReportRes, error) {
|
||||
if len(req) == 0 {
|
||||
return nil, errors.New("批量创建数据不能为空")
|
||||
}
|
||||
|
||||
successCount, failCount, failedIndexes, err := dao.PopulationReport.BatchInsert(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &dto.BatchCreatePopulationReportRes{
|
||||
SuccessCount: successCount,
|
||||
FailCount: failCount,
|
||||
FailedItems: failedIndexes,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// List 查询人群报表数据列表
|
||||
func (s *populationReportService) List(ctx context.Context, req *dto.ListPopulationReportReq) (*dto.ListPopulationReportRes, error) {
|
||||
list, total, err := dao.PopulationReport.List(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 转换为 DTO
|
||||
items := s.convertToDTOs(list)
|
||||
|
||||
return &dto.ListPopulationReportRes{
|
||||
List: items,
|
||||
Total: total,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// convertToDTOs 将实体列表转换为 DTO 列表
|
||||
func (s *populationReportService) convertToDTOs(entities []*entity.PopulationReport) []*dto.PopulationReportItem {
|
||||
items := make([]*dto.PopulationReportItem, 0, len(entities))
|
||||
for _, e := range entities {
|
||||
items = append(items, s.convertToDTO(e))
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
// convertToDTO 将实体转换为 DTO
|
||||
func (s *populationReportService) convertToDTO(e *entity.PopulationReport) *dto.PopulationReportItem {
|
||||
return &dto.PopulationReportItem{
|
||||
PhotoName: e.PhotoName,
|
||||
PhotoIdStr: e.PhotoIdStr,
|
||||
PhotoId: e.PhotoId,
|
||||
ModPriceSegment: e.ModPriceSegment,
|
||||
AgeSegment: e.AgeSegment,
|
||||
Province: e.Province,
|
||||
Gender: e.Gender,
|
||||
MerchantProductId: e.MerchantProductId,
|
||||
ReportDateStr: e.ReportDateStr,
|
||||
CampaignId: e.CampaignId,
|
||||
CampaignName: e.CampaignName,
|
||||
UnitId: e.UnitId,
|
||||
UnitName: e.UnitName,
|
||||
CreativeId: e.CreativeId,
|
||||
CreativeName: e.CreativeName,
|
||||
AdPhotoPlayedFiveRatio: e.AdPhotoPlayedFiveRatio,
|
||||
AdPhotoPlayedThreeRatio: e.AdPhotoPlayedThreeRatio,
|
||||
OrderSubmitRoi: e.OrderSubmitRoi,
|
||||
OrderSubmitAmt: e.OrderSubmitAmt,
|
||||
EventOrderSubmitCost: e.EventOrderSubmitCost,
|
||||
EventOrderSubmit: e.EventOrderSubmit,
|
||||
EventOrderPaidRoi: e.EventOrderPaidRoi,
|
||||
EventAppInvoked: e.EventAppInvoked,
|
||||
EventAddShoppingCart: e.EventAddShoppingCart,
|
||||
ConversionNumCost: e.ConversionNumCost,
|
||||
AdEffectivePlayNum: e.AdEffectivePlayNum,
|
||||
AdItemClick: e.AdItemClick,
|
||||
CostTotal: e.CostTotal,
|
||||
AdShow: e.AdShow,
|
||||
AdShow1kCost: e.AdShow1kCost,
|
||||
Impression: e.Impression,
|
||||
PhotoClick: e.PhotoClick,
|
||||
PhotoClickRatio: e.PhotoClickRatio,
|
||||
Click: e.Click,
|
||||
ActionbarClick: e.ActionbarClick,
|
||||
ActionbarClickCost: e.ActionbarClickCost,
|
||||
EspClickRatio: e.EspClickRatio,
|
||||
ActionRatio: e.ActionRatio,
|
||||
AdItemClickCount: e.AdItemClickCount,
|
||||
EspLivePlayedSeconds: e.EspLivePlayedSeconds,
|
||||
PlayedThreeSeconds: e.PlayedThreeSeconds,
|
||||
Play3sRatio: e.Play3sRatio,
|
||||
PlayedFiveSeconds: e.PlayedFiveSeconds,
|
||||
Play5sRatio: e.Play5sRatio,
|
||||
PlayedEnd: e.PlayedEnd,
|
||||
PlayEndRatio: e.PlayEndRatio,
|
||||
Share: e.Share,
|
||||
Comment: e.Comment,
|
||||
Likes: e.Likes,
|
||||
Report: e.Report,
|
||||
Block: e.Block,
|
||||
ItemNegative: e.ItemNegative,
|
||||
LiveShare: e.LiveShare,
|
||||
LiveComment: e.LiveComment,
|
||||
LiveReward: e.LiveReward,
|
||||
EffectivePlayCount: e.EffectivePlayCount,
|
||||
EffectivePlayRatio: e.EffectivePlayRatio,
|
||||
ConversionNum: e.ConversionNum,
|
||||
ConversionCostEsp: e.ConversionCostEsp,
|
||||
Roi: e.Roi,
|
||||
Gmv: e.Gmv,
|
||||
T0Gmv: e.T0Gmv,
|
||||
T1Gmv: e.T1Gmv,
|
||||
T3Gmv: e.T3Gmv,
|
||||
T7Gmv: e.T7Gmv,
|
||||
T15Gmv: e.T15Gmv,
|
||||
T30Gmv: e.T30Gmv,
|
||||
T0Roi: e.T0Roi,
|
||||
T1Roi: e.T1Roi,
|
||||
T3Roi: e.T3Roi,
|
||||
T7Roi: e.T7Roi,
|
||||
T15Roi: e.T15Roi,
|
||||
T30Roi: e.T30Roi,
|
||||
PaiedOrder: e.PaiedOrder,
|
||||
OrderRatio: e.OrderRatio,
|
||||
T0OrderCnt: e.T0OrderCnt,
|
||||
T0OrderCntCost: e.T0OrderCntCost,
|
||||
T0OrderCntRatio: e.T0OrderCntRatio,
|
||||
T1OrderCnt: e.T1OrderCnt,
|
||||
T7OrderCnt: e.T7OrderCnt,
|
||||
T15OrderCnt: e.T15OrderCnt,
|
||||
T30OrderCnt: e.T30OrderCnt,
|
||||
MerchantRecoFans: e.MerchantRecoFans,
|
||||
T1Retention: e.T1Retention,
|
||||
T7Retention: e.T7Retention,
|
||||
T15Retention: e.T15Retention,
|
||||
T30Retention: e.T30Retention,
|
||||
T1RetentionRatio: e.T1RetentionRatio,
|
||||
T7RetentionRatio: e.T7RetentionRatio,
|
||||
T15RetentionRatio: e.T15RetentionRatio,
|
||||
T30RetentionRatio: e.T30RetentionRatio,
|
||||
ReservationSuccess: e.ReservationSuccess,
|
||||
ReservationCost: e.ReservationCost,
|
||||
StandardLivePlayedStarted: e.StandardLivePlayedStarted,
|
||||
AdLivePlayCnt: e.AdLivePlayCnt,
|
||||
AdLivePlayCntCost: e.AdLivePlayCntCost,
|
||||
LiveAudienceCost: e.LiveAudienceCost,
|
||||
LiveEventGoodsView: e.LiveEventGoodsView,
|
||||
GoodsClickRatio: e.GoodsClickRatio,
|
||||
DirectAttrPlatNewBuyerCnt: e.DirectAttrPlatNewBuyerCnt,
|
||||
T30AttrPlatTotalBuyerCnt: e.T30AttrPlatTotalBuyerCnt,
|
||||
DirectAttrSellerNewBuyerCnt: e.DirectAttrSellerNewBuyerCnt,
|
||||
T30AttrSellerTotalBuyerCnt: e.T30AttrSellerTotalBuyerCnt,
|
||||
T7IndirectOrderAmt: e.T7IndirectOrderAmt,
|
||||
T7IndirectOrderCnt: e.T7IndirectOrderCnt,
|
||||
FansT0GmvPerFans: e.FansT0GmvPerFans,
|
||||
FansT3GmvPerFans: e.FansT3GmvPerFans,
|
||||
FansT7GmvPerFans: e.FansT7GmvPerFans,
|
||||
FansT15GmvPerFans: e.FansT15GmvPerFans,
|
||||
FansT30GmvPerFans: e.FansT30GmvPerFans,
|
||||
RecoFansCost: e.RecoFansCost,
|
||||
QcpxWhiteboxDirectOrderPaymentAmt: e.QcpxWhiteboxDirectOrderPaymentAmt,
|
||||
QcpxWhiteboxDirectOrderCnt: e.QcpxWhiteboxDirectOrderCnt,
|
||||
FansT0Gmv: e.FansT0Gmv,
|
||||
FansT1Gmv: e.FansT1Gmv,
|
||||
FansT7Gmv: e.FansT7Gmv,
|
||||
FansT15Gmv: e.FansT15Gmv,
|
||||
FansT30Gmv: e.FansT30Gmv,
|
||||
FansT0Roi: e.FansT0Roi,
|
||||
FansT1Roi: e.FansT1Roi,
|
||||
FansT7Roi: e.FansT7Roi,
|
||||
FansT15Roi: e.FansT15Roi,
|
||||
FansT30Roi: e.FansT30Roi,
|
||||
T0ShopNewBuyerOrderPaymentAmt: e.T0ShopNewBuyerOrderPaymentAmt,
|
||||
T1ShopNewBuyerOrderPaymentAmt: e.T1ShopNewBuyerOrderPaymentAmt,
|
||||
T3ShopNewBuyerOrderPaymentAmt: e.T3ShopNewBuyerOrderPaymentAmt,
|
||||
T7ShopNewBuyerOrderPaymentAmt: e.T7ShopNewBuyerOrderPaymentAmt,
|
||||
T15ShopNewBuyerOrderPaymentAmt: e.T15ShopNewBuyerOrderPaymentAmt,
|
||||
T30ShopNewBuyerOrderPaymentAmt: e.T30ShopNewBuyerOrderPaymentAmt,
|
||||
T0ShopNewBuyerOrderCnt: e.T0ShopNewBuyerOrderCnt,
|
||||
T1ShopNewBuyerOrderCnt: e.T1ShopNewBuyerOrderCnt,
|
||||
T3ShopNewBuyerOrderCnt: e.T3ShopNewBuyerOrderCnt,
|
||||
T7ShopNewBuyerOrderCnt: e.T7ShopNewBuyerOrderCnt,
|
||||
T15ShopNewBuyerOrderCnt: e.T15ShopNewBuyerOrderCnt,
|
||||
T30ShopNewBuyerOrderCnt: e.T30ShopNewBuyerOrderCnt,
|
||||
T1NewBuyerRepurchaseRatio: e.T1NewBuyerRepurchaseRatio,
|
||||
T3NewBuyerRepurchaseRatio: e.T3NewBuyerRepurchaseRatio,
|
||||
T7NewBuyerRepurchaseRatio: e.T7NewBuyerRepurchaseRatio,
|
||||
T15NewBuyerRepurchaseRatio: e.T15NewBuyerRepurchaseRatio,
|
||||
T30NewBuyerRepurchaseRatio: e.T30NewBuyerRepurchaseRatio,
|
||||
T0ShopNewBuyerRoi: e.T0ShopNewBuyerRoi,
|
||||
T1ShopNewBuyerRoi: e.T1ShopNewBuyerRoi,
|
||||
T3ShopNewBuyerRoi: e.T3ShopNewBuyerRoi,
|
||||
T7ShopNewBuyerRoi: e.T7ShopNewBuyerRoi,
|
||||
T15ShopNewBuyerRoi: e.T15ShopNewBuyerRoi,
|
||||
T30ShopNewBuyerRoi: e.T30ShopNewBuyerRoi,
|
||||
CreateCardOrderCnt: e.CreateCardOrderCnt,
|
||||
ForwardTsCreateCardOrderCnt: e.ForwardTsCreateCardOrderCnt,
|
||||
CreateCardOrderCost: e.CreateCardOrderCost,
|
||||
ForwardTsCreateCardOrderCost: e.ForwardTsCreateCardOrderCost,
|
||||
ActivateCardOrderCnt: e.ActivateCardOrderCnt,
|
||||
ForwardTsActivateCardOrderCnt: e.ForwardTsActivateCardOrderCnt,
|
||||
ActivateCardOrderCost: e.ActivateCardOrderCost,
|
||||
ForwardTsActivateCardOrderCost: e.ForwardTsActivateCardOrderCost,
|
||||
CreateCardOrderRatio: e.CreateCardOrderRatio,
|
||||
ForwardTsCreateCardOrderRatio: e.ForwardTsCreateCardOrderRatio,
|
||||
ActivateCardOrderCntRatio: e.ActivateCardOrderCntRatio,
|
||||
ForwardTsActivateCardOrderRatio: e.ForwardTsActivateCardOrderRatio,
|
||||
LivePlayCnt: e.LivePlayCnt,
|
||||
ItemEntranceClkCnt: e.ItemEntranceClkCnt,
|
||||
ShowCnt: e.ShowCnt,
|
||||
}
|
||||
}
|
||||
@@ -1,97 +0,0 @@
|
||||
package copydata
|
||||
|
||||
import (
|
||||
"context"
|
||||
dao "dataengine/dao/copydata"
|
||||
dto "dataengine/model/dto/copydata"
|
||||
"errors"
|
||||
|
||||
"gitea.com/red-future/common/beans"
|
||||
)
|
||||
|
||||
type storewideReportService struct{}
|
||||
|
||||
// StorewideReportSum 广告效果指标表服务
|
||||
var StorewideReportSum = new(storewideReportService)
|
||||
|
||||
// Create 创建广告效果指标表
|
||||
func (s *storewideReportService) Create(ctx context.Context, req *dto.StorewideReportSumItem) (res *dto.CreateStorewideReportSumRes, err error) {
|
||||
// 验证必要字段
|
||||
if req.ReportDateStr == "" {
|
||||
return nil, errors.New("报告日期不能为空")
|
||||
}
|
||||
|
||||
// 插入数据库
|
||||
id, err := dao.StorewideReportSum.Insert(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res = &dto.CreateStorewideReportSumRes{
|
||||
Id: id,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// BatchCreate 批量创建广告效果指标表
|
||||
func (s *storewideReportService) BatchCreate(ctx context.Context, req *dto.BatchCreateStorewideReportSumReq) (res *dto.BatchCreateStorewideReportSumRes, err error) {
|
||||
ctx = context.WithValue(ctx, "user", &beans.User{UserName: "admin"})
|
||||
// 验证数据
|
||||
if len(req.Items) == 0 {
|
||||
return nil, errors.New("批量创建数据不能为空")
|
||||
}
|
||||
|
||||
// 批量插入数据库
|
||||
successCount, failCount, failedIndexes, err := dao.StorewideReportSum.BatchInsert(ctx, req.Items)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res = &dto.BatchCreateStorewideReportSumRes{
|
||||
SuccessCount: successCount,
|
||||
FailCount: failCount,
|
||||
FailedItems: failedIndexes,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Create 创建广告效果指标表
|
||||
func (s *storewideReportService) CreateDetail(ctx context.Context, req *dto.StorewideReportDetailItem) (res *dto.CreateStorewideReportDetailRes, err error) {
|
||||
// 验证必要字段
|
||||
if req.ReportDateStr == "" {
|
||||
return nil, errors.New("报告日期不能为空")
|
||||
}
|
||||
|
||||
// 插入数据库
|
||||
id, err := dao.StorewideReportDetail.Insert(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res = &dto.CreateStorewideReportDetailRes{
|
||||
Id: id,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// BatchCreate 批量创建广告效果指标表
|
||||
func (s *storewideReportService) BatchCreateDetail(ctx context.Context, req *dto.BatchCreateStorewideReportDetailReq) (res *dto.BatchCreateStorewideReportDetailRes, err error) {
|
||||
ctx = context.WithValue(ctx, "user", &beans.User{UserName: "admin"})
|
||||
// 验证数据
|
||||
if len(req.Items) == 0 {
|
||||
return nil, errors.New("批量创建数据不能为空")
|
||||
}
|
||||
|
||||
// 批量插入数据库
|
||||
successCount, failCount, failedIndexes, err := dao.StorewideReportDetail.BatchInsert(ctx, req.Items)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res = &dto.BatchCreateStorewideReportDetailRes{
|
||||
SuccessCount: successCount,
|
||||
FailCount: failCount,
|
||||
FailedItems: failedIndexes,
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -1,113 +0,0 @@
|
||||
package copydata
|
||||
|
||||
import (
|
||||
"context"
|
||||
dao "dataengine/dao/copydata"
|
||||
dto "dataengine/model/dto/copydata"
|
||||
entity "dataengine/model/entity/copydata"
|
||||
"errors"
|
||||
|
||||
"gitea.com/red-future/common/beans"
|
||||
)
|
||||
|
||||
type taskReportService struct{}
|
||||
|
||||
// TaskReport 调控任务数据服务
|
||||
var TaskReport = new(taskReportService)
|
||||
|
||||
// Create 创建调控任务数据
|
||||
func (s *taskReportService) Create(ctx context.Context, req *dto.TaskReportItem) (res *dto.CreateTaskReportRes, err error) {
|
||||
// 验证必要字段
|
||||
if req.ReportDateStr == "" {
|
||||
return nil, errors.New("报告日期不能为空")
|
||||
}
|
||||
|
||||
// 插入数据库
|
||||
id, err := dao.TaskReport.Insert(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res = &dto.CreateTaskReportRes{
|
||||
Id: id,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// BatchCreate 批量创建调控任务数据
|
||||
func (s *taskReportService) BatchCreate(ctx context.Context, req *dto.BatchCreateTaskReportReq) (res *dto.BatchCreateTaskReportRes, err error) {
|
||||
ctx = context.WithValue(ctx, "user", &beans.User{UserName: "admin"})
|
||||
// 验证数据
|
||||
if len(req.Items) == 0 {
|
||||
return nil, errors.New("批量创建数据不能为空")
|
||||
}
|
||||
|
||||
// 批量插入数据库
|
||||
successCount, failCount, failedIndexes, err := dao.TaskReport.BatchInsert(ctx, req.Items)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res = &dto.BatchCreateTaskReportRes{
|
||||
SuccessCount: successCount,
|
||||
FailCount: failCount,
|
||||
FailedItems: failedIndexes,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// List 获取调控任务数据列表
|
||||
func (s *taskReportService) List(ctx context.Context, req *dto.ListTaskReportReq) (res *dto.ListTaskReportRes, err error) {
|
||||
list, total, err := dao.TaskReport.List(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 转换为 DTO 格式
|
||||
items := make([]*dto.TaskReportItem, len(list))
|
||||
for i, item := range list {
|
||||
items[i] = convertEntityToDTO(item)
|
||||
}
|
||||
|
||||
res = &dto.ListTaskReportRes{
|
||||
List: items,
|
||||
Total: total,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// convertEntityToDTO 将实体转换为 DTO
|
||||
func convertEntityToDTO(entity *entity.TaskReport) *dto.TaskReportItem {
|
||||
return &dto.TaskReportItem{
|
||||
ItemOrderConversionRatio: entity.ItemOrderConversionRatio,
|
||||
ItemCardClickRatio: entity.ItemCardClickRatio,
|
||||
ItemCardClkCnt: entity.ItemCardClkCnt,
|
||||
LivePlayCntCost: entity.LivePlayCntCost,
|
||||
AdMerchantFollowCost: entity.AdMerchantFollowCost,
|
||||
AdMerchantFollow: entity.AdMerchantFollow,
|
||||
NetT0OrderCnt: entity.NetT0OrderCnt,
|
||||
NetT0Roi: entity.NetT0Roi,
|
||||
NetT0Gmv: entity.NetT0Gmv,
|
||||
PhotoName: entity.PhotoName,
|
||||
PhotoId: entity.PhotoId,
|
||||
CostTotal: entity.CostTotal,
|
||||
T0Gmv: entity.T0Gmv,
|
||||
T0Roi: entity.T0Roi,
|
||||
T0OrderCnt: entity.T0OrderCnt,
|
||||
T0OrderCntCost: entity.T0OrderCntCost,
|
||||
FansT0Gmv: entity.FansT0Gmv,
|
||||
FansT1Gmv: entity.FansT1Gmv,
|
||||
FansT7Gmv: entity.FansT7Gmv,
|
||||
FansT15Gmv: entity.FansT15Gmv,
|
||||
FansT30Gmv: entity.FansT30Gmv,
|
||||
FansT0Roi: entity.FansT0Roi,
|
||||
FansT1Roi: entity.FansT1Roi,
|
||||
FansT7Roi: entity.FansT7Roi,
|
||||
FansT15Roi: entity.FansT15Roi,
|
||||
FansT30Roi: entity.FansT30Roi,
|
||||
LivePlayCnt: entity.LivePlayCnt,
|
||||
ItemEntranceClkCnt: entity.ItemEntranceClkCnt,
|
||||
ShowCnt: entity.ShowCnt,
|
||||
ReportDateStr: entity.ReportDateStr,
|
||||
}
|
||||
}
|
||||
@@ -1,90 +0,0 @@
|
||||
package copydata
|
||||
|
||||
import (
|
||||
"context"
|
||||
dao "dataengine/dao/copydata"
|
||||
dto "dataengine/model/dto/copydata"
|
||||
"errors"
|
||||
|
||||
"gitea.com/red-future/common/beans"
|
||||
)
|
||||
|
||||
type unitReportSumService struct{}
|
||||
|
||||
// UnitReportSumService 广告效果指标服务
|
||||
var UnitReportSumService = new(unitReportSumService)
|
||||
|
||||
// Create 创建广告效果指标
|
||||
func (s *unitReportSumService) Create(ctx context.Context, req *dto.UnitReportSumItem) (res *dto.CreateUnitReportSumRes, err error) {
|
||||
ctx = context.WithValue(ctx, "user", &beans.User{UserName: "admin"})
|
||||
if req.ReportDateStr == "" {
|
||||
return nil, errors.New("报告日期不能为空")
|
||||
}
|
||||
|
||||
id, err := dao.UnitReportSum.Insert(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res = &dto.CreateUnitReportSumRes{
|
||||
Id: id,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// BatchCreate 批量创建广告效果指标
|
||||
func (s *unitReportSumService) BatchCreate(ctx context.Context, req *dto.BatchCreateUnitReportSumReq) (res *dto.BatchCreateUnitReportSumRes, err error) {
|
||||
ctx = context.WithValue(ctx, "user", &beans.User{UserName: "admin"})
|
||||
if len(req.Items) == 0 {
|
||||
return nil, errors.New("批量创建数据不能为空")
|
||||
}
|
||||
|
||||
successCount, failCount, failedIndexes, err := dao.UnitReportSum.BatchInsert(ctx, req.Items)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res = &dto.BatchCreateUnitReportSumRes{
|
||||
SuccessCount: successCount,
|
||||
FailCount: failCount,
|
||||
FailedItems: failedIndexes,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Create 创建广告效果指标详情
|
||||
func (s *unitReportSumService) CreateDetail(ctx context.Context, req *dto.UnitReportDetailItem) (res *dto.CreateUnitReportDetailRes, err error) {
|
||||
if req.ReportDateStr == "" {
|
||||
return nil, errors.New("报告日期不能为空")
|
||||
}
|
||||
|
||||
id, err := dao.UnitReportDetail.Insert(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res = &dto.CreateUnitReportDetailRes{
|
||||
Id: id,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// BatchCreate 批量创建广告效果指标详情
|
||||
func (s *unitReportSumService) BatchCreateDetail(ctx context.Context, req *dto.BatchCreateUnitReportDetailReq) (res *dto.BatchCreateUnitReportDetailRes, err error) {
|
||||
ctx = context.WithValue(ctx, "user", &beans.User{UserName: "admin"})
|
||||
if len(req.Items) == 0 {
|
||||
return nil, errors.New("批量创建数据不能为空")
|
||||
}
|
||||
|
||||
successCount, failCount, failedIndexes, err := dao.UnitReportDetail.BatchInsert(ctx, req.Items)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res = &dto.BatchCreateUnitReportDetailRes{
|
||||
SuccessCount: successCount,
|
||||
FailCount: failCount,
|
||||
FailedItems: failedIndexes,
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -1,396 +0,0 @@
|
||||
package dict
|
||||
|
||||
import (
|
||||
"context"
|
||||
consts "dataengine/consts/dict"
|
||||
"dataengine/dao/dict"
|
||||
dto "dataengine/model/dto/dict"
|
||||
entity "dataengine/model/entity/dict"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.com/red-future/common/beans"
|
||||
)
|
||||
|
||||
type fieldMappingConfigService struct{}
|
||||
|
||||
// FieldMappingConfig 字段映射配置服务
|
||||
var FieldMappingConfig = new(fieldMappingConfigService)
|
||||
|
||||
// Create 创建字段映射配置
|
||||
func (s *fieldMappingConfigService) Create(ctx context.Context, req *dto.CreateFieldMappingConfigReq) (res *dto.CreateFieldMappingConfigRes, err error) {
|
||||
// 验证必填字段
|
||||
if err = s.validateRequiredFields(req); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 检查重复配置
|
||||
exists, err := dict.FieldMappingConfig.CheckDuplicate(ctx, req.VendorName, req.ApiName, req.SourceField, req.TargetField, 0)
|
||||
if err != nil {
|
||||
return nil, errors.New("检查配置重复性失败")
|
||||
}
|
||||
if exists {
|
||||
return nil, errors.New("相同厂商、接口、源字段和目标字段的配置已存在")
|
||||
}
|
||||
|
||||
// 验证转换参数
|
||||
if err = s.validateTransformParams(req.TransformType, req.TransformParams); err != nil {
|
||||
return nil, fmt.Errorf("转换参数验证失败: %v", err)
|
||||
}
|
||||
|
||||
// 验证业务域
|
||||
if req.BusinessDomain != "" && !s.isValidBusinessDomain(req.BusinessDomain) {
|
||||
return nil, errors.New("无效的业务域")
|
||||
}
|
||||
|
||||
// 验证生效时间和失效时间
|
||||
if req.EffectiveDate != nil && req.ExpiryDate != nil && req.EffectiveDate.After(*req.ExpiryDate) {
|
||||
return nil, errors.New("生效时间不能晚于失效时间")
|
||||
}
|
||||
|
||||
// 插入数据库
|
||||
id, err := dict.FieldMappingConfig.Insert(ctx, req)
|
||||
if err != nil {
|
||||
return nil, errors.New("创建配置失败")
|
||||
}
|
||||
|
||||
res = &dto.CreateFieldMappingConfigRes{
|
||||
Id: id,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// List 获取字段映射配置列表
|
||||
func (s *fieldMappingConfigService) List(ctx context.Context, req *dto.ListFieldMappingConfigReq) (res *dto.ListFieldMappingConfigRes, err error) {
|
||||
configs, total, err := dict.FieldMappingConfig.List(ctx, req)
|
||||
if err != nil {
|
||||
return nil, errors.New("查询配置列表失败")
|
||||
}
|
||||
|
||||
// 组装响应数据
|
||||
list := make([]dto.FieldMappingConfigItem, 0, len(configs))
|
||||
for _, item := range configs {
|
||||
list = append(list, dto.FieldMappingConfigItem{
|
||||
Id: item.Id,
|
||||
ConfigName: item.ConfigName,
|
||||
VendorName: item.VendorName,
|
||||
ApiName: item.ApiName,
|
||||
ApiVersion: item.ApiVersion,
|
||||
SourceField: item.SourceField,
|
||||
TargetField: item.TargetField,
|
||||
TargetFieldType: item.TargetFieldType,
|
||||
TransformType: item.TransformType,
|
||||
TransformTypeName: s.getTransformTypeName(item.TransformType),
|
||||
IsActive: item.IsActive,
|
||||
Priority: item.Priority,
|
||||
BusinessDomain: item.BusinessDomain,
|
||||
BusinessDomainName: s.getBusinessDomainName(item.BusinessDomain),
|
||||
FieldGroup: item.FieldGroup,
|
||||
ConfigVersion: item.ConfigVersion,
|
||||
CreatedBy: item.CreatedBy,
|
||||
CreatedTime: item.CreatedTime,
|
||||
UpdatedBy: item.UpdatedBy,
|
||||
UpdatedTime: item.UpdatedTime,
|
||||
})
|
||||
}
|
||||
|
||||
res = &dto.ListFieldMappingConfigRes{
|
||||
List: list,
|
||||
Total: total,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// GetOne 获取单个字段映射配置
|
||||
func (s *fieldMappingConfigService) GetOne(ctx context.Context, req *dto.GetFieldMappingConfigReq) (res *dto.GetFieldMappingConfigRes, err error) {
|
||||
config, err := dict.FieldMappingConfig.GetOne(ctx, req)
|
||||
if err != nil {
|
||||
return nil, errors.New("获取配置详情失败")
|
||||
}
|
||||
if config == nil {
|
||||
return nil, errors.New("配置不存在")
|
||||
}
|
||||
|
||||
return &dto.GetFieldMappingConfigRes{
|
||||
FieldMappingConfig: config,
|
||||
TransformTypeName: s.getTransformTypeName(config.TransformType),
|
||||
BusinessDomainName: s.getBusinessDomainName(config.BusinessDomain),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Update 更新字段映射配置
|
||||
func (s *fieldMappingConfigService) Update(ctx context.Context, req *dto.UpdateFieldMappingConfigReq) (err error) {
|
||||
// 检查配置是否存在
|
||||
exist, err := dict.FieldMappingConfig.GetOne(ctx, &dto.GetFieldMappingConfigReq{Id: req.Id})
|
||||
if err != nil || exist == nil {
|
||||
return errors.New("配置不存在")
|
||||
}
|
||||
|
||||
// 如果修改了关键字段，检查重复性
|
||||
if (req.VendorName != "" && req.VendorName != exist.VendorName) ||
|
||||
(req.ApiName != "" && req.ApiName != exist.ApiName) ||
|
||||
(req.SourceField != "" && req.SourceField != exist.SourceField) ||
|
||||
(req.TargetField != "" && req.TargetField != exist.TargetField) {
|
||||
|
||||
vendorName := req.VendorName
|
||||
if vendorName == "" {
|
||||
vendorName = exist.VendorName
|
||||
}
|
||||
apiName := req.ApiName
|
||||
if apiName == "" {
|
||||
apiName = exist.ApiName
|
||||
}
|
||||
sourceField := req.SourceField
|
||||
if sourceField == "" {
|
||||
sourceField = exist.SourceField
|
||||
}
|
||||
targetField := req.TargetField
|
||||
if targetField == "" {
|
||||
targetField = exist.TargetField
|
||||
}
|
||||
|
||||
exists, err := dict.FieldMappingConfig.CheckDuplicate(ctx, vendorName, apiName, sourceField, targetField, req.Id)
|
||||
if err != nil {
|
||||
return errors.New("检查配置重复性失败")
|
||||
}
|
||||
if exists {
|
||||
return errors.New("相同厂商、接口、源字段和目标字段的配置已存在")
|
||||
}
|
||||
}
|
||||
|
||||
// 验证转换参数
|
||||
if req.TransformType != "" && req.TransformParams != nil {
|
||||
if err = s.validateTransformParams(req.TransformType, req.TransformParams); err != nil {
|
||||
return fmt.Errorf("转换参数验证失败: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 验证生效时间和失效时间
|
||||
if req.EffectiveDate != nil && req.ExpiryDate != nil && req.EffectiveDate.After(*req.ExpiryDate) {
|
||||
return errors.New("生效时间不能晚于失效时间")
|
||||
}
|
||||
|
||||
// 更新数据库
|
||||
_, err = dict.FieldMappingConfig.Update(ctx, req)
|
||||
if err != nil {
|
||||
return errors.New("更新配置失败")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateStatus 更新字段映射配置状态
|
||||
func (s *fieldMappingConfigService) UpdateStatus(ctx context.Context, req *dto.UpdateFieldMappingConfigStatusReq) (err error) {
|
||||
_, err = dict.FieldMappingConfig.UpdateStatus(ctx, req.Id, req.IsActive)
|
||||
if err != nil {
|
||||
return errors.New("更新配置状态失败")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete 删除字段映射配置
|
||||
func (s *fieldMappingConfigService) Delete(ctx context.Context, req *dto.DeleteFieldMappingConfigReq) (err error) {
|
||||
_, err = dict.FieldMappingConfig.Delete(ctx, req)
|
||||
if err != nil {
|
||||
return errors.New("删除配置失败")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// QueryByVendorApi 根据厂商和接口查询字段映射
|
||||
func (s *fieldMappingConfigService) QueryByVendorApi(ctx context.Context, req *dto.QueryFieldMappingByVendorApiReq) (res *dto.QueryFieldMappingByVendorApiRes, err error) {
|
||||
configs, err := dict.FieldMappingConfig.GetByVendorAndApi(ctx, req.VendorName, req.ApiName, req.ApiVersion, req.IsActive)
|
||||
if err != nil {
|
||||
return nil, errors.New("查询字段映射配置失败")
|
||||
}
|
||||
|
||||
// 过滤掉已过期的配置
|
||||
var validConfigs []*entity.FieldMappingConfig
|
||||
now := time.Now()
|
||||
for _, config := range configs {
|
||||
if (config.EffectiveDate == nil || !config.EffectiveDate.After(now)) &&
|
||||
(config.ExpiryDate == nil || config.ExpiryDate.After(now)) {
|
||||
validConfigs = append(validConfigs, config)
|
||||
}
|
||||
}
|
||||
|
||||
res = &dto.QueryFieldMappingByVendorApiRes{
|
||||
List: validConfigs,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Validate 验证字段映射配置
|
||||
func (s *fieldMappingConfigService) Validate(ctx context.Context, req *dto.ValidateFieldMappingReq) (res *dto.ValidateFieldMappingRes, err error) {
|
||||
ctx = context.WithValue(ctx, "user", &beans.User{UserName: "admin"})
|
||||
res = &dto.ValidateFieldMappingRes{
|
||||
IsValid: false,
|
||||
}
|
||||
|
||||
// 验证基本配置
|
||||
if req.ConfigName == "" || req.VendorName == "" || req.ApiName == "" || req.SourceField == "" || req.TargetField == "" {
|
||||
res.Error = "配置名称、厂商名称、接口名称、源字段和目标字段不能为空"
|
||||
return
|
||||
}
|
||||
|
||||
// 检查重复配置
|
||||
exists, err := dict.FieldMappingConfig.CheckDuplicate(ctx, req.VendorName, req.ApiName, req.SourceField, req.TargetField, 0)
|
||||
if err != nil {
|
||||
res.Error = "检查配置重复性失败"
|
||||
return
|
||||
}
|
||||
if exists {
|
||||
res.Error = "相同厂商、接口、源字段和目标字段的配置已存在"
|
||||
return
|
||||
}
|
||||
|
||||
// 如果有测试值，尝试转换
|
||||
if req.TestValue != nil {
|
||||
// 这里可以实现具体的转换逻辑
|
||||
// 例如：根据转换类型和参数进行值转换
|
||||
res.TransformedValue = req.TestValue
|
||||
res.Warnings = []string{"转换逻辑需要根据具体业务实现"}
|
||||
}
|
||||
|
||||
res.IsValid = true
|
||||
return
|
||||
}
|
||||
|
||||
// validateRequiredFields 验证必填字段
|
||||
func (s *fieldMappingConfigService) validateRequiredFields(req *dto.CreateFieldMappingConfigReq) error {
|
||||
if req.ConfigName == "" {
|
||||
return errors.New("配置名称不能为空")
|
||||
}
|
||||
if req.VendorName == "" {
|
||||
return errors.New("厂商名称不能为空")
|
||||
}
|
||||
if req.ApiName == "" {
|
||||
return errors.New("接口名称不能为空")
|
||||
}
|
||||
if req.SourceField == "" {
|
||||
return errors.New("源字段不能为空")
|
||||
}
|
||||
if req.TargetField == "" {
|
||||
return errors.New("目标字段不能为空")
|
||||
}
|
||||
if req.TargetFieldType == "" {
|
||||
return errors.New("目标字段类型不能为空")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateRequiredFieldsForBatch 批量创建的字段验证
|
||||
func (s *fieldMappingConfigService) validateRequiredFieldsForBatch(config *dto.BatchFieldMappingConfigItem) error {
|
||||
if config.ConfigName == "" {
|
||||
return errors.New("配置名称不能为空")
|
||||
}
|
||||
if config.VendorName == "" {
|
||||
return errors.New("厂商名称不能为空")
|
||||
}
|
||||
if config.ApiName == "" {
|
||||
return errors.New("接口名称不能为空")
|
||||
}
|
||||
if config.SourceField == "" {
|
||||
return errors.New("源字段不能为空")
|
||||
}
|
||||
if config.TargetField == "" {
|
||||
return errors.New("目标字段不能为空")
|
||||
}
|
||||
if config.TargetFieldType == "" {
|
||||
return errors.New("目标字段类型不能为空")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateTransformParams 验证转换参数
|
||||
func (s *fieldMappingConfigService) validateTransformParams(transformType string, params map[string]interface{}) error {
|
||||
if params == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
switch transformType {
|
||||
case consts.TransformTypeFormatDate:
|
||||
if _, ok := params["source_format"]; !ok {
|
||||
return errors.New("日期格式化转换需要source_format参数")
|
||||
}
|
||||
if _, ok := params["target_format"]; !ok {
|
||||
return errors.New("日期格式化转换需要target_format参数")
|
||||
}
|
||||
case consts.TransformTypeMapValue:
|
||||
if len(params) == 0 {
|
||||
return errors.New("值映射转换需要映射规则参数")
|
||||
}
|
||||
case consts.TransformTypeConcat:
|
||||
if _, ok := params["fields"]; !ok {
|
||||
return errors.New("拼接转换需要fields参数")
|
||||
}
|
||||
case consts.TransformTypeRegex:
|
||||
if _, ok := params["pattern"]; !ok {
|
||||
return errors.New("正则提取转换需要pattern参数")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// isValidBusinessDomain 验证业务域是否有效
|
||||
func (s *fieldMappingConfigService) isValidBusinessDomain(domain string) bool {
|
||||
validDomains := []string{
|
||||
consts.BusinessDomainUser,
|
||||
consts.BusinessDomainOrder,
|
||||
consts.BusinessDomainProduct,
|
||||
consts.BusinessDomainPayment,
|
||||
}
|
||||
for _, valid := range validDomains {
|
||||
if domain == valid {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// getTransformTypeName 获取转换类型名称
|
||||
func (s *fieldMappingConfigService) getTransformTypeName(transformType string) string {
|
||||
typeNames := map[string]string{
|
||||
consts.TransformTypeDirect: "直接映射",
|
||||
consts.TransformTypeFormatDate: "日期格式化",
|
||||
consts.TransformTypeMapValue: "值映射",
|
||||
consts.TransformTypeConcat: "拼接",
|
||||
consts.TransformTypeCalc: "计算",
|
||||
consts.TransformTypeRegex: "正则提取",
|
||||
}
|
||||
if name, ok := typeNames[transformType]; ok {
|
||||
return name
|
||||
}
|
||||
return transformType
|
||||
}
|
||||
|
||||
// getBusinessDomainName 获取业务域名称
|
||||
func (s *fieldMappingConfigService) getBusinessDomainName(domain string) string {
|
||||
domainNames := map[string]string{
|
||||
consts.BusinessDomainUser: "用户",
|
||||
consts.BusinessDomainOrder: "订单",
|
||||
consts.BusinessDomainProduct: "商品",
|
||||
consts.BusinessDomainPayment: "支付",
|
||||
}
|
||||
if name, ok := domainNames[domain]; ok {
|
||||
return name
|
||||
}
|
||||
return domain
|
||||
}
|
||||
|
||||
// GetActiveConfigsByBusinessDomain 根据业务域获取启用的配置
|
||||
func (s *fieldMappingConfigService) GetActiveConfigsByBusinessDomain(ctx context.Context, businessDomain string) ([]entity.FieldMappingConfig, error) {
|
||||
return dict.FieldMappingConfig.GetActiveConfigsByBusinessDomain(ctx, businessDomain)
|
||||
}
|
||||
|
||||
// GetFieldGroupsByVendorApi 获取指定厂商接口的字段分组
|
||||
func (s *fieldMappingConfigService) GetFieldGroupsByVendorApi(ctx context.Context, vendorName, apiName string) ([]string, error) {
|
||||
return dict.FieldMappingConfig.GetFieldGroupsByVendorApi(ctx, vendorName, apiName)
|
||||
}
|
||||
|
||||
// CleanExpiredConfigs 清理过期配置
|
||||
func (s *fieldMappingConfigService) CleanExpiredConfigs(ctx context.Context) (int64, error) {
|
||||
return dict.FieldMappingConfig.DeleteExpiredConfigs(ctx)
|
||||
}
|
||||
252
service/sync/api_client.go
Normal file
252
service/sync/api_client.go
Normal file
@@ -0,0 +1,252 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/big"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// ApiResult API 调用结果
|
||||
type ApiResult struct {
|
||||
Body []byte
|
||||
DurationMs int64
|
||||
}
|
||||
|
||||
// ApiClient 通用 API 客户端
|
||||
type ApiClient struct {
|
||||
config *PlatformConfig
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// NewApiClient 创建客户端
|
||||
func NewApiClient(config *PlatformConfig) *ApiClient {
|
||||
timeout := 30 * time.Second
|
||||
if config.RequestTimeoutMs > 0 {
|
||||
timeout = time.Duration(config.RequestTimeoutMs) * time.Millisecond
|
||||
}
|
||||
return &ApiClient{
|
||||
config: config,
|
||||
client: &http.Client{Timeout: timeout},
|
||||
}
|
||||
}
|
||||
|
||||
// Get 发送 GET 请求（无参数）
|
||||
func (c *ApiClient) Get(ctx context.Context, path string) (*ApiResult, error) {
|
||||
return c.doRequest(ctx, "GET", path, nil, false)
|
||||
}
|
||||
|
||||
// PostJSON 发送 POST JSON 请求
|
||||
func (c *ApiClient) PostJSON(ctx context.Context, path string, body interface{}) (*ApiResult, error) {
|
||||
return c.doRequest(ctx, "POST", path, body, false)
|
||||
}
|
||||
|
||||
// Request 通用请求方法（支持 GET/POST，支持参数在 query 或 body）
|
||||
func (c *ApiClient) Request(ctx context.Context, method, path string, params map[string]interface{}, paramsInQuery bool) (*ApiResult, error) {
|
||||
if paramsInQuery {
|
||||
return c.doRequest(ctx, method, path, params, true)
|
||||
}
|
||||
if method == "GET" {
|
||||
return c.doRequest(ctx, "GET", path, params, true)
|
||||
}
|
||||
return c.doRequest(ctx, method, path, params, false)
|
||||
}
|
||||
|
||||
func (c *ApiClient) doRequest(ctx context.Context, method, path string, body interface{}, paramsInQuery bool) (result *ApiResult, err error) {
|
||||
maxRetries := c.config.MaxRetries
|
||||
if maxRetries <= 0 {
|
||||
maxRetries = 3
|
||||
}
|
||||
retryDelay := time.Duration(c.config.RetryDelayMs) * time.Millisecond
|
||||
if retryDelay <= 0 {
|
||||
retryDelay = 1 * time.Second
|
||||
}
|
||||
|
||||
for attempt := 0; attempt <= maxRetries; attempt++ {
|
||||
result, err = c.execute(ctx, method, path, body, paramsInQuery)
|
||||
if err == nil {
|
||||
return result, nil
|
||||
}
|
||||
logrus.Warnf("请求失败 (attempt %d/%d): %v", attempt+1, maxRetries+1, err)
|
||||
if attempt < maxRetries {
|
||||
time.Sleep(retryDelay * time.Duration(attempt+1))
|
||||
}
|
||||
}
|
||||
return result, fmt.Errorf("请求已重试 %d 次仍失败: %w", maxRetries, err)
|
||||
}
|
||||
|
||||
func (c *ApiClient) execute(ctx context.Context, method, path string, body interface{}, paramsInQuery bool) (*ApiResult, error) {
|
||||
start := time.Now()
|
||||
fullURL := c.config.GetApiUrl(path)
|
||||
|
||||
// 先注入认证参数
|
||||
fullURL = c.applyAuthURL(fullURL)
|
||||
|
||||
var reqBody io.Reader
|
||||
if body != nil && !paramsInQuery {
|
||||
b, _ := json.Marshal(body)
|
||||
reqBody = bytes.NewBuffer(b)
|
||||
}
|
||||
|
||||
// 如果参数在查询字符串中，拼接到 URL
|
||||
if body != nil && paramsInQuery {
|
||||
if paramsMap, ok := body.(map[string]interface{}); ok {
|
||||
fullURL = c.buildQueryURL(fullURL, paramsMap)
|
||||
}
|
||||
}
|
||||
|
||||
logrus.Infof("请求 URL: %s", fullURL)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, method, fullURL, reqBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("创建请求失败: %w", err)
|
||||
}
|
||||
|
||||
c.applyAuthHeader(req)
|
||||
req.Header.Set("User-Agent", "data-engine/1.0")
|
||||
if body != nil && !paramsInQuery {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
|
||||
resp, err := c.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("请求失败: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
result := &ApiResult{Body: respBody, DurationMs: time.Since(start).Milliseconds()}
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
return result, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(respBody))
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// buildQueryURL 将 params 拼接到 URL 查询参数中
|
||||
// 支持数组/对象类型的值自动 JSON 序列化 + URL 编码
|
||||
func (c *ApiClient) buildQueryURL(rawURL string, params map[string]interface{}) string {
|
||||
parsed, _ := url.Parse(rawURL)
|
||||
q := parsed.Query()
|
||||
|
||||
for k, v := range params {
|
||||
switch val := v.(type) {
|
||||
case string:
|
||||
q.Set(k, val)
|
||||
case bool:
|
||||
if val {
|
||||
q.Set(k, "true")
|
||||
} else {
|
||||
q.Set(k, "false")
|
||||
}
|
||||
case float64:
|
||||
// JSON 数字反序列化默认是 float64，转 int 避免科学计数法
|
||||
if val == float64(int64(val)) {
|
||||
q.Set(k, fmt.Sprintf("%d", int64(val)))
|
||||
} else {
|
||||
q.Set(k, fmt.Sprintf("%v", val))
|
||||
}
|
||||
case float32:
|
||||
q.Set(k, fmt.Sprintf("%v", val))
|
||||
case int, int8, int16, int32, int64:
|
||||
q.Set(k, fmt.Sprintf("%d", val))
|
||||
case uint, uint8, uint16, uint32, uint64:
|
||||
q.Set(k, fmt.Sprintf("%d", val))
|
||||
case []interface{}, map[string]interface{}:
|
||||
// 数组或对象需要 JSON 序列化后 URL 编码
|
||||
b, _ := json.Marshal(v)
|
||||
q.Set(k, string(b))
|
||||
default:
|
||||
q.Set(k, fmt.Sprintf("%v", v))
|
||||
}
|
||||
}
|
||||
|
||||
parsed.RawQuery = q.Encode()
|
||||
return parsed.String()
|
||||
}
|
||||
|
||||
func (c *ApiClient) applyAuthURL(rawURL string) string {
|
||||
cfg := c.config.AuthConfig
|
||||
token := c.config.AccessToken
|
||||
if cfg == nil {
|
||||
return rawURL
|
||||
}
|
||||
|
||||
tokenInQuery, _ := cfg["token_in_query"].(bool)
|
||||
queryKey, _ := cfg["query_key"].(string)
|
||||
if queryKey == "" {
|
||||
queryKey = "access_token"
|
||||
}
|
||||
|
||||
extraParams := make(map[string]string)
|
||||
if eq, ok := cfg["extra_query_params"].(map[string]interface{}); ok {
|
||||
for k, v := range eq {
|
||||
val := fmt.Sprintf("%v", v)
|
||||
val = strings.ReplaceAll(val, "{timestamp}", fmt.Sprintf("%d", time.Now().Unix()))
|
||||
val = strings.ReplaceAll(val, "{nonce}", generateNonce())
|
||||
extraParams[k] = val
|
||||
}
|
||||
}
|
||||
|
||||
if !tokenInQuery && len(extraParams) == 0 {
|
||||
return rawURL
|
||||
}
|
||||
|
||||
parsed, _ := url.Parse(rawURL)
|
||||
q := parsed.Query()
|
||||
if tokenInQuery && token != "" {
|
||||
q.Set(queryKey, token)
|
||||
}
|
||||
for k, v := range extraParams {
|
||||
q.Set(k, v)
|
||||
}
|
||||
parsed.RawQuery = q.Encode()
|
||||
return parsed.String()
|
||||
}
|
||||
|
||||
func (c *ApiClient) applyAuthHeader(req *http.Request) {
|
||||
cfg := c.config.AuthConfig
|
||||
token := c.config.AccessToken
|
||||
|
||||
if cfg != nil {
|
||||
if tiq, _ := cfg["token_in_query"].(bool); tiq {
|
||||
return
|
||||
}
|
||||
}
|
||||
if token == "" {
|
||||
return
|
||||
}
|
||||
|
||||
if cfg != nil {
|
||||
if h, ok := cfg["header_name"].(string); ok {
|
||||
f := cfg["header_format"].(string)
|
||||
if f == "" {
|
||||
f = "{token}"
|
||||
}
|
||||
req.Header.Set(h, strings.ReplaceAll(f, "{token}", token))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
switch c.config.AuthType {
|
||||
case "OAUTH2", "TOKEN":
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
case "API_KEY":
|
||||
req.Header.Set("X-API-Key", token)
|
||||
}
|
||||
}
|
||||
|
||||
func generateNonce() string {
|
||||
nanoPart := time.Now().UnixNano() % 1000000000000
|
||||
r, _ := rand.Int(rand.Reader, big.NewInt(10000))
|
||||
return fmt.Sprintf("%012d%04d", nanoPart, r.Int64())
|
||||
}
|
||||
116
service/sync/compensation.go
Normal file
116
service/sync/compensation.go
Normal file
@@ -0,0 +1,116 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
dao "dataengine/dao/copydata"
|
||||
taskDto "dataengine/model/dto/copydata"
|
||||
|
||||
"gitea.com/red-future/common/beans"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// StartCompensation 启动补偿调度器（在后台循环执行）
|
||||
func StartCompensation(ctx context.Context) {
|
||||
sec := g.Cfg().MustGet(ctx, "sync.compensation_interval_seconds", 300).Int()
|
||||
if sec < 10 {
|
||||
sec = 300
|
||||
}
|
||||
interval := time.Duration(sec) * time.Second
|
||||
logrus.Infof("补偿调度器启动，间隔: %v", interval)
|
||||
|
||||
ctx = context.WithValue(ctx, "user", &beans.User{UserName: "admin", TenantId: 1})
|
||||
|
||||
runCompensation(ctx)
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
runCompensation(ctx)
|
||||
case <-ctx.Done():
|
||||
logrus.Info("补偿调度器已停止")
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func runCompensation(ctx context.Context) {
|
||||
logrus.Info("=== 开始补偿扫描 ===")
|
||||
|
||||
tasks, err := dao.SyncTaskLog.QueryFailedTasks(ctx, &taskDto.QueryFailedTasksReq{
|
||||
Status: []string{"failed"},
|
||||
Limit: 50,
|
||||
})
|
||||
if err != nil {
|
||||
logrus.Errorf("查询失败任务异常: %v", err)
|
||||
return
|
||||
}
|
||||
if len(tasks) == 0 {
|
||||
logrus.Info("当前没有需要补偿的任务")
|
||||
return
|
||||
}
|
||||
|
||||
logrus.Infof("发现 %d 个失败任务", len(tasks))
|
||||
|
||||
for _, task := range tasks {
|
||||
if task.RetryCount >= task.MaxRetry {
|
||||
logrus.Warnf("任务 %s 已达最大重试次数 %d", task.TaskID, task.MaxRetry)
|
||||
dao.SyncTaskLog.Update(ctx, &taskDto.UpdateSyncTaskLogReq{
|
||||
ID: task.Id,
|
||||
Status: "manual_review",
|
||||
ErrorMessage: fmt.Sprintf("已达最大重试次数 %d", task.MaxRetry),
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
platformCode := task.PlatformCode
|
||||
interfaceCode := task.InterfaceCode
|
||||
if platformCode == "" || interfaceCode == "" {
|
||||
logrus.Warnf("任务 %s 缺少 platform_code 或 interface_code，跳过", task.TaskID)
|
||||
continue
|
||||
}
|
||||
|
||||
logrus.Infof("补偿: %s/%s (第 %d 次)", platformCode, interfaceCode, task.RetryCount+1)
|
||||
|
||||
retryCount := task.RetryCount + 1
|
||||
dao.SyncTaskLog.Update(ctx, &taskDto.UpdateSyncTaskLogReq{
|
||||
ID: task.Id,
|
||||
Status: "retrying",
|
||||
RetryCount: &retryCount,
|
||||
})
|
||||
|
||||
_, err := SyncByConfig(ctx, platformCode, interfaceCode, false)
|
||||
if err != nil {
|
||||
logrus.Errorf("补偿失败: %v", err)
|
||||
backoff := []int{5, 15, 30, 60, 120}
|
||||
waitMin := 5
|
||||
if retryCount <= len(backoff) {
|
||||
waitMin = backoff[retryCount-1]
|
||||
} else {
|
||||
waitMin = backoff[len(backoff)-1]
|
||||
}
|
||||
nextRetry := time.Now().Add(time.Duration(waitMin) * time.Minute)
|
||||
dao.SyncTaskLog.Update(ctx, &taskDto.UpdateSyncTaskLogReq{
|
||||
ID: task.Id,
|
||||
Status: "failed",
|
||||
ErrorMessage: err.Error(),
|
||||
ErrorCode: "COMPENSATION_FAILED",
|
||||
NextRetryTime: nextRetry,
|
||||
})
|
||||
} else {
|
||||
logrus.Infof("补偿成功: %s/%s", platformCode, interfaceCode)
|
||||
now := time.Now()
|
||||
dao.SyncTaskLog.Update(ctx, &taskDto.UpdateSyncTaskLogReq{
|
||||
ID: task.Id,
|
||||
Status: "success",
|
||||
CompletedAt: now,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
logrus.Info("=== 补偿扫描完成 ===")
|
||||
}
|
||||
70
service/sync/data_writer.go
Normal file
70
service/sync/data_writer.go
Normal file
@@ -0,0 +1,70 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"gitea.com/red-future/common/db/gfdb"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// InsertRows 批量写入数据
|
||||
func InsertRows(ctx context.Context, tableName string, conflictKeys []string, rows []map[string]interface{}) (int, error) {
|
||||
if len(rows) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
for i := range rows {
|
||||
if rows[i] == nil {
|
||||
rows[i] = make(map[string]interface{})
|
||||
}
|
||||
if _, ok := rows[i]["created_at"]; !ok {
|
||||
rows[i]["created_at"] = now
|
||||
}
|
||||
if _, ok := rows[i]["updated_at"]; !ok {
|
||||
rows[i]["updated_at"] = now
|
||||
}
|
||||
}
|
||||
|
||||
batchSize := 100
|
||||
total := 0
|
||||
for i := 0; i < len(rows); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(rows) {
|
||||
end = len(rows)
|
||||
}
|
||||
batch := rows[i:end]
|
||||
|
||||
m := gfdb.DB(ctx).Model(ctx, tableName).Data(batch)
|
||||
if len(conflictKeys) > 0 {
|
||||
keys := make([]interface{}, len(conflictKeys))
|
||||
for j, k := range conflictKeys {
|
||||
keys[j] = k
|
||||
}
|
||||
m = m.OnConflict(keys...)
|
||||
}
|
||||
_, err := m.Save()
|
||||
if err != nil {
|
||||
logrus.Errorf("批量写入 %s 失败: %v", tableName, err)
|
||||
for _, row := range batch {
|
||||
mm := gfdb.DB(ctx).Model(ctx, tableName).Data(row)
|
||||
if len(conflictKeys) > 0 {
|
||||
keys := make([]interface{}, len(conflictKeys))
|
||||
for j, k := range conflictKeys {
|
||||
keys[j] = k
|
||||
}
|
||||
mm = mm.OnConflict(keys...)
|
||||
}
|
||||
if _, e := mm.Save(); e != nil {
|
||||
logrus.Errorf("逐条写入失败: %v", e)
|
||||
} else {
|
||||
total++
|
||||
}
|
||||
}
|
||||
} else {
|
||||
total += len(batch)
|
||||
}
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
658
service/sync/dynamic_sync.go
Normal file
658
service/sync/dynamic_sync.go
Normal file
@@ -0,0 +1,658 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
consts "dataengine/consts/public"
|
||||
dao "dataengine/dao/copydata"
|
||||
taskDto "dataengine/model/dto/copydata"
|
||||
entity "dataengine/model/entity/dict"
|
||||
|
||||
"gitea.com/red-future/common/db/gfdb"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// SyncResult 同步结果
|
||||
type SyncResult struct {
|
||||
TableName string
|
||||
TotalPages int
|
||||
TotalRows int
|
||||
InsertedRows int
|
||||
Duration string
|
||||
}
|
||||
|
||||
// PrefetchConfig 预取配置
|
||||
type PrefetchConfig struct {
|
||||
URL string `json:"url"`
|
||||
Method string `json:"method"`
|
||||
ResponsePath string `json:"response_path"`
|
||||
TargetParam string `json:"target_param"`
|
||||
ValueField string `json:"value_field"`
|
||||
}
|
||||
|
||||
// SyncByConfig 执行同步
|
||||
func SyncByConfig(ctx context.Context, platformCode, interfaceCode string, isFullSync bool) (*SyncResult, error) {
|
||||
start := time.Now()
|
||||
pm := &PlatformManager{}
|
||||
|
||||
platform, ifaces, err := pm.GetPlatformWithInterfaces(ctx, platformCode)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("读取平台配置失败: %w", err)
|
||||
}
|
||||
|
||||
var iface *entity.ApiInterface
|
||||
for i := range ifaces {
|
||||
if ifaces[i].Code == interfaceCode {
|
||||
iface = &ifaces[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if iface == nil {
|
||||
return nil, fmt.Errorf("未找到接口 [%s]", interfaceCode)
|
||||
}
|
||||
if iface.TableDefinition == nil || len(iface.TableDefinition) == 0 {
|
||||
return nil, fmt.Errorf("接口 [%s] 未配置 table_definition", interfaceCode)
|
||||
}
|
||||
|
||||
td, err := ParseTableDefinition(iface.TableDefinition)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("解析表结构失败: %w", err)
|
||||
}
|
||||
if err := EnsureTable(ctx, td); err != nil {
|
||||
return nil, fmt.Errorf("建表失败: %w", err)
|
||||
}
|
||||
|
||||
// 检查上次同步状态（在标记 running 之前检查）
|
||||
prevStatus := getSyncStatus(ctx, platformCode, interfaceCode)
|
||||
lastSyncTime := int64(0)
|
||||
if !isFullSync {
|
||||
lastSyncTime = getLastSyncTime(ctx, platformCode, interfaceCode)
|
||||
}
|
||||
if prevStatus == "running" {
|
||||
logrus.Warnf("检测到上次同步异常中断 [%s/%s]，将重新全量同步", platformCode, interfaceCode)
|
||||
lastSyncTime = 0
|
||||
}
|
||||
|
||||
// 标记同步开始（保留 last_sync_time 不变，状态设为 running）
|
||||
markSyncRunning(ctx, platformCode, interfaceCode, lastSyncTime)
|
||||
|
||||
api := NewApiClient(platform)
|
||||
|
||||
prefetch := parsePrefetchConfig(iface.RequestConfig)
|
||||
if prefetch != nil {
|
||||
return syncWithPrefetch(ctx, api, platform, iface, ifaces, td, prefetch, isFullSync, lastSyncTime, start)
|
||||
}
|
||||
return syncSingleAPI(ctx, api, platform, iface, td, lastSyncTime, start)
|
||||
}
|
||||
|
||||
// paramsInQuery 判断参数是否应放在 URL 查询字符串中
|
||||
func paramsInQuery(iface *entity.ApiInterface) bool {
|
||||
if iface.Method == "GET" {
|
||||
return true
|
||||
}
|
||||
if iface.RequestConfig != nil {
|
||||
if loc, _ := iface.RequestConfig["parameters_location"].(string); loc == "query" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// syncSingleAPI 单接口分页同步
|
||||
func syncSingleAPI(ctx context.Context, api *ApiClient, platform *PlatformConfig, iface *entity.ApiInterface, td *TableDefinition, lastSyncTime int64, start time.Time) (*SyncResult, error) {
|
||||
pageSize := GetSyncPageSize(ctx)
|
||||
if ps, ok := iface.RequestConfig["page_size"].(float64); ok {
|
||||
pageSize = int(ps)
|
||||
}
|
||||
|
||||
inQuery := paramsInQuery(iface)
|
||||
method := string(iface.Method)
|
||||
|
||||
body := buildReqBody(iface, 1, pageSize, lastSyncTime, nil)
|
||||
resp, err := api.Request(ctx, method, iface.Url, body, inQuery)
|
||||
if err != nil {
|
||||
recordFailure(ctx, platform.PlatformCode, iface.Code, err.Error())
|
||||
return nil, fmt.Errorf("获取第一页失败: %w", err)
|
||||
}
|
||||
|
||||
rows, totalPages, maxTime, err := parseResp(resp.Body, iface.ResponseConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
result := &SyncResult{TableName: td.TableName, TotalPages: totalPages}
|
||||
inserted, _ := savePage(ctx, td, rows)
|
||||
result.InsertedRows += inserted
|
||||
result.TotalRows += len(rows)
|
||||
|
||||
for page := 2; page <= totalPages; page++ {
|
||||
body := buildReqBody(iface, page, pageSize, lastSyncTime, nil)
|
||||
resp, err := api.Request(ctx, method, iface.Url, body, inQuery)
|
||||
if err != nil {
|
||||
logrus.Errorf("第 %d 页失败: %v", page, err)
|
||||
continue
|
||||
}
|
||||
rows, _, mt, err := parseResp(resp.Body, iface.ResponseConfig)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
inserted, _ = savePage(ctx, td, rows)
|
||||
result.InsertedRows += inserted
|
||||
result.TotalRows += len(rows)
|
||||
if mt > maxTime {
|
||||
maxTime = mt
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
|
||||
if maxTime <= 0 {
|
||||
maxTime = time.Now().Unix()
|
||||
}
|
||||
updateSyncTime(ctx, platform.PlatformCode, iface.Code, maxTime)
|
||||
|
||||
result.Duration = fmt.Sprintf("%.1fs", time.Since(start).Seconds())
|
||||
logrus.Infof("同步完成 - 表:%s, %d条, 写入%d条, 耗时%s", td.TableName, result.TotalRows, result.InsertedRows, result.Duration)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// syncWithPrefetch 预取模式同步（先分页拉取全部实体列表，再并发处理每个实体）
|
||||
func syncWithPrefetch(ctx context.Context, api *ApiClient, platform *PlatformConfig, iface *entity.ApiInterface, allIfaces []entity.ApiInterface, td *TableDefinition, prefetch *PrefetchConfig, isFullSync bool, lastSyncTime int64, start time.Time) (*SyncResult, error) {
|
||||
logrus.Infof("预取模式: %s -> %s", prefetch.URL, iface.Url)
|
||||
|
||||
// 1. 查找匹配 prefetch URL 的接口配置（用于获取正确的请求参数）
|
||||
prefetchIface := findInterfaceByURL(allIfaces, prefetch.URL)
|
||||
prefetchParams := buildPrefetchParams(iface)
|
||||
if prefetchIface != nil && prefetchIface.RequestConfig != nil {
|
||||
// 使用 prefetch 目标接口的 request_config 重建参数（覆盖默认值）
|
||||
for k, v := range prefetchIface.RequestConfig {
|
||||
if k == "headers" || k == "prefetch" || k == "page_param" ||
|
||||
k == "page_size_param" || k == "time_field" || k == "parameters_location" ||
|
||||
k == "filtering" || k == "group_by" || k == "date_range" {
|
||||
continue
|
||||
}
|
||||
prefetchParams[k] = v
|
||||
}
|
||||
}
|
||||
method := strings.ToUpper(prefetch.Method)
|
||||
inQuery := paramsInQuery(iface)
|
||||
|
||||
allEntities := make([]interface{}, 0)
|
||||
allRows := make([]map[string]interface{}, 0)
|
||||
prefetchPage := 1
|
||||
prefetchTotalPages := 1
|
||||
|
||||
for prefetchPage <= prefetchTotalPages {
|
||||
params := make(map[string]interface{})
|
||||
for k, v := range prefetchParams {
|
||||
params[k] = v
|
||||
}
|
||||
pageParam := "page"
|
||||
if p, ok := iface.RequestConfig["page_param"].(string); ok {
|
||||
pageParam = p
|
||||
}
|
||||
params[pageParam] = prefetchPage
|
||||
|
||||
resp, err := api.Request(ctx, method, prefetch.URL, params, true)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("预取第 %d 页失败: %w", prefetchPage, err)
|
||||
}
|
||||
|
||||
entities, _, _, err := parseResp(resp.Body, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("解析预取第 %d 页响应失败: %w", prefetchPage, err)
|
||||
}
|
||||
|
||||
// 收集完整数据行（用于存库）和提取的 ID 值（用于遍历）
|
||||
for _, item := range entities {
|
||||
allRows = append(allRows, item)
|
||||
if prefetch.ValueField == "" {
|
||||
allEntities = append(allEntities, item)
|
||||
} else if v, ok := item[prefetch.ValueField]; ok {
|
||||
// 将 float64 转 int64，避免后续 URL 参数中出现科学计数法
|
||||
if f, ok := v.(float64); ok {
|
||||
allEntities = append(allEntities, int64(f))
|
||||
} else {
|
||||
allEntities = append(allEntities, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if prefetchPage == 1 {
|
||||
if tp := getTotalPages(resp.Body); tp > 0 {
|
||||
prefetchTotalPages = tp
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
prefetchPage++
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
|
||||
if len(allEntities) == 0 {
|
||||
logrus.Warn("预取结果为空列表，跳过同步")
|
||||
return &SyncResult{TableName: td.TableName, Duration: fmt.Sprintf("%.1fs", time.Since(start).Seconds())}, nil
|
||||
}
|
||||
logrus.Infof("预取到 %d 个实体（共 %d 页）", len(allEntities), prefetchPage-1)
|
||||
|
||||
// 2. 将预取的数据也存入库（如账户列表存入 tencent_account_relation）
|
||||
if prefetchIface != nil && prefetchIface.TableDefinition != nil {
|
||||
prefetchTd, err := ParseTableDefinition(prefetchIface.TableDefinition)
|
||||
if err == nil {
|
||||
if ensureErr := EnsureTable(ctx, prefetchTd); ensureErr == nil {
|
||||
saved, _ := savePage(ctx, prefetchTd, allRows)
|
||||
logrus.Infof("预取数据已存库: %s, %d 条", prefetchTd.TableName, saved)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 2. 并发处理每个实体的数据
|
||||
result := &SyncResult{TableName: td.TableName}
|
||||
pageSize := GetSyncPageSize(ctx)
|
||||
if ps, ok := iface.RequestConfig["page_size"].(float64); ok {
|
||||
pageSize = int(ps)
|
||||
}
|
||||
|
||||
dataMethod := string(iface.Method)
|
||||
concurrency := GetSyncConcurrency(ctx)
|
||||
|
||||
var mu sync.Mutex
|
||||
var wg sync.WaitGroup
|
||||
sem := make(chan struct{}, concurrency)
|
||||
globalMaxTime := lastSyncTime
|
||||
|
||||
for idx, entityVal := range allEntities {
|
||||
wg.Add(1)
|
||||
sem <- struct{}{}
|
||||
|
||||
go func(idx int, val interface{}) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
|
||||
logrus.Infof(" 处理实体 [%d/%d]: %v", idx+1, len(allEntities), val)
|
||||
|
||||
page := 1
|
||||
totalPages := 1
|
||||
entityMaxTime := int64(0)
|
||||
|
||||
for page <= totalPages {
|
||||
body := buildReqBody(iface, page, pageSize, lastSyncTime, map[string]interface{}{
|
||||
prefetch.TargetParam: val,
|
||||
})
|
||||
|
||||
resp, err := api.Request(ctx, dataMethod, iface.Url, body, inQuery)
|
||||
if err != nil {
|
||||
logrus.Errorf(" 实体 %v 第 %d 页失败: %v", val, page, err)
|
||||
page++
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
continue
|
||||
}
|
||||
|
||||
rows, tp, mt, parseErr := parseResp(resp.Body, iface.ResponseConfig)
|
||||
if parseErr != nil {
|
||||
logrus.Errorf(" 解析响应失败: %v", parseErr)
|
||||
page++
|
||||
continue
|
||||
}
|
||||
|
||||
if page == 1 {
|
||||
totalPages = tp
|
||||
}
|
||||
|
||||
for i := range rows {
|
||||
rows[i][prefetch.TargetParam] = val
|
||||
}
|
||||
|
||||
inserted, _ := savePage(ctx, td, rows)
|
||||
|
||||
mu.Lock()
|
||||
result.InsertedRows += inserted
|
||||
result.TotalRows += len(rows)
|
||||
mu.Unlock()
|
||||
|
||||
if mt > entityMaxTime {
|
||||
entityMaxTime = mt
|
||||
}
|
||||
|
||||
page++
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
|
||||
if entityMaxTime > 0 {
|
||||
mu.Lock()
|
||||
if entityMaxTime > globalMaxTime {
|
||||
globalMaxTime = entityMaxTime
|
||||
}
|
||||
mu.Unlock()
|
||||
}
|
||||
}(idx, entityVal)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
if globalMaxTime <= 0 {
|
||||
globalMaxTime = time.Now().Unix()
|
||||
}
|
||||
updateSyncTime(ctx, platform.PlatformCode, iface.Code, globalMaxTime)
|
||||
|
||||
result.Duration = fmt.Sprintf("%.1fs", time.Since(start).Seconds())
|
||||
logrus.Infof("同步完成 - 表:%s, %d条, 写入%d条, 耗时%s", td.TableName, result.TotalRows, result.InsertedRows, result.Duration)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// getTotalPages 从响应中提取总页数
|
||||
func getTotalPages(raw []byte) int {
|
||||
var r struct {
|
||||
Data map[string]interface{} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &r); err != nil {
|
||||
return 0
|
||||
}
|
||||
if r.Data == nil {
|
||||
return 0
|
||||
}
|
||||
if pi, ok := r.Data["page_info"].(map[string]interface{}); ok {
|
||||
if tp, ok := pi["total_page"].(float64); ok {
|
||||
return int(tp)
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// buildPrefetchParams 构建预取接口的请求参数
|
||||
func buildPrefetchParams(iface *entity.ApiInterface) map[string]interface{} {
|
||||
params := make(map[string]interface{})
|
||||
|
||||
if iface.RequestConfig != nil {
|
||||
pageParam := "page"
|
||||
psParam := "page_size"
|
||||
if p, ok := iface.RequestConfig["page_param"].(string); ok {
|
||||
pageParam = p
|
||||
}
|
||||
if p, ok := iface.RequestConfig["page_size_param"].(string); ok {
|
||||
psParam = p
|
||||
}
|
||||
params[pageParam] = 1
|
||||
params[psParam] = 100
|
||||
|
||||
for k, v := range iface.RequestConfig {
|
||||
if k == "headers" || k == "prefetch" || k == "page_param" ||
|
||||
k == "page_size_param" || k == "time_field" || k == "parameters_location" ||
|
||||
k == "filtering" || k == "group_by" || k == "date_range" {
|
||||
continue
|
||||
}
|
||||
if k == pageParam || k == psParam {
|
||||
continue
|
||||
}
|
||||
params[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
return params
|
||||
}
|
||||
|
||||
// parsePrefetchConfig 解析预取配置
|
||||
func parsePrefetchConfig(requestConfig map[string]interface{}) *PrefetchConfig {
|
||||
if requestConfig == nil {
|
||||
return nil
|
||||
}
|
||||
raw, ok := requestConfig["prefetch"]
|
||||
if !ok || raw == nil {
|
||||
return nil
|
||||
}
|
||||
m, ok := raw.(map[string]interface{})
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
pc := &PrefetchConfig{}
|
||||
if u, _ := m["url"].(string); u != "" {
|
||||
pc.URL = u
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
if method, _ := m["method"].(string); method != "" {
|
||||
pc.Method = method
|
||||
} else {
|
||||
pc.Method = "GET"
|
||||
}
|
||||
pc.ResponsePath, _ = m["response_path"].(string)
|
||||
pc.TargetParam, _ = m["target_param"].(string)
|
||||
pc.ValueField, _ = m["value_field"].(string)
|
||||
return pc
|
||||
}
|
||||
|
||||
// extractValues 从 JSON 响应中提取值列表
|
||||
func extractValues(raw []byte, path, valueField string) ([]interface{}, error) {
|
||||
var resp map[string]interface{}
|
||||
if err := json.Unmarshal(raw, &resp); err != nil {
|
||||
return nil, fmt.Errorf("JSON解析失败: %w", err)
|
||||
}
|
||||
|
||||
parts := strings.Split(path, ".")
|
||||
current := resp
|
||||
for i, part := range parts {
|
||||
if i == len(parts)-1 {
|
||||
list, ok := current[part].([]interface{})
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("路径 %s 不是数组", path)
|
||||
}
|
||||
var values []interface{}
|
||||
for _, item := range list {
|
||||
if valueField == "" {
|
||||
values = append(values, item)
|
||||
} else if m, ok := item.(map[string]interface{}); ok {
|
||||
if v, exists := m[valueField]; exists {
|
||||
values = append(values, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
return values, nil
|
||||
}
|
||||
next, ok := current[part].(map[string]interface{})
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("路径 %s 在 %s 处中断", path, part)
|
||||
}
|
||||
current = next
|
||||
}
|
||||
return nil, fmt.Errorf("路径 %s 不完整", path)
|
||||
}
|
||||
|
||||
// buildReqBody 构建请求参数
|
||||
func buildReqBody(iface *entity.ApiInterface, page, pageSize int, lastSyncTime int64, extraParams map[string]interface{}) map[string]interface{} {
|
||||
body := make(map[string]interface{})
|
||||
if iface.RequestConfig != nil {
|
||||
for k, v := range iface.RequestConfig {
|
||||
if k == "time_field" || k == "headers" || k == "prefetch" ||
|
||||
k == "page_param" || k == "page_size_param" || k == "parameters_location" {
|
||||
continue
|
||||
}
|
||||
body[k] = v
|
||||
}
|
||||
}
|
||||
pageParam := "page"
|
||||
psParam := "page_size"
|
||||
if iface.RequestConfig != nil {
|
||||
if p, ok := iface.RequestConfig["page_param"].(string); ok {
|
||||
pageParam = p
|
||||
}
|
||||
if p, ok := iface.RequestConfig["page_size_param"].(string); ok {
|
||||
psParam = p
|
||||
}
|
||||
}
|
||||
body[pageParam] = page
|
||||
body[psParam] = pageSize
|
||||
// 增量同步：将 time_field 转为 API 期望的 filtering 格式
|
||||
// 如 filtering=[{"field":"last_modified_time","operator":"GREATER_EQUALS","values":["1780037982"]}]
|
||||
if lastSyncTime > 0 {
|
||||
if tf, ok := iface.RequestConfig["time_field"].(string); ok && tf != "" {
|
||||
timeFilter := map[string]interface{}{
|
||||
"field": tf,
|
||||
"operator": "GREATER_EQUALS",
|
||||
"values": []interface{}{fmt.Sprintf("%d", lastSyncTime)},
|
||||
}
|
||||
// 合并已有的 filtering（如果 request_config 中已定义其他过滤条件）
|
||||
if existing, ok := body["filtering"].([]interface{}); ok {
|
||||
body["filtering"] = append(existing, timeFilter)
|
||||
} else {
|
||||
body["filtering"] = []interface{}{timeFilter}
|
||||
}
|
||||
}
|
||||
}
|
||||
for k, v := range extraParams {
|
||||
body[k] = v
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
// parseResp 解析同步接口返回值
|
||||
func parseResp(raw []byte, responseConfig map[string]interface{}) ([]map[string]interface{}, int, int64, error) {
|
||||
var r struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data map[string]interface{} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &r); err != nil {
|
||||
return nil, 0, 0, fmt.Errorf("解析响应失败: %w", err)
|
||||
}
|
||||
if r.Code != 0 {
|
||||
return nil, 0, 0, fmt.Errorf("API错误: code=%d, message=%s", r.Code, r.Message)
|
||||
}
|
||||
|
||||
var rows []map[string]interface{}
|
||||
totalPages := 1
|
||||
maxTime := int64(0)
|
||||
|
||||
var listData []interface{}
|
||||
if lp, ok := r.Data["list"]; ok {
|
||||
listData, _ = lp.([]interface{})
|
||||
} else if lp, ok := r.Data["data"]; ok {
|
||||
if m, ok := lp.(map[string]interface{}); ok {
|
||||
if l, ok := m["list"].([]interface{}); ok {
|
||||
listData = l
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, item := range listData {
|
||||
if m, ok := item.(map[string]interface{}); ok {
|
||||
j, _ := json.Marshal(m)
|
||||
m["raw_data"] = string(j)
|
||||
if t, ok := m["last_modified_time"].(float64); ok && int64(t) > maxTime {
|
||||
maxTime = int64(t)
|
||||
}
|
||||
if t, ok := m["created_time"].(float64); ok && int64(t) > maxTime {
|
||||
maxTime = int64(t)
|
||||
}
|
||||
rows = append(rows, m)
|
||||
}
|
||||
}
|
||||
|
||||
if pi, ok := r.Data["page_info"].(map[string]interface{}); ok {
|
||||
if tp, ok := pi["total_page"].(float64); ok {
|
||||
totalPages = int(tp)
|
||||
} else if tp, ok := pi["total_page"].(int); ok {
|
||||
totalPages = tp
|
||||
}
|
||||
}
|
||||
|
||||
return rows, totalPages, maxTime, nil
|
||||
}
|
||||
|
||||
func savePage(ctx context.Context, td *TableDefinition, rows []map[string]interface{}) (int, error) {
|
||||
if len(rows) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
colSet := make(map[string]bool)
|
||||
for _, c := range td.Columns {
|
||||
colSet[c.Name] = true
|
||||
}
|
||||
var clean []map[string]interface{}
|
||||
for _, row := range rows {
|
||||
c := make(map[string]interface{})
|
||||
for k, v := range row {
|
||||
if colSet[k] {
|
||||
c[k] = v
|
||||
}
|
||||
}
|
||||
if r, ok := row["raw_data"]; ok {
|
||||
c["raw_data"] = r
|
||||
}
|
||||
clean = append(clean, c)
|
||||
}
|
||||
return InsertRows(ctx, td.TableName, td.ConflictKeys, clean)
|
||||
}
|
||||
|
||||
func getLastSyncTime(ctx context.Context, platformCode, interfaceCode string) int64 {
|
||||
var t int64
|
||||
gfdb.DB(ctx).Model(ctx, consts.SyncTrackerTable).
|
||||
Fields("last_sync_time").
|
||||
Where("platform_code", platformCode).
|
||||
Where("interface_code", interfaceCode).
|
||||
Scan(&t)
|
||||
return t
|
||||
}
|
||||
|
||||
func getSyncStatus(ctx context.Context, platformCode, interfaceCode string) string {
|
||||
var s string
|
||||
gfdb.DB(ctx).Model(ctx, consts.SyncTrackerTable).
|
||||
Fields("sync_status").
|
||||
Where("platform_code", platformCode).
|
||||
Where("interface_code", interfaceCode).
|
||||
Scan(&s)
|
||||
return s
|
||||
}
|
||||
|
||||
func markSyncRunning(ctx context.Context, platformCode, interfaceCode string, lastSyncTime int64) {
|
||||
gfdb.DB(ctx).Model(ctx, consts.SyncTrackerTable).
|
||||
Data(map[string]interface{}{
|
||||
"platform_code": platformCode,
|
||||
"interface_code": interfaceCode,
|
||||
"last_sync_time": lastSyncTime,
|
||||
"sync_status": "running",
|
||||
}).
|
||||
OnConflict("platform_code", "interface_code").
|
||||
Save()
|
||||
}
|
||||
|
||||
func updateSyncTime(ctx context.Context, platformCode, interfaceCode string, t int64) {
|
||||
gfdb.DB(ctx).Model(ctx, consts.SyncTrackerTable).
|
||||
Data(map[string]interface{}{
|
||||
"platform_code": platformCode,
|
||||
"interface_code": interfaceCode,
|
||||
"last_sync_time": t,
|
||||
"last_sync_at": time.Now(),
|
||||
"sync_status": "success",
|
||||
}).
|
||||
OnConflict("platform_code", "interface_code").
|
||||
Save()
|
||||
}
|
||||
|
||||
func recordFailure(ctx context.Context, platformCode, interfaceCode, errMsg string) {
|
||||
dao.SyncTaskLog.Create(ctx, &taskDto.CreateSyncTaskLogReq{
|
||||
TaskID: fmt.Sprintf("%s_%s_%d", platformCode, interfaceCode, time.Now().UnixNano()),
|
||||
TaskType: fmt.Sprintf("%s_%s", platformCode, interfaceCode),
|
||||
PlatformCode: platformCode,
|
||||
InterfaceCode: interfaceCode,
|
||||
Status: "failed",
|
||||
MaxRetry: 3,
|
||||
RequestParams: map[string]interface{}{
|
||||
"platform_code": platformCode,
|
||||
"interface_code": interfaceCode,
|
||||
"error": errMsg,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// findInterfaceByURL 在所有接口中查找匹配 URL 的接口
|
||||
func findInterfaceByURL(ifaces []entity.ApiInterface, url string) *entity.ApiInterface {
|
||||
for i := range ifaces {
|
||||
if ifaces[i].Url == url {
|
||||
return &ifaces[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
43
service/sync/helpers.go
Normal file
43
service/sync/helpers.go
Normal file
@@ -0,0 +1,43 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
)
|
||||
|
||||
// GetSyncPageSize 获取分页大小（默认100）
|
||||
func GetSyncPageSize(ctx context.Context) int {
|
||||
ps := g.Cfg().MustGet(ctx, "sync.page_size", 100).Int()
|
||||
if ps < 1 || ps > 100 {
|
||||
return 100
|
||||
}
|
||||
return ps
|
||||
}
|
||||
|
||||
// GetSyncConcurrency 获取并发数（默认5）
|
||||
func GetSyncConcurrency(ctx context.Context) int {
|
||||
c := g.Cfg().MustGet(ctx, "sync.concurrency", 5).Int()
|
||||
if c < 1 {
|
||||
return 1
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// GetSyncInterval 获取同步间隔（分钟，默认60）
|
||||
func GetSyncInterval(ctx context.Context) int {
|
||||
m := g.Cfg().MustGet(ctx, "sync.sync_interval_minutes", 60).Int()
|
||||
if m < 5 {
|
||||
return 60
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// GetRetryCount 获取重试次数（默认3）
|
||||
func GetRetryCount(ctx context.Context) int {
|
||||
r := g.Cfg().MustGet(ctx, "sync.retry_count", 3).Int()
|
||||
if r < 0 {
|
||||
return 3
|
||||
}
|
||||
return r
|
||||
}
|
||||
111
service/sync/platform_manager.go
Normal file
111
service/sync/platform_manager.go
Normal file
@@ -0,0 +1,111 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
dao "dataengine/dao/dict"
|
||||
dto "dataengine/model/dto/dict"
|
||||
entity "dataengine/model/entity/dict"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// PlatformConfig 运行时平台配置
|
||||
type PlatformConfig struct {
|
||||
*entity.DatasourcePlatform
|
||||
AccessToken string
|
||||
}
|
||||
|
||||
// GetApiUrl 拼接完整 API URL
|
||||
func (c *PlatformConfig) GetApiUrl(apiPath string) string {
|
||||
if c.ApiBaseUrl == "" {
|
||||
return apiPath
|
||||
}
|
||||
return c.ApiBaseUrl + apiPath
|
||||
}
|
||||
|
||||
// PlatformManager 平台配置管理器
|
||||
type PlatformManager struct{}
|
||||
|
||||
// GetPlatform 根据平台编码获取配置
|
||||
func (m *PlatformManager) GetPlatform(ctx context.Context, platformCode string) (*PlatformConfig, error) {
|
||||
platform, err := dao.DatasourcePlatform.GetByPlatformCode(ctx, platformCode)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("查询平台配置失败 [%s]: %w", platformCode, err)
|
||||
}
|
||||
if platform == nil {
|
||||
return nil, fmt.Errorf("平台不存在 [%s]", platformCode)
|
||||
}
|
||||
if platform.Status != "ACTIVE" {
|
||||
return nil, fmt.Errorf("平台 [%s] 未启用", platformCode)
|
||||
}
|
||||
|
||||
cfg := &PlatformConfig{DatasourcePlatform: platform}
|
||||
|
||||
switch platform.AuthType {
|
||||
case "TOKEN":
|
||||
cfg.AccessToken = platform.Token
|
||||
case "OAUTH2":
|
||||
if platform.Token != "" {
|
||||
cfg.AccessToken = platform.Token
|
||||
}
|
||||
case "API_KEY":
|
||||
cfg.AccessToken = platform.ApiKey
|
||||
default:
|
||||
logrus.Warnf("平台 %s 认证类型 %s 未处理", platformCode, platform.AuthType)
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// GetInterfaces 获取平台下的活跃接口列表
|
||||
func (m *PlatformManager) GetInterfaces(ctx context.Context, platformId int64) ([]entity.ApiInterface, error) {
|
||||
interfaces, _, err := dao.ApiInterface.List(ctx, &dto.ListApiInterfaceReq{
|
||||
PlatformId: platformId,
|
||||
Status: "active",
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return interfaces, nil
|
||||
}
|
||||
|
||||
// GetInterfaceByCode 根据编码获取接口定义
|
||||
func (m *PlatformManager) GetInterfaceByCode(ctx context.Context, platformId int64, code string) (*entity.ApiInterface, error) {
|
||||
all, _, err := dao.ApiInterface.List(ctx, &dto.ListApiInterfaceReq{
|
||||
PlatformId: platformId,
|
||||
Code: code,
|
||||
Status: "active",
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(all) == 0 {
|
||||
return nil, fmt.Errorf("未找到接口 [code=%s]", code)
|
||||
}
|
||||
return &all[0], nil
|
||||
}
|
||||
|
||||
// GetPlatformWithInterfaces 获取平台及所有接口
|
||||
func (m *PlatformManager) GetPlatformWithInterfaces(ctx context.Context, platformCode string) (*PlatformConfig, []entity.ApiInterface, error) {
|
||||
cfg, err := m.GetPlatform(ctx, platformCode)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
interfaces, err := m.GetInterfaces(ctx, cfg.ID)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return cfg, interfaces, nil
|
||||
}
|
||||
|
||||
// FindInterfaceUrl 在接口列表中查找指定编码的 URL
|
||||
func FindInterfaceUrl(ifaces []entity.ApiInterface, code string) string {
|
||||
for _, iface := range ifaces {
|
||||
if iface.Code == code {
|
||||
return iface.Url
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
93
service/sync/sync_scheduler.go
Normal file
93
service/sync/sync_scheduler.go
Normal file
@@ -0,0 +1,93 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
dao "dataengine/dao/dict"
|
||||
dto "dataengine/model/dto/dict"
|
||||
|
||||
"gitea.com/red-future/common/beans"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// StartAutoSync 启动自动同步（独立 goroutine，启动后自动循环执行）
|
||||
func StartAutoSync(ctx context.Context) {
|
||||
interval := GetSyncInterval(ctx)
|
||||
logrus.Infof("自动同步调度器启动，间隔: %d 分钟", interval)
|
||||
|
||||
// 首次执行：根据 sync_tracker 是否有记录自动判断全量/增量
|
||||
// 无记录 → 全量，有记录 → 增量
|
||||
runAutoSync(ctx)
|
||||
|
||||
ticker := time.NewTicker(time.Duration(interval) * time.Minute)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
runAutoSync(ctx)
|
||||
case <-ctx.Done():
|
||||
logrus.Info("自动同步调度器已停止")
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func runAutoSync(ctx context.Context) {
|
||||
logrus.Info("=== 开始自动同步 ===")
|
||||
|
||||
// 注入用户上下文（ORM 框架需要用于租户隔离）
|
||||
ctx = context.WithValue(ctx, "user", &beans.User{UserName: "admin", TenantId: 1})
|
||||
|
||||
// 查询所有 ACTIVE 平台
|
||||
platforms, _, err := dao.DatasourcePlatform.List(ctx, &dto.ListDatasourcePlatformReq{
|
||||
Status: "ACTIVE",
|
||||
})
|
||||
if err != nil {
|
||||
logrus.Errorf("查询平台列表失败: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, p := range platforms {
|
||||
// 查询该平台下有 table_definition 的接口
|
||||
interfaces, _, err := dao.ApiInterface.List(ctx, &dto.ListApiInterfaceReq{
|
||||
PlatformId: p.ID,
|
||||
Status: "active",
|
||||
})
|
||||
if err != nil {
|
||||
logrus.Errorf("查询接口列表失败 [platform=%s]: %v", p.PlatformCode, err)
|
||||
continue
|
||||
}
|
||||
|
||||
for _, iface := range interfaces {
|
||||
if iface.TableDefinition == nil || len(iface.TableDefinition) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
logrus.Infof("自动同步: %s / %s", p.PlatformCode, iface.Code)
|
||||
// isFullSync=false 表示去查 sync_tracker：
|
||||
// 有记录 → 增量，无记录 → lastSyncTime=0 → 全量
|
||||
_, err := SyncByConfig(ctx, p.PlatformCode, iface.Code, false)
|
||||
if err != nil {
|
||||
logrus.Errorf("自动同步失败 [%s/%s]: %v", p.PlatformCode, iface.Code, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logrus.Info("=== 自动同步完成 ===")
|
||||
}
|
||||
|
||||
// InitAndStartAutoSync 在 main 中调用：初始化配置后启动自动同步和补偿
|
||||
func InitAndStartAutoSync(ctx context.Context) {
|
||||
// 读取配置中的同步开关
|
||||
enabled := g.Cfg().MustGet(ctx, "sync.auto_sync_enabled", true).Bool()
|
||||
if enabled {
|
||||
go StartAutoSync(ctx)
|
||||
} else {
|
||||
logrus.Info("自动同步已关闭")
|
||||
}
|
||||
// 补偿调度器独立启动，不受 auto_sync_enabled 控制
|
||||
go StartCompensation(ctx)
|
||||
}
|
||||
103
service/sync/table_manager.go
Normal file
103
service/sync/table_manager.go
Normal file
@@ -0,0 +1,103 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.com/red-future/common/db/gfdb"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// ColumnDef 列定义
|
||||
type ColumnDef struct {
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
Comment string `json:"comment,omitempty"`
|
||||
}
|
||||
|
||||
// TableDefinition 表结构定义
|
||||
type TableDefinition struct {
|
||||
TableName string `json:"table_name"`
|
||||
Columns []ColumnDef `json:"columns"`
|
||||
ConflictKeys []string `json:"conflict_keys,omitempty"`
|
||||
}
|
||||
|
||||
// ParseTableDefinition 解析 table_definition JSON
|
||||
func ParseTableDefinition(raw map[string]interface{}) (*TableDefinition, error) {
|
||||
td := &TableDefinition{}
|
||||
name, _ := raw["table_name"].(string)
|
||||
if name == "" {
|
||||
return nil, fmt.Errorf("table_definition 缺少 table_name")
|
||||
}
|
||||
td.TableName = name
|
||||
|
||||
colsRaw, _ := raw["columns"].([]interface{})
|
||||
for _, c := range colsRaw {
|
||||
cm, _ := c.(map[string]interface{})
|
||||
if cm == nil {
|
||||
continue
|
||||
}
|
||||
n, _ := cm["name"].(string)
|
||||
t, _ := cm["type"].(string)
|
||||
comment, _ := cm["comment"].(string)
|
||||
if n == "" || t == "" {
|
||||
continue
|
||||
}
|
||||
td.Columns = append(td.Columns, ColumnDef{Name: n, Type: t, Comment: comment})
|
||||
}
|
||||
if keys, _ := raw["conflict_keys"].([]interface{}); keys != nil {
|
||||
for _, k := range keys {
|
||||
if s, ok := k.(string); ok {
|
||||
td.ConflictKeys = append(td.ConflictKeys, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(td.Columns) == 0 {
|
||||
return nil, fmt.Errorf("table_definition 列定义为空")
|
||||
}
|
||||
return td, nil
|
||||
}
|
||||
|
||||
// EnsureTable 确保表存在
|
||||
func EnsureTable(ctx context.Context, td *TableDefinition) error {
|
||||
sql := buildCreateSQL(td)
|
||||
logrus.Infof("建表: %s", td.TableName)
|
||||
_, err := gfdb.DB(ctx).Exec(ctx, sql)
|
||||
if err != nil {
|
||||
return fmt.Errorf("建表失败 [%s]: %w", td.TableName, err)
|
||||
}
|
||||
logrus.Infof("表 %s 已就绪", td.TableName)
|
||||
return nil
|
||||
}
|
||||
|
||||
func buildCreateSQL(td *TableDefinition) string {
|
||||
cols := []string{
|
||||
"id BIGSERIAL PRIMARY KEY",
|
||||
"tenant_id BIGINT NOT NULL DEFAULT 0",
|
||||
"creator VARCHAR(64) DEFAULT ''",
|
||||
"created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()",
|
||||
"updater VARCHAR(64) DEFAULT ''",
|
||||
"updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()",
|
||||
"deleted_at TIMESTAMP WITH TIME ZONE",
|
||||
}
|
||||
for _, c := range td.Columns {
|
||||
cols = append(cols, fmt.Sprintf("%s %s", c.Name, c.Type))
|
||||
}
|
||||
cols = append(cols, "raw_data JSONB DEFAULT '{}'")
|
||||
|
||||
// 添加复合唯一索引（用于 ON CONFLICT upsert，所有 conflict_keys 作为一个复合索引）
|
||||
var constraints []string
|
||||
if len(td.ConflictKeys) > 0 {
|
||||
cols := strings.Join(td.ConflictKeys, ", ")
|
||||
indexName := fmt.Sprintf("uq_%s_conflict", td.TableName)
|
||||
constraints = append(constraints,
|
||||
fmt.Sprintf("CREATE UNIQUE INDEX IF NOT EXISTS %s ON %s (%s)", indexName, td.TableName, cols))
|
||||
}
|
||||
|
||||
sql := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (\n %s\n);\n", td.TableName, strings.Join(cols, ",\n "))
|
||||
if len(constraints) > 0 {
|
||||
sql += strings.Join(constraints, ";\n") + ";"
|
||||
}
|
||||
return sql
|
||||
}
|
||||
@@ -1,186 +0,0 @@
|
||||
package tencent
|
||||
|
||||
import (
|
||||
"context"
|
||||
dao "dataengine/dao/tencent"
|
||||
dto "dataengine/model/dto/tencent"
|
||||
entity "dataengine/model/entity/tencent"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type accountRelationService struct{}
|
||||
|
||||
var AccountRelationService = new(accountRelationService)
|
||||
|
||||
// API响应结构
|
||||
type accountRelationResponse struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data struct {
|
||||
List []struct {
|
||||
AccountID int64 `json:"account_id"`
|
||||
CorporationName string `json:"corporation_name"`
|
||||
CommentDataList json.RawMessage `json:"comment_data_list"`
|
||||
IsAdx bool `json:"is_adx"`
|
||||
IsBid bool `json:"is_bid"`
|
||||
IsMp bool `json:"is_mp"`
|
||||
} `json:"list"`
|
||||
PageInfo struct {
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"page_size"`
|
||||
TotalNumber int `json:"total_number"`
|
||||
TotalPage int `json:"total_page"`
|
||||
} `json:"page_info"`
|
||||
} `json:"data"`
|
||||
TraceID string `json:"trace_id"`
|
||||
}
|
||||
|
||||
// SyncAll 同步所有账户关系数据（自动分页）
|
||||
func (s *accountRelationService) SyncAll(ctx context.Context, req *dto.SyncAccountRelationReq) (res *dto.SyncAccountRelationRes, err error) {
|
||||
// 获取access_token
|
||||
accessToken := req.AccessToken
|
||||
if accessToken == "" {
|
||||
accessToken = g.Cfg().MustGet(ctx, "tencent.oauth.access_token").String()
|
||||
}
|
||||
|
||||
if accessToken == "" {
|
||||
return nil, fmt.Errorf("access_token不能为空")
|
||||
}
|
||||
|
||||
res = &dto.SyncAccountRelationRes{}
|
||||
totalSynced := 0
|
||||
|
||||
// 先获取第一页，得到总页数
|
||||
firstPageData, err := s.fetchPage(ctx, accessToken, 1, 100)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("获取第一页数据失败: %w", err)
|
||||
}
|
||||
|
||||
totalPage := firstPageData.Data.PageInfo.TotalPage
|
||||
res.TotalNumber = firstPageData.Data.PageInfo.TotalNumber
|
||||
res.TotalPage = totalPage
|
||||
|
||||
logrus.Infof("开始同步腾讯广告账户关系 - 总页数: %d, 总记录数: %d", totalPage, res.TotalNumber)
|
||||
|
||||
// 处理第一页数据
|
||||
synced, err := s.savePageData(ctx, firstPageData)
|
||||
if err != nil {
|
||||
logrus.Errorf("保存第一页数据失败: %v", err)
|
||||
}
|
||||
totalSynced += synced
|
||||
|
||||
// 循环获取剩余页
|
||||
for page := 2; page <= totalPage; page++ {
|
||||
logrus.Infof("正在获取第 %d/%d 页...", page, totalPage)
|
||||
|
||||
pageData, err := s.fetchPage(ctx, accessToken, page, 100)
|
||||
if err != nil {
|
||||
logrus.Errorf("获取第 %d 页失败: %v，继续下一页", page, err)
|
||||
continue
|
||||
}
|
||||
|
||||
synced, err := s.savePageData(ctx, pageData)
|
||||
if err != nil {
|
||||
logrus.Errorf("保存第 %d 页数据失败: %v", page, err)
|
||||
continue
|
||||
}
|
||||
totalSynced += synced
|
||||
|
||||
// 避免请求过快，休眠100ms
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
|
||||
res.SyncedCount = totalSynced
|
||||
res.Message = fmt.Sprintf("同步完成，共处理 %d 条记录", totalSynced)
|
||||
|
||||
logrus.Infof("同步完成 - 总页数: %d, 总记录数: %d, 成功同步: %d", totalPage, res.TotalNumber, totalSynced)
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// fetchPage 获取单页数据
|
||||
func (s *accountRelationService) fetchPage(ctx context.Context, accessToken string, page, pageSize int) (*accountRelationResponse, error) {
|
||||
timestamp := time.Now().Unix()
|
||||
// 使用时间戳+随机数生成唯一的nonce
|
||||
nonce := fmt.Sprintf("%d_%d", timestamp, time.Now().UnixNano())
|
||||
|
||||
url := fmt.Sprintf("https://api.e.qq.com/v3.0/organization_account_relation/get?"+
|
||||
"access_token=%s×tamp=%d&nonce=%s&pagination_mode=PAGINATION_MODE_NORMAL&page=%d&page_size=%d",
|
||||
accessToken, timestamp, nonce, page, pageSize)
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("创建请求失败: %w", err)
|
||||
}
|
||||
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
resp, err := client.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("请求失败: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("读取响应失败: %w", err)
|
||||
}
|
||||
|
||||
var result accountRelationResponse
|
||||
if err := json.Unmarshal(body, &result); err != nil {
|
||||
return nil, fmt.Errorf("解析响应失败: %w", err)
|
||||
}
|
||||
|
||||
if result.Code != 0 {
|
||||
return nil, fmt.Errorf("API错误: code=%d, message=%s", result.Code, result.Message)
|
||||
}
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// savePageData 保存单页数据到数据库
|
||||
func (s *accountRelationService) savePageData(ctx context.Context, data *accountRelationResponse) (int, error) {
|
||||
if len(data.Data.List) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
logrus.Infof("准备保存 %d 条账户关系数据", len(data.Data.List))
|
||||
|
||||
var items []*entity.AccountRelation
|
||||
for _, item := range data.Data.List {
|
||||
commentJSON := "{}"
|
||||
if len(item.CommentDataList) > 0 {
|
||||
commentJSON = string(item.CommentDataList)
|
||||
}
|
||||
|
||||
accountRelation := &entity.AccountRelation{
|
||||
AccountID: item.AccountID,
|
||||
CorporationName: item.CorporationName,
|
||||
CommentDataList: commentJSON,
|
||||
IsAdx: item.IsAdx,
|
||||
IsBid: item.IsBid,
|
||||
IsMp: item.IsMp,
|
||||
}
|
||||
// 设置 TenantID（框架将0视为空值，所以使用1）
|
||||
accountRelation.TenantId = 1
|
||||
|
||||
items = append(items, accountRelation)
|
||||
}
|
||||
|
||||
logrus.Infof("调用 BatchUpsert...")
|
||||
successCount, err := dao.AccountRelation.BatchUpsert(ctx, items)
|
||||
logrus.Infof("BatchUpsert 返回: successCount=%d, err=%v", successCount, err)
|
||||
|
||||
return successCount, err
|
||||
}
|
||||
|
||||
// ListAll 获取所有账户关系
|
||||
func (s *accountRelationService) ListAll(ctx context.Context) ([]entity.AccountRelation, error) {
|
||||
return dao.AccountRelation.ListAll(ctx)
|
||||
}
|
||||
@@ -1,214 +0,0 @@
|
||||
package tencent
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
dao "dataengine/dao/tencent"
|
||||
dto "dataengine/model/dto/tencent"
|
||||
entity "dataengine/model/entity/tencent"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type audioService struct{}
|
||||
|
||||
var AudioService = new(audioService)
|
||||
|
||||
// API响应结构
|
||||
type audioResponse struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data struct {
|
||||
List []struct {
|
||||
AudioId string `json:"audio_id"`
|
||||
CoverImageUrl string `json:"cover_image_url"`
|
||||
AudioName string `json:"audio_name"`
|
||||
Author string `json:"author"`
|
||||
Duration float64 `json:"duration"`
|
||||
ExpireTime int64 `json:"expire_time"`
|
||||
FeelTags []string `json:"feel_tags"`
|
||||
GenreTags []string `json:"genre_tags"`
|
||||
} `json:"list"`
|
||||
PageInfo struct {
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"page_size"`
|
||||
TotalNumber int `json:"total_number"`
|
||||
TotalPage int `json:"total_page"`
|
||||
} `json:"page_info"`
|
||||
} `json:"data"`
|
||||
TraceId string `json:"trace_id"`
|
||||
}
|
||||
|
||||
// SyncAll 同步所有音乐素材数据（自动分页）
|
||||
func (s *audioService) SyncAll(ctx context.Context, req *dto.SyncAudioReq) (res *dto.SyncAudioRes, err error) {
|
||||
// 获取access_token
|
||||
accessToken := req.AccessToken
|
||||
if accessToken == "" {
|
||||
accessToken = g.Cfg().MustGet(ctx, "tencent.oauth.access_token").String()
|
||||
}
|
||||
|
||||
if accessToken == "" {
|
||||
return nil, fmt.Errorf("access_token不能为空")
|
||||
}
|
||||
|
||||
res = &dto.SyncAudioRes{}
|
||||
totalSynced := 0
|
||||
|
||||
// 先获取第一页，得到总页数
|
||||
firstPageData, err := s.fetchPage(ctx, accessToken, 1, 100)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("获取第一页数据失败: %w", err)
|
||||
}
|
||||
|
||||
totalPage := firstPageData.Data.PageInfo.TotalPage
|
||||
res.TotalNumber = firstPageData.Data.PageInfo.TotalNumber
|
||||
res.TotalPage = totalPage
|
||||
|
||||
logrus.Infof("开始同步腾讯广告音乐素材 - 总页数: %d, 总记录数: %d", totalPage, res.TotalNumber)
|
||||
|
||||
// 处理第一页数据
|
||||
synced, err := s.savePageData(ctx, firstPageData)
|
||||
if err != nil {
|
||||
logrus.Errorf("保存第一页数据失败: %v", err)
|
||||
}
|
||||
totalSynced += synced
|
||||
|
||||
// 循环获取剩余页
|
||||
for page := 2; page <= totalPage; page++ {
|
||||
logrus.Infof("正在获取第 %d/%d 页...", page, totalPage)
|
||||
|
||||
pageData, err := s.fetchPage(ctx, accessToken, page, 100)
|
||||
if err != nil {
|
||||
logrus.Errorf("获取第 %d 页失败: %v，继续下一页", page, err)
|
||||
continue
|
||||
}
|
||||
|
||||
synced, err := s.savePageData(ctx, pageData)
|
||||
if err != nil {
|
||||
logrus.Errorf("保存第 %d 页数据失败: %v", page, err)
|
||||
continue
|
||||
}
|
||||
totalSynced += synced
|
||||
|
||||
// 避免请求过快，休眠100ms
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
|
||||
res.SyncedCount = totalSynced
|
||||
res.Message = fmt.Sprintf("同步完成，共处理 %d 条记录", totalSynced)
|
||||
|
||||
logrus.Infof("同步完成 - 总页数: %d, 总记录数: %d, 成功同步: %d", totalPage, res.TotalNumber, totalSynced)
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// fetchPage 获取单页数据
|
||||
func (s *audioService) fetchPage(ctx context.Context, accessToken string, page, pageSize int) (*audioResponse, error) {
|
||||
timestamp := time.Now().Unix()
|
||||
nonce := fmt.Sprintf("%d_%d", timestamp, time.Now().UnixNano())
|
||||
|
||||
url := fmt.Sprintf("https://api.e.qq.com/v3.0/muse_audios/get?access_token=%s×tamp=%d&nonce=%s",
|
||||
accessToken, timestamp, nonce)
|
||||
|
||||
// 构建请求体
|
||||
requestBody := map[string]interface{}{
|
||||
"fields": []string{
|
||||
"audio_id",
|
||||
"cover_image_url",
|
||||
"audio_name",
|
||||
"author",
|
||||
"duration",
|
||||
"expire_time",
|
||||
"feel_tags",
|
||||
"genre_tags",
|
||||
},
|
||||
"page": page,
|
||||
"page_size": pageSize,
|
||||
}
|
||||
|
||||
jsonBody, err := json.Marshal(requestBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("序列化请求体失败: %w", err)
|
||||
}
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonBody))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("创建请求失败: %w", err)
|
||||
}
|
||||
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
resp, err := client.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("请求失败: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("读取响应失败: %w", err)
|
||||
}
|
||||
|
||||
var result audioResponse
|
||||
if err := json.Unmarshal(body, &result); err != nil {
|
||||
return nil, fmt.Errorf("解析响应失败: %w", err)
|
||||
}
|
||||
|
||||
if result.Code != 0 {
|
||||
return nil, fmt.Errorf("API错误: code=%d, message=%s", result.Code, result.Message)
|
||||
}
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// savePageData 保存单页数据到数据库
|
||||
func (s *audioService) savePageData(ctx context.Context, data *audioResponse) (int, error) {
|
||||
if len(data.Data.List) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
logrus.Infof("准备保存 %d 条音乐素材数据", len(data.Data.List))
|
||||
|
||||
var items []*entity.Audio
|
||||
for _, item := range data.Data.List {
|
||||
// 序列化标签数组
|
||||
feelTagsJSON, _ := json.Marshal(item.FeelTags)
|
||||
genreTagsJSON, _ := json.Marshal(item.GenreTags)
|
||||
|
||||
audio := &entity.Audio{
|
||||
TenantId: 1,
|
||||
Creator: "system",
|
||||
Updater: "system",
|
||||
AudioId: item.AudioId,
|
||||
CoverImageUrl: item.CoverImageUrl,
|
||||
AudioName: item.AudioName,
|
||||
Author: item.Author,
|
||||
Duration: item.Duration,
|
||||
ExpireTime: item.ExpireTime,
|
||||
FeelTags: string(feelTagsJSON),
|
||||
GenreTags: string(genreTagsJSON),
|
||||
// 设置默认校验状态为待校验
|
||||
VerifyStatus: "PENDING",
|
||||
}
|
||||
|
||||
items = append(items, audio)
|
||||
}
|
||||
|
||||
logrus.Infof("调用 BatchUpsert...")
|
||||
successCount, err := dao.Audio.BatchUpsert(ctx, items)
|
||||
logrus.Infof("BatchUpsert 返回: successCount=%d, err=%v", successCount, err)
|
||||
|
||||
return successCount, err
|
||||
}
|
||||
|
||||
// ListAll 获取所有音乐素材
|
||||
func (s *audioService) ListAll(ctx context.Context) ([]entity.Audio, error) {
|
||||
return dao.Audio.ListAll(ctx)
|
||||
}
|
||||
@@ -1,365 +0,0 @@
|
||||
package tencent
|
||||
|
||||
import (
|
||||
"context"
|
||||
dao "dataengine/dao/tencent"
|
||||
dto "dataengine/model/dto/tencent"
|
||||
entity "dataengine/model/entity/tencent"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"gitea.com/red-future/common/db/gfdb"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type imageService struct{}
|
||||
|
||||
var ImageService = new(imageService)
|
||||
|
||||
// API响应结构
|
||||
type imageResponse struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data struct {
|
||||
List []struct {
|
||||
ImageId string `json:"image_id"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
FileSize int64 `json:"file_size"`
|
||||
Type string `json:"type"`
|
||||
Signature string `json:"signature"`
|
||||
Description string `json:"description"`
|
||||
SourceSignature string `json:"source_signature"`
|
||||
PreviewUrl string `json:"preview_url"`
|
||||
ThumbPreviewUrl string `json:"thumb_preview_url"`
|
||||
SourceType string `json:"source_type"`
|
||||
ImageUsage string `json:"image_usage"`
|
||||
CreatedTime int64 `json:"created_time"`
|
||||
LastModifiedTime int64 `json:"last_modified_time"`
|
||||
ProductCatalogId int64 `json:"product_catalog_id"`
|
||||
ProductOuterId string `json:"product_outer_id"`
|
||||
SourceReferenceId string `json:"source_reference_id"`
|
||||
OwnerAccountId string `json:"owner_account_id"`
|
||||
Status string `json:"status"`
|
||||
SampleAspectRatio string `json:"sample_aspect_ratio"`
|
||||
SourceMaterialId string `json:"source_material_id"`
|
||||
NewSourceType string `json:"new_source_type"`
|
||||
FirstPublicationStatus string `json:"first_publication_status"`
|
||||
QualityStatus string `json:"quality_status"`
|
||||
SimilarityStatus string `json:"similarity_status"`
|
||||
UserAigcStatus string `json:"user_aigc_status"`
|
||||
SystemAigcStatus string `json:"system_aigc_status"`
|
||||
AigcSource string `json:"aigc_source"`
|
||||
AigcFlag string `json:"aigc_flag"`
|
||||
MuseAigcVersion int `json:"muse_aigc_version"`
|
||||
AigcType int `json:"aigc_type"`
|
||||
} `json:"list"`
|
||||
PageInfo struct {
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"page_size"`
|
||||
TotalNumber int `json:"total_number"`
|
||||
TotalPage int `json:"total_page"`
|
||||
} `json:"page_info"`
|
||||
} `json:"data"`
|
||||
TraceId string `json:"trace_id"`
|
||||
}
|
||||
|
||||
// SyncAll 同步所有图片素材数据（遍历所有账户，自动分页）
|
||||
func (s *imageService) SyncAll(ctx context.Context, req *dto.SyncImageReq) (res *dto.SyncImageRes, err error) {
|
||||
// 创建独立的context，避免HTTP请求超时导致context被取消
|
||||
// 设置30分钟超时，足够完成421个账户的同步任务
|
||||
independentCtx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
// 保留原context中的user信息，供数据库中间件使用
|
||||
if user := ctx.Value("user"); user != nil {
|
||||
independentCtx = context.WithValue(independentCtx, "user", user)
|
||||
}
|
||||
|
||||
// 获取access_token
|
||||
accessToken := req.AccessToken
|
||||
if accessToken == "" {
|
||||
accessToken = g.Cfg().MustGet(independentCtx, "tencent.oauth.access_token").String()
|
||||
}
|
||||
|
||||
if accessToken == "" {
|
||||
return nil, fmt.Errorf("access_token不能为空")
|
||||
}
|
||||
|
||||
res = &dto.SyncImageRes{}
|
||||
totalSynced := 0
|
||||
totalImages := 0
|
||||
|
||||
// 获取所有账户列表
|
||||
accounts, err := s.getAccountList(independentCtx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("获取账户列表失败: %w", err)
|
||||
}
|
||||
|
||||
res.TotalAccounts = len(accounts)
|
||||
logrus.Infof("开始同步腾讯广告图片素材 - 账户数: %d", len(accounts))
|
||||
|
||||
// 遍历每个账户
|
||||
for _, account := range accounts {
|
||||
logrus.Infof("========== 开始处理账户: %d (%s) ==========", account.AccountID, account.CorporationName)
|
||||
|
||||
// 获取该账户的所有图片（分页）
|
||||
accountImages, err := s.syncAccountImages(independentCtx, accessToken, account.AccountID)
|
||||
if err != nil {
|
||||
logrus.Errorf("账户 %d 同步失败: %v，继续下一个账户", account.AccountID, err)
|
||||
continue
|
||||
}
|
||||
|
||||
totalImages += accountImages
|
||||
totalSynced += accountImages
|
||||
|
||||
// 避免请求过快，休眠200ms
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
|
||||
res.TotalImages = totalImages
|
||||
res.SyncedCount = totalSynced
|
||||
res.Message = fmt.Sprintf("同步完成，共处理 %d 个账户，%d 条图片记录", res.TotalAccounts, totalSynced)
|
||||
|
||||
logrus.Infof("同步完成 - 账户数: %d, 总图片数: %d, 成功同步: %d", res.TotalAccounts, totalImages, totalSynced)
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// getAccountList 获取所有账户列表
|
||||
func (s *imageService) getAccountList(ctx context.Context) ([]entity.AccountRelation, error) {
|
||||
var accounts []entity.AccountRelation
|
||||
err := gfdb.DB(ctx).Model(ctx, "tencent_account_relation").
|
||||
WhereNull("deleted_at").
|
||||
Scan(&accounts)
|
||||
|
||||
return accounts, err
|
||||
}
|
||||
|
||||
// syncAccountImages 同步单个账户的图片数据
|
||||
func (s *imageService) syncAccountImages(ctx context.Context, accessToken string, accountId int64) (int, error) {
|
||||
totalSynced := 0
|
||||
|
||||
// 先获取第一页，得到总页数
|
||||
firstPageData, err := s.fetchPage(ctx, accessToken, accountId, 1, 100)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("获取第一页数据失败: %w", err)
|
||||
}
|
||||
|
||||
totalPage := firstPageData.Data.PageInfo.TotalPage
|
||||
logrus.Infof("账户 %d - 总页数: %d, 总记录数: %d", accountId, totalPage, firstPageData.Data.PageInfo.TotalNumber)
|
||||
|
||||
// 处理第一页数据
|
||||
synced, err := s.savePageData(ctx, firstPageData, accountId)
|
||||
if err != nil {
|
||||
logrus.Errorf("保存第一页数据失败: %v", err)
|
||||
}
|
||||
totalSynced += synced
|
||||
|
||||
// 循环获取剩余页
|
||||
for page := 2; page <= totalPage; page++ {
|
||||
logrus.Infof("账户 %d - 正在获取第 %d/%d 页...", accountId, page, totalPage)
|
||||
|
||||
pageData, err := s.fetchPage(ctx, accessToken, accountId, page, 100)
|
||||
if err != nil {
|
||||
logrus.Errorf("账户 %d - 获取第 %d 页失败: %v，继续下一页", accountId, page, err)
|
||||
continue
|
||||
}
|
||||
|
||||
synced, err := s.savePageData(ctx, pageData, accountId)
|
||||
if err != nil {
|
||||
logrus.Errorf("账户 %d - 保存第 %d 页数据失败: %v", accountId, page, err)
|
||||
continue
|
||||
}
|
||||
totalSynced += synced
|
||||
|
||||
// 避免请求过快，休眠100ms
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
|
||||
logrus.Infof("账户 %d - 同步完成，共 %d 条记录", accountId, totalSynced)
|
||||
return totalSynced, nil
|
||||
}
|
||||
|
||||
// fetchPage 获取单页数据
|
||||
func (s *imageService) fetchPage(ctx context.Context, accessToken string, accountId int64, page, pageSize int) (*imageResponse, error) {
|
||||
// 构建filtering参数：状态为正常
|
||||
filtering := `[{"field":"status","operator":"EQUALS","values":["ADSTATUS_NORMAL"]}]`
|
||||
|
||||
// URL编码filtering参数
|
||||
encodedFiltering := url.QueryEscape(filtering)
|
||||
|
||||
// 在发送请求前生成最新的时间戳和nonce，避免时间戳过期
|
||||
timestamp := time.Now().Unix()
|
||||
// 使用时间戳+纳秒后6位+随机数，确保唯一性且不超过32字符
|
||||
nanoSuffix := time.Now().UnixNano() % 1000000 // 取纳秒的后6位
|
||||
nonce := fmt.Sprintf("%d%06d%d", timestamp, nanoSuffix, rand.Intn(1000))
|
||||
|
||||
urlStr := fmt.Sprintf("https://api.e.qq.com/v3.0/images/get?access_token=%s&nonce=%s×tamp=%d&account_id=%d&filtering=%s&page=%d&page_size=%d",
|
||||
accessToken, nonce, timestamp, accountId, encodedFiltering, page, pageSize)
|
||||
|
||||
logrus.Debugf("请求URL: %s", urlStr)
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("创建请求失败: %w", err)
|
||||
}
|
||||
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
resp, err := client.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("请求失败: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("读取响应失败: %w", err)
|
||||
}
|
||||
|
||||
logrus.Debugf("API响应: %s", string(body))
|
||||
|
||||
var result imageResponse
|
||||
if err := json.Unmarshal(body, &result); err != nil {
|
||||
return nil, fmt.Errorf("解析响应失败: %w", err)
|
||||
}
|
||||
|
||||
if result.Code != 0 {
|
||||
return nil, fmt.Errorf("API错误: code=%d, message=%s", result.Code, result.Message)
|
||||
}
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// savePageData 保存单页数据到数据库
|
||||
func (s *imageService) savePageData(ctx context.Context, data *imageResponse, accountId int64) (int, error) {
|
||||
if len(data.Data.List) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
logrus.Infof("准备保存 %d 条图片素材数据", len(data.Data.List))
|
||||
|
||||
var items []*entity.Image
|
||||
for _, item := range data.Data.List {
|
||||
image := &entity.Image{
|
||||
ImageId: item.ImageId,
|
||||
AccountId: accountId,
|
||||
Width: item.Width,
|
||||
Height: item.Height,
|
||||
FileSize: item.FileSize,
|
||||
Type: item.Type,
|
||||
Signature: item.Signature,
|
||||
Description: item.Description,
|
||||
SourceSignature: item.SourceSignature,
|
||||
PreviewUrl: item.PreviewUrl,
|
||||
ThumbPreviewUrl: item.ThumbPreviewUrl,
|
||||
SourceType: item.SourceType,
|
||||
ImageUsage: item.ImageUsage,
|
||||
CreatedTime: item.CreatedTime,
|
||||
LastModifiedTime: item.LastModifiedTime,
|
||||
ProductCatalogId: item.ProductCatalogId,
|
||||
ProductOuterId: item.ProductOuterId,
|
||||
SourceReferenceId: item.SourceReferenceId,
|
||||
OwnerAccountId: item.OwnerAccountId,
|
||||
Status: item.Status,
|
||||
SampleAspectRatio: item.SampleAspectRatio,
|
||||
SourceMaterialId: item.SourceMaterialId,
|
||||
NewSourceType: item.NewSourceType,
|
||||
FirstPublicationStatus: item.FirstPublicationStatus,
|
||||
QualityStatus: item.QualityStatus,
|
||||
SimilarityStatus: item.SimilarityStatus,
|
||||
UserAigcStatus: item.UserAigcStatus,
|
||||
SystemAigcStatus: item.SystemAigcStatus,
|
||||
AigcSource: item.AigcSource,
|
||||
AigcFlag: item.AigcFlag,
|
||||
MuseAigcVersion: item.MuseAigcVersion,
|
||||
AigcType: item.AigcType,
|
||||
}
|
||||
// 设置 TenantID（框架将0视为空值，所以使用1）
|
||||
image.TenantId = 1
|
||||
// 设置默认校验状态为待校验
|
||||
image.VerifyStatus = "PENDING"
|
||||
|
||||
items = append(items, image)
|
||||
}
|
||||
|
||||
logrus.Infof("调用 BatchUpsert...")
|
||||
successCount, err := dao.Image.BatchUpsert(ctx, items)
|
||||
logrus.Infof("BatchUpsert 返回: successCount=%d, err=%v", successCount, err)
|
||||
|
||||
return successCount, err
|
||||
}
|
||||
|
||||
// ListAll 获取所有图片素材
|
||||
func (s *imageService) ListAll(ctx context.Context) ([]entity.Image, error) {
|
||||
return dao.Image.ListAll(ctx)
|
||||
}
|
||||
|
||||
// ListWithPage 分页查询图片素材（支持时间过滤）
|
||||
func (s *imageService) ListWithPage(ctx context.Context, req *dto.ListImageQueryReq) (*dto.ListImageRes, error) {
|
||||
// 设置默认值
|
||||
page := req.Page
|
||||
if page <= 0 {
|
||||
page = 1
|
||||
}
|
||||
pageSize := req.PageSize
|
||||
if pageSize <= 0 {
|
||||
pageSize = 20
|
||||
}
|
||||
if pageSize > 100 {
|
||||
pageSize = 100 // 限制最大每页数量
|
||||
}
|
||||
|
||||
// 调用DAO层查询
|
||||
list, total, err := dao.Image.ListWithPage(ctx, page, pageSize, req.AccountId, req.StartTime, req.EndTime, req.Status)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("查询图片素材失败: %w", err)
|
||||
}
|
||||
|
||||
// 计算总页数
|
||||
totalPages := (total + pageSize - 1) / pageSize
|
||||
if totalPages == 0 && total > 0 {
|
||||
totalPages = 1
|
||||
}
|
||||
|
||||
// 转换为DTO
|
||||
items := make([]dto.ImageItem, 0, len(list))
|
||||
for _, item := range list {
|
||||
items = append(items, dto.ImageItem{
|
||||
Id: item.Id,
|
||||
ImageId: item.ImageId,
|
||||
AccountId: item.AccountId,
|
||||
Width: item.Width,
|
||||
Height: item.Height,
|
||||
FileSize: item.FileSize,
|
||||
Type: item.Type,
|
||||
Signature: item.Signature,
|
||||
Description: item.Description,
|
||||
PreviewUrl: item.PreviewUrl,
|
||||
ThumbPreviewUrl: item.ThumbPreviewUrl,
|
||||
Status: item.Status,
|
||||
CreatedTime: item.CreatedTime,
|
||||
LastModifiedTime: item.LastModifiedTime,
|
||||
CreatedAt: item.CreatedAt.Format("2006-01-02 15:04:05"),
|
||||
UpdatedAt: item.UpdatedAt.Format("2006-01-02 15:04:05"),
|
||||
})
|
||||
}
|
||||
|
||||
res := &dto.ListImageRes{
|
||||
List: items,
|
||||
Total: total,
|
||||
Page: page,
|
||||
PageSize: pageSize,
|
||||
TotalPages: totalPages,
|
||||
}
|
||||
|
||||
logrus.Infof("查询图片素材 - 页码: %d, 每页: %d, 总数: %d, 总页数: %d", page, pageSize, total, totalPages)
|
||||
|
||||
return res, nil
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package tencent
|
||||
|
||||
import (
|
||||
"context"
|
||||
dto "dataengine/model/dto/tencent"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
)
|
||||
|
||||
type oauthService struct{}
|
||||
|
||||
var OauthService = new(oauthService)
|
||||
|
||||
// RefreshToken 刷新腾讯广告Token
|
||||
func (s *oauthService) RefreshToken(ctx context.Context, req *dto.RefreshTokenReq) (res *dto.RefreshTokenRes, err error) {
|
||||
// 如果请求中没有提供参数，则从配置文件读取
|
||||
clientID := req.ClientID
|
||||
clientSecret := req.ClientSecret
|
||||
refreshToken := req.RefreshToken
|
||||
|
||||
if clientID == "" || clientSecret == "" || refreshToken == "" {
|
||||
clientID = g.Cfg().MustGet(ctx, "tencent.oauth.client_id").String()
|
||||
clientSecret = g.Cfg().MustGet(ctx, "tencent.oauth.client_secret").String()
|
||||
refreshToken = g.Cfg().MustGet(ctx, "tencent.oauth.refresh_token").String()
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("https://api.e.qq.com/oauth/refresh_token?client_id=%s&client_secret=%s&refresh_token=%s",
|
||||
clientID, clientSecret, refreshToken)
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("创建请求失败: %w", err)
|
||||
}
|
||||
|
||||
client := &http.Client{}
|
||||
resp, err := client.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("请求失败: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("读取响应失败: %w", err)
|
||||
}
|
||||
|
||||
var result struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data struct {
|
||||
AccessToken string `json:"access_token"`
|
||||
RefreshToken string `json:"refresh_token"`
|
||||
AccessTokenExpiresIn int64 `json:"access_token_expires_in"`
|
||||
RefreshTokenExpiresIn int64 `json:"refresh_token_expires_in"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(body, &result); err != nil {
|
||||
return nil, fmt.Errorf("解析响应失败: %w", err)
|
||||
}
|
||||
|
||||
if result.Code != 0 {
|
||||
return nil, fmt.Errorf("API错误: code=%d, message=%s", result.Code, result.Message)
|
||||
}
|
||||
|
||||
res = &dto.RefreshTokenRes{
|
||||
AccessToken: result.Data.AccessToken,
|
||||
RefreshToken: result.Data.RefreshToken,
|
||||
AccessTokenExpiresIn: result.Data.AccessTokenExpiresIn,
|
||||
RefreshTokenExpiresIn: result.Data.RefreshTokenExpiresIn,
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
@@ -1,419 +0,0 @@
|
||||
package tencent
|
||||
|
||||
import (
|
||||
"context"
|
||||
dao "dataengine/dao/tencent"
|
||||
dto "dataengine/model/dto/tencent"
|
||||
entity "dataengine/model/entity/tencent"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"gitea.com/red-future/common/db/gfdb"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type videoService struct{}
|
||||
|
||||
var VideoService = new(videoService)
|
||||
|
||||
// API响应结构
|
||||
type videoResponse struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data struct {
|
||||
List []struct {
|
||||
VideoId int64 `json:"video_id"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
VideoFrames int `json:"video_frames"`
|
||||
VideoFps int `json:"video_fps"`
|
||||
VideoCodec string `json:"video_codec"`
|
||||
VideoBitRate int64 `json:"video_bit_rate"`
|
||||
AudioCodec string `json:"audio_codec"`
|
||||
AudioBitRate int64 `json:"audio_bit_rate"`
|
||||
FileSize int64 `json:"file_size"`
|
||||
Type string `json:"type"`
|
||||
Signature string `json:"signature"`
|
||||
SystemStatus string `json:"system_status"`
|
||||
Description string `json:"description"`
|
||||
PreviewUrl string `json:"preview_url"`
|
||||
KeyFrameImageUrl string `json:"key_frame_image_url"`
|
||||
CreatedTime int64 `json:"created_time"`
|
||||
LastModifiedTime int64 `json:"last_modified_time"`
|
||||
VideoProfileName string `json:"video_profile_name"`
|
||||
AudioSampleRate int `json:"audio_sample_rate"`
|
||||
MaxKeyframeInterval int `json:"max_keyframe_interval"`
|
||||
MinKeyframeInterval int `json:"min_keyframe_interval"`
|
||||
SampleAspectRatio string `json:"sample_aspect_ratio"`
|
||||
AudioProfileName string `json:"audio_profile_name"`
|
||||
ScanType string `json:"scan_type"`
|
||||
ImageDurationMs int64 `json:"image_duration_millisecond"`
|
||||
AudioDurationMs int64 `json:"audio_duration_millisecond"`
|
||||
SourceType string `json:"source_type"`
|
||||
ProductCatalogId string `json:"product_catalog_id"`
|
||||
ProductOuterId string `json:"product_outer_id"`
|
||||
SourceReferenceId string `json:"source_reference_id"`
|
||||
OwnerAccountId string `json:"owner_account_id"`
|
||||
Status string `json:"status"`
|
||||
SourceMaterialId string `json:"source_material_id"`
|
||||
NewSourceType string `json:"new_source_type"`
|
||||
AigcType int `json:"aigc_type"`
|
||||
FirstPublicationStatus string `json:"first_publication_status"`
|
||||
QualityStatus string `json:"quality_status"`
|
||||
CoverId string `json:"cover_id"`
|
||||
SimilarityStatus string `json:"similarity_status"`
|
||||
UserAigcStatus string `json:"user_aigc_status"`
|
||||
SystemAigcStatus string `json:"system_aigc_status"`
|
||||
AigcSource string `json:"aigc_source"`
|
||||
AigcFlag string `json:"aigc_flag"`
|
||||
MuseAigcVersion int `json:"muse_aigc_version"`
|
||||
} `json:"list"`
|
||||
PageInfo struct {
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"page_size"`
|
||||
TotalNumber int `json:"total_number"`
|
||||
TotalPage int `json:"total_page"`
|
||||
} `json:"page_info"`
|
||||
} `json:"data"`
|
||||
TraceId string `json:"trace_id"`
|
||||
}
|
||||
|
||||
// SyncAll 同步所有视频素材数据（遍历所有账户，自动分页）
|
||||
func (s *videoService) SyncAll(ctx context.Context, req *dto.SyncVideoReq) (res *dto.SyncVideoRes, err error) {
|
||||
// 创建独立的context，避免HTTP请求超时导致context被取消
|
||||
// 设置30分钟超时，足够完成所有账户的同步任务
|
||||
independentCtx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
// 保留原context中的user信息，供数据库中间件使用
|
||||
if user := ctx.Value("user"); user != nil {
|
||||
independentCtx = context.WithValue(independentCtx, "user", user)
|
||||
}
|
||||
|
||||
// 获取access_token
|
||||
accessToken := req.AccessToken
|
||||
if accessToken == "" {
|
||||
accessToken = g.Cfg().MustGet(independentCtx, "tencent.oauth.access_token").String()
|
||||
}
|
||||
|
||||
if accessToken == "" {
|
||||
return nil, fmt.Errorf("access_token不能为空")
|
||||
}
|
||||
|
||||
res = &dto.SyncVideoRes{}
|
||||
totalSynced := 0
|
||||
totalVideos := 0
|
||||
|
||||
// 获取所有账户列表
|
||||
accounts, err := s.getAccountList(independentCtx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("获取账户列表失败: %w", err)
|
||||
}
|
||||
|
||||
res.TotalAccounts = len(accounts)
|
||||
logrus.Infof("开始同步腾讯广告视频素材 - 账户数: %d", len(accounts))
|
||||
|
||||
// 遍历每个账户
|
||||
for _, account := range accounts {
|
||||
logrus.Infof("========== 开始处理账户: %d (%s) ==========", account.AccountID, account.CorporationName)
|
||||
|
||||
// 获取该账户的所有视频（分页）
|
||||
accountVideos, err := s.syncAccountVideos(independentCtx, accessToken, account.AccountID)
|
||||
if err != nil {
|
||||
logrus.Errorf("账户 %d 同步失败: %v，继续下一个账户", account.AccountID, err)
|
||||
continue
|
||||
}
|
||||
|
||||
totalVideos += accountVideos
|
||||
totalSynced += accountVideos
|
||||
|
||||
// 避免请求过快，休眠200ms
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
|
||||
res.TotalVideos = totalVideos
|
||||
res.SyncedCount = totalSynced
|
||||
res.Message = fmt.Sprintf("同步完成，共处理 %d 个账户，%d 条视频记录", res.TotalAccounts, totalSynced)
|
||||
|
||||
logrus.Infof("同步完成 - 账户数: %d, 总视频数: %d, 成功同步: %d", res.TotalAccounts, totalVideos, totalSynced)
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// getAccountList 获取所有账户列表
|
||||
func (s *videoService) getAccountList(ctx context.Context) ([]entity.AccountRelation, error) {
|
||||
var accounts []entity.AccountRelation
|
||||
err := gfdb.DB(ctx).Model(ctx, "tencent_account_relation").
|
||||
WhereNull("deleted_at").
|
||||
Scan(&accounts)
|
||||
|
||||
return accounts, err
|
||||
}
|
||||
|
||||
// syncAccountVideos 同步单个账户的视频数据
|
||||
func (s *videoService) syncAccountVideos(ctx context.Context, accessToken string, accountId int64) (int, error) {
|
||||
totalSynced := 0
|
||||
|
||||
// 先获取第一页，得到总页数
|
||||
firstPageData, err := s.fetchPage(ctx, accessToken, accountId, 1, 100)
|
||||
if err != nil {
|
||||
// 如果是请求失败或API错误，返回友好的提示
|
||||
errMsg := err.Error()
|
||||
if contains(errMsg, "请求失败") || contains(errMsg, "API错误") {
|
||||
return 0, fmt.Errorf("该账户没有视频或无法访问")
|
||||
}
|
||||
return 0, fmt.Errorf("获取第一页数据失败: %w", err)
|
||||
}
|
||||
|
||||
totalPage := firstPageData.Data.PageInfo.TotalPage
|
||||
logrus.Infof("账户 %d - 总页数: %d, 总记录数: %d", accountId, totalPage, firstPageData.Data.PageInfo.TotalNumber)
|
||||
|
||||
// 如果没有数据，直接返回
|
||||
if totalPage == 0 || firstPageData.Data.PageInfo.TotalNumber == 0 {
|
||||
logrus.Infof("账户 %d - 没有视频数据", accountId)
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// 处理第一页数据
|
||||
synced, err := s.savePageData(ctx, firstPageData, accountId)
|
||||
if err != nil {
|
||||
logrus.Errorf("保存第一页数据失败: %v", err)
|
||||
}
|
||||
totalSynced += synced
|
||||
|
||||
// 循环获取剩余页
|
||||
for page := 2; page <= totalPage; page++ {
|
||||
logrus.Infof("账户 %d - 正在获取第 %d/%d 页...", accountId, page, totalPage)
|
||||
|
||||
pageData, err := s.fetchPage(ctx, accessToken, accountId, page, 100)
|
||||
if err != nil {
|
||||
logrus.Errorf("账户 %d - 获取第 %d 页失败: %v，继续下一页", accountId, page, err)
|
||||
continue
|
||||
}
|
||||
|
||||
synced, err := s.savePageData(ctx, pageData, accountId)
|
||||
if err != nil {
|
||||
logrus.Errorf("账户 %d - 保存第 %d 页数据失败: %v", accountId, page, err)
|
||||
continue
|
||||
}
|
||||
totalSynced += synced
|
||||
|
||||
// 避免请求过快，休眠100ms
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
|
||||
logrus.Infof("账户 %d - 同步完成，共 %d 条记录", accountId, totalSynced)
|
||||
return totalSynced, nil
|
||||
}
|
||||
|
||||
// fetchPage 获取单页数据
|
||||
func (s *videoService) fetchPage(ctx context.Context, accessToken string, accountId int64, page, pageSize int) (*videoResponse, error) {
|
||||
// 构建filtering参数：状态为正常
|
||||
filtering := `[{"field":"status","operator":"EQUALS","values":["ADSTATUS_NORMAL"]}]`
|
||||
|
||||
// URL编码filtering参数
|
||||
encodedFiltering := url.QueryEscape(filtering)
|
||||
|
||||
// 在发送请求前生成最新的时间戳和nonce，避免时间戳过期
|
||||
timestamp := time.Now().Unix()
|
||||
// 使用时间戳+纳秒后6位+随机数，确保唯一性且不超过32字符
|
||||
nanoSuffix := time.Now().UnixNano() % 1000000 // 取纳秒的后6位
|
||||
nonce := fmt.Sprintf("%d%06d%d", timestamp, nanoSuffix, rand.Intn(1000))
|
||||
|
||||
urlStr := fmt.Sprintf("https://api.e.qq.com/v3.0/videos/get?access_token=%s&nonce=%s×tamp=%d&account_id=%d&filtering=%s&page=%d&page_size=%d",
|
||||
accessToken, nonce, timestamp, accountId, encodedFiltering, page, pageSize)
|
||||
|
||||
logrus.Debugf("请求URL: %s", urlStr)
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("创建请求失败: %w", err)
|
||||
}
|
||||
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
resp, err := client.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("请求失败: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("读取响应失败: %w", err)
|
||||
}
|
||||
|
||||
logrus.Debugf("API响应: %s", string(body))
|
||||
|
||||
var result videoResponse
|
||||
if err := json.Unmarshal(body, &result); err != nil {
|
||||
return nil, fmt.Errorf("解析响应失败: %w", err)
|
||||
}
|
||||
|
||||
if result.Code != 0 {
|
||||
return nil, fmt.Errorf("API错误: code=%d, message=%s", result.Code, result.Message)
|
||||
}
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// savePageData 保存单页数据到数据库
|
||||
func (s *videoService) savePageData(ctx context.Context, data *videoResponse, accountId int64) (int, error) {
|
||||
if len(data.Data.List) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
logrus.Infof("准备保存 %d 条视频素材数据", len(data.Data.List))
|
||||
|
||||
var items []*entity.Video
|
||||
for _, item := range data.Data.List {
|
||||
video := &entity.Video{
|
||||
VideoId: fmt.Sprintf("%d", item.VideoId),
|
||||
AccountId: accountId,
|
||||
Width: item.Width,
|
||||
Height: item.Height,
|
||||
VideoFrames: item.VideoFrames,
|
||||
VideoFps: item.VideoFps,
|
||||
VideoCodec: item.VideoCodec,
|
||||
VideoBitRate: item.VideoBitRate,
|
||||
AudioCodec: item.AudioCodec,
|
||||
AudioBitRate: item.AudioBitRate,
|
||||
FileSize: item.FileSize,
|
||||
Type: item.Type,
|
||||
Signature: item.Signature,
|
||||
SystemStatus: item.SystemStatus,
|
||||
Description: item.Description,
|
||||
PreviewUrl: item.PreviewUrl,
|
||||
KeyFrameImageUrl: item.KeyFrameImageUrl,
|
||||
CreatedTime: item.CreatedTime,
|
||||
LastModifiedTime: item.LastModifiedTime,
|
||||
VideoProfileName: item.VideoProfileName,
|
||||
AudioSampleRate: item.AudioSampleRate,
|
||||
MaxKeyframeInterval: item.MaxKeyframeInterval,
|
||||
MinKeyframeInterval: item.MinKeyframeInterval,
|
||||
SampleAspectRatio: item.SampleAspectRatio,
|
||||
AudioProfileName: item.AudioProfileName,
|
||||
ScanType: item.ScanType,
|
||||
ImageDurationMillisecond: item.ImageDurationMs,
|
||||
AudioDurationMillisecond: item.AudioDurationMs,
|
||||
SourceType: item.SourceType,
|
||||
ProductCatalogId: item.ProductCatalogId,
|
||||
ProductOuterId: item.ProductOuterId,
|
||||
SourceReferenceId: item.SourceReferenceId,
|
||||
OwnerAccountId: item.OwnerAccountId,
|
||||
Status: item.Status,
|
||||
SourceMaterialId: item.SourceMaterialId,
|
||||
NewSourceType: item.NewSourceType,
|
||||
AigcType: item.AigcType,
|
||||
FirstPublicationStatus: item.FirstPublicationStatus,
|
||||
QualityStatus: item.QualityStatus,
|
||||
CoverId: item.CoverId,
|
||||
SimilarityStatus: item.SimilarityStatus,
|
||||
UserAigcStatus: item.UserAigcStatus,
|
||||
SystemAigcStatus: item.SystemAigcStatus,
|
||||
AigcSource: item.AigcSource,
|
||||
AigcFlag: item.AigcFlag,
|
||||
MuseAigcVersion: item.MuseAigcVersion,
|
||||
}
|
||||
// 设置 TenantID（框架将0视为空值，所以使用1）
|
||||
video.TenantId = 1
|
||||
// 设置默认校验状态为待校验
|
||||
video.VerifyStatus = "PENDING"
|
||||
|
||||
items = append(items, video)
|
||||
}
|
||||
|
||||
logrus.Infof("调用 BatchUpsert...")
|
||||
successCount, err := dao.Video.BatchUpsert(ctx, items)
|
||||
logrus.Infof("BatchUpsert 返回: successCount=%d, err=%v", successCount, err)
|
||||
|
||||
return successCount, err
|
||||
}
|
||||
|
||||
// ListAll 获取所有视频素材
|
||||
func (s *videoService) ListAll(ctx context.Context) ([]entity.Video, error) {
|
||||
return dao.Video.ListAll(ctx)
|
||||
}
|
||||
|
||||
// ListWithPage 分页查询视频素材（支持时间过滤）
|
||||
func (s *videoService) ListWithPage(ctx context.Context, req *dto.ListVideoQueryReq) (*dto.ListVideoRes, error) {
|
||||
// 设置默认值
|
||||
page := req.Page
|
||||
if page <= 0 {
|
||||
page = 1
|
||||
}
|
||||
pageSize := req.PageSize
|
||||
if pageSize <= 0 {
|
||||
pageSize = 20
|
||||
}
|
||||
if pageSize > 100 {
|
||||
pageSize = 100 // 限制最大每页数量
|
||||
}
|
||||
|
||||
// 调用DAO层查询
|
||||
list, total, err := dao.Video.ListWithPage(ctx, page, pageSize, req.AccountId, req.StartTime, req.EndTime, req.Status)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("查询视频素材失败: %w", err)
|
||||
}
|
||||
|
||||
// 计算总页数
|
||||
totalPages := (total + pageSize - 1) / pageSize
|
||||
if totalPages == 0 && total > 0 {
|
||||
totalPages = 1
|
||||
}
|
||||
|
||||
// 转换为DTO
|
||||
items := make([]dto.VideoItem, 0, len(list))
|
||||
for _, item := range list {
|
||||
items = append(items, dto.VideoItem{
|
||||
Id: item.Id,
|
||||
VideoId: item.VideoId,
|
||||
AccountId: item.AccountId,
|
||||
Width: item.Width,
|
||||
Height: item.Height,
|
||||
VideoFrames: item.VideoFrames,
|
||||
VideoFps: item.VideoFps,
|
||||
FileSize: item.FileSize,
|
||||
Type: item.Type,
|
||||
Description: item.Description,
|
||||
PreviewUrl: item.PreviewUrl,
|
||||
KeyFrameImageUrl: item.KeyFrameImageUrl,
|
||||
Status: item.Status,
|
||||
CreatedTime: item.CreatedTime,
|
||||
LastModifiedTime: item.LastModifiedTime,
|
||||
CreatedAt: item.CreatedAt.Format("2006-01-02 15:04:05"),
|
||||
UpdatedAt: item.UpdatedAt.Format("2006-01-02 15:04:05"),
|
||||
})
|
||||
}
|
||||
|
||||
res := &dto.ListVideoRes{
|
||||
List: items,
|
||||
Total: total,
|
||||
Page: page,
|
||||
PageSize: pageSize,
|
||||
TotalPages: totalPages,
|
||||
}
|
||||
|
||||
logrus.Infof("查询视频素材 - 页码: %d, 每页: %d, 总数: %d, 总页数: %d", page, pageSize, total, totalPages)
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// contains 检查字符串是否包含子串
|
||||
func contains(s, substr string) bool {
|
||||
return len(s) >= len(substr) && (s == substr || len(s) > len(substr) && (s[:len(substr)] == substr || s[len(s)-len(substr):] == substr || findSubstring(s, substr)))
|
||||
}
|
||||
|
||||
func findSubstring(s, substr string) bool {
|
||||
for i := 0; i <= len(s)-len(substr); i++ {
|
||||
if s[i:i+len(substr)] == substr {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
Reference in New Issue
Block a user