366 lines
12 KiB
Go
366 lines
12 KiB
Go
package tencent
|
||
|
||
import (
|
||
"context"
|
||
dao "dataengine/dao/tencent"
|
||
dto "dataengine/model/dto/tencent"
|
||
entity "dataengine/model/entity/tencent"
|
||
"encoding/json"
|
||
"fmt"
|
||
"io"
|
||
"math/rand"
|
||
"net/http"
|
||
"net/url"
|
||
"time"
|
||
|
||
"gitea.redpowerfuture.com/red-future/common/db/gfdb"
|
||
"github.com/gogf/gf/v2/frame/g"
|
||
"github.com/sirupsen/logrus"
|
||
)
|
||
|
||
type imageService struct{}
|
||
|
||
var ImageService = new(imageService)
|
||
|
||
// API响应结构
|
||
type imageResponse struct {
|
||
Code int `json:"code"`
|
||
Message string `json:"message"`
|
||
Data struct {
|
||
List []struct {
|
||
ImageId string `json:"image_id"`
|
||
Width int `json:"width"`
|
||
Height int `json:"height"`
|
||
FileSize int64 `json:"file_size"`
|
||
Type string `json:"type"`
|
||
Signature string `json:"signature"`
|
||
Description string `json:"description"`
|
||
SourceSignature string `json:"source_signature"`
|
||
PreviewUrl string `json:"preview_url"`
|
||
ThumbPreviewUrl string `json:"thumb_preview_url"`
|
||
SourceType string `json:"source_type"`
|
||
ImageUsage string `json:"image_usage"`
|
||
CreatedTime int64 `json:"created_time"`
|
||
LastModifiedTime int64 `json:"last_modified_time"`
|
||
ProductCatalogId int64 `json:"product_catalog_id"`
|
||
ProductOuterId string `json:"product_outer_id"`
|
||
SourceReferenceId string `json:"source_reference_id"`
|
||
OwnerAccountId string `json:"owner_account_id"`
|
||
Status string `json:"status"`
|
||
SampleAspectRatio string `json:"sample_aspect_ratio"`
|
||
SourceMaterialId string `json:"source_material_id"`
|
||
NewSourceType string `json:"new_source_type"`
|
||
FirstPublicationStatus string `json:"first_publication_status"`
|
||
QualityStatus string `json:"quality_status"`
|
||
SimilarityStatus string `json:"similarity_status"`
|
||
UserAigcStatus string `json:"user_aigc_status"`
|
||
SystemAigcStatus string `json:"system_aigc_status"`
|
||
AigcSource string `json:"aigc_source"`
|
||
AigcFlag string `json:"aigc_flag"`
|
||
MuseAigcVersion int `json:"muse_aigc_version"`
|
||
AigcType int `json:"aigc_type"`
|
||
} `json:"list"`
|
||
PageInfo struct {
|
||
Page int `json:"page"`
|
||
PageSize int `json:"page_size"`
|
||
TotalNumber int `json:"total_number"`
|
||
TotalPage int `json:"total_page"`
|
||
} `json:"page_info"`
|
||
} `json:"data"`
|
||
TraceId string `json:"trace_id"`
|
||
}
|
||
|
||
// SyncAll 同步所有图片素材数据（遍历所有账户，自动分页）
|
||
func (s *imageService) SyncAll(ctx context.Context, req *dto.SyncImageReq) (res *dto.SyncImageRes, err error) {
|
||
// 创建独立的context，避免HTTP请求超时导致context被取消
|
||
// 设置30分钟超时，足够完成421个账户的同步任务
|
||
independentCtx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||
defer cancel()
|
||
|
||
// 保留原context中的user信息，供数据库中间件使用
|
||
if user := ctx.Value("user"); user != nil {
|
||
independentCtx = context.WithValue(independentCtx, "user", user)
|
||
}
|
||
|
||
// 获取access_token
|
||
accessToken := req.AccessToken
|
||
if accessToken == "" {
|
||
accessToken = g.Cfg().MustGet(independentCtx, "tencent.oauth.access_token").String()
|
||
}
|
||
|
||
if accessToken == "" {
|
||
return nil, fmt.Errorf("access_token不能为空")
|
||
}
|
||
|
||
res = &dto.SyncImageRes{}
|
||
totalSynced := 0
|
||
totalImages := 0
|
||
|
||
// 获取所有账户列表
|
||
accounts, err := s.getAccountList(independentCtx)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("获取账户列表失败: %w", err)
|
||
}
|
||
|
||
res.TotalAccounts = len(accounts)
|
||
logrus.Infof("开始同步腾讯广告图片素材 - 账户数: %d", len(accounts))
|
||
|
||
// 遍历每个账户
|
||
for _, account := range accounts {
|
||
logrus.Infof("========== 开始处理账户: %d (%s) ==========", account.AccountID, account.CorporationName)
|
||
|
||
// 获取该账户的所有图片（分页）
|
||
accountImages, err := s.syncAccountImages(independentCtx, accessToken, account.AccountID)
|
||
if err != nil {
|
||
logrus.Errorf("账户 %d 同步失败: %v，继续下一个账户", account.AccountID, err)
|
||
continue
|
||
}
|
||
|
||
totalImages += accountImages
|
||
totalSynced += accountImages
|
||
|
||
// 避免请求过快，休眠200ms
|
||
time.Sleep(200 * time.Millisecond)
|
||
}
|
||
|
||
res.TotalImages = totalImages
|
||
res.SyncedCount = totalSynced
|
||
res.Message = fmt.Sprintf("同步完成，共处理 %d 个账户，%d 条图片记录", res.TotalAccounts, totalSynced)
|
||
|
||
logrus.Infof("同步完成 - 账户数: %d, 总图片数: %d, 成功同步: %d", res.TotalAccounts, totalImages, totalSynced)
|
||
|
||
return res, nil
|
||
}
|
||
|
||
// getAccountList 获取所有账户列表
|
||
func (s *imageService) getAccountList(ctx context.Context) ([]entity.AccountRelation, error) {
|
||
var accounts []entity.AccountRelation
|
||
err := gfdb.DB(ctx).Model(ctx, "tencent_account_relation").
|
||
WhereNull("deleted_at").
|
||
Scan(&accounts)
|
||
|
||
return accounts, err
|
||
}
|
||
|
||
// syncAccountImages 同步单个账户的图片数据
|
||
func (s *imageService) syncAccountImages(ctx context.Context, accessToken string, accountId int64) (int, error) {
|
||
totalSynced := 0
|
||
|
||
// 先获取第一页，得到总页数
|
||
firstPageData, err := s.fetchPage(ctx, accessToken, accountId, 1, 100)
|
||
if err != nil {
|
||
return 0, fmt.Errorf("获取第一页数据失败: %w", err)
|
||
}
|
||
|
||
totalPage := firstPageData.Data.PageInfo.TotalPage
|
||
logrus.Infof("账户 %d - 总页数: %d, 总记录数: %d", accountId, totalPage, firstPageData.Data.PageInfo.TotalNumber)
|
||
|
||
// 处理第一页数据
|
||
synced, err := s.savePageData(ctx, firstPageData, accountId)
|
||
if err != nil {
|
||
logrus.Errorf("保存第一页数据失败: %v", err)
|
||
}
|
||
totalSynced += synced
|
||
|
||
// 循环获取剩余页
|
||
for page := 2; page <= totalPage; page++ {
|
||
logrus.Infof("账户 %d - 正在获取第 %d/%d 页...", accountId, page, totalPage)
|
||
|
||
pageData, err := s.fetchPage(ctx, accessToken, accountId, page, 100)
|
||
if err != nil {
|
||
logrus.Errorf("账户 %d - 获取第 %d 页失败: %v，继续下一页", accountId, page, err)
|
||
continue
|
||
}
|
||
|
||
synced, err := s.savePageData(ctx, pageData, accountId)
|
||
if err != nil {
|
||
logrus.Errorf("账户 %d - 保存第 %d 页数据失败: %v", accountId, page, err)
|
||
continue
|
||
}
|
||
totalSynced += synced
|
||
|
||
// 避免请求过快，休眠100ms
|
||
time.Sleep(100 * time.Millisecond)
|
||
}
|
||
|
||
logrus.Infof("账户 %d - 同步完成，共 %d 条记录", accountId, totalSynced)
|
||
return totalSynced, nil
|
||
}
|
||
|
||
// fetchPage 获取单页数据
|
||
func (s *imageService) fetchPage(ctx context.Context, accessToken string, accountId int64, page, pageSize int) (*imageResponse, error) {
|
||
// 构建filtering参数：状态为正常
|
||
filtering := `[{"field":"status","operator":"EQUALS","values":["ADSTATUS_NORMAL"]}]`
|
||
|
||
// URL编码filtering参数
|
||
encodedFiltering := url.QueryEscape(filtering)
|
||
|
||
// 在发送请求前生成最新的时间戳和nonce，避免时间戳过期
|
||
timestamp := time.Now().Unix()
|
||
// 使用时间戳+纳秒后6位+随机数，确保唯一性且不超过32字符
|
||
nanoSuffix := time.Now().UnixNano() % 1000000 // 取纳秒的后6位
|
||
nonce := fmt.Sprintf("%d%06d%d", timestamp, nanoSuffix, rand.Intn(1000))
|
||
|
||
urlStr := fmt.Sprintf("https://api.e.qq.com/v3.0/images/get?access_token=%s&nonce=%s×tamp=%d&account_id=%d&filtering=%s&page=%d&page_size=%d",
|
||
accessToken, nonce, timestamp, accountId, encodedFiltering, page, pageSize)
|
||
|
||
logrus.Debugf("请求URL: %s", urlStr)
|
||
|
||
httpReq, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("创建请求失败: %w", err)
|
||
}
|
||
|
||
client := &http.Client{Timeout: 30 * time.Second}
|
||
resp, err := client.Do(httpReq)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("请求失败: %w", err)
|
||
}
|
||
defer resp.Body.Close()
|
||
|
||
body, err := io.ReadAll(resp.Body)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("读取响应失败: %w", err)
|
||
}
|
||
|
||
logrus.Debugf("API响应: %s", string(body))
|
||
|
||
var result imageResponse
|
||
if err := json.Unmarshal(body, &result); err != nil {
|
||
return nil, fmt.Errorf("解析响应失败: %w", err)
|
||
}
|
||
|
||
if result.Code != 0 {
|
||
return nil, fmt.Errorf("API错误: code=%d, message=%s", result.Code, result.Message)
|
||
}
|
||
|
||
return &result, nil
|
||
}
|
||
|
||
// savePageData 保存单页数据到数据库
|
||
func (s *imageService) savePageData(ctx context.Context, data *imageResponse, accountId int64) (int, error) {
|
||
if len(data.Data.List) == 0 {
|
||
return 0, nil
|
||
}
|
||
|
||
logrus.Infof("准备保存 %d 条图片素材数据", len(data.Data.List))
|
||
|
||
var items []*entity.Image
|
||
for _, item := range data.Data.List {
|
||
image := &entity.Image{
|
||
ImageId: item.ImageId,
|
||
AccountId: accountId,
|
||
Width: item.Width,
|
||
Height: item.Height,
|
||
FileSize: item.FileSize,
|
||
Type: item.Type,
|
||
Signature: item.Signature,
|
||
Description: item.Description,
|
||
SourceSignature: item.SourceSignature,
|
||
PreviewUrl: item.PreviewUrl,
|
||
ThumbPreviewUrl: item.ThumbPreviewUrl,
|
||
SourceType: item.SourceType,
|
||
ImageUsage: item.ImageUsage,
|
||
CreatedTime: item.CreatedTime,
|
||
LastModifiedTime: item.LastModifiedTime,
|
||
ProductCatalogId: item.ProductCatalogId,
|
||
ProductOuterId: item.ProductOuterId,
|
||
SourceReferenceId: item.SourceReferenceId,
|
||
OwnerAccountId: item.OwnerAccountId,
|
||
Status: item.Status,
|
||
SampleAspectRatio: item.SampleAspectRatio,
|
||
SourceMaterialId: item.SourceMaterialId,
|
||
NewSourceType: item.NewSourceType,
|
||
FirstPublicationStatus: item.FirstPublicationStatus,
|
||
QualityStatus: item.QualityStatus,
|
||
SimilarityStatus: item.SimilarityStatus,
|
||
UserAigcStatus: item.UserAigcStatus,
|
||
SystemAigcStatus: item.SystemAigcStatus,
|
||
AigcSource: item.AigcSource,
|
||
AigcFlag: item.AigcFlag,
|
||
MuseAigcVersion: item.MuseAigcVersion,
|
||
AigcType: item.AigcType,
|
||
}
|
||
// 设置 TenantID（框架将0视为空值，所以使用1）
|
||
image.TenantId = 1
|
||
// 设置默认校验状态为待校验
|
||
image.VerifyStatus = "PENDING"
|
||
|
||
items = append(items, image)
|
||
}
|
||
|
||
logrus.Infof("调用 BatchUpsert...")
|
||
successCount, err := dao.Image.BatchUpsert(ctx, items)
|
||
logrus.Infof("BatchUpsert 返回: successCount=%d, err=%v", successCount, err)
|
||
|
||
return successCount, err
|
||
}
|
||
|
||
// ListAll 获取所有图片素材
|
||
func (s *imageService) ListAll(ctx context.Context) ([]entity.Image, error) {
|
||
return dao.Image.ListAll(ctx)
|
||
}
|
||
|
||
// ListWithPage 分页查询图片素材（支持时间过滤）
|
||
func (s *imageService) ListWithPage(ctx context.Context, req *dto.ListImageQueryReq) (*dto.ListImageRes, error) {
|
||
// 设置默认值
|
||
page := req.Page
|
||
if page <= 0 {
|
||
page = 1
|
||
}
|
||
pageSize := req.PageSize
|
||
if pageSize <= 0 {
|
||
pageSize = 20
|
||
}
|
||
if pageSize > 100 {
|
||
pageSize = 100 // 限制最大每页数量
|
||
}
|
||
|
||
// 调用DAO层查询
|
||
list, total, err := dao.Image.ListWithPage(ctx, page, pageSize, req.AccountId, req.StartTime, req.EndTime, req.Status)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("查询图片素材失败: %w", err)
|
||
}
|
||
|
||
// 计算总页数
|
||
totalPages := (total + pageSize - 1) / pageSize
|
||
if totalPages == 0 && total > 0 {
|
||
totalPages = 1
|
||
}
|
||
|
||
// 转换为DTO
|
||
items := make([]dto.ImageItem, 0, len(list))
|
||
for _, item := range list {
|
||
items = append(items, dto.ImageItem{
|
||
Id: item.Id,
|
||
ImageId: item.ImageId,
|
||
AccountId: item.AccountId,
|
||
Width: item.Width,
|
||
Height: item.Height,
|
||
FileSize: item.FileSize,
|
||
Type: item.Type,
|
||
Signature: item.Signature,
|
||
Description: item.Description,
|
||
PreviewUrl: item.PreviewUrl,
|
||
ThumbPreviewUrl: item.ThumbPreviewUrl,
|
||
Status: item.Status,
|
||
CreatedTime: item.CreatedTime,
|
||
LastModifiedTime: item.LastModifiedTime,
|
||
CreatedAt: item.CreatedAt.Format("2006-01-02 15:04:05"),
|
||
UpdatedAt: item.UpdatedAt.Format("2006-01-02 15:04:05"),
|
||
})
|
||
}
|
||
|
||
res := &dto.ListImageRes{
|
||
List: items,
|
||
Total: total,
|
||
Page: page,
|
||
PageSize: pageSize,
|
||
TotalPages: totalPages,
|
||
}
|
||
|
||
logrus.Infof("查询图片素材 - 页码: %d, 每页: %d, 总数: %d, 总页数: %d", page, pageSize, total, totalPages)
|
||
|
||
return res, nil
|
||
}
|