同步音频和图片
This commit is contained in:
186
service/tencent/account_relation_service.go
Normal file
186
service/tencent/account_relation_service.go
Normal file
@@ -0,0 +1,186 @@
|
||||
package tencent
|
||||
|
||||
import (
|
||||
"context"
|
||||
dao "dataengine/dao/tencent"
|
||||
dto "dataengine/model/dto/tencent"
|
||||
entity "dataengine/model/entity/tencent"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type accountRelationService struct{}
|
||||
|
||||
var AccountRelationService = new(accountRelationService)
|
||||
|
||||
// API响应结构
|
||||
type accountRelationResponse struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data struct {
|
||||
List []struct {
|
||||
AccountID int64 `json:"account_id"`
|
||||
CorporationName string `json:"corporation_name"`
|
||||
CommentDataList json.RawMessage `json:"comment_data_list"`
|
||||
IsAdx bool `json:"is_adx"`
|
||||
IsBid bool `json:"is_bid"`
|
||||
IsMp bool `json:"is_mp"`
|
||||
} `json:"list"`
|
||||
PageInfo struct {
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"page_size"`
|
||||
TotalNumber int `json:"total_number"`
|
||||
TotalPage int `json:"total_page"`
|
||||
} `json:"page_info"`
|
||||
} `json:"data"`
|
||||
TraceID string `json:"trace_id"`
|
||||
}
|
||||
|
||||
// SyncAll 同步所有账户关系数据（自动分页）
|
||||
func (s *accountRelationService) SyncAll(ctx context.Context, req *dto.SyncAccountRelationReq) (res *dto.SyncAccountRelationRes, err error) {
|
||||
// 获取access_token
|
||||
accessToken := req.AccessToken
|
||||
if accessToken == "" {
|
||||
accessToken = g.Cfg().MustGet(ctx, "tencent.oauth.access_token").String()
|
||||
}
|
||||
|
||||
if accessToken == "" {
|
||||
return nil, fmt.Errorf("access_token不能为空")
|
||||
}
|
||||
|
||||
res = &dto.SyncAccountRelationRes{}
|
||||
totalSynced := 0
|
||||
|
||||
// 先获取第一页，得到总页数
|
||||
firstPageData, err := s.fetchPage(ctx, accessToken, 1, 100)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("获取第一页数据失败: %w", err)
|
||||
}
|
||||
|
||||
totalPage := firstPageData.Data.PageInfo.TotalPage
|
||||
res.TotalNumber = firstPageData.Data.PageInfo.TotalNumber
|
||||
res.TotalPage = totalPage
|
||||
|
||||
logrus.Infof("开始同步腾讯广告账户关系 - 总页数: %d, 总记录数: %d", totalPage, res.TotalNumber)
|
||||
|
||||
// 处理第一页数据
|
||||
synced, err := s.savePageData(ctx, firstPageData)
|
||||
if err != nil {
|
||||
logrus.Errorf("保存第一页数据失败: %v", err)
|
||||
}
|
||||
totalSynced += synced
|
||||
|
||||
// 循环获取剩余页
|
||||
for page := 2; page <= totalPage; page++ {
|
||||
logrus.Infof("正在获取第 %d/%d 页...", page, totalPage)
|
||||
|
||||
pageData, err := s.fetchPage(ctx, accessToken, page, 100)
|
||||
if err != nil {
|
||||
logrus.Errorf("获取第 %d 页失败: %v，继续下一页", page, err)
|
||||
continue
|
||||
}
|
||||
|
||||
synced, err := s.savePageData(ctx, pageData)
|
||||
if err != nil {
|
||||
logrus.Errorf("保存第 %d 页数据失败: %v", page, err)
|
||||
continue
|
||||
}
|
||||
totalSynced += synced
|
||||
|
||||
// 避免请求过快，休眠100ms
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
|
||||
res.SyncedCount = totalSynced
|
||||
res.Message = fmt.Sprintf("同步完成，共处理 %d 条记录", totalSynced)
|
||||
|
||||
logrus.Infof("同步完成 - 总页数: %d, 总记录数: %d, 成功同步: %d", totalPage, res.TotalNumber, totalSynced)
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// fetchPage 获取单页数据
|
||||
func (s *accountRelationService) fetchPage(ctx context.Context, accessToken string, page, pageSize int) (*accountRelationResponse, error) {
|
||||
timestamp := time.Now().Unix()
|
||||
// 使用时间戳+随机数生成唯一的nonce
|
||||
nonce := fmt.Sprintf("%d_%d", timestamp, time.Now().UnixNano())
|
||||
|
||||
url := fmt.Sprintf("https://api.e.qq.com/v3.0/organization_account_relation/get?"+
|
||||
"access_token=%s×tamp=%d&nonce=%s&pagination_mode=PAGINATION_MODE_NORMAL&page=%d&page_size=%d",
|
||||
accessToken, timestamp, nonce, page, pageSize)
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("创建请求失败: %w", err)
|
||||
}
|
||||
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
resp, err := client.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("请求失败: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("读取响应失败: %w", err)
|
||||
}
|
||||
|
||||
var result accountRelationResponse
|
||||
if err := json.Unmarshal(body, &result); err != nil {
|
||||
return nil, fmt.Errorf("解析响应失败: %w", err)
|
||||
}
|
||||
|
||||
if result.Code != 0 {
|
||||
return nil, fmt.Errorf("API错误: code=%d, message=%s", result.Code, result.Message)
|
||||
}
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// savePageData 保存单页数据到数据库
|
||||
func (s *accountRelationService) savePageData(ctx context.Context, data *accountRelationResponse) (int, error) {
|
||||
if len(data.Data.List) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
logrus.Infof("准备保存 %d 条账户关系数据", len(data.Data.List))
|
||||
|
||||
var items []*entity.AccountRelation
|
||||
for _, item := range data.Data.List {
|
||||
commentJSON := "{}"
|
||||
if len(item.CommentDataList) > 0 {
|
||||
commentJSON = string(item.CommentDataList)
|
||||
}
|
||||
|
||||
accountRelation := &entity.AccountRelation{
|
||||
AccountID: item.AccountID,
|
||||
CorporationName: item.CorporationName,
|
||||
CommentDataList: commentJSON,
|
||||
IsAdx: item.IsAdx,
|
||||
IsBid: item.IsBid,
|
||||
IsMp: item.IsMp,
|
||||
}
|
||||
// 设置 TenantID（框架将0视为空值，所以使用1）
|
||||
accountRelation.TenantId = 1
|
||||
|
||||
items = append(items, accountRelation)
|
||||
}
|
||||
|
||||
logrus.Infof("调用 BatchUpsert...")
|
||||
successCount, err := dao.AccountRelation.BatchUpsert(ctx, items)
|
||||
logrus.Infof("BatchUpsert 返回: successCount=%d, err=%v", successCount, err)
|
||||
|
||||
return successCount, err
|
||||
}
|
||||
|
||||
// ListAll 获取所有账户关系
|
||||
func (s *accountRelationService) ListAll(ctx context.Context) ([]entity.AccountRelation, error) {
|
||||
return dao.AccountRelation.ListAll(ctx)
|
||||
}
|
||||
212
service/tencent/audio_service.go
Normal file
212
service/tencent/audio_service.go
Normal file
@@ -0,0 +1,212 @@
|
||||
package tencent
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
dao "dataengine/dao/tencent"
|
||||
dto "dataengine/model/dto/tencent"
|
||||
entity "dataengine/model/entity/tencent"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type audioService struct{}
|
||||
|
||||
var AudioService = new(audioService)
|
||||
|
||||
// API响应结构
|
||||
type audioResponse struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data struct {
|
||||
List []struct {
|
||||
AudioId string `json:"audio_id"`
|
||||
CoverImageUrl string `json:"cover_image_url"`
|
||||
AudioName string `json:"audio_name"`
|
||||
Author string `json:"author"`
|
||||
Duration float64 `json:"duration"`
|
||||
ExpireTime int64 `json:"expire_time"`
|
||||
FeelTags []string `json:"feel_tags"`
|
||||
GenreTags []string `json:"genre_tags"`
|
||||
} `json:"list"`
|
||||
PageInfo struct {
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"page_size"`
|
||||
TotalNumber int `json:"total_number"`
|
||||
TotalPage int `json:"total_page"`
|
||||
} `json:"page_info"`
|
||||
} `json:"data"`
|
||||
TraceId string `json:"trace_id"`
|
||||
}
|
||||
|
||||
// SyncAll 同步所有音乐素材数据（自动分页）
|
||||
func (s *audioService) SyncAll(ctx context.Context, req *dto.SyncAudioReq) (res *dto.SyncAudioRes, err error) {
|
||||
// 获取access_token
|
||||
accessToken := req.AccessToken
|
||||
if accessToken == "" {
|
||||
accessToken = g.Cfg().MustGet(ctx, "tencent.oauth.access_token").String()
|
||||
}
|
||||
|
||||
if accessToken == "" {
|
||||
return nil, fmt.Errorf("access_token不能为空")
|
||||
}
|
||||
|
||||
res = &dto.SyncAudioRes{}
|
||||
totalSynced := 0
|
||||
|
||||
// 先获取第一页，得到总页数
|
||||
firstPageData, err := s.fetchPage(ctx, accessToken, 1, 100)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("获取第一页数据失败: %w", err)
|
||||
}
|
||||
|
||||
totalPage := firstPageData.Data.PageInfo.TotalPage
|
||||
res.TotalNumber = firstPageData.Data.PageInfo.TotalNumber
|
||||
res.TotalPage = totalPage
|
||||
|
||||
logrus.Infof("开始同步腾讯广告音乐素材 - 总页数: %d, 总记录数: %d", totalPage, res.TotalNumber)
|
||||
|
||||
// 处理第一页数据
|
||||
synced, err := s.savePageData(ctx, firstPageData)
|
||||
if err != nil {
|
||||
logrus.Errorf("保存第一页数据失败: %v", err)
|
||||
}
|
||||
totalSynced += synced
|
||||
|
||||
// 循环获取剩余页
|
||||
for page := 2; page <= totalPage; page++ {
|
||||
logrus.Infof("正在获取第 %d/%d 页...", page, totalPage)
|
||||
|
||||
pageData, err := s.fetchPage(ctx, accessToken, page, 100)
|
||||
if err != nil {
|
||||
logrus.Errorf("获取第 %d 页失败: %v，继续下一页", page, err)
|
||||
continue
|
||||
}
|
||||
|
||||
synced, err := s.savePageData(ctx, pageData)
|
||||
if err != nil {
|
||||
logrus.Errorf("保存第 %d 页数据失败: %v", page, err)
|
||||
continue
|
||||
}
|
||||
totalSynced += synced
|
||||
|
||||
// 避免请求过快，休眠100ms
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
|
||||
res.SyncedCount = totalSynced
|
||||
res.Message = fmt.Sprintf("同步完成，共处理 %d 条记录", totalSynced)
|
||||
|
||||
logrus.Infof("同步完成 - 总页数: %d, 总记录数: %d, 成功同步: %d", totalPage, res.TotalNumber, totalSynced)
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// fetchPage 获取单页数据
|
||||
func (s *audioService) fetchPage(ctx context.Context, accessToken string, page, pageSize int) (*audioResponse, error) {
|
||||
timestamp := time.Now().Unix()
|
||||
nonce := fmt.Sprintf("%d_%d", timestamp, time.Now().UnixNano())
|
||||
|
||||
url := fmt.Sprintf("https://api.e.qq.com/v3.0/muse_audios/get?access_token=%s×tamp=%d&nonce=%s",
|
||||
accessToken, timestamp, nonce)
|
||||
|
||||
// 构建请求体
|
||||
requestBody := map[string]interface{}{
|
||||
"fields": []string{
|
||||
"audio_id",
|
||||
"cover_image_url",
|
||||
"audio_name",
|
||||
"author",
|
||||
"duration",
|
||||
"expire_time",
|
||||
"feel_tags",
|
||||
"genre_tags",
|
||||
},
|
||||
"page": page,
|
||||
"page_size": pageSize,
|
||||
}
|
||||
|
||||
jsonBody, err := json.Marshal(requestBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("序列化请求体失败: %w", err)
|
||||
}
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonBody))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("创建请求失败: %w", err)
|
||||
}
|
||||
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
resp, err := client.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("请求失败: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("读取响应失败: %w", err)
|
||||
}
|
||||
|
||||
var result audioResponse
|
||||
if err := json.Unmarshal(body, &result); err != nil {
|
||||
return nil, fmt.Errorf("解析响应失败: %w", err)
|
||||
}
|
||||
|
||||
if result.Code != 0 {
|
||||
return nil, fmt.Errorf("API错误: code=%d, message=%s", result.Code, result.Message)
|
||||
}
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// savePageData 保存单页数据到数据库
|
||||
func (s *audioService) savePageData(ctx context.Context, data *audioResponse) (int, error) {
|
||||
if len(data.Data.List) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
logrus.Infof("准备保存 %d 条音乐素材数据", len(data.Data.List))
|
||||
|
||||
var items []*entity.Audio
|
||||
for _, item := range data.Data.List {
|
||||
// 序列化标签数组
|
||||
feelTagsJSON, _ := json.Marshal(item.FeelTags)
|
||||
genreTagsJSON, _ := json.Marshal(item.GenreTags)
|
||||
|
||||
audio := &entity.Audio{
|
||||
TenantId: 1,
|
||||
Creator: "system",
|
||||
Updater: "system",
|
||||
AudioId: item.AudioId,
|
||||
CoverImageUrl: item.CoverImageUrl,
|
||||
AudioName: item.AudioName,
|
||||
Author: item.Author,
|
||||
Duration: item.Duration,
|
||||
ExpireTime: item.ExpireTime,
|
||||
FeelTags: string(feelTagsJSON),
|
||||
GenreTags: string(genreTagsJSON),
|
||||
}
|
||||
|
||||
items = append(items, audio)
|
||||
}
|
||||
|
||||
logrus.Infof("调用 BatchUpsert...")
|
||||
successCount, err := dao.Audio.BatchUpsert(ctx, items)
|
||||
logrus.Infof("BatchUpsert 返回: successCount=%d, err=%v", successCount, err)
|
||||
|
||||
return successCount, err
|
||||
}
|
||||
|
||||
// ListAll 获取所有音乐素材
|
||||
func (s *audioService) ListAll(ctx context.Context) ([]entity.Audio, error) {
|
||||
return dao.Audio.ListAll(ctx)
|
||||
}
|
||||
363
service/tencent/image_service.go
Normal file
363
service/tencent/image_service.go
Normal file
@@ -0,0 +1,363 @@
|
||||
package tencent
|
||||
|
||||
import (
|
||||
"context"
|
||||
dao "dataengine/dao/tencent"
|
||||
dto "dataengine/model/dto/tencent"
|
||||
entity "dataengine/model/entity/tencent"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"gitea.com/red-future/common/db/gfdb"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type imageService struct{}
|
||||
|
||||
var ImageService = new(imageService)
|
||||
|
||||
// API响应结构
|
||||
type imageResponse struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data struct {
|
||||
List []struct {
|
||||
ImageId string `json:"image_id"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
FileSize int64 `json:"file_size"`
|
||||
Type string `json:"type"`
|
||||
Signature string `json:"signature"`
|
||||
Description string `json:"description"`
|
||||
SourceSignature string `json:"source_signature"`
|
||||
PreviewUrl string `json:"preview_url"`
|
||||
ThumbPreviewUrl string `json:"thumb_preview_url"`
|
||||
SourceType string `json:"source_type"`
|
||||
ImageUsage string `json:"image_usage"`
|
||||
CreatedTime int64 `json:"created_time"`
|
||||
LastModifiedTime int64 `json:"last_modified_time"`
|
||||
ProductCatalogId int64 `json:"product_catalog_id"`
|
||||
ProductOuterId string `json:"product_outer_id"`
|
||||
SourceReferenceId string `json:"source_reference_id"`
|
||||
OwnerAccountId string `json:"owner_account_id"`
|
||||
Status string `json:"status"`
|
||||
SampleAspectRatio string `json:"sample_aspect_ratio"`
|
||||
SourceMaterialId string `json:"source_material_id"`
|
||||
NewSourceType string `json:"new_source_type"`
|
||||
FirstPublicationStatus string `json:"first_publication_status"`
|
||||
QualityStatus string `json:"quality_status"`
|
||||
SimilarityStatus string `json:"similarity_status"`
|
||||
UserAigcStatus string `json:"user_aigc_status"`
|
||||
SystemAigcStatus string `json:"system_aigc_status"`
|
||||
AigcSource string `json:"aigc_source"`
|
||||
AigcFlag string `json:"aigc_flag"`
|
||||
MuseAigcVersion int `json:"muse_aigc_version"`
|
||||
AigcType int `json:"aigc_type"`
|
||||
} `json:"list"`
|
||||
PageInfo struct {
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"page_size"`
|
||||
TotalNumber int `json:"total_number"`
|
||||
TotalPage int `json:"total_page"`
|
||||
} `json:"page_info"`
|
||||
} `json:"data"`
|
||||
TraceId string `json:"trace_id"`
|
||||
}
|
||||
|
||||
// SyncAll 同步所有图片素材数据（遍历所有账户，自动分页）
|
||||
func (s *imageService) SyncAll(ctx context.Context, req *dto.SyncImageReq) (res *dto.SyncImageRes, err error) {
|
||||
// 创建独立的context，避免HTTP请求超时导致context被取消
|
||||
// 设置30分钟超时，足够完成421个账户的同步任务
|
||||
independentCtx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
// 保留原context中的user信息，供数据库中间件使用
|
||||
if user := ctx.Value("user"); user != nil {
|
||||
independentCtx = context.WithValue(independentCtx, "user", user)
|
||||
}
|
||||
|
||||
// 获取access_token
|
||||
accessToken := req.AccessToken
|
||||
if accessToken == "" {
|
||||
accessToken = g.Cfg().MustGet(independentCtx, "tencent.oauth.access_token").String()
|
||||
}
|
||||
|
||||
if accessToken == "" {
|
||||
return nil, fmt.Errorf("access_token不能为空")
|
||||
}
|
||||
|
||||
res = &dto.SyncImageRes{}
|
||||
totalSynced := 0
|
||||
totalImages := 0
|
||||
|
||||
// 获取所有账户列表
|
||||
accounts, err := s.getAccountList(independentCtx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("获取账户列表失败: %w", err)
|
||||
}
|
||||
|
||||
res.TotalAccounts = len(accounts)
|
||||
logrus.Infof("开始同步腾讯广告图片素材 - 账户数: %d", len(accounts))
|
||||
|
||||
// 遍历每个账户
|
||||
for _, account := range accounts {
|
||||
logrus.Infof("========== 开始处理账户: %d (%s) ==========", account.AccountID, account.CorporationName)
|
||||
|
||||
// 获取该账户的所有图片（分页）
|
||||
accountImages, err := s.syncAccountImages(independentCtx, accessToken, account.AccountID)
|
||||
if err != nil {
|
||||
logrus.Errorf("账户 %d 同步失败: %v，继续下一个账户", account.AccountID, err)
|
||||
continue
|
||||
}
|
||||
|
||||
totalImages += accountImages
|
||||
totalSynced += accountImages
|
||||
|
||||
// 避免请求过快，休眠200ms
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
|
||||
res.TotalImages = totalImages
|
||||
res.SyncedCount = totalSynced
|
||||
res.Message = fmt.Sprintf("同步完成，共处理 %d 个账户，%d 条图片记录", res.TotalAccounts, totalSynced)
|
||||
|
||||
logrus.Infof("同步完成 - 账户数: %d, 总图片数: %d, 成功同步: %d", res.TotalAccounts, totalImages, totalSynced)
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// getAccountList 获取所有账户列表
|
||||
func (s *imageService) getAccountList(ctx context.Context) ([]entity.AccountRelation, error) {
|
||||
var accounts []entity.AccountRelation
|
||||
err := gfdb.DB(ctx).Model(ctx, "tencent_account_relation").
|
||||
WhereNull("deleted_at").
|
||||
Scan(&accounts)
|
||||
|
||||
return accounts, err
|
||||
}
|
||||
|
||||
// syncAccountImages 同步单个账户的图片数据
|
||||
func (s *imageService) syncAccountImages(ctx context.Context, accessToken string, accountId int64) (int, error) {
|
||||
totalSynced := 0
|
||||
|
||||
// 先获取第一页，得到总页数
|
||||
firstPageData, err := s.fetchPage(ctx, accessToken, accountId, 1, 100)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("获取第一页数据失败: %w", err)
|
||||
}
|
||||
|
||||
totalPage := firstPageData.Data.PageInfo.TotalPage
|
||||
logrus.Infof("账户 %d - 总页数: %d, 总记录数: %d", accountId, totalPage, firstPageData.Data.PageInfo.TotalNumber)
|
||||
|
||||
// 处理第一页数据
|
||||
synced, err := s.savePageData(ctx, firstPageData, accountId)
|
||||
if err != nil {
|
||||
logrus.Errorf("保存第一页数据失败: %v", err)
|
||||
}
|
||||
totalSynced += synced
|
||||
|
||||
// 循环获取剩余页
|
||||
for page := 2; page <= totalPage; page++ {
|
||||
logrus.Infof("账户 %d - 正在获取第 %d/%d 页...", accountId, page, totalPage)
|
||||
|
||||
pageData, err := s.fetchPage(ctx, accessToken, accountId, page, 100)
|
||||
if err != nil {
|
||||
logrus.Errorf("账户 %d - 获取第 %d 页失败: %v，继续下一页", accountId, page, err)
|
||||
continue
|
||||
}
|
||||
|
||||
synced, err := s.savePageData(ctx, pageData, accountId)
|
||||
if err != nil {
|
||||
logrus.Errorf("账户 %d - 保存第 %d 页数据失败: %v", accountId, page, err)
|
||||
continue
|
||||
}
|
||||
totalSynced += synced
|
||||
|
||||
// 避免请求过快，休眠100ms
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
|
||||
logrus.Infof("账户 %d - 同步完成，共 %d 条记录", accountId, totalSynced)
|
||||
return totalSynced, nil
|
||||
}
|
||||
|
||||
// fetchPage 获取单页数据
|
||||
func (s *imageService) fetchPage(ctx context.Context, accessToken string, accountId int64, page, pageSize int) (*imageResponse, error) {
|
||||
// 构建filtering参数：状态为正常
|
||||
filtering := `[{"field":"status","operator":"EQUALS","values":["ADSTATUS_NORMAL"]}]`
|
||||
|
||||
// URL编码filtering参数
|
||||
encodedFiltering := url.QueryEscape(filtering)
|
||||
|
||||
// 在发送请求前生成最新的时间戳和nonce，避免时间戳过期
|
||||
timestamp := time.Now().Unix()
|
||||
// 使用时间戳+纳秒后6位+随机数，确保唯一性且不超过32字符
|
||||
nanoSuffix := time.Now().UnixNano() % 1000000 // 取纳秒的后6位
|
||||
nonce := fmt.Sprintf("%d%06d%d", timestamp, nanoSuffix, rand.Intn(1000))
|
||||
|
||||
urlStr := fmt.Sprintf("https://api.e.qq.com/v3.0/images/get?access_token=%s&nonce=%s×tamp=%d&account_id=%d&filtering=%s&page=%d&page_size=%d",
|
||||
accessToken, nonce, timestamp, accountId, encodedFiltering, page, pageSize)
|
||||
|
||||
logrus.Debugf("请求URL: %s", urlStr)
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("创建请求失败: %w", err)
|
||||
}
|
||||
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
resp, err := client.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("请求失败: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("读取响应失败: %w", err)
|
||||
}
|
||||
|
||||
logrus.Debugf("API响应: %s", string(body))
|
||||
|
||||
var result imageResponse
|
||||
if err := json.Unmarshal(body, &result); err != nil {
|
||||
return nil, fmt.Errorf("解析响应失败: %w", err)
|
||||
}
|
||||
|
||||
if result.Code != 0 {
|
||||
return nil, fmt.Errorf("API错误: code=%d, message=%s", result.Code, result.Message)
|
||||
}
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// savePageData 保存单页数据到数据库
|
||||
func (s *imageService) savePageData(ctx context.Context, data *imageResponse, accountId int64) (int, error) {
|
||||
if len(data.Data.List) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
logrus.Infof("准备保存 %d 条图片素材数据", len(data.Data.List))
|
||||
|
||||
var items []*entity.Image
|
||||
for _, item := range data.Data.List {
|
||||
image := &entity.Image{
|
||||
ImageId: item.ImageId,
|
||||
AccountId: accountId,
|
||||
Width: item.Width,
|
||||
Height: item.Height,
|
||||
FileSize: item.FileSize,
|
||||
Type: item.Type,
|
||||
Signature: item.Signature,
|
||||
Description: item.Description,
|
||||
SourceSignature: item.SourceSignature,
|
||||
PreviewUrl: item.PreviewUrl,
|
||||
ThumbPreviewUrl: item.ThumbPreviewUrl,
|
||||
SourceType: item.SourceType,
|
||||
ImageUsage: item.ImageUsage,
|
||||
CreatedTime: item.CreatedTime,
|
||||
LastModifiedTime: item.LastModifiedTime,
|
||||
ProductCatalogId: item.ProductCatalogId,
|
||||
ProductOuterId: item.ProductOuterId,
|
||||
SourceReferenceId: item.SourceReferenceId,
|
||||
OwnerAccountId: item.OwnerAccountId,
|
||||
Status: item.Status,
|
||||
SampleAspectRatio: item.SampleAspectRatio,
|
||||
SourceMaterialId: item.SourceMaterialId,
|
||||
NewSourceType: item.NewSourceType,
|
||||
FirstPublicationStatus: item.FirstPublicationStatus,
|
||||
QualityStatus: item.QualityStatus,
|
||||
SimilarityStatus: item.SimilarityStatus,
|
||||
UserAigcStatus: item.UserAigcStatus,
|
||||
SystemAigcStatus: item.SystemAigcStatus,
|
||||
AigcSource: item.AigcSource,
|
||||
AigcFlag: item.AigcFlag,
|
||||
MuseAigcVersion: item.MuseAigcVersion,
|
||||
AigcType: item.AigcType,
|
||||
}
|
||||
// 设置 TenantID（框架将0视为空值，所以使用1）
|
||||
image.TenantId = 1
|
||||
|
||||
items = append(items, image)
|
||||
}
|
||||
|
||||
logrus.Infof("调用 BatchUpsert...")
|
||||
successCount, err := dao.Image.BatchUpsert(ctx, items)
|
||||
logrus.Infof("BatchUpsert 返回: successCount=%d, err=%v", successCount, err)
|
||||
|
||||
return successCount, err
|
||||
}
|
||||
|
||||
// ListAll 获取所有图片素材
|
||||
func (s *imageService) ListAll(ctx context.Context) ([]entity.Image, error) {
|
||||
return dao.Image.ListAll(ctx)
|
||||
}
|
||||
|
||||
// ListWithPage 分页查询图片素材（支持时间过滤）
|
||||
func (s *imageService) ListWithPage(ctx context.Context, req *dto.ListImageQueryReq) (*dto.ListImageRes, error) {
|
||||
// 设置默认值
|
||||
page := req.Page
|
||||
if page <= 0 {
|
||||
page = 1
|
||||
}
|
||||
pageSize := req.PageSize
|
||||
if pageSize <= 0 {
|
||||
pageSize = 20
|
||||
}
|
||||
if pageSize > 100 {
|
||||
pageSize = 100 // 限制最大每页数量
|
||||
}
|
||||
|
||||
// 调用DAO层查询
|
||||
list, total, err := dao.Image.ListWithPage(ctx, page, pageSize, req.AccountId, req.StartTime, req.EndTime, req.Status)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("查询图片素材失败: %w", err)
|
||||
}
|
||||
|
||||
// 计算总页数
|
||||
totalPages := (total + pageSize - 1) / pageSize
|
||||
if totalPages == 0 && total > 0 {
|
||||
totalPages = 1
|
||||
}
|
||||
|
||||
// 转换为DTO
|
||||
items := make([]dto.ImageItem, 0, len(list))
|
||||
for _, item := range list {
|
||||
items = append(items, dto.ImageItem{
|
||||
Id: item.Id,
|
||||
ImageId: item.ImageId,
|
||||
AccountId: item.AccountId,
|
||||
Width: item.Width,
|
||||
Height: item.Height,
|
||||
FileSize: item.FileSize,
|
||||
Type: item.Type,
|
||||
Signature: item.Signature,
|
||||
Description: item.Description,
|
||||
PreviewUrl: item.PreviewUrl,
|
||||
ThumbPreviewUrl: item.ThumbPreviewUrl,
|
||||
Status: item.Status,
|
||||
CreatedTime: item.CreatedTime,
|
||||
LastModifiedTime: item.LastModifiedTime,
|
||||
CreatedAt: item.CreatedAt.Format("2006-01-02 15:04:05"),
|
||||
UpdatedAt: item.UpdatedAt.Format("2006-01-02 15:04:05"),
|
||||
})
|
||||
}
|
||||
|
||||
res := &dto.ListImageRes{
|
||||
List: items,
|
||||
Total: total,
|
||||
Page: page,
|
||||
PageSize: pageSize,
|
||||
TotalPages: totalPages,
|
||||
}
|
||||
|
||||
logrus.Infof("查询图片素材 - 页码: %d, 每页: %d, 总数: %d, 总页数: %d", page, pageSize, total, totalPages)
|
||||
|
||||
return res, nil
|
||||
}
|
||||
78
service/tencent/oauth_service.go
Normal file
78
service/tencent/oauth_service.go
Normal file
@@ -0,0 +1,78 @@
|
||||
package tencent
|
||||
|
||||
import (
|
||||
"context"
|
||||
dto "dataengine/model/dto/tencent"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
)
|
||||
|
||||
type oauthService struct{}
|
||||
|
||||
var OauthService = new(oauthService)
|
||||
|
||||
// RefreshToken 刷新腾讯广告Token
|
||||
func (s *oauthService) RefreshToken(ctx context.Context, req *dto.RefreshTokenReq) (res *dto.RefreshTokenRes, err error) {
|
||||
// 如果请求中没有提供参数，则从配置文件读取
|
||||
clientID := req.ClientID
|
||||
clientSecret := req.ClientSecret
|
||||
refreshToken := req.RefreshToken
|
||||
|
||||
if clientID == "" || clientSecret == "" || refreshToken == "" {
|
||||
clientID = g.Cfg().MustGet(ctx, "tencent.oauth.client_id").String()
|
||||
clientSecret = g.Cfg().MustGet(ctx, "tencent.oauth.client_secret").String()
|
||||
refreshToken = g.Cfg().MustGet(ctx, "tencent.oauth.refresh_token").String()
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("https://api.e.qq.com/oauth/refresh_token?client_id=%s&client_secret=%s&refresh_token=%s",
|
||||
clientID, clientSecret, refreshToken)
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("创建请求失败: %w", err)
|
||||
}
|
||||
|
||||
client := &http.Client{}
|
||||
resp, err := client.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("请求失败: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("读取响应失败: %w", err)
|
||||
}
|
||||
|
||||
var result struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data struct {
|
||||
AccessToken string `json:"access_token"`
|
||||
RefreshToken string `json:"refresh_token"`
|
||||
AccessTokenExpiresIn int64 `json:"access_token_expires_in"`
|
||||
RefreshTokenExpiresIn int64 `json:"refresh_token_expires_in"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(body, &result); err != nil {
|
||||
return nil, fmt.Errorf("解析响应失败: %w", err)
|
||||
}
|
||||
|
||||
if result.Code != 0 {
|
||||
return nil, fmt.Errorf("API错误: code=%d, message=%s", result.Code, result.Message)
|
||||
}
|
||||
|
||||
res = &dto.RefreshTokenRes{
|
||||
AccessToken: result.Data.AccessToken,
|
||||
RefreshToken: result.Data.RefreshToken,
|
||||
AccessTokenExpiresIn: result.Data.AccessTokenExpiresIn,
|
||||
RefreshTokenExpiresIn: result.Data.RefreshTokenExpiresIn,
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
Reference in New Issue
Block a user