package tencent import ( "context" dao "dataengine/dao/tencent" dto "dataengine/model/dto/tencent" entity "dataengine/model/entity/tencent" "encoding/json" "fmt" "io" "math/rand" "net/http" "net/url" "time" "gitea.com/red-future/common/db/gfdb" "github.com/gogf/gf/v2/frame/g" "github.com/sirupsen/logrus" ) type imageService struct{} var ImageService = new(imageService) // API响应结构 type imageResponse struct { Code int `json:"code"` Message string `json:"message"` Data struct { List []struct { ImageId string `json:"image_id"` Width int `json:"width"` Height int `json:"height"` FileSize int64 `json:"file_size"` Type string `json:"type"` Signature string `json:"signature"` Description string `json:"description"` SourceSignature string `json:"source_signature"` PreviewUrl string `json:"preview_url"` ThumbPreviewUrl string `json:"thumb_preview_url"` SourceType string `json:"source_type"` ImageUsage string `json:"image_usage"` CreatedTime int64 `json:"created_time"` LastModifiedTime int64 `json:"last_modified_time"` ProductCatalogId int64 `json:"product_catalog_id"` ProductOuterId string `json:"product_outer_id"` SourceReferenceId string `json:"source_reference_id"` OwnerAccountId string `json:"owner_account_id"` Status string `json:"status"` SampleAspectRatio string `json:"sample_aspect_ratio"` SourceMaterialId string `json:"source_material_id"` NewSourceType string `json:"new_source_type"` FirstPublicationStatus string `json:"first_publication_status"` QualityStatus string `json:"quality_status"` SimilarityStatus string `json:"similarity_status"` UserAigcStatus string `json:"user_aigc_status"` SystemAigcStatus string `json:"system_aigc_status"` AigcSource string `json:"aigc_source"` AigcFlag string `json:"aigc_flag"` MuseAigcVersion int `json:"muse_aigc_version"` AigcType int `json:"aigc_type"` } `json:"list"` PageInfo struct { Page int `json:"page"` PageSize int `json:"page_size"` TotalNumber int `json:"total_number"` TotalPage int `json:"total_page"` } `json:"page_info"` } `json:"data"` TraceId string `json:"trace_id"` } // SyncAll 同步所有图片素材数据(遍历所有账户,自动分页) func (s *imageService) SyncAll(ctx context.Context, req *dto.SyncImageReq) (res *dto.SyncImageRes, err error) { // 创建独立的context,避免HTTP请求超时导致context被取消 // 设置30分钟超时,足够完成421个账户的同步任务 independentCtx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() // 保留原context中的user信息,供数据库中间件使用 if user := ctx.Value("user"); user != nil { independentCtx = context.WithValue(independentCtx, "user", user) } // 获取access_token accessToken := req.AccessToken if accessToken == "" { accessToken = g.Cfg().MustGet(independentCtx, "tencent.oauth.access_token").String() } if accessToken == "" { return nil, fmt.Errorf("access_token不能为空") } res = &dto.SyncImageRes{} totalSynced := 0 totalImages := 0 // 获取所有账户列表 accounts, err := s.getAccountList(independentCtx) if err != nil { return nil, fmt.Errorf("获取账户列表失败: %w", err) } res.TotalAccounts = len(accounts) logrus.Infof("开始同步腾讯广告图片素材 - 账户数: %d", len(accounts)) // 遍历每个账户 for _, account := range accounts { logrus.Infof("========== 开始处理账户: %d (%s) ==========", account.AccountID, account.CorporationName) // 获取该账户的所有图片(分页) accountImages, err := s.syncAccountImages(independentCtx, accessToken, account.AccountID) if err != nil { logrus.Errorf("账户 %d 同步失败: %v,继续下一个账户", account.AccountID, err) continue } totalImages += accountImages totalSynced += accountImages // 避免请求过快,休眠200ms time.Sleep(200 * time.Millisecond) } res.TotalImages = totalImages res.SyncedCount = totalSynced res.Message = fmt.Sprintf("同步完成,共处理 %d 个账户,%d 条图片记录", res.TotalAccounts, totalSynced) logrus.Infof("同步完成 - 账户数: %d, 总图片数: %d, 成功同步: %d", res.TotalAccounts, totalImages, totalSynced) return res, nil } // getAccountList 获取所有账户列表 func (s *imageService) getAccountList(ctx context.Context) ([]entity.AccountRelation, error) { var accounts []entity.AccountRelation err := gfdb.DB(ctx).Model(ctx, "tencent_account_relation"). WhereNull("deleted_at"). Scan(&accounts) return accounts, err } // syncAccountImages 同步单个账户的图片数据 func (s *imageService) syncAccountImages(ctx context.Context, accessToken string, accountId int64) (int, error) { totalSynced := 0 // 先获取第一页,得到总页数 firstPageData, err := s.fetchPage(ctx, accessToken, accountId, 1, 100) if err != nil { return 0, fmt.Errorf("获取第一页数据失败: %w", err) } totalPage := firstPageData.Data.PageInfo.TotalPage logrus.Infof("账户 %d - 总页数: %d, 总记录数: %d", accountId, totalPage, firstPageData.Data.PageInfo.TotalNumber) // 处理第一页数据 synced, err := s.savePageData(ctx, firstPageData, accountId) if err != nil { logrus.Errorf("保存第一页数据失败: %v", err) } totalSynced += synced // 循环获取剩余页 for page := 2; page <= totalPage; page++ { logrus.Infof("账户 %d - 正在获取第 %d/%d 页...", accountId, page, totalPage) pageData, err := s.fetchPage(ctx, accessToken, accountId, page, 100) if err != nil { logrus.Errorf("账户 %d - 获取第 %d 页失败: %v,继续下一页", accountId, page, err) continue } synced, err := s.savePageData(ctx, pageData, accountId) if err != nil { logrus.Errorf("账户 %d - 保存第 %d 页数据失败: %v", accountId, page, err) continue } totalSynced += synced // 避免请求过快,休眠100ms time.Sleep(100 * time.Millisecond) } logrus.Infof("账户 %d - 同步完成,共 %d 条记录", accountId, totalSynced) return totalSynced, nil } // fetchPage 获取单页数据 func (s *imageService) fetchPage(ctx context.Context, accessToken string, accountId int64, page, pageSize int) (*imageResponse, error) { // 构建filtering参数:状态为正常 filtering := `[{"field":"status","operator":"EQUALS","values":["ADSTATUS_NORMAL"]}]` // URL编码filtering参数 encodedFiltering := url.QueryEscape(filtering) // 在发送请求前生成最新的时间戳和nonce,避免时间戳过期 timestamp := time.Now().Unix() // 使用时间戳+纳秒后6位+随机数,确保唯一性且不超过32字符 nanoSuffix := time.Now().UnixNano() % 1000000 // 取纳秒的后6位 nonce := fmt.Sprintf("%d%06d%d", timestamp, nanoSuffix, rand.Intn(1000)) urlStr := fmt.Sprintf("https://api.e.qq.com/v3.0/images/get?access_token=%s&nonce=%s×tamp=%d&account_id=%d&filtering=%s&page=%d&page_size=%d", accessToken, nonce, timestamp, accountId, encodedFiltering, page, pageSize) logrus.Debugf("请求URL: %s", urlStr) httpReq, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil) if err != nil { return nil, fmt.Errorf("创建请求失败: %w", err) } client := &http.Client{Timeout: 30 * time.Second} resp, err := client.Do(httpReq) if err != nil { return nil, fmt.Errorf("请求失败: %w", err) } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("读取响应失败: %w", err) } logrus.Debugf("API响应: %s", string(body)) var result imageResponse if err := json.Unmarshal(body, &result); err != nil { return nil, fmt.Errorf("解析响应失败: %w", err) } if result.Code != 0 { return nil, fmt.Errorf("API错误: code=%d, message=%s", result.Code, result.Message) } return &result, nil } // savePageData 保存单页数据到数据库 func (s *imageService) savePageData(ctx context.Context, data *imageResponse, accountId int64) (int, error) { if len(data.Data.List) == 0 { return 0, nil } logrus.Infof("准备保存 %d 条图片素材数据", len(data.Data.List)) var items []*entity.Image for _, item := range data.Data.List { image := &entity.Image{ ImageId: item.ImageId, AccountId: accountId, Width: item.Width, Height: item.Height, FileSize: item.FileSize, Type: item.Type, Signature: item.Signature, Description: item.Description, SourceSignature: item.SourceSignature, PreviewUrl: item.PreviewUrl, ThumbPreviewUrl: item.ThumbPreviewUrl, SourceType: item.SourceType, ImageUsage: item.ImageUsage, CreatedTime: item.CreatedTime, LastModifiedTime: item.LastModifiedTime, ProductCatalogId: item.ProductCatalogId, ProductOuterId: item.ProductOuterId, SourceReferenceId: item.SourceReferenceId, OwnerAccountId: item.OwnerAccountId, Status: item.Status, SampleAspectRatio: item.SampleAspectRatio, SourceMaterialId: item.SourceMaterialId, NewSourceType: item.NewSourceType, FirstPublicationStatus: item.FirstPublicationStatus, QualityStatus: item.QualityStatus, SimilarityStatus: item.SimilarityStatus, UserAigcStatus: item.UserAigcStatus, SystemAigcStatus: item.SystemAigcStatus, AigcSource: item.AigcSource, AigcFlag: item.AigcFlag, MuseAigcVersion: item.MuseAigcVersion, AigcType: item.AigcType, } // 设置 TenantID(框架将0视为空值,所以使用1) image.TenantId = 1 // 设置默认校验状态为待校验 image.VerifyStatus = "PENDING" items = append(items, image) } logrus.Infof("调用 BatchUpsert...") successCount, err := dao.Image.BatchUpsert(ctx, items) logrus.Infof("BatchUpsert 返回: successCount=%d, err=%v", successCount, err) return successCount, err } // ListAll 获取所有图片素材 func (s *imageService) ListAll(ctx context.Context) ([]entity.Image, error) { return dao.Image.ListAll(ctx) } // ListWithPage 分页查询图片素材(支持时间过滤) func (s *imageService) ListWithPage(ctx context.Context, req *dto.ListImageQueryReq) (*dto.ListImageRes, error) { // 设置默认值 page := req.Page if page <= 0 { page = 1 } pageSize := req.PageSize if pageSize <= 0 { pageSize = 20 } if pageSize > 100 { pageSize = 100 // 限制最大每页数量 } // 调用DAO层查询 list, total, err := dao.Image.ListWithPage(ctx, page, pageSize, req.AccountId, req.StartTime, req.EndTime, req.Status) if err != nil { return nil, fmt.Errorf("查询图片素材失败: %w", err) } // 计算总页数 totalPages := (total + pageSize - 1) / pageSize if totalPages == 0 && total > 0 { totalPages = 1 } // 转换为DTO items := make([]dto.ImageItem, 0, len(list)) for _, item := range list { items = append(items, dto.ImageItem{ Id: item.Id, ImageId: item.ImageId, AccountId: item.AccountId, Width: item.Width, Height: item.Height, FileSize: item.FileSize, Type: item.Type, Signature: item.Signature, Description: item.Description, PreviewUrl: item.PreviewUrl, ThumbPreviewUrl: item.ThumbPreviewUrl, Status: item.Status, CreatedTime: item.CreatedTime, LastModifiedTime: item.LastModifiedTime, CreatedAt: item.CreatedAt.Format("2006-01-02 15:04:05"), UpdatedAt: item.UpdatedAt.Format("2006-01-02 15:04:05"), }) } res := &dto.ListImageRes{ List: items, Total: total, Page: page, PageSize: pageSize, TotalPages: totalPages, } logrus.Infof("查询图片素材 - 页码: %d, 每页: %d, 总数: %d, 总页数: %d", page, pageSize, total, totalPages) return res, nil }