red-future/data-engine
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

同步视频

Browse Source
This commit is contained in:
李明康
2026-05-06 17:29:09 +08:00
parent 162bab15e6
commit a1a259733d
7 changed files with 958 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
consts/public/collections.go
View File

@@ -22,4 +22,5 @@ const (
TencentAccountRelationTable = "tencent_account_relation" // 腾讯广告账户关系表
TencentAudioTable = "tencent_audio" // 腾讯广告音乐素材表
TencentImageTable = "tencent_image" // 腾讯广告图片素材表
TencentVideoTable = "tencent_video" // 腾讯广告视频素材表
)

27
controller/tencent/oauth_controller.go
View File

@@ -92,3 +92,30 @@ func (c *oauthController) ListImagePage(ctx context.Context, req *dto.ListImageP
}
return service.ImageService.ListWithPage(ctx, queryReq)
}
// SyncVideo 同步视频素材（遍历所有账户，自动分页）
func (c *oauthController) SyncVideo(ctx context.Context, req *dto.SyncVideoReq) (res *dto.SyncVideoRes, err error) {
ctx = context.WithValue(ctx, "user", &beans.User{UserName: "admin"})
return service.VideoService.SyncAll(ctx, req)
}
// ListVideo 获取所有视频素材（旧接口，保留兼容）
func (c *oauthController) ListVideo(ctx context.Context, req *dto.ListVideoReq) (res []entity.Video, err error) {
ctx = context.WithValue(ctx, "user", &beans.User{UserName: "admin"})
return service.VideoService.ListAll(ctx)
}
// ListVideoPage 分页查询视频素材（支持时间过滤）
func (c *oauthController) ListVideoPage(ctx context.Context, req *dto.ListVideoPageReq) (res *dto.ListVideoRes, err error) {
ctx = context.WithValue(ctx, "user", &beans.User{UserName: "admin"})
// 转换请求参数为Service层使用的类型
queryReq := &dto.ListVideoQueryReq{
Page: req.Page,
PageSize: req.PageSize,
AccountId: req.AccountId,
StartTime: req.StartTime,
EndTime: req.EndTime,
Status: req.Status,
}
return service.VideoService.ListWithPage(ctx, queryReq)
}

158
dao/tencent/video_dao.go Normal file
View File

@@ -0,0 +1,158 @@
package tencent
import (
"context"
consts "dataengine/consts/public"
entity "dataengine/model/entity/tencent"
"gitea.com/red-future/common/db/gfdb"
"github.com/gogf/gf/v2/frame/g"
"github.com/sirupsen/logrus"
)
type videoDao struct{}
var Video = new(videoDao)
// BatchUpsert 批量插入或更新（使用 OnConflict 实现 Upsert）
func (d *videoDao) BatchUpsert(ctx context.Context, items []*entity.Video) (successCount int, err error) {
if len(items) == 0 {
return 0, nil
}
logrus.Infof("开始批量Upsert视频素材: %d 条记录", len(items))
// 分批处理，每批100条
batchSize := 100
successCount = 0
for i := 0; i < len(items); i += batchSize {
end := i + batchSize
if end > len(items) {
end = len(items)
}
batch := items[i:end]
logrus.Infof("处理第 %d-%d 条视频素材记录", i+1, end)
// 执行批量插入，使用 OnConflict 实现 Upsert
result, err := gfdb.DB(ctx).Model(ctx, consts.TencentVideoTable).
Data(batch).
OnConflict("(video_id, account_id)").
Save()
if err != nil {
logrus.Errorf("批量Upsert视频素材失败: %v，尝试逐条处理", err)
// 批量失败，逐条处理
for _, item := range batch {
if upsertErr := d.upsertSingle(ctx, item); upsertErr != nil {
logrus.Errorf("逐条Upsert视频素材失败: video_id=%s, account_id=%d, err=%v", item.VideoId, item.AccountId, upsertErr)
} else {
successCount++
}
}
} else {
affected, _ := result.RowsAffected()
successCount += int(affected)
logrus.Infof("批量Upsert视频素材成功: 影响 %d 条记录", affected)
}
}
logrus.Infof("批量Upsert视频素材完成: 成功 %d 条", successCount)
return successCount, nil
}
// upsertSingle 单条插入或更新
func (d *videoDao) upsertSingle(ctx context.Context, item *entity.Video) error {
var existing entity.Video
err := gfdb.DB(ctx).Model(ctx, consts.TencentVideoTable).
Where(entity.VideoCols.VideoId, item.VideoId).
Where(entity.VideoCols.AccountId, item.AccountId).
WhereNull("deleted_at").
Scan(&existing)
if err != nil && existing.Id == 0 {
// 记录不存在，执行插入
_, err = gfdb.DB(ctx).Model(ctx, consts.TencentVideoTable).
Data(item).
Insert()
return err
}
// 记录存在，执行更新
_, err = gfdb.DB(ctx).Model(ctx, consts.TencentVideoTable).
Where("id", existing.Id).
Data(g.Map{
entity.VideoCols.Width: item.Width,
entity.VideoCols.Height: item.Height,
entity.VideoCols.FileSize: item.FileSize,
entity.VideoCols.Type: item.Type,
entity.VideoCols.Signature: item.Signature,
entity.VideoCols.Description: item.Description,
entity.VideoCols.PreviewUrl: item.PreviewUrl,
entity.VideoCols.Status: item.Status,
entity.VideoCols.LastModifiedTime: item.LastModifiedTime,
}).
Update()
return err
}
// ListAll 获取所有视频素材
func (d *videoDao) ListAll(ctx context.Context) ([]entity.Video, error) {
var list []entity.Video
err := gfdb.DB(ctx).Model(ctx, consts.TencentVideoTable).
WhereNull("deleted_at").
OrderAsc(entity.VideoCols.VideoId).
Scan(&list)
return list, err
}
// ListWithPage 分页查询视频素材（支持时间过滤）
func (d *videoDao) ListWithPage(ctx context.Context, page, pageSize int, accountId *int64, startTime, endTime *int64, status string) ([]entity.Video, int, error) {
model := gfdb.DB(ctx).Model(ctx, consts.TencentVideoTable).
WhereNull("deleted_at")
// 账户ID过滤
if accountId != nil && *accountId > 0 {
model = model.Where(entity.VideoCols.AccountId, *accountId)
}
// 状态过滤
if status != "" {
model = model.Where(entity.VideoCols.Status, status)
}
// 时间范围过滤（根据 last_modified_time）
if startTime != nil && *startTime > 0 {
model = model.WhereGTE(entity.VideoCols.LastModifiedTime, *startTime)
}
if endTime != nil && *endTime > 0 {
model = model.WhereLTE(entity.VideoCols.LastModifiedTime, *endTime)
}
// 设置排序（按最后修改时间降序）
model = model.OrderDesc(entity.VideoCols.LastModifiedTime)
// 获取总数
total, err := model.Count()
if err != nil {
return nil, 0, err
}
// 分页查询
var list []entity.Video
if page > 0 && pageSize > 0 {
err = model.Page(page, pageSize).Scan(&list)
} else {
err = model.Scan(&list)
}
if err != nil {
return nil, 0, err
}
return list, total, nil
}

73
model/dto/tencent/video_dto.go Normal file
View File

@@ -0,0 +1,73 @@
package tencent
import "github.com/gogf/gf/v2/frame/g"
// SyncVideoReq 同步视频素材请求
type SyncVideoReq struct {
g.Meta `path:"/syncVideo" method:"post" tags:"腾讯广告视频素材" summary:"同步视频素材" dc:"遍历所有账户，自动分页获取视频素材并保存到数据库"`
AccessToken string `json:"access_token" dc:"访问令牌（可选，不传则从配置读取）"`
}
// SyncVideoRes 同步视频素材响应
type SyncVideoRes struct {
TotalAccounts int `json:"total_accounts" dc:"处理的账户数"`
TotalVideos int `json:"total_videos" dc:"总视频数"`
SyncedCount int `json:"synced_count" dc:"同步成功数量"`
Message string `json:"message" dc:"消息"`
}
// ListVideoReq 获取视频素材列表请求（旧接口，无分页）
type ListVideoReq struct {
g.Meta `path:"/listVideo" method:"post" tags:"腾讯广告视频素材" summary:"获取视频素材列表" dc:"从本地数据库查询所有视频素材（无分页）"`
}
// ListVideoPageReq 分页查询视频素材请求
type ListVideoPageReq struct {
g.Meta `path:"/listVideoPage" method:"post" tags:"腾讯广告视频素材" summary:"分页查询视频素材" dc:"支持分页、时间过滤、账户过滤等条件查询"`
Page int `json:"page" dc:"页码" d:"1"`
PageSize int `json:"page_size" dc:"每页数量" d:"20"`
AccountId *int64 `json:"account_id,omitempty" dc:"账户ID（可选）"`
StartTime *int64 `json:"start_time,omitempty" dc:"开始时间戳（秒，可选）"`
EndTime *int64 `json:"end_time,omitempty" dc:"结束时间戳（秒，可选）"`
Status string `json:"status,omitempty" dc:"状态筛选（可选）"`
}
// ListVideoQueryReq 视频素材查询请求（Service层使用）
type ListVideoQueryReq struct {
Page int `json:"page" dc:"页码"`
PageSize int `json:"page_size" dc:"每页数量"`
AccountId *int64 `json:"account_id,omitempty" dc:"账户ID（可选）"`
StartTime *int64 `json:"start_time,omitempty" dc:"开始时间戳（秒，可选）"`
EndTime *int64 `json:"end_time,omitempty" dc:"结束时间戳（秒，可选）"`
Status string `json:"status,omitempty" dc:"状态筛选（可选）"`
}
// ListVideoRes 获取视频素材列表响应
type ListVideoRes struct {
List []VideoItem `json:"list" dc:"视频素材列表"`
Total int `json:"total" dc:"总记录数"`
Page int `json:"page" dc:"当前页码"`
PageSize int `json:"page_size" dc:"每页数量"`
TotalPages int `json:"total_pages" dc:"总页数"`
}
// VideoItem 视频素材项
type VideoItem struct {
Id int64 `json:"id" dc:"主键ID"`
VideoId string `json:"video_id" dc:"视频ID"`
AccountId int64 `json:"account_id" dc:"账户ID"`
Width int `json:"width" dc:"宽度"`
Height int `json:"height" dc:"高度"`
VideoFrames int `json:"video_frames" dc:"视频帧数"`
VideoFps int `json:"video_fps" dc:"帧率"`
FileSize int64 `json:"file_size" dc:"文件大小"`
Type string `json:"type" dc:"媒体类型"`
Description string `json:"description" dc:"描述"`
PreviewUrl string `json:"preview_url" dc:"预览URL"`
KeyFrameImageUrl string `json:"key_frame_image_url" dc:"关键帧图片URL"`
Status string `json:"status" dc:"状态"`
CreatedTime int64 `json:"created_time" dc:"创建时间戳"`
LastModifiedTime int64 `json:"last_modified_time" dc:"最后修改时间戳"`
CreatedAt string `json:"created_at" dc:"数据库创建时间"`
UpdatedAt string `json:"updated_at" dc:"数据库更新时间"`
}

159
model/entity/tencent/video.go Normal file
View File

@@ -0,0 +1,159 @@
package tencent
import (
"gitea.com/red-future/common/beans"
)
// Video 腾讯广告视频素材实体
type Video struct {
beans.SQLBaseDO `orm:",inherit"`
VideoId string `orm:"video_id" json:"videoId" description:"视频ID"`
AccountId int64 `orm:"account_id" json:"accountId" description:"账户ID"`
Width int `orm:"width" json:"width" description:"宽度"`
Height int `orm:"height" json:"height" description:"高度"`
VideoFrames int `orm:"video_frames" json:"videoFrames" description:"视频帧数"`
VideoFps int `orm:"video_fps" json:"videoFps" description:"帧率"`
VideoCodec string `orm:"video_codec" json:"videoCodec" description:"视频编码"`
VideoBitRate int64 `orm:"video_bit_rate" json:"videoBitRate" description:"视频码率"`
AudioCodec string `orm:"audio_codec" json:"audioCodec" description:"音频编码"`
AudioBitRate int64 `orm:"audio_bit_rate" json:"audioBitRate" description:"音频码率"`
FileSize int64 `orm:"file_size" json:"fileSize" description:"文件大小"`
Type string `orm:"type" json:"type" description:"媒体类型"`
Signature string `orm:"signature" json:"signature" description:"签名"`
SystemStatus string `orm:"system_status" json:"systemStatus" description:"系统状态"`
Description string `orm:"description" json:"description" description:"描述"`
PreviewUrl string `orm:"preview_url" json:"previewUrl" description:"预览URL"`
KeyFrameImageUrl string `orm:"key_frame_image_url" json:"keyFrameImageUrl" description:"关键帧图片URL"`
CreatedTime int64 `orm:"created_time" json:"createdTime" description:"创建时间戳"`
LastModifiedTime int64 `orm:"last_modified_time" json:"lastModifiedTime" description:"最后修改时间戳"`
VideoProfileName string `orm:"video_profile_name" json:"videoProfileName" description:"视频配置名称"`
AudioSampleRate int `orm:"audio_sample_rate" json:"audioSampleRate" description:"音频采样率"`
MaxKeyframeInterval int `orm:"max_keyframe_interval" json:"maxKeyframeInterval" description:"最大关键帧间隔"`
MinKeyframeInterval int `orm:"min_keyframe_interval" json:"minKeyframeInterval" description:"最小关键帧间隔"`
SampleAspectRatio string `orm:"sample_aspect_ratio" json:"sampleAspectRatio" description:"示例宽高比"`
AudioProfileName string `orm:"audio_profile_name" json:"audioProfileName" description:"音频配置名称"`
ScanType string `orm:"scan_type" json:"scanType" description:"扫描类型"`
ImageDurationMillisecond int64 `orm:"image_duration_millisecond" json:"imageDurationMillisecond" description:"图片时长（毫秒）"`
AudioDurationMillisecond int64 `orm:"audio_duration_millisecond" json:"audioDurationMillisecond" description:"音频时长（毫秒）"`
SourceType string `orm:"source_type" json:"sourceType" description:"来源类型"`
ProductCatalogId string `orm:"product_catalog_id" json:"productCatalogId" description:"产品目录ID"`
ProductOuterId string `orm:"product_outer_id" json:"productOuterId" description:"产品外部ID"`
SourceReferenceId string `orm:"source_reference_id" json:"sourceReferenceId" description:"源引用ID"`
OwnerAccountId string `orm:"owner_account_id" json:"ownerAccountId" description:"所有者账户ID"`
Status string `orm:"status" json:"status" description:"状态"`
SourceMaterialId string `orm:"source_material_id" json:"sourceMaterialId" description:"源素材ID"`
NewSourceType string `orm:"new_source_type" json:"newSourceType" description:"新来源类型"`
AigcType int `orm:"aigc_type" json:"aigcType" description:"AIGC类型"`
FirstPublicationStatus string `orm:"first_publication_status" json:"firstPublicationStatus" description:"首次发布状态"`
QualityStatus string `orm:"quality_status" json:"qualityStatus" description:"质量状态"`
CoverId string `orm:"cover_id" json:"coverId" description:"封面ID"`
SimilarityStatus string `orm:"similarity_status" json:"similarityStatus" description:"相似度状态"`
UserAigcStatus string `orm:"user_aigc_status" json:"userAigcStatus" description:"用户AIGC状态"`
SystemAigcStatus string `orm:"system_aigc_status" json:"systemAigcStatus" description:"系统AIGC状态"`
AigcSource string `orm:"aigc_source" json:"aigcSource" description:"AIGC来源"`
AigcFlag string `orm:"aigc_flag" json:"aigcFlag" description:"AIGC标志"`
MuseAigcVersion int `orm:"muse_aigc_version" json:"museAigcVersion" description:"Muse AIGC版本"`
}
// VideoCol 视频素材表字段定义
type VideoCol struct {
beans.SQLBaseCol
VideoId string
AccountId string
Width string
Height string
VideoFrames string
VideoFps string
VideoCodec string
VideoBitRate string
AudioCodec string
AudioBitRate string
FileSize string
Type string
Signature string
SystemStatus string
Description string
PreviewUrl string
KeyFrameImageUrl string
CreatedTime string
LastModifiedTime string
VideoProfileName string
AudioSampleRate string
MaxKeyframeInterval string
MinKeyframeInterval string
SampleAspectRatio string
AudioProfileName string
ScanType string
ImageDurationMillisecond string
AudioDurationMillisecond string
SourceType string
ProductCatalogId string
ProductOuterId string
SourceReferenceId string
OwnerAccountId string
Status string
SourceMaterialId string
NewSourceType string
AigcType string
FirstPublicationStatus string
QualityStatus string
CoverId string
SimilarityStatus string
UserAigcStatus string
SystemAigcStatus string
AigcSource string
AigcFlag string
MuseAigcVersion string
}
// VideoCols 视频素材表字段常量
var VideoCols = VideoCol{
SQLBaseCol: beans.DefSQLBaseCol,
VideoId: "video_id",
AccountId: "account_id",
Width: "width",
Height: "height",
VideoFrames: "video_frames",
VideoFps: "video_fps",
VideoCodec: "video_codec",
VideoBitRate: "video_bit_rate",
AudioCodec: "audio_codec",
AudioBitRate: "audio_bit_rate",
FileSize: "file_size",
Type: "type",
Signature: "signature",
SystemStatus: "system_status",
Description: "description",
PreviewUrl: "preview_url",
KeyFrameImageUrl: "key_frame_image_url",
CreatedTime: "created_time",
LastModifiedTime: "last_modified_time",
VideoProfileName: "video_profile_name",
AudioSampleRate: "audio_sample_rate",
MaxKeyframeInterval: "max_keyframe_interval",
MinKeyframeInterval: "min_keyframe_interval",
SampleAspectRatio: "sample_aspect_ratio",
AudioProfileName: "audio_profile_name",
ScanType: "scan_type",
ImageDurationMillisecond: "image_duration_millisecond",
AudioDurationMillisecond: "audio_duration_millisecond",
SourceType: "source_type",
ProductCatalogId: "product_catalog_id",
ProductOuterId: "product_outer_id",
SourceReferenceId: "source_reference_id",
OwnerAccountId: "owner_account_id",
Status: "status",
SourceMaterialId: "source_material_id",
NewSourceType: "new_source_type",
AigcType: "aigc_type",
FirstPublicationStatus: "first_publication_status",
QualityStatus: "quality_status",
CoverId: "cover_id",
SimilarityStatus: "similarity_status",
UserAigcStatus: "user_aigc_status",
SystemAigcStatus: "system_aigc_status",
AigcSource: "aigc_source",
AigcFlag: "aigc_flag",
MuseAigcVersion: "muse_aigc_version",
}

417
service/tencent/video_service.go Normal file
View File

@@ -0,0 +1,417 @@
package tencent
import (
"context"
dao "dataengine/dao/tencent"
dto "dataengine/model/dto/tencent"
entity "dataengine/model/entity/tencent"
"encoding/json"
"fmt"
"io"
"math/rand"
"net/http"
"net/url"
"time"
"gitea.com/red-future/common/db/gfdb"
"github.com/gogf/gf/v2/frame/g"
"github.com/sirupsen/logrus"
)
type videoService struct{}
var VideoService = new(videoService)
// API响应结构
type videoResponse struct {
Code int `json:"code"`
Message string `json:"message"`
Data struct {
List []struct {
VideoId int64 `json:"video_id"`
Width int `json:"width"`
Height int `json:"height"`
VideoFrames int `json:"video_frames"`
VideoFps int `json:"video_fps"`
VideoCodec string `json:"video_codec"`
VideoBitRate int64 `json:"video_bit_rate"`
AudioCodec string `json:"audio_codec"`
AudioBitRate int64 `json:"audio_bit_rate"`
FileSize int64 `json:"file_size"`
Type string `json:"type"`
Signature string `json:"signature"`
SystemStatus string `json:"system_status"`
Description string `json:"description"`
PreviewUrl string `json:"preview_url"`
KeyFrameImageUrl string `json:"key_frame_image_url"`
CreatedTime int64 `json:"created_time"`
LastModifiedTime int64 `json:"last_modified_time"`
VideoProfileName string `json:"video_profile_name"`
AudioSampleRate int `json:"audio_sample_rate"`
MaxKeyframeInterval int `json:"max_keyframe_interval"`
MinKeyframeInterval int `json:"min_keyframe_interval"`
SampleAspectRatio string `json:"sample_aspect_ratio"`
AudioProfileName string `json:"audio_profile_name"`
ScanType string `json:"scan_type"`
ImageDurationMs int64 `json:"image_duration_millisecond"`
AudioDurationMs int64 `json:"audio_duration_millisecond"`
SourceType string `json:"source_type"`
ProductCatalogId string `json:"product_catalog_id"`
ProductOuterId string `json:"product_outer_id"`
SourceReferenceId string `json:"source_reference_id"`
OwnerAccountId string `json:"owner_account_id"`
Status string `json:"status"`
SourceMaterialId string `json:"source_material_id"`
NewSourceType string `json:"new_source_type"`
AigcType int `json:"aigc_type"`
FirstPublicationStatus string `json:"first_publication_status"`
QualityStatus string `json:"quality_status"`
CoverId string `json:"cover_id"`
SimilarityStatus string `json:"similarity_status"`
UserAigcStatus string `json:"user_aigc_status"`
SystemAigcStatus string `json:"system_aigc_status"`
AigcSource string `json:"aigc_source"`
AigcFlag string `json:"aigc_flag"`
MuseAigcVersion int `json:"muse_aigc_version"`
} `json:"list"`
PageInfo struct {
Page int `json:"page"`
PageSize int `json:"page_size"`
TotalNumber int `json:"total_number"`
TotalPage int `json:"total_page"`
} `json:"page_info"`
} `json:"data"`
TraceId string `json:"trace_id"`
}
// SyncAll 同步所有视频素材数据（遍历所有账户，自动分页）
func (s *videoService) SyncAll(ctx context.Context, req *dto.SyncVideoReq) (res *dto.SyncVideoRes, err error) {
// 创建独立的context，避免HTTP请求超时导致context被取消
// 设置30分钟超时，足够完成所有账户的同步任务
independentCtx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
defer cancel()
// 保留原context中的user信息，供数据库中间件使用
if user := ctx.Value("user"); user != nil {
independentCtx = context.WithValue(independentCtx, "user", user)
}
// 获取access_token
accessToken := req.AccessToken
if accessToken == "" {
accessToken = g.Cfg().MustGet(independentCtx, "tencent.oauth.access_token").String()
}
if accessToken == "" {
return nil, fmt.Errorf("access_token不能为空")
}
res = &dto.SyncVideoRes{}
totalSynced := 0
totalVideos := 0
// 获取所有账户列表
accounts, err := s.getAccountList(independentCtx)
if err != nil {
return nil, fmt.Errorf("获取账户列表失败: %w", err)
}
res.TotalAccounts = len(accounts)
logrus.Infof("开始同步腾讯广告视频素材 - 账户数: %d", len(accounts))
// 遍历每个账户
for _, account := range accounts {
logrus.Infof("========== 开始处理账户: %d (%s) ==========", account.AccountID, account.CorporationName)
// 获取该账户的所有视频（分页）
accountVideos, err := s.syncAccountVideos(independentCtx, accessToken, account.AccountID)
if err != nil {
logrus.Errorf("账户 %d 同步失败: %v，继续下一个账户", account.AccountID, err)
continue
}
totalVideos += accountVideos
totalSynced += accountVideos
// 避免请求过快，休眠200ms
time.Sleep(200 * time.Millisecond)
}
res.TotalVideos = totalVideos
res.SyncedCount = totalSynced
res.Message = fmt.Sprintf("同步完成，共处理 %d 个账户，%d 条视频记录", res.TotalAccounts, totalSynced)
logrus.Infof("同步完成 - 账户数: %d, 总视频数: %d, 成功同步: %d", res.TotalAccounts, totalVideos, totalSynced)
return res, nil
}
// getAccountList 获取所有账户列表
func (s *videoService) getAccountList(ctx context.Context) ([]entity.AccountRelation, error) {
var accounts []entity.AccountRelation
err := gfdb.DB(ctx).Model(ctx, "tencent_account_relation").
WhereNull("deleted_at").
Scan(&accounts)
return accounts, err
}
// syncAccountVideos 同步单个账户的视频数据
func (s *videoService) syncAccountVideos(ctx context.Context, accessToken string, accountId int64) (int, error) {
totalSynced := 0
// 先获取第一页，得到总页数
firstPageData, err := s.fetchPage(ctx, accessToken, accountId, 1, 100)
if err != nil {
// 如果是请求失败或API错误，返回友好的提示
errMsg := err.Error()
if contains(errMsg, "请求失败") || contains(errMsg, "API错误") {
return 0, fmt.Errorf("该账户没有视频或无法访问")
}
return 0, fmt.Errorf("获取第一页数据失败: %w", err)
}
totalPage := firstPageData.Data.PageInfo.TotalPage
logrus.Infof("账户 %d - 总页数: %d, 总记录数: %d", accountId, totalPage, firstPageData.Data.PageInfo.TotalNumber)
// 如果没有数据，直接返回
if totalPage == 0 || firstPageData.Data.PageInfo.TotalNumber == 0 {
logrus.Infof("账户 %d - 没有视频数据", accountId)
return 0, nil
}
// 处理第一页数据
synced, err := s.savePageData(ctx, firstPageData, accountId)
if err != nil {
logrus.Errorf("保存第一页数据失败: %v", err)
}
totalSynced += synced
// 循环获取剩余页
for page := 2; page <= totalPage; page++ {
logrus.Infof("账户 %d - 正在获取第 %d/%d 页...", accountId, page, totalPage)
pageData, err := s.fetchPage(ctx, accessToken, accountId, page, 100)
if err != nil {
logrus.Errorf("账户 %d - 获取第 %d 页失败: %v，继续下一页", accountId, page, err)
continue
}
synced, err := s.savePageData(ctx, pageData, accountId)
if err != nil {
logrus.Errorf("账户 %d - 保存第 %d 页数据失败: %v", accountId, page, err)
continue
}
totalSynced += synced
// 避免请求过快，休眠100ms
time.Sleep(100 * time.Millisecond)
}
logrus.Infof("账户 %d - 同步完成，共 %d 条记录", accountId, totalSynced)
return totalSynced, nil
}
// fetchPage 获取单页数据
func (s *videoService) fetchPage(ctx context.Context, accessToken string, accountId int64, page, pageSize int) (*videoResponse, error) {
// 构建filtering参数：状态为正常
filtering := `[{"field":"status","operator":"EQUALS","values":["ADSTATUS_NORMAL"]}]`
// URL编码filtering参数
encodedFiltering := url.QueryEscape(filtering)
// 在发送请求前生成最新的时间戳和nonce，避免时间戳过期
timestamp := time.Now().Unix()
// 使用时间戳+纳秒后6位+随机数，确保唯一性且不超过32字符
nanoSuffix := time.Now().UnixNano() % 1000000 // 取纳秒的后6位
nonce := fmt.Sprintf("%d%06d%d", timestamp, nanoSuffix, rand.Intn(1000))
urlStr := fmt.Sprintf("https://api.e.qq.com/v3.0/videos/get?access_token=%s&nonce=%s&timestamp=%d&account_id=%d&filtering=%s&page=%d&page_size=%d",
accessToken, nonce, timestamp, accountId, encodedFiltering, page, pageSize)
logrus.Debugf("请求URL: %s", urlStr)
httpReq, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
if err != nil {
return nil, fmt.Errorf("创建请求失败: %w", err)
}
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Do(httpReq)
if err != nil {
return nil, fmt.Errorf("请求失败: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("读取响应失败: %w", err)
}
logrus.Debugf("API响应: %s", string(body))
var result videoResponse
if err := json.Unmarshal(body, &result); err != nil {
return nil, fmt.Errorf("解析响应失败: %w", err)
}
if result.Code != 0 {
return nil, fmt.Errorf("API错误: code=%d, message=%s", result.Code, result.Message)
}
return &result, nil
}
// savePageData 保存单页数据到数据库
func (s *videoService) savePageData(ctx context.Context, data *videoResponse, accountId int64) (int, error) {
if len(data.Data.List) == 0 {
return 0, nil
}
logrus.Infof("准备保存 %d 条视频素材数据", len(data.Data.List))
var items []*entity.Video
for _, item := range data.Data.List {
video := &entity.Video{
VideoId: fmt.Sprintf("%d", item.VideoId),
AccountId: accountId,
Width: item.Width,
Height: item.Height,
VideoFrames: item.VideoFrames,
VideoFps: item.VideoFps,
VideoCodec: item.VideoCodec,
VideoBitRate: item.VideoBitRate,
AudioCodec: item.AudioCodec,
AudioBitRate: item.AudioBitRate,
FileSize: item.FileSize,
Type: item.Type,
Signature: item.Signature,
SystemStatus: item.SystemStatus,
Description: item.Description,
PreviewUrl: item.PreviewUrl,
KeyFrameImageUrl: item.KeyFrameImageUrl,
CreatedTime: item.CreatedTime,
LastModifiedTime: item.LastModifiedTime,
VideoProfileName: item.VideoProfileName,
AudioSampleRate: item.AudioSampleRate,
MaxKeyframeInterval: item.MaxKeyframeInterval,
MinKeyframeInterval: item.MinKeyframeInterval,
SampleAspectRatio: item.SampleAspectRatio,
AudioProfileName: item.AudioProfileName,
ScanType: item.ScanType,
ImageDurationMillisecond: item.ImageDurationMs,
AudioDurationMillisecond: item.AudioDurationMs,
SourceType: item.SourceType,
ProductCatalogId: item.ProductCatalogId,
ProductOuterId: item.ProductOuterId,
SourceReferenceId: item.SourceReferenceId,
OwnerAccountId: item.OwnerAccountId,
Status: item.Status,
SourceMaterialId: item.SourceMaterialId,
NewSourceType: item.NewSourceType,
AigcType: item.AigcType,
FirstPublicationStatus: item.FirstPublicationStatus,
QualityStatus: item.QualityStatus,
CoverId: item.CoverId,
SimilarityStatus: item.SimilarityStatus,
UserAigcStatus: item.UserAigcStatus,
SystemAigcStatus: item.SystemAigcStatus,
AigcSource: item.AigcSource,
AigcFlag: item.AigcFlag,
MuseAigcVersion: item.MuseAigcVersion,
}
// 设置 TenantID（框架将0视为空值，所以使用1）
video.TenantId = 1
items = append(items, video)
}
logrus.Infof("调用 BatchUpsert...")
successCount, err := dao.Video.BatchUpsert(ctx, items)
logrus.Infof("BatchUpsert 返回: successCount=%d, err=%v", successCount, err)
return successCount, err
}
// ListAll 获取所有视频素材
func (s *videoService) ListAll(ctx context.Context) ([]entity.Video, error) {
return dao.Video.ListAll(ctx)
}
// ListWithPage 分页查询视频素材（支持时间过滤）
func (s *videoService) ListWithPage(ctx context.Context, req *dto.ListVideoQueryReq) (*dto.ListVideoRes, error) {
// 设置默认值
page := req.Page
if page <= 0 {
page = 1
}
pageSize := req.PageSize
if pageSize <= 0 {
pageSize = 20
}
if pageSize > 100 {
pageSize = 100 // 限制最大每页数量
}
// 调用DAO层查询
list, total, err := dao.Video.ListWithPage(ctx, page, pageSize, req.AccountId, req.StartTime, req.EndTime, req.Status)
if err != nil {
return nil, fmt.Errorf("查询视频素材失败: %w", err)
}
// 计算总页数
totalPages := (total + pageSize - 1) / pageSize
if totalPages == 0 && total > 0 {
totalPages = 1
}
// 转换为DTO
items := make([]dto.VideoItem, 0, len(list))
for _, item := range list {
items = append(items, dto.VideoItem{
Id: item.Id,
VideoId: item.VideoId,
AccountId: item.AccountId,
Width: item.Width,
Height: item.Height,
VideoFrames: item.VideoFrames,
VideoFps: item.VideoFps,
FileSize: item.FileSize,
Type: item.Type,
Description: item.Description,
PreviewUrl: item.PreviewUrl,
KeyFrameImageUrl: item.KeyFrameImageUrl,
Status: item.Status,
CreatedTime: item.CreatedTime,
LastModifiedTime: item.LastModifiedTime,
CreatedAt: item.CreatedAt.Format("2006-01-02 15:04:05"),
UpdatedAt: item.UpdatedAt.Format("2006-01-02 15:04:05"),
})
}
res := &dto.ListVideoRes{
List: items,
Total: total,
Page: page,
PageSize: pageSize,
TotalPages: totalPages,
}
logrus.Infof("查询视频素材 - 页码: %d, 每页: %d, 总数: %d, 总页数: %d", page, pageSize, total, totalPages)
return res, nil
}
// contains 检查字符串是否包含子串
func contains(s, substr string) bool {
return len(s) >= len(substr) && (s == substr || len(s) > len(substr) && (s[:len(substr)] == substr || s[len(s)-len(substr):] == substr || findSubstring(s, substr)))
}
func findSubstring(s, substr string) bool {
for i := 0; i <= len(s)-len(substr); i++ {
if s[i:i+len(substr)] == substr {
return true
}
}
return false
}

123
sql/12_tencent_video.sql Normal file
View File

@@ -0,0 +1,123 @@
-- 腾讯广告视频素材表
CREATE SEQUENCE IF NOT EXISTS tencent_video_id_seq START WITH 1 INCREMENT BY 1;
CREATE TABLE IF NOT EXISTS tencent_video (
id BIGINT NOT NULL DEFAULT nextval('tencent_video_id_seq'::regclass),
tenant_id BIGINT NOT NULL DEFAULT 0,
creator VARCHAR(100) DEFAULT '',
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
updater VARCHAR(100) DEFAULT '',
updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
deleted_at TIMESTAMP WITH TIME ZONE,
-- 业务字段
video_id VARCHAR(100) NOT NULL,
account_id BIGINT NOT NULL,
width INT,
height INT,
video_frames INT,
video_fps INT,
video_codec VARCHAR(50),
video_bit_rate BIGINT,
audio_codec VARCHAR(50),
audio_bit_rate BIGINT,
file_size BIGINT,
type VARCHAR(50),
signature VARCHAR(200),
system_status VARCHAR(50),
description TEXT,
preview_url TEXT,
key_frame_image_url TEXT,
created_time BIGINT,
last_modified_time BIGINT,
video_profile_name VARCHAR(50),
audio_sample_rate INT,
max_keyframe_interval INT,
min_keyframe_interval INT,
sample_aspect_ratio VARCHAR(50),
audio_profile_name VARCHAR(50),
scan_type VARCHAR(50),
image_duration_millisecond BIGINT,
audio_duration_millisecond BIGINT,
source_type VARCHAR(100),
product_catalog_id VARCHAR(200),
product_outer_id VARCHAR(200),
source_reference_id VARCHAR(200),
owner_account_id VARCHAR(100),
status VARCHAR(50),
source_material_id VARCHAR(100),
new_source_type VARCHAR(100),
aigc_type INT,
first_publication_status VARCHAR(100),
quality_status VARCHAR(100),
cover_id VARCHAR(100),
similarity_status VARCHAR(100),
user_aigc_status VARCHAR(100),
system_aigc_status VARCHAR(100),
aigc_source VARCHAR(200),
aigc_flag VARCHAR(50),
muse_aigc_version INT,
PRIMARY KEY (id)
);
COMMENT ON TABLE tencent_video IS '腾讯广告视频素材表';
COMMENT ON COLUMN tencent_video.id IS '主键ID';
COMMENT ON COLUMN tencent_video.tenant_id IS '租户ID';
COMMENT ON COLUMN tencent_video.creator IS '创建人';
COMMENT ON COLUMN tencent_video.created_at IS '创建时间';
COMMENT ON COLUMN tencent_video.updater IS '更新人';
COMMENT ON COLUMN tencent_video.updated_at IS '更新时间';
COMMENT ON COLUMN tencent_video.deleted_at IS '软删除时间';
COMMENT ON COLUMN tencent_video.video_id IS '视频ID';
COMMENT ON COLUMN tencent_video.account_id IS '账户ID';
COMMENT ON COLUMN tencent_video.width IS '宽度';
COMMENT ON COLUMN tencent_video.height IS '高度';
COMMENT ON COLUMN tencent_video.video_frames IS '视频帧数';
COMMENT ON COLUMN tencent_video.video_fps IS '帧率';
COMMENT ON COLUMN tencent_video.video_codec IS '视频编码';
COMMENT ON COLUMN tencent_video.video_bit_rate IS '视频码率';
COMMENT ON COLUMN tencent_video.audio_codec IS '音频编码';
COMMENT ON COLUMN tencent_video.audio_bit_rate IS '音频码率';
COMMENT ON COLUMN tencent_video.file_size IS '文件大小';
COMMENT ON COLUMN tencent_video.type IS '媒体类型';
COMMENT ON COLUMN tencent_video.signature IS '签名';
COMMENT ON COLUMN tencent_video.system_status IS '系统状态';
COMMENT ON COLUMN tencent_video.description IS '描述';
COMMENT ON COLUMN tencent_video.preview_url IS '预览URL';
COMMENT ON COLUMN tencent_video.key_frame_image_url IS '关键帧图片URL';
COMMENT ON COLUMN tencent_video.created_time IS '创建时间戳';
COMMENT ON COLUMN tencent_video.last_modified_time IS '最后修改时间戳';
COMMENT ON COLUMN tencent_video.video_profile_name IS '视频配置名称';
COMMENT ON COLUMN tencent_video.audio_sample_rate IS '音频采样率';
COMMENT ON COLUMN tencent_video.max_keyframe_interval IS '最大关键帧间隔';
COMMENT ON COLUMN tencent_video.min_keyframe_interval IS '最小关键帧间隔';
COMMENT ON COLUMN tencent_video.sample_aspect_ratio IS '示例宽高比';
COMMENT ON COLUMN tencent_video.audio_profile_name IS '音频配置名称';
COMMENT ON COLUMN tencent_video.scan_type IS '扫描类型';
COMMENT ON COLUMN tencent_video.image_duration_millisecond IS '图片时长（毫秒）';
COMMENT ON COLUMN tencent_video.audio_duration_millisecond IS '音频时长（毫秒）';
COMMENT ON COLUMN tencent_video.source_type IS '来源类型';
COMMENT ON COLUMN tencent_video.product_catalog_id IS '产品目录ID';
COMMENT ON COLUMN tencent_video.product_outer_id IS '产品外部ID';
COMMENT ON COLUMN tencent_video.source_reference_id IS '源引用ID';
COMMENT ON COLUMN tencent_video.owner_account_id IS '所有者账户ID';
COMMENT ON COLUMN tencent_video.status IS '状态';
COMMENT ON COLUMN tencent_video.source_material_id IS '源素材ID';
COMMENT ON COLUMN tencent_video.new_source_type IS '新来源类型';
COMMENT ON COLUMN tencent_video.aigc_type IS 'AIGC类型';
COMMENT ON COLUMN tencent_video.first_publication_status IS '首次发布状态';
COMMENT ON COLUMN tencent_video.quality_status IS '质量状态';
COMMENT ON COLUMN tencent_video.cover_id IS '封面ID';
COMMENT ON COLUMN tencent_video.similarity_status IS '相似度状态';
COMMENT ON COLUMN tencent_video.user_aigc_status IS '用户AIGC状态';
COMMENT ON COLUMN tencent_video.system_aigc_status IS '系统AIGC状态';
COMMENT ON COLUMN tencent_video.aigc_source IS 'AIGC来源';
COMMENT ON COLUMN tencent_video.aigc_flag IS 'AIGC标志';
COMMENT ON COLUMN tencent_video.muse_aigc_version IS 'Muse AIGC版本';
-- 唯一索引：根据video_id和account_id判断是否存在
CREATE UNIQUE INDEX idx_tencent_video_video_account ON tencent_video(tenant_id, video_id, account_id);
CREATE INDEX idx_tencent_video_account_id ON tencent_video(account_id);
CREATE INDEX idx_tencent_video_last_modified ON tencent_video(last_modified_time);
CREATE INDEX idx_tencent_video_status ON tencent_video(status);