red-future/data-engine
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

重构数据引擎

Browse Source
This commit is contained in:
李明康
2026-05-29 18:39:32 +08:00
parent 3ced686cb5
commit 15db71b7ba
132 changed files with 2534 additions and 26198 deletions
Download Patch File Download Diff File Expand all files Collapse all files

160
dao/copydata/api_account_report_detail_dao.go
View File

@@ -1,160 +0,0 @@
package copydata
import (
"context"
consts "dataengine/consts/public"
dto "dataengine/model/dto/copydata"
entity "dataengine/model/entity/copydata"
"errors"
"gitea.com/red-future/common/db/gfdb"
"github.com/gogf/gf/v2/util/gconv"
"github.com/sirupsen/logrus"
)
var CidAccountReportDetail = new(cidAccountReportDetailDao)
type cidAccountReportDetailDao struct{}
// Insert 插入广告数据报表详情
func (d *cidAccountReportDetailDao) Insert(ctx context.Context, req *dto.CidAccountReportDetailItem) (id int64, err error) {
var entityData *entity.CidAccountReportDetail
if err = gconv.Struct(req, &entityData); err != nil {
return
}
r, err := gfdb.DB(ctx).Model(ctx, consts.CidAccountReportDetailTable).Data(&entityData).Insert()
if err != nil {
return
}
return r.LastInsertId()
}
// BatchInsert 批量插入广告数据报表详情（使用 OnConflict 实现幂等性）
func (d *cidAccountReportDetailDao) BatchInsert(ctx context.Context, reqs []*dto.CidAccountReportDetailItem) (successCount int64, failCount int64, failedIndexes []int64, err error) {
if len(reqs) == 0 {
return 0, 0, nil, errors.New("批量插入数据不能为空")
}
// 分批处理，每批100条
batchSize := 100
successCount = 0
failCount = 0
failedIndexes = make([]int64, 0)
for i := 0; i < len(reqs); i += batchSize {
end := i + batchSize
if end > len(reqs) {
end = len(reqs)
}
batch := reqs[i:end]
entityList := make([]*entity.CidAccountReportDetail, len(batch))
for j, req := range batch {
var entityData entity.CidAccountReportDetail
if err = gconv.Struct(req, &entityData); err != nil {
failCount++
failedIndexes = append(failedIndexes, int64(i+j))
continue
}
entityList[j] = &entityData
}
if len(entityList) == 0 {
continue
}
// 执行批量插入，使用 OnConflict 实现幂等性
_, err = gfdb.DB(ctx).Model(ctx, consts.CidAccountReportDetailTable).
Data(entityList).
OnConflict(
"report_date_str",
"page_number",
"campaign_id",
"creative_id",
).
Save()
if err != nil {
logrus.Warnf("批量插入失败，尝试逐条插入: %v", err)
// 批量插入失败，尝试逐条插入
for k := range batch {
_, singleErr := d.Insert(ctx, batch[k])
if singleErr != nil {
failCount++
failedIndexes = append(failedIndexes, int64(i+k))
} else {
successCount++
}
}
} else {
successCount += int64(len(entityList))
}
}
return successCount, failCount, failedIndexes, nil
}
// DeleteByDateRange 按日期范围删除数据（用于补偿前去重）
func (d *cidAccountReportDetailDao) DeleteByDateRange(ctx context.Context, advertiserID int64, startDateStr, endDateStr string) (int64, error) {
cols := (&entity.CidAccountReportDetail{}).GetCols()
result, err := gfdb.DB(ctx).Model(ctx, consts.CidAccountReportDetailTable).
Where(cols.ReportDateStr+" >= ? AND "+cols.ReportDateStr+" <= ?", startDateStr, endDateStr).
Delete()
if err != nil {
return 0, err
}
affected, _ := result.RowsAffected()
return affected, nil
}
// BatchInsertInTx 在事务中批量插入
func (d *cidAccountReportDetailDao) BatchInsertInTx(ctx context.Context, tx interface{}, reqs []*dto.CidAccountReportDetailItem) (successCount int64, failCount int64, err error) {
if len(reqs) == 0 {
return 0, 0, errors.New("批量插入数据不能为空")
}
batchSize := 100
successCount = 0
failCount = 0
for i := 0; i < len(reqs); i += batchSize {
end := i + batchSize
if end > len(reqs) {
end = len(reqs)
}
batch := reqs[i:end]
entityList := make([]*entity.CidAccountReportDetail, 0, len(batch))
for _, req := range batch {
var entityData entity.CidAccountReportDetail
if err = gconv.Struct(req, &entityData); err != nil {
failCount++
logrus.Errorf("数据转换失败: %v", err)
continue
}
entityList = append(entityList, &entityData)
}
if len(entityList) == 0 {
continue
}
_, txErr := gfdb.DB(ctx).Model(ctx, consts.CidAccountReportDetailTable).Data(entityList).Insert()
if txErr != nil {
logrus.Errorf("批量插入失败 batch[%d:%d]: %v", i, end, txErr)
failCount += int64(len(entityList))
err = txErr
continue
}
successCount += int64(len(entityList))
}
return successCount, failCount, err
}

113
dao/copydata/api_account_report_sum_dao.go
View File

@@ -1,113 +0,0 @@
package copydata
import (
"context"
consts "dataengine/consts/public"
dto "dataengine/model/dto/copydata"
entity "dataengine/model/entity/copydata"
"errors"
"gitea.com/red-future/common/db/gfdb"
"github.com/gogf/gf/v2/util/gconv"
"github.com/sirupsen/logrus"
)
var CidAccountReportSum = new(CidAccountReportSumDao)
type CidAccountReportSumDao struct{}
// Insert 插入广告数据报表汇总
func (d *CidAccountReportSumDao) Insert(ctx context.Context, req *dto.CidAccountReportSumItem) (id int64, err error) {
var entityData *entity.CidAccountReportSum
if err = gconv.Struct(req, &entityData); err != nil {
return
}
r, err := gfdb.DB(ctx).Model(ctx, consts.CidAccountReportSumTable).Data(&entityData).Insert()
if err != nil {
return
}
return r.LastInsertId()
}
// BatchInsert 批量插入广告数据报表汇总（使用 OnConflict 实现幂等性）
func (d *CidAccountReportSumDao) BatchInsert(ctx context.Context, reqs []*dto.CidAccountReportSumItem) (successCount int64, failCount int64, failedIndexes []int64, err error) {
if len(reqs) == 0 {
return 0, 0, nil, errors.New("批量插入数据不能为空")
}
// 分批处理，每批100条
batchSize := 100
successCount = 0
failCount = 0
failedIndexes = make([]int64, 0)
for i := 0; i < len(reqs); i += batchSize {
end := i + batchSize
if end > len(reqs) {
end = len(reqs)
}
batch := reqs[i:end]
entityList := make([]*entity.CidAccountReportSum, len(batch))
for j, req := range batch {
var entityData entity.CidAccountReportSum
if err = gconv.Struct(req, &entityData); err != nil {
failCount++
failedIndexes = append(failedIndexes, int64(i+j))
continue
}
entityList[j] = &entityData
}
if len(entityList) == 0 {
continue
}
// 执行批量插入，使用 OnConflict 实现幂等性
_, err = gfdb.DB(ctx).Model(ctx, consts.CidAccountReportSumTable).
Data(entityList).
OnConflict(
"report_date_str",
"page_number",
"campaign_id",
"creative_id",
).
Save()
if err != nil {
logrus.Warnf("批量插入失败，尝试逐条插入: %v", err)
// 批量插入失败，尝试逐条插入
for k := range batch {
_, singleErr := d.Insert(ctx, batch[k])
if singleErr != nil {
failCount++
failedIndexes = append(failedIndexes, int64(i+k))
} else {
successCount++
}
}
} else {
successCount += int64(len(entityList))
}
}
return successCount, failCount, failedIndexes, nil
}
// DeleteByDateRange 按日期范围删除数据（用于补偿前去重）
func (d *CidAccountReportSumDao) DeleteByDateRange(ctx context.Context, advertiserID int64, startDateStr, endDateStr string) (int64, error) {
cols := (&entity.CidAccountReportSum{}).GetCols()
result, err := gfdb.DB(ctx).Model(ctx, consts.CidAccountReportSumTable).
Where(cols.ReportDateStr+" >= ? AND "+cols.ReportDateStr+" <= ?", startDateStr, endDateStr).
Delete()
if err != nil {
return 0, err
}
affected, _ := result.RowsAffected()
return affected, nil
}

81
dao/copydata/campaign_report_detail_dao.go
View File

@@ -1,81 +0,0 @@
package copydata
import (
"context"
consts "dataengine/consts/public"
dto "dataengine/model/dto/copydata"
entity "dataengine/model/entity/copydata"
"errors"
"gitea.com/red-future/common/db/gfdb"
"github.com/gogf/gf/v2/util/gconv"
)
var CampaignReportDetail = new(CampaignReportDetailDao)
type CampaignReportDetailDao struct{}
func (d *CampaignReportDetailDao) Insert(ctx context.Context, req *dto.CampaignReportDetailItem) (id int64, err error) {
var entityData *entity.CampaignReportDetail
if err = gconv.Struct(req, &entityData); err != nil {
return
}
r, err := gfdb.DB(ctx).Model(ctx, consts.CampaignReportDetailTable).Data(&entityData).Insert()
if err != nil {
return
}
return r.LastInsertId()
}
func (d *CampaignReportDetailDao) BatchInsert(ctx context.Context, reqs []*dto.CampaignReportDetailItem) (successCount int64, failCount int64, failedIndexes []int64, err error) {
if len(reqs) == 0 {
return 0, 0, nil, errors.New("批量插入数据不能为空")
}
batchSize := 100
successCount = 0
failCount = 0
failedIndexes = make([]int64, 0)
for i := 0; i < len(reqs); i += batchSize {
end := i + batchSize
if end > len(reqs) {
end = len(reqs)
}
batch := reqs[i:end]
entityList := make([]*entity.CampaignReportDetail, len(batch))
for j, req := range batch {
var entityData entity.CampaignReportDetail
if err = gconv.Struct(req, &entityData); err != nil {
failCount++
failedIndexes = append(failedIndexes, int64(i+j))
continue
}
entityList[j] = &entityData
}
if len(entityList) == 0 {
continue
}
_, err = gfdb.DB(ctx).Model(ctx, consts.CampaignReportDetailTable).Data(entityList).Insert()
if err != nil {
for k := range batch {
_, singleErr := d.Insert(ctx, batch[k])
if singleErr != nil {
failCount++
failedIndexes = append(failedIndexes, int64(i+k))
} else {
successCount++
}
}
} else {
successCount += int64(len(entityList))
}
}
return successCount, failCount, failedIndexes, nil
}

86
dao/copydata/campaign_report_sum_dao.go
View File

@@ -1,86 +0,0 @@
package copydata
import (
"context"
consts "dataengine/consts/public"
dto "dataengine/model/dto/copydata"
entity "dataengine/model/entity/copydata"
"errors"
"gitea.com/red-future/common/db/gfdb"
"github.com/gogf/gf/v2/util/gconv"
)
var CampaignReportSum = new(CampaignReportSumDao)
type CampaignReportSumDao struct{}
// Insert 插入广告计划效果指标表
func (d *CampaignReportSumDao) Insert(ctx context.Context, req *dto.CampaignReportSumItem) (id int64, err error) {
var entityData *entity.CampaignReportSum
if err = gconv.Struct(req, &entityData); err != nil {
return
}
r, err := gfdb.DB(ctx).Model(ctx, consts.CampaignReportSumTable).Data(&entityData).Insert()
if err != nil {
return
}
return r.LastInsertId()
}
// BatchInsert 批量插入广告计划效果指标表
func (d *CampaignReportSumDao) BatchInsert(ctx context.Context, reqs []*dto.CampaignReportSumItem) (successCount int64, failCount int64, failedIndexes []int64, err error) {
if len(reqs) == 0 {
return 0, 0, nil, errors.New("批量插入数据不能为空")
}
// 分批处理，每批 100 条
batchSize := 100
successCount = 0
failCount = 0
failedIndexes = make([]int64, 0)
for i := 0; i < len(reqs); i += batchSize {
end := i + batchSize
if end > len(reqs) {
end = len(reqs)
}
batch := reqs[i:end]
entityList := make([]*entity.CampaignReportSum, len(batch))
for j, req := range batch {
var entityData entity.CampaignReportSum
if err = gconv.Struct(req, &entityData); err != nil {
failCount++
failedIndexes = append(failedIndexes, int64(i+j))
continue
}
entityList[j] = &entityData
}
if len(entityList) == 0 {
continue
}
// 执行批量插入
_, err = gfdb.DB(ctx).Model(ctx, consts.CampaignReportSumTable).Data(entityList).Insert()
if err != nil {
// 批量插入失败，尝试逐条插入
for k := range batch {
_, singleErr := d.Insert(ctx, batch[k])
if singleErr != nil {
failCount++
failedIndexes = append(failedIndexes, int64(i+k))
} else {
successCount++
}
}
} else {
successCount += int64(len(entityList))
}
}
return successCount, failCount, failedIndexes, nil
}

81
dao/copydata/creative_report_detail_dao.go
View File

@@ -1,81 +0,0 @@
package copydata
import (
"context"
consts "dataengine/consts/public"
dto "dataengine/model/dto/copydata"
entity "dataengine/model/entity/copydata"
"errors"
"gitea.com/red-future/common/db/gfdb"
"github.com/gogf/gf/v2/util/gconv"
)
var CreativeReportDetail = new(CreativeReportDetailDao)
type CreativeReportDetailDao struct{}
func (d *CreativeReportDetailDao) Insert(ctx context.Context, req *dto.CreativeReportDetailItem) (id int64, err error) {
var entityData *entity.CreativeReportDetail
if err = gconv.Struct(req, &entityData); err != nil {
return
}
r, err := gfdb.DB(ctx).Model(ctx, consts.CreativeReportDetailTable).Data(&entityData).Insert()
if err != nil {
return
}
return r.LastInsertId()
}
func (d *CreativeReportDetailDao) BatchInsert(ctx context.Context, reqs []*dto.CreativeReportDetailItem) (successCount int64, failCount int64, failedIndexes []int64, err error) {
if len(reqs) == 0 {
return 0, 0, nil, errors.New("批量插入数据不能为空")
}
batchSize := 100
successCount = 0
failCount = 0
failedIndexes = make([]int64, 0)
for i := 0; i < len(reqs); i += batchSize {
end := i + batchSize
if end > len(reqs) {
end = len(reqs)
}
batch := reqs[i:end]
entityList := make([]*entity.CreativeReportDetail, len(batch))
for j, req := range batch {
var entityData entity.CreativeReportDetail
if err = gconv.Struct(req, &entityData); err != nil {
failCount++
failedIndexes = append(failedIndexes, int64(i+j))
continue
}
entityList[j] = &entityData
}
if len(entityList) == 0 {
continue
}
_, err = gfdb.DB(ctx).Model(ctx, consts.CreativeReportDetailTable).Data(entityList).Insert()
if err != nil {
for k := range batch {
_, singleErr := d.Insert(ctx, batch[k])
if singleErr != nil {
failCount++
failedIndexes = append(failedIndexes, int64(i+k))
} else {
successCount++
}
}
} else {
successCount += int64(len(entityList))
}
}
return successCount, failCount, failedIndexes, nil
}

86
dao/copydata/creative_report_sum_dao.go
View File

@@ -1,86 +0,0 @@
package copydata
import (
"context"
consts "dataengine/consts/public"
dto "dataengine/model/dto/copydata"
entity "dataengine/model/entity/copydata"
"errors"
"gitea.com/red-future/common/db/gfdb"
"github.com/gogf/gf/v2/util/gconv"
)
var CreativeReportSum = new(CreativeReportSumDao)
type CreativeReportSumDao struct{}
// Insert 插入广告效果指标表
func (d *CreativeReportSumDao) Insert(ctx context.Context, req *dto.CreativeReportSumItem) (id int64, err error) {
var entityData *entity.CreativeReportSum
if err = gconv.Struct(req, &entityData); err != nil {
return
}
r, err := gfdb.DB(ctx).Model(ctx, consts.CreativeReportSumTable).Data(&entityData).Insert()
if err != nil {
return
}
return r.LastInsertId()
}
// BatchInsert 批量插入广告效果指标表
func (d *CreativeReportSumDao) BatchInsert(ctx context.Context, reqs []*dto.CreativeReportSumItem) (successCount int64, failCount int64, failedIndexes []int64, err error) {
if len(reqs) == 0 {
return 0, 0, nil, errors.New("批量插入数据不能为空")
}
// 分批处理，每批 100 条
batchSize := 100
successCount = 0
failCount = 0
failedIndexes = make([]int64, 0)
for i := 0; i < len(reqs); i += batchSize {
end := i + batchSize
if end > len(reqs) {
end = len(reqs)
}
batch := reqs[i:end]
entityList := make([]*entity.CreativeReportSum, len(batch))
for j, req := range batch {
var entityData entity.CreativeReportSum
if err = gconv.Struct(req, &entityData); err != nil {
failCount++
failedIndexes = append(failedIndexes, int64(i+j))
continue
}
entityList[j] = &entityData
}
if len(entityList) == 0 {
continue
}
// 执行批量插入
_, err = gfdb.DB(ctx).Model(ctx, consts.CreativeReportSumTable).Data(entityList).Insert()
if err != nil {
// 批量插入失败，尝试逐条插入
for k := range batch {
_, singleErr := d.Insert(ctx, batch[k])
if singleErr != nil {
failCount++
failedIndexes = append(failedIndexes, int64(i+k))
} else {
successCount++
}
}
} else {
successCount += int64(len(entityList))
}
}
return successCount, failCount, failedIndexes, nil
}

128
dao/copydata/material_report_dao.go
View File

@@ -1,128 +0,0 @@
package copydata
import (
"context"
consts "dataengine/consts/public"
dto "dataengine/model/dto/copydata"
entity "dataengine/model/entity/copydata"
"errors"
"gitea.com/red-future/common/db/gfdb"
"github.com/gogf/gf/v2/util/gconv"
)
var MaterialReport = new(materialReportDao)
type materialReportDao struct{}
// Insert 插入素材报表数据
func (d *materialReportDao) Insert(ctx context.Context, req *dto.MaterialReportItem) (id int64, err error) {
var entityData *entity.MaterialReport
if err = gconv.Struct(req, &entityData); err != nil {
return
}
r, err := gfdb.DB(ctx).Model(ctx, consts.MaterialReportTable).Data(&entityData).Insert()
if err != nil {
return
}
return r.LastInsertId()
}
// BatchInsert 批量插入素材报表数据
func (d *materialReportDao) BatchInsert(ctx context.Context, reqs []*dto.MaterialReportItem) (successCount int64, failCount int64, failedIndexes []int64, err error) {
if len(reqs) == 0 {
return 0, 0, nil, errors.New("批量插入数据不能为空")
}
batchSize := 100
successCount = 0
failCount = 0
failedIndexes = make([]int64, 0)
for i := 0; i < len(reqs); i += batchSize {
end := i + batchSize
if end > len(reqs) {
end = len(reqs)
}
batch := reqs[i:end]
entityList := make([]*entity.MaterialReport, len(batch))
for j, req := range batch {
var entityData entity.MaterialReport
if err = gconv.Struct(req, &entityData); err != nil {
failCount++
failedIndexes = append(failedIndexes, int64(i+j))
continue
}
entityList[j] = &entityData
}
if len(entityList) == 0 {
continue
}
_, err = gfdb.DB(ctx).Model(ctx, consts.MaterialReportTable).Data(entityList).Insert()
if err != nil {
for k := range batch {
_, singleErr := d.Insert(ctx, batch[k])
if singleErr != nil {
failCount++
failedIndexes = append(failedIndexes, int64(i+k))
} else {
successCount++
}
}
} else {
successCount += int64(len(entityList))
}
}
return successCount, failCount, failedIndexes, nil
}
// List 查询素材报表数据列表
func (d *materialReportDao) List(ctx context.Context, req *dto.ListMaterialReportReq) ([]*entity.MaterialReport, int, error) {
model := gfdb.DB(ctx).Model(ctx, consts.MaterialReportTable).Model
if req.ReportDateStr != "" {
model = model.Where("report_date_str", req.ReportDateStr)
}
if req.PhotoId != "" {
model = model.Where("photo_id", req.PhotoId)
}
if req.CampaignId != nil {
model = model.Where("campaign_id", req.CampaignId)
}
if req.UnitId != nil {
model = model.Where("unit_id", req.UnitId)
}
if req.CreativeId != nil {
model = model.Where("creative_id", req.CreativeId)
}
if req.Keyword != "" {
model = model.Where("(photo_name LIKE ? OR campaign_name LIKE ? OR unit_name LIKE ? OR creative_name LIKE ?)",
"%"+req.Keyword+"%", "%"+req.Keyword+"%", "%"+req.Keyword+"%", "%"+req.Keyword+"%")
}
model = model.OrderDesc("created_at")
total, err := model.Count()
if err != nil {
return nil, 0, err
}
var list []*entity.MaterialReport
if req.Page != nil {
err = model.Page(int(req.Page.PageNum), int(req.Page.PageSize)).Scan(&list)
} else {
err = model.Scan(&list)
}
if err != nil {
return nil, 0, err
}
return list, total, nil
}

124
dao/copydata/population_report_dao.go
View File

@@ -1,124 +0,0 @@
package copydata
import (
"context"
consts "dataengine/consts/public"
dto "dataengine/model/dto/copydata"
entity "dataengine/model/entity/copydata"
"errors"
"gitea.com/red-future/common/db/gfdb"
"github.com/gogf/gf/v2/util/gconv"
)
var PopulationReport = new(populationReportDao)
type populationReportDao struct{}
// Insert 插入人群报表数据
func (d *populationReportDao) Insert(ctx context.Context, req *dto.PopulationReportItem) (id int64, err error) {
var entityData *entity.PopulationReport
if err = gconv.Struct(req, &entityData); err != nil {
return
}
r, err := gfdb.DB(ctx).Model(ctx, consts.PopulationReportTable).Data(&entityData).Insert()
if err != nil {
return
}
return r.LastInsertId()
}
// BatchInsert 批量插入人群报表数据
func (d *populationReportDao) BatchInsert(ctx context.Context, reqs []*dto.PopulationReportItem) (successCount int64, failCount int64, failedIndexes []int64, err error) {
if len(reqs) == 0 {
return 0, 0, nil, errors.New("批量插入数据不能为空")
}
// 分批处理，每批 100 条
batchSize := 100
successCount = 0
failCount = 0
failedIndexes = make([]int64, 0)
for i := 0; i < len(reqs); i += batchSize {
end := i + batchSize
if end > len(reqs) {
end = len(reqs)
}
batch := reqs[i:end]
entityList := make([]*entity.PopulationReport, len(batch))
for j, req := range batch {
var entityData entity.PopulationReport
if err = gconv.Struct(req, &entityData); err != nil {
failCount++
failedIndexes = append(failedIndexes, int64(i+j))
continue
}
entityList[j] = &entityData
}
if len(entityList) == 0 {
continue
}
// 执行批量插入
_, err = gfdb.DB(ctx).Model(ctx, consts.PopulationReportTable).Data(entityList).Insert()
if err != nil {
// 批量插入失败，尝试逐条插入
for k := range batch {
_, singleErr := d.Insert(ctx, batch[k])
if singleErr != nil {
failCount++
failedIndexes = append(failedIndexes, int64(i+k))
} else {
successCount++
}
}
} else {
successCount += int64(len(entityList))
}
}
return successCount, failCount, failedIndexes, nil
}
// List 查询人群报表数据列表
func (d *populationReportDao) List(ctx context.Context, req *dto.ListPopulationReportReq) ([]*entity.PopulationReport, int, error) {
model := gfdb.DB(ctx).Model(ctx, consts.PopulationReportTable).Model
// 构建查询条件
if req.ReportDateStr != "" {
model = model.Where("report_date_str", req.ReportDateStr)
}
if req.PhotoId != "" {
model = model.Where("photo_id", req.PhotoId)
}
if req.Keyword != "" {
model = model.Where("photo_name LIKE ?", "%"+req.Keyword+"%")
}
// 设置排序
model = model.OrderDesc("created_at")
// 分页查询并获取总数
total, err := model.Count()
if err != nil {
return nil, 0, err
}
var list []*entity.PopulationReport
if req.Page != nil {
err = model.Page(int(req.Page.PageNum), int(req.Page.PageSize)).Scan(&list)
} else {
err = model.Scan(&list)
}
if err != nil {
return nil, 0, err
}
return list, total, nil
}

86
dao/copydata/storewide_report_detail_dao.go
View File

@@ -1,86 +0,0 @@
package copydata
import (
"context"
consts "dataengine/consts/public"
dto "dataengine/model/dto/copydata"
entity "dataengine/model/entity/copydata"
"errors"
"gitea.com/red-future/common/db/gfdb"
"github.com/gogf/gf/v2/util/gconv"
)
var StorewideReportDetail = new(StorewideReportDetailDao)
type StorewideReportDetailDao struct{}
// Insert 插入广告效果指标表
func (d *StorewideReportDetailDao) Insert(ctx context.Context, req *dto.StorewideReportDetailItem) (id int64, err error) {
var entityData *entity.StorewideReportDetail
if err = gconv.Struct(req, &entityData); err != nil {
return
}
r, err := gfdb.DB(ctx).Model(ctx, consts.StorewideReportDetailTable).Data(&entityData).Insert()
if err != nil {
return
}
return r.LastInsertId()
}
// BatchInsert 批量插入广告效果指标表
func (d *StorewideReportDetailDao) BatchInsert(ctx context.Context, reqs []*dto.StorewideReportDetailItem) (successCount int64, failCount int64, failedIndexes []int64, err error) {
if len(reqs) == 0 {
return 0, 0, nil, errors.New("批量插入数据不能为空")
}
// 分批处理，每批 100 条
batchSize := 100
successCount = 0
failCount = 0
failedIndexes = make([]int64, 0)
for i := 0; i < len(reqs); i += batchSize {
end := i + batchSize
if end > len(reqs) {
end = len(reqs)
}
batch := reqs[i:end]
entityList := make([]*entity.StorewideReportDetail, len(batch))
for j, req := range batch {
var entityData entity.StorewideReportDetail
if err = gconv.Struct(req, &entityData); err != nil {
failCount++
failedIndexes = append(failedIndexes, int64(i+j))
continue
}
entityList[j] = &entityData
}
if len(entityList) == 0 {
continue
}
// 执行批量插入
_, err = gfdb.DB(ctx).Model(ctx, consts.StorewideReportDetailTable).Data(entityList).Insert()
if err != nil {
// 批量插入失败，尝试逐条插入
for k := range batch {
_, singleErr := d.Insert(ctx, batch[k])
if singleErr != nil {
failCount++
failedIndexes = append(failedIndexes, int64(i+k))
} else {
successCount++
}
}
} else {
successCount += int64(len(entityList))
}
}
return successCount, failCount, failedIndexes, nil
}

86
dao/copydata/storewide_report_sum_dao.go
View File

@@ -1,86 +0,0 @@
package copydata
import (
"context"
consts "dataengine/consts/public"
dto "dataengine/model/dto/copydata"
entity "dataengine/model/entity/copydata"
"errors"
"gitea.com/red-future/common/db/gfdb"
"github.com/gogf/gf/v2/util/gconv"
)
var StorewideReportSum = new(StorewideReportSumDao)
type StorewideReportSumDao struct{}
// Insert 插入广告效果指标表
func (d *StorewideReportSumDao) Insert(ctx context.Context, req *dto.StorewideReportSumItem) (id int64, err error) {
var entityData *entity.StorewideReportSum
if err = gconv.Struct(req, &entityData); err != nil {
return
}
r, err := gfdb.DB(ctx).Model(ctx, consts.StorewideReportSumTable).Data(&entityData).Insert()
if err != nil {
return
}
return r.LastInsertId()
}
// BatchInsert 批量插入广告效果指标表
func (d *StorewideReportSumDao) BatchInsert(ctx context.Context, reqs []*dto.StorewideReportSumItem) (successCount int64, failCount int64, failedIndexes []int64, err error) {
if len(reqs) == 0 {
return 0, 0, nil, errors.New("批量插入数据不能为空")
}
// 分批处理，每批 100 条
batchSize := 100
successCount = 0
failCount = 0
failedIndexes = make([]int64, 0)
for i := 0; i < len(reqs); i += batchSize {
end := i + batchSize
if end > len(reqs) {
end = len(reqs)
}
batch := reqs[i:end]
entityList := make([]*entity.StorewideReportSum, len(batch))
for j, req := range batch {
var entityData entity.StorewideReportSum
if err = gconv.Struct(req, &entityData); err != nil {
failCount++
failedIndexes = append(failedIndexes, int64(i+j))
continue
}
entityList[j] = &entityData
}
if len(entityList) == 0 {
continue
}
// 执行批量插入
_, err = gfdb.DB(ctx).Model(ctx, consts.StorewideReportSumTable).Data(entityList).Insert()
if err != nil {
// 批量插入失败，尝试逐条插入
for k := range batch {
_, singleErr := d.Insert(ctx, batch[k])
if singleErr != nil {
failCount++
failedIndexes = append(failedIndexes, int64(i+k))
} else {
successCount++
}
}
} else {
successCount += int64(len(entityList))
}
}
return successCount, failCount, failedIndexes, nil
}

2
dao/copydata/sync_task_log_dao.go
View File

@@ -28,6 +28,8 @@ func (d *SyncTaskLogDao) Create(ctx context.Context, req *dto.CreateSyncTaskLogR
data := map[string]interface{}{
"task_id": req.TaskID,
"task_type": req.TaskType,
"platform_code": req.PlatformCode,
"interface_code": req.InterfaceCode,
"advertiser_id": req.AdvertiserID,
"start_time": req.StartTime,
"end_time": req.EndTime,

124
dao/copydata/task_report_dao.go
View File

@@ -1,124 +0,0 @@
package copydata
import (
"context"
consts "dataengine/consts/public"
dto "dataengine/model/dto/copydata"
entity "dataengine/model/entity/copydata"
"errors"
"gitea.com/red-future/common/db/gfdb"
"github.com/gogf/gf/v2/util/gconv"
)
var TaskReport = new(taskReportDao)
type taskReportDao struct{}
// Insert 插入调控任务数据
func (d *taskReportDao) Insert(ctx context.Context, req *dto.TaskReportItem) (id int64, err error) {
var entityData *entity.TaskReport
if err = gconv.Struct(req, &entityData); err != nil {
return
}
r, err := gfdb.DB(ctx).Model(ctx, consts.TaskReportTable).Data(&entityData).Insert()
if err != nil {
return
}
return r.LastInsertId()
}
// BatchInsert 批量插入调控任务数据
func (d *taskReportDao) BatchInsert(ctx context.Context, reqs []*dto.TaskReportItem) (successCount int64, failCount int64, failedIndexes []int64, err error) {
if len(reqs) == 0 {
return 0, 0, nil, errors.New("批量插入数据不能为空")
}
// 分批处理，每批 100 条
batchSize := 100
successCount = 0
failCount = 0
failedIndexes = make([]int64, 0)
for i := 0; i < len(reqs); i += batchSize {
end := i + batchSize
if end > len(reqs) {
end = len(reqs)
}
batch := reqs[i:end]
entityList := make([]*entity.TaskReport, len(batch))
for j, req := range batch {
var entityData entity.TaskReport
if err = gconv.Struct(req, &entityData); err != nil {
failCount++
failedIndexes = append(failedIndexes, int64(i+j))
continue
}
entityList[j] = &entityData
}
if len(entityList) == 0 {
continue
}
// 执行批量插入
_, err = gfdb.DB(ctx).Model(ctx, consts.TaskReportTable).Data(entityList).Insert()
if err != nil {
// 批量插入失败，尝试逐条插入
for k := range batch {
_, singleErr := d.Insert(ctx, batch[k])
if singleErr != nil {
failCount++
failedIndexes = append(failedIndexes, int64(i+k))
} else {
successCount++
}
}
} else {
successCount += int64(len(entityList))
}
}
return successCount, failCount, failedIndexes, nil
}
// List 查询调控任务数据列表
func (d *taskReportDao) List(ctx context.Context, req *dto.ListTaskReportReq) ([]*entity.TaskReport, int, error) {
model := gfdb.DB(ctx).Model(ctx, consts.TaskReportTable).Model
// 构建查询条件
if req.ReportDateStr != "" {
model = model.Where("report_date_str", req.ReportDateStr)
}
if req.PhotoId != "" {
model = model.Where("photo_id", req.PhotoId)
}
if req.Keyword != "" {
model = model.Where("photo_name LIKE ?", "%"+req.Keyword+"%")
}
// 设置排序
model = model.OrderDesc("created_at")
// 分页查询并获取总数
total, err := model.Count()
if err != nil {
return nil, 0, err
}
var list []*entity.TaskReport
if req.Page != nil {
err = model.Page(int(req.Page.PageNum), int(req.Page.PageSize)).Scan(&list)
} else {
err = model.Scan(&list)
}
if err != nil {
return nil, 0, err
}
return list, total, nil
}

83
dao/copydata/unit_report_detail_dao.go
View File

@@ -1,83 +0,0 @@
package copydata
import (
"context"
consts "dataengine/consts/public"
dto "dataengine/model/dto/copydata"
entity "dataengine/model/entity/copydata"
"errors"
"gitea.com/red-future/common/db/gfdb"
"github.com/gogf/gf/v2/util/gconv"
)
var UnitReportDetail = new(UnitReportDetailDao)
type UnitReportDetailDao struct{}
// Insert 插入广告效果指标详情
func (d *UnitReportDetailDao) Insert(ctx context.Context, req *dto.UnitReportDetailItem) (id int64, err error) {
var entityData *entity.UnitReportDetail
if err = gconv.Struct(req, &entityData); err != nil {
return
}
r, err := gfdb.DB(ctx).Model(ctx, consts.UnitReportDetailTable).Data(&entityData).Insert()
if err != nil {
return
}
return r.LastInsertId()
}
// BatchInsert 批量插入广告效果指标详情
func (d *UnitReportDetailDao) BatchInsert(ctx context.Context, reqs []*dto.UnitReportDetailItem) (successCount int64, failCount int64, failedIndexes []int64, err error) {
if len(reqs) == 0 {
return 0, 0, nil, errors.New("批量插入数据不能为空")
}
batchSize := 100
successCount = 0
failCount = 0
failedIndexes = make([]int64, 0)
for i := 0; i < len(reqs); i += batchSize {
end := i + batchSize
if end > len(reqs) {
end = len(reqs)
}
batch := reqs[i:end]
entityList := make([]*entity.UnitReportDetail, len(batch))
for j, req := range batch {
var entityData entity.UnitReportDetail
if err = gconv.Struct(req, &entityData); err != nil {
failCount++
failedIndexes = append(failedIndexes, int64(i+j))
continue
}
entityList[j] = &entityData
}
if len(entityList) == 0 {
continue
}
_, err = gfdb.DB(ctx).Model(ctx, consts.UnitReportDetailTable).Data(entityList).Insert()
if err != nil {
for k := range batch {
_, singleErr := d.Insert(ctx, batch[k])
if singleErr != nil {
failCount++
failedIndexes = append(failedIndexes, int64(i+k))
} else {
successCount++
}
}
} else {
successCount += int64(len(entityList))
}
}
return successCount, failCount, failedIndexes, nil
}

83
dao/copydata/unit_report_sum_dao.go
View File

@@ -1,83 +0,0 @@
package copydata
import (
"context"
consts "dataengine/consts/public"
dto "dataengine/model/dto/copydata"
entity "dataengine/model/entity/copydata"
"errors"
"gitea.com/red-future/common/db/gfdb"
"github.com/gogf/gf/v2/util/gconv"
)
var UnitReportSum = new(UnitReportSumDao)
type UnitReportSumDao struct{}
// Insert 插入广告效果指标
func (d *UnitReportSumDao) Insert(ctx context.Context, req *dto.UnitReportSumItem) (id int64, err error) {
var entityData *entity.UnitReportSum
if err = gconv.Struct(req, &entityData); err != nil {
return
}
r, err := gfdb.DB(ctx).Model(ctx, consts.UnitReportSumTable).Data(&entityData).Insert()
if err != nil {
return
}
return r.LastInsertId()
}
// BatchInsert 批量插入广告效果指标
func (d *UnitReportSumDao) BatchInsert(ctx context.Context, reqs []*dto.UnitReportSumItem) (successCount int64, failCount int64, failedIndexes []int64, err error) {
if len(reqs) == 0 {
return 0, 0, nil, errors.New("批量插入数据不能为空")
}
batchSize := 100
successCount = 0
failCount = 0
failedIndexes = make([]int64, 0)
for i := 0; i < len(reqs); i += batchSize {
end := i + batchSize
if end > len(reqs) {
end = len(reqs)
}
batch := reqs[i:end]
entityList := make([]*entity.UnitReportSum, len(batch))
for j, req := range batch {
var entityData entity.UnitReportSum
if err = gconv.Struct(req, &entityData); err != nil {
failCount++
failedIndexes = append(failedIndexes, int64(i+j))
continue
}
entityList[j] = &entityData
}
if len(entityList) == 0 {
continue
}
_, err = gfdb.DB(ctx).Model(ctx, consts.UnitReportSumTable).Data(entityList).Insert()
if err != nil {
for k := range batch {
_, singleErr := d.Insert(ctx, batch[k])
if singleErr != nil {
failCount++
failedIndexes = append(failedIndexes, int64(i+k))
} else {
successCount++
}
}
} else {
successCount += int64(len(entityList))
}
}
return successCount, failCount, failedIndexes, nil
}

274
dao/dict/api_field_mapping_config_dao.go
View File

@@ -1,274 +0,0 @@
package dict
import (
"context"
consts "dataengine/consts/public"
dto "dataengine/model/dto/dict"
entity "dataengine/model/entity/dict"
"time"
"gitea.com/red-future/common/beans"
"gitea.com/red-future/common/db/gfdb"
"github.com/gogf/gf/v2/database/gdb"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/util/gconv"
)
var FieldMappingConfig = new(fieldMappingConfigDao)
type fieldMappingConfigDao struct{}
// Insert 插入字段映射配置
func (d *fieldMappingConfigDao) Insert(ctx context.Context, req *dto.CreateFieldMappingConfigReq) (id int64, err error) {
var config entity.FieldMappingConfig
if err = gconv.Struct(req, &config); err != nil {
return
}
now := time.Now()
config.CreatedTime = now
config.UpdatedTime = now
r, err := gfdb.DB(ctx).Model(ctx, consts.FieldMappingConfigTable).Data(&config).Insert()
if err != nil {
return
}
return r.LastInsertId()
}
// Update 更新字段映射配置
func (d *fieldMappingConfigDao) Update(ctx context.Context, req *dto.UpdateFieldMappingConfigReq) (rows int64, err error) {
r, err := gfdb.DB(ctx).Model(ctx, consts.FieldMappingConfigTable).Data(&req).OmitEmpty().Where(entity.FieldMappingConfigCol.Id, req.Id).Update()
if err != nil {
return
}
return r.RowsAffected()
}
// Delete 删除字段映射配置
func (d *fieldMappingConfigDao) Delete(ctx context.Context, req *dto.DeleteFieldMappingConfigReq) (rows int64, err error) {
r, err := gfdb.DB(ctx).Model(ctx, consts.FieldMappingConfigTable).
Where(entity.FieldMappingConfigCol.Id, req.Id).
Delete()
if err != nil {
return
}
return r.RowsAffected()
}
// GetOne 获取单个字段映射配置
func (d *fieldMappingConfigDao) GetOne(ctx context.Context, req *dto.GetFieldMappingConfigReq) (res *entity.FieldMappingConfig, err error) {
r, err := gfdb.DB(ctx).Model(ctx, consts.FieldMappingConfigTable).
Where(entity.FieldMappingConfigCol.Id, req.Id).
One()
if err != nil {
return
}
err = r.Struct(&res)
return
}
// Count 获取字段映射配置数量
func (d *fieldMappingConfigDao) Count(ctx context.Context, req *dto.ListFieldMappingConfigReq) (count int, err error) {
return d.buildListFilter(ctx, req).Count()
}
// List 获取字段映射配置列表
func (d *fieldMappingConfigDao) List(ctx context.Context, req *dto.ListFieldMappingConfigReq) (res []entity.FieldMappingConfig, total int, err error) {
model := d.buildListFilter(ctx, req)
model.OrderDesc(entity.FieldMappingConfigCol.CreatedTime)
if req.Page != nil {
model.Page(int(req.Page.PageNum), int(req.Page.PageSize))
}
r, total, err := model.AllAndCount(false)
if err != nil {
return
}
err = r.Structs(&res)
return
}
// buildListFilter 构建列表查询的过滤条件
func (d *fieldMappingConfigDao) buildListFilter(ctx context.Context, req *dto.ListFieldMappingConfigReq) *gdb.Model {
model := gfdb.DB(ctx).Model(ctx, consts.FieldMappingConfigTable).Model
if !g.IsEmpty(req.Keyword) {
model.WhereLike(entity.FieldMappingConfigCol.ConfigName, "%"+req.Keyword+"%").
WhereOrLike(entity.FieldMappingConfigCol.SourceFieldDesc, "%"+req.Keyword+"%").
WhereOrLike(entity.FieldMappingConfigCol.TargetFieldDesc, "%"+req.Keyword+"%")
}
if !g.IsEmpty(req.ConfigName) {
model.Where(entity.FieldMappingConfigCol.ConfigName, req.ConfigName)
}
if !g.IsEmpty(req.VendorName) {
model.Where(entity.FieldMappingConfigCol.VendorName, req.VendorName)
}
if !g.IsEmpty(req.ApiName) {
model.Where(entity.FieldMappingConfigCol.ApiName, req.ApiName)
}
if !g.IsEmpty(req.ApiVersion) {
model.Where(entity.FieldMappingConfigCol.ApiVersion, req.ApiVersion)
}
if !g.IsEmpty(req.SourceField) {
model.Where(entity.FieldMappingConfigCol.SourceField, req.SourceField)
}
if !g.IsEmpty(req.TargetField) {
model.Where(entity.FieldMappingConfigCol.TargetField, req.TargetField)
}
if !g.IsEmpty(req.TransformType) {
model.Where(entity.FieldMappingConfigCol.TransformType, req.TransformType)
}
if !g.IsEmpty(req.BusinessDomain) {
model.Where(entity.FieldMappingConfigCol.BusinessDomain, req.BusinessDomain)
}
if !g.IsEmpty(req.FieldGroup) {
model.Where(entity.FieldMappingConfigCol.FieldGroup, req.FieldGroup)
}
if req.IsActive != nil {
model.Where(entity.FieldMappingConfigCol.IsActive, *req.IsActive)
}
model.OmitEmptyWhere()
return model
}
// UpdateStatus 更新配置状态
func (d *fieldMappingConfigDao) UpdateStatus(ctx context.Context, id int64, isActive bool) (rows int64, err error) {
r, err := gfdb.DB(ctx).Model(ctx, consts.FieldMappingConfigTable).
Data(g.Map{
"is_active": isActive,
"updated_time": time.Now(),
}).
Where(entity.FieldMappingConfigCol.Id, id).
Update()
if err != nil {
return
}
return r.RowsAffected()
}
// GetByIds 根据ID列表获取配置列表
func (d *fieldMappingConfigDao) GetByIds(ctx context.Context, ids []int64) (res []entity.FieldMappingConfig, err error) {
r, err := gfdb.DB(ctx).Model(ctx, consts.FieldMappingConfigTable).
WhereIn(entity.FieldMappingConfigCol.Id, ids).
All()
if err != nil {
return
}
err = r.Structs(&res)
return
}
// GetByVendorAndApi 根据厂商和接口获取字段映射配置
func (d *fieldMappingConfigDao) GetByVendorAndApi(ctx context.Context, vendorName, apiName, apiVersion string, isActive *bool) (res []*entity.FieldMappingConfig, err error) {
model := gfdb.DB(ctx).Model(ctx, consts.FieldMappingConfigTable).
Where(entity.FieldMappingConfigCol.VendorName, vendorName).
Where(entity.FieldMappingConfigCol.ApiName, apiName)
if !g.IsEmpty(apiVersion) {
model.Where(entity.FieldMappingConfigCol.ApiVersion, apiVersion)
}
if isActive != nil {
model.Where(entity.FieldMappingConfigCol.IsActive, *isActive)
}
model.OrderDesc(entity.FieldMappingConfigCol.Priority).
OrderAsc(entity.FieldMappingConfigCol.Id)
r, err := model.All()
if err != nil {
return
}
err = r.Structs(&res)
return
}
// CheckDuplicate 检查重复配置
func (d *fieldMappingConfigDao) CheckDuplicate(ctx context.Context, vendorName, apiName, sourceField, targetField string, excludeId int64) (exists bool, err error) {
ctx = context.WithValue(ctx, "user", &beans.User{UserName: "admin"})
model := gfdb.DB(ctx).Model(ctx, consts.FieldMappingConfigTable).
Where(entity.FieldMappingConfigCol.VendorName, vendorName).
Where(entity.FieldMappingConfigCol.ApiName, apiName).
Where(entity.FieldMappingConfigCol.SourceField, sourceField).
Where(entity.FieldMappingConfigCol.TargetField, targetField)
if excludeId > 0 {
model.WhereNot(entity.FieldMappingConfigCol.Id, excludeId)
}
count, err := model.Count()
if err != nil {
return
}
exists = count > 0
return
}
// GetActiveConfigsByBusinessDomain 根据业务域获取启用的配置
func (d *fieldMappingConfigDao) GetActiveConfigsByBusinessDomain(ctx context.Context, businessDomain string) (res []entity.FieldMappingConfig, err error) {
r, err := gfdb.DB(ctx).Model(ctx, consts.FieldMappingConfigTable).
Where(entity.FieldMappingConfigCol.BusinessDomain, businessDomain).
Where(entity.FieldMappingConfigCol.IsActive, true).
OrderDesc(entity.FieldMappingConfigCol.Priority).
OrderAsc(entity.FieldMappingConfigCol.Id).
All()
if err != nil {
return
}
err = r.Structs(&res)
return
}
// GetFieldGroupsByVendorApi 获取指定厂商接口的字段分组
func (d *fieldMappingConfigDao) GetFieldGroupsByVendorApi(ctx context.Context, vendorName, apiName string) (groups []string, err error) {
r, err := gfdb.DB(ctx).Model(ctx, consts.FieldMappingConfigTable).
Fields(entity.FieldMappingConfigCol.FieldGroup).
Where(entity.FieldMappingConfigCol.VendorName, vendorName).
Where(entity.FieldMappingConfigCol.ApiName, apiName).
Where(entity.FieldMappingConfigCol.IsActive, true).
Group(entity.FieldMappingConfigCol.FieldGroup).
All()
if err != nil {
return
}
for _, record := range r {
group := record.Map()[entity.FieldMappingConfigCol.FieldGroup]
if groupStr, ok := group.(string); ok && groupStr != "" {
groups = append(groups, groupStr)
}
}
return
}
// DeleteExpiredConfigs 删除已过期的配置
func (d *fieldMappingConfigDao) DeleteExpiredConfigs(ctx context.Context) (rows int64, err error) {
r, err := gfdb.DB(ctx).Model(ctx, consts.FieldMappingConfigTable).
Where("expiry_date IS NOT NULL AND expiry_date < ?", time.Now()).
Delete()
if err != nil {
return
}
return r.RowsAffected()
}
// GetConfigsEffectiveNow 获取当前生效的配置
func (d *fieldMappingConfigDao) GetConfigsEffectiveNow(ctx context.Context) (res []entity.FieldMappingConfig, err error) {
now := time.Now()
r, err := gfdb.DB(ctx).Model(ctx, consts.FieldMappingConfigTable).
Where(entity.FieldMappingConfigCol.IsActive, true).
Where("(effective_date IS NULL OR effective_date <= ?)", now).
Where("(expiry_date IS NULL OR expiry_date > ?)", now).
OrderDesc(entity.FieldMappingConfigCol.Priority).
OrderAsc(entity.FieldMappingConfigCol.Id).
All()
if err != nil {
return
}
err = r.Structs(&res)
return
}

118
dao/tencent/account_relation_dao.go
View File

@@ -1,118 +0,0 @@
package tencent
import (
"context"
consts "dataengine/consts/public"
entity "dataengine/model/entity/tencent"
"time"
"gitea.com/red-future/common/db/gfdb"
"github.com/gogf/gf/v2/frame/g"
"github.com/sirupsen/logrus"
)
type accountRelationDao struct{}
var AccountRelation = new(accountRelationDao)
// Upsert 插入或更新账户关系（根据account_id判断）
func (d *accountRelationDao) Upsert(ctx context.Context, item *entity.AccountRelation) error {
now := time.Now()
// 检查是否已存在
var existing entity.AccountRelation
err := gfdb.DB(ctx).Model(ctx, consts.TencentAccountRelationTable).
Where("tenant_id", item.TenantId).
Where(entity.AccountRelationCols.AccountID, item.AccountID).
WhereNull(entity.AccountRelationCols.DeletedAt).
Scan(&existing)
// Scan找不到记录时err不为nil，但这是正常情况，需要继续执行插入
if err != nil && existing.Id == 0 {
// 记录不存在，执行插入
item.CreatedAt = &now
item.UpdatedAt = &now
_, err = gfdb.DB(ctx).Model(ctx, consts.TencentAccountRelationTable).
Data(item).
Insert()
return err
}
// 记录存在，执行更新
item.UpdatedAt = &now
_, err = gfdb.DB(ctx).Model(ctx, consts.TencentAccountRelationTable).
Where("id", existing.Id).
Data(g.Map{
entity.AccountRelationCols.CorporationName: item.CorporationName,
entity.AccountRelationCols.CommentDataList: item.CommentDataList,
entity.AccountRelationCols.IsAdx: item.IsAdx,
entity.AccountRelationCols.IsBid: item.IsBid,
entity.AccountRelationCols.IsMp: item.IsMp,
entity.AccountRelationCols.UpdatedAt: now,
}).
Update()
return err
}
// BatchUpsert 批量插入或更新（使用 OnConflict 实现 Upsert）
func (d *accountRelationDao) BatchUpsert(ctx context.Context, items []*entity.AccountRelation) (successCount int, err error) {
if len(items) == 0 {
return 0, nil
}
logrus.Infof("开始批量Upsert: %d 条记录", len(items))
// 分批处理，每批100条
batchSize := 100
successCount = 0
for i := 0; i < len(items); i += batchSize {
end := i + batchSize
if end > len(items) {
end = len(items)
}
batch := items[i:end]
logrus.Infof("处理第 %d-%d 条记录", i+1, end)
// 执行批量插入，使用 OnConflict 实现 Upsert
result, err := gfdb.DB(ctx).Model(ctx, consts.TencentAccountRelationTable).
Data(batch).
OnConflict(entity.AccountRelationCols.AccountID).
Save()
if err != nil {
logrus.Errorf("批量Upsert失败: %v，尝试逐条处理", err)
// 批量失败，逐条处理
for _, item := range batch {
if upsertErr := d.Upsert(ctx, item); upsertErr != nil {
logrus.Errorf("逐条Upsert失败: account_id=%d, err=%v", item.AccountID, upsertErr)
continue
}
successCount++
}
} else {
affected, _ := result.RowsAffected()
successCount += int(affected)
logrus.Infof("批量Upsert成功: 影响 %d 条记录", affected)
}
}
logrus.Infof("批量Upsert完成: 成功 %d 条", successCount)
return successCount, nil
}
// ListAll 获取所有账户关系
func (d *accountRelationDao) ListAll(ctx context.Context) ([]entity.AccountRelation, error) {
var list []entity.AccountRelation
err := gfdb.DB(ctx).Model(ctx, consts.TencentAccountRelationTable).
WhereNull(entity.AccountRelationCols.DeletedAt).
OrderAsc(entity.AccountRelationCols.AccountID).
Scan(&list)
return list, err
}

111
dao/tencent/audio_dao.go
View File

@@ -1,111 +0,0 @@
package tencent
import (
"context"
consts "dataengine/consts/public"
entity "dataengine/model/entity/tencent"
"gitea.com/red-future/common/db/gfdb"
"github.com/gogf/gf/v2/frame/g"
"github.com/sirupsen/logrus"
)
type audioDao struct{}
var Audio = new(audioDao)
// BatchUpsert 批量插入或更新（使用 OnConflict 实现 Upsert）
func (d *audioDao) BatchUpsert(ctx context.Context, items []*entity.Audio) (successCount int, err error) {
if len(items) == 0 {
return 0, nil
}
logrus.Infof("开始批量Upsert音乐素材: %d 条记录", len(items))
// 分批处理，每批100条
batchSize := 100
successCount = 0
for i := 0; i < len(items); i += batchSize {
end := i + batchSize
if end > len(items) {
end = len(items)
}
batch := items[i:end]
logrus.Infof("处理第 %d-%d 条音乐素材记录", i+1, end)
// 执行批量插入，使用 OnConflict 实现 Upsert
result, err := gfdb.DB(ctx).Model(ctx, consts.TencentAudioTable).
Data(batch).
OnConflict(entity.AudioCols.AudioId).
Save()
if err != nil {
logrus.Errorf("批量Upsert音乐素材失败: %v，尝试逐条处理", err)
// 批量失败，逐条处理
for _, item := range batch {
if upsertErr := d.upsertSingle(ctx, item); upsertErr != nil {
logrus.Errorf("逐条Upsert音乐素材失败: audio_id=%s, err=%v", item.AudioId, upsertErr)
continue
}
successCount++
}
} else {
affected, _ := result.RowsAffected()
successCount += int(affected)
logrus.Infof("批量Upsert音乐素材成功: 影响 %d 条记录", affected)
}
}
logrus.Infof("批量Upsert音乐素材完成: 成功 %d 条", successCount)
return successCount, nil
}
// upsertSingle 单条插入或更新
func (d *audioDao) upsertSingle(ctx context.Context, item *entity.Audio) error {
var existing entity.Audio
err := gfdb.DB(ctx).Model(ctx, consts.TencentAudioTable).
Where("tenant_id", item.TenantId).
Where(entity.AudioCols.AudioId, item.AudioId).
WhereNull(entity.AudioCols.DeletedAt).
Scan(&existing)
if err != nil && existing.Id == 0 {
// 记录不存在，执行插入
_, err = gfdb.DB(ctx).Model(ctx, consts.TencentAudioTable).
Data(item).
Insert()
return err
}
// 记录存在，执行更新
_, err = gfdb.DB(ctx).Model(ctx, consts.TencentAudioTable).
Where("id", existing.Id).
Data(g.Map{
entity.AudioCols.CoverImageUrl: item.CoverImageUrl,
entity.AudioCols.AudioName: item.AudioName,
entity.AudioCols.Author: item.Author,
entity.AudioCols.Duration: item.Duration,
entity.AudioCols.ExpireTime: item.ExpireTime,
entity.AudioCols.FeelTags: item.FeelTags,
entity.AudioCols.GenreTags: item.GenreTags,
entity.AudioCols.Updater: item.Updater,
entity.AudioCols.UpdatedAt: item.UpdatedAt,
}).
Update()
return err
}
// ListAll 获取所有音乐素材
func (d *audioDao) ListAll(ctx context.Context) ([]entity.Audio, error) {
var list []entity.Audio
err := gfdb.DB(ctx).Model(ctx, consts.TencentAudioTable).
WhereNull(entity.AudioCols.DeletedAt).
OrderAsc(entity.AudioCols.AudioId).
Scan(&list)
return list, err
}

159
dao/tencent/image_dao.go
View File

@@ -1,159 +0,0 @@
package tencent
import (
"context"
consts "dataengine/consts/public"
entity "dataengine/model/entity/tencent"
"gitea.com/red-future/common/db/gfdb"
"github.com/gogf/gf/v2/frame/g"
"github.com/sirupsen/logrus"
)
type imageDao struct{}
var Image = new(imageDao)
// BatchUpsert 批量插入或更新（使用 OnConflict 实现 Upsert）
func (d *imageDao) BatchUpsert(ctx context.Context, items []*entity.Image) (successCount int, err error) {
if len(items) == 0 {
return 0, nil
}
logrus.Infof("开始批量Upsert图片素材: %d 条记录", len(items))
// 分批处理，每批100条
batchSize := 100
successCount = 0
for i := 0; i < len(items); i += batchSize {
end := i + batchSize
if end > len(items) {
end = len(items)
}
batch := items[i:end]
logrus.Infof("处理第 %d-%d 条图片素材记录", i+1, end)
// 执行批量插入，使用 OnConflict 实现 Upsert
result, err := gfdb.DB(ctx).Model(ctx, consts.TencentImageTable).
Data(batch).
OnConflict("(image_id, account_id)").
Save()
if err != nil {
logrus.Errorf("批量Upsert图片素材失败: %v，尝试逐条处理", err)
// 批量失败，逐条处理
for _, item := range batch {
if upsertErr := d.upsertSingle(ctx, item); upsertErr != nil {
logrus.Errorf("逐条Upsert图片素材失败: image_id=%s, account_id=%d, err=%v", item.ImageId, item.AccountId, upsertErr)
} else {
successCount++
}
}
} else {
affected, _ := result.RowsAffected()
successCount += int(affected)
logrus.Infof("批量Upsert图片素材成功: 影响 %d 条记录", affected)
}
}
logrus.Infof("批量Upsert图片素材完成: 成功 %d 条", successCount)
return successCount, nil
}
// upsertSingle 单条插入或更新
func (d *imageDao) upsertSingle(ctx context.Context, item *entity.Image) error {
var existing entity.Image
err := gfdb.DB(ctx).Model(ctx, consts.TencentImageTable).
Where(entity.ImageCols.ImageId, item.ImageId).
Where(entity.ImageCols.AccountId, item.AccountId).
WhereNull("deleted_at").
Scan(&existing)
if err != nil && existing.Id == 0 {
// 记录不存在，执行插入
_, err = gfdb.DB(ctx).Model(ctx, consts.TencentImageTable).
Data(item).
Insert()
return err
}
// 记录存在，执行更新
_, err = gfdb.DB(ctx).Model(ctx, consts.TencentImageTable).
Where("id", existing.Id).
Data(g.Map{
entity.ImageCols.Width: item.Width,
entity.ImageCols.Height: item.Height,
entity.ImageCols.FileSize: item.FileSize,
entity.ImageCols.Type: item.Type,
entity.ImageCols.Signature: item.Signature,
entity.ImageCols.Description: item.Description,
entity.ImageCols.PreviewUrl: item.PreviewUrl,
entity.ImageCols.ThumbPreviewUrl: item.ThumbPreviewUrl,
entity.ImageCols.Status: item.Status,
entity.ImageCols.LastModifiedTime: item.LastModifiedTime,
}).
Update()
return err
}
// ListAll 获取所有图片素材
func (d *imageDao) ListAll(ctx context.Context) ([]entity.Image, error) {
var list []entity.Image
err := gfdb.DB(ctx).Model(ctx, consts.TencentImageTable).
WhereNull("deleted_at").
OrderAsc(entity.ImageCols.ImageId).
Scan(&list)
return list, err
}
// ListWithPage 分页查询图片素材（支持时间过滤）
func (d *imageDao) ListWithPage(ctx context.Context, page, pageSize int, accountId *int64, startTime, endTime *int64, status string) ([]entity.Image, int, error) {
model := gfdb.DB(ctx).Model(ctx, consts.TencentImageTable).
WhereNull("deleted_at")
// 账户ID过滤
if accountId != nil && *accountId > 0 {
model = model.Where(entity.ImageCols.AccountId, *accountId)
}
// 状态过滤
if status != "" {
model = model.Where(entity.ImageCols.Status, status)
}
// 时间范围过滤（根据 last_modified_time）
if startTime != nil && *startTime > 0 {
model = model.WhereGTE(entity.ImageCols.LastModifiedTime, *startTime)
}
if endTime != nil && *endTime > 0 {
model = model.WhereLTE(entity.ImageCols.LastModifiedTime, *endTime)
}
// 设置排序（按最后修改时间降序）
model = model.OrderDesc(entity.ImageCols.LastModifiedTime)
// 获取总数
total, err := model.Count()
if err != nil {
return nil, 0, err
}
// 分页查询
var list []entity.Image
if page > 0 && pageSize > 0 {
err = model.Page(page, pageSize).Scan(&list)
} else {
err = model.Scan(&list)
}
if err != nil {
return nil, 0, err
}
return list, total, nil
}

158
dao/tencent/video_dao.go
View File

@@ -1,158 +0,0 @@
package tencent
import (
"context"
consts "dataengine/consts/public"
entity "dataengine/model/entity/tencent"
"gitea.com/red-future/common/db/gfdb"
"github.com/gogf/gf/v2/frame/g"
"github.com/sirupsen/logrus"
)
type videoDao struct{}
var Video = new(videoDao)
// BatchUpsert 批量插入或更新（使用 OnConflict 实现 Upsert）
func (d *videoDao) BatchUpsert(ctx context.Context, items []*entity.Video) (successCount int, err error) {
if len(items) == 0 {
return 0, nil
}
logrus.Infof("开始批量Upsert视频素材: %d 条记录", len(items))
// 分批处理，每批100条
batchSize := 100
successCount = 0
for i := 0; i < len(items); i += batchSize {
end := i + batchSize
if end > len(items) {
end = len(items)
}
batch := items[i:end]
logrus.Infof("处理第 %d-%d 条视频素材记录", i+1, end)
// 执行批量插入，使用 OnConflict 实现 Upsert
result, err := gfdb.DB(ctx).Model(ctx, consts.TencentVideoTable).
Data(batch).
OnConflict("(video_id, account_id)").
Save()
if err != nil {
logrus.Errorf("批量Upsert视频素材失败: %v，尝试逐条处理", err)
// 批量失败，逐条处理
for _, item := range batch {
if upsertErr := d.upsertSingle(ctx, item); upsertErr != nil {
logrus.Errorf("逐条Upsert视频素材失败: video_id=%s, account_id=%d, err=%v", item.VideoId, item.AccountId, upsertErr)
} else {
successCount++
}
}
} else {
affected, _ := result.RowsAffected()
successCount += int(affected)
logrus.Infof("批量Upsert视频素材成功: 影响 %d 条记录", affected)
}
}
logrus.Infof("批量Upsert视频素材完成: 成功 %d 条", successCount)
return successCount, nil
}
// upsertSingle 单条插入或更新
func (d *videoDao) upsertSingle(ctx context.Context, item *entity.Video) error {
var existing entity.Video
err := gfdb.DB(ctx).Model(ctx, consts.TencentVideoTable).
Where(entity.VideoCols.VideoId, item.VideoId).
Where(entity.VideoCols.AccountId, item.AccountId).
WhereNull("deleted_at").
Scan(&existing)
if err != nil && existing.Id == 0 {
// 记录不存在，执行插入
_, err = gfdb.DB(ctx).Model(ctx, consts.TencentVideoTable).
Data(item).
Insert()
return err
}
// 记录存在，执行更新
_, err = gfdb.DB(ctx).Model(ctx, consts.TencentVideoTable).
Where("id", existing.Id).
Data(g.Map{
entity.VideoCols.Width: item.Width,
entity.VideoCols.Height: item.Height,
entity.VideoCols.FileSize: item.FileSize,
entity.VideoCols.Type: item.Type,
entity.VideoCols.Signature: item.Signature,
entity.VideoCols.Description: item.Description,
entity.VideoCols.PreviewUrl: item.PreviewUrl,
entity.VideoCols.Status: item.Status,
entity.VideoCols.LastModifiedTime: item.LastModifiedTime,
}).
Update()
return err
}
// ListAll 获取所有视频素材
func (d *videoDao) ListAll(ctx context.Context) ([]entity.Video, error) {
var list []entity.Video
err := gfdb.DB(ctx).Model(ctx, consts.TencentVideoTable).
WhereNull("deleted_at").
OrderAsc(entity.VideoCols.VideoId).
Scan(&list)
return list, err
}
// ListWithPage 分页查询视频素材（支持时间过滤）
func (d *videoDao) ListWithPage(ctx context.Context, page, pageSize int, accountId *int64, startTime, endTime *int64, status string) ([]entity.Video, int, error) {
model := gfdb.DB(ctx).Model(ctx, consts.TencentVideoTable).
WhereNull("deleted_at")
// 账户ID过滤
if accountId != nil && *accountId > 0 {
model = model.Where(entity.VideoCols.AccountId, *accountId)
}
// 状态过滤
if status != "" {
model = model.Where(entity.VideoCols.Status, status)
}
// 时间范围过滤（根据 last_modified_time）
if startTime != nil && *startTime > 0 {
model = model.WhereGTE(entity.VideoCols.LastModifiedTime, *startTime)
}
if endTime != nil && *endTime > 0 {
model = model.WhereLTE(entity.VideoCols.LastModifiedTime, *endTime)
}
// 设置排序（按最后修改时间降序）
model = model.OrderDesc(entity.VideoCols.LastModifiedTime)
// 获取总数
total, err := model.Count()
if err != nil {
return nil, 0, err
}
// 分页查询
var list []entity.Video
if page > 0 && pageSize > 0 {
err = model.Page(page, pageSize).Scan(&list)
} else {
err = model.Scan(&list)
}
if err != nil {
return nil, 0, err
}
return list, total, nil
}