重构数据引擎和报表引擎

This commit is contained in:
2026-06-11 13:06:54 +08:00
parent 285a0fc632
commit 419473f266
53 changed files with 8434 additions and 375 deletions

View File

@@ -29,7 +29,7 @@ type ApiResult struct {
type ApiClient struct {
config *PlatformConfig
client *http.Client
rateLimiter <-chan time.Time // 限流 ticker
rateLimiter *time.Ticker // 限流 ticker,可被 GC
}
// NewApiClient 创建客户端
@@ -38,14 +38,22 @@ func NewApiClient(config *PlatformConfig) *ApiClient {
if config.RequestTimeoutMs > 0 {
timeout = time.Duration(config.RequestTimeoutMs) * time.Millisecond
}
transport := &http.Transport{
MaxIdleConns: 100,
MaxIdleConnsPerHost: 20,
IdleConnTimeout: 90 * time.Second,
}
ac := &ApiClient{
config: config,
client: &http.Client{Timeout: timeout},
client: &http.Client{
Timeout: timeout,
Transport: transport,
},
}
// 初始化限流
if config.RateLimitPerMinute > 0 {
interval := time.Minute / time.Duration(config.RateLimitPerMinute)
ac.rateLimiter = time.Tick(interval)
ac.rateLimiter = time.NewTicker(interval)
logrus.Infof("限流已启用: %d 次/分钟, 间隔 %v", config.RateLimitPerMinute, interval)
}
return ac
@@ -61,6 +69,13 @@ func (c *ApiClient) PostJSON(ctx context.Context, path string, body interface{})
return c.doRequest(ctx, "POST", path, body, false)
}
// Close 释放客户端资源(限流 ticker
func (c *ApiClient) Close() {
if c.rateLimiter != nil {
c.rateLimiter.Stop()
}
}
// Request 通用请求方法(支持 GET/POST支持参数在 query 或 body
func (c *ApiClient) Request(ctx context.Context, method, path string, params map[string]interface{}, paramsInQuery bool) (*ApiResult, error) {
if paramsInQuery {
@@ -99,8 +114,9 @@ func (c *ApiClient) execute(ctx context.Context, method, path string, body inter
// 限流等待
if c.rateLimiter != nil {
select {
case <-c.rateLimiter:
case <-c.rateLimiter.C:
case <-ctx.Done():
c.rateLimiter.Stop()
return nil, ctx.Err()
}
}
@@ -112,8 +128,13 @@ func (c *ApiClient) execute(ctx context.Context, method, path string, body inter
fullURL = c.applyAuthURL(fullURL)
var reqBody io.Reader
var reqBodyBytes []byte
if body != nil && !paramsInQuery {
b, _ := json.Marshal(body)
b, err := json.Marshal(body)
if err != nil {
return nil, fmt.Errorf("JSON序列化请求体失败: %w", err)
}
reqBodyBytes = b
reqBody = bytes.NewBuffer(b)
}
@@ -133,7 +154,7 @@ func (c *ApiClient) execute(ctx context.Context, method, path string, body inter
return nil, fmt.Errorf("创建请求失败: %w", err)
}
c.applyAuthHeader(req)
c.applyAuthHeader(req, reqBodyBytes)
req.Header.Set("User-Agent", "data-engine/1.0")
if body != nil && !paramsInQuery {
req.Header.Set("Content-Type", "application/json")
@@ -145,7 +166,10 @@ func (c *ApiClient) execute(ctx context.Context, method, path string, body inter
}
defer resp.Body.Close()
respBody, _ := io.ReadAll(resp.Body)
respBody, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("读取响应体失败: %w", err)
}
result := &ApiResult{Body: respBody, DurationMs: time.Since(start).Milliseconds()}
if resp.StatusCode >= 400 {
@@ -157,7 +181,11 @@ func (c *ApiClient) execute(ctx context.Context, method, path string, body inter
// buildQueryURL 将 params 拼接到 URL 查询参数中
// 支持数组/对象类型的值自动 JSON 序列化 + URL 编码
func (c *ApiClient) buildQueryURL(rawURL string, params map[string]interface{}) string {
parsed, _ := url.Parse(rawURL)
parsed, err := url.Parse(rawURL)
if err != nil || parsed == nil {
logrus.Errorf("buildQueryURL: 解析 URL 失败: %v", err)
return rawURL
}
q := parsed.Query()
for k, v := range params {
@@ -224,7 +252,11 @@ func (c *ApiClient) applyAuthURL(rawURL string) string {
return rawURL
}
parsed, _ := url.Parse(rawURL)
parsed, err := url.Parse(rawURL)
if err != nil || parsed == nil {
logrus.Errorf("applyAuthURL: 解析 URL 失败: %v", err)
return rawURL
}
q := parsed.Query()
if tokenInQuery && token != "" {
q.Set(queryKey, token)
@@ -236,10 +268,16 @@ func (c *ApiClient) applyAuthURL(rawURL string) string {
return parsed.String()
}
func (c *ApiClient) applyAuthHeader(req *http.Request) {
func (c *ApiClient) applyAuthHeader(req *http.Request, bodyBytes []byte) {
cfg := c.config.AuthConfig
token := c.config.AccessToken
// APP_SIGNATURE 认证app-id + signature 头部(如钉钉智能薪酬)
if c.config.AuthType == "APP_SIGNATURE" {
c.applyAppSignatureAuth(req, bodyBytes)
return
}
if cfg != nil {
if tiq, _ := cfg["token_in_query"].(bool); tiq {
return
@@ -251,9 +289,9 @@ func (c *ApiClient) applyAuthHeader(req *http.Request) {
if cfg != nil {
if h, ok := cfg["header_name"].(string); ok {
f := cfg["header_format"].(string)
if f == "" {
f = "{token}"
f := "{token}"
if fv, ok2 := cfg["header_format"].(string); ok2 {
f = fv
}
req.Header.Set(h, strings.ReplaceAll(f, "{token}", token))
return
@@ -268,6 +306,73 @@ func (c *ApiClient) applyAuthHeader(req *http.Request) {
}
}
// applyAppSignatureAuth 设置 app-id + signature 认证头部
func (c *ApiClient) applyAppSignatureAuth(req *http.Request, bodyBytes []byte) {
cfg := c.config.AuthConfig
if cfg == nil {
return
}
// 1. 设置 app-id 头部
appIdHeader := "app-id"
if h, _ := cfg["app_id_header"].(string); h != "" {
appIdHeader = h
}
appId := c.config.AppKey
if appId == "" {
if aid, _ := cfg["app_id"].(string); aid != "" {
appId = aid
}
}
if appId != "" {
req.Header.Set(appIdHeader, appId)
}
// 2. 计算签名并设置 signature 头部
signHeader := "signature"
if h, _ := cfg["sign_header"].(string); h != "" {
signHeader = h
}
secret := c.config.AppSecret
signAlgo := "md5_upper_body"
if a, _ := cfg["sign_algorithm"].(string); a != "" {
signAlgo = a
}
sig := computeBodySignature(bodyBytes, secret, signAlgo)
if sig != "" {
req.Header.Set(signHeader, sig)
}
}
// computeBodySignature 计算基于请求体的签名
// 支持的算法:
// - md5_upper_body: MD5(body_string + secret) 大写(默认,钉钉智能薪酬)
// - md5_body: MD5(body_string + secret) 小写
func computeBodySignature(bodyBytes []byte, secret, algo string) string {
if secret == "" {
return ""
}
bodyStr := ""
if len(bodyBytes) > 0 {
bodyStr = string(bodyBytes)
}
switch algo {
case "md5_body", "md5_upper_body":
h := md5.Sum([]byte(bodyStr + secret))
sig := hex.EncodeToString(h[:])
if algo == "md5_upper_body" {
sig = strings.ToUpper(sig)
}
return sig
default:
logrus.Warnf("未知签名算法: %s", algo)
return ""
}
}
func generateNonce() string {
nanoPart := time.Now().UnixNano() % 1000000000000
r, _ := rand.Int(rand.Reader, big.NewInt(10000))
@@ -293,7 +398,11 @@ func (c *ApiClient) applySignature(rawURL string, body interface{}, paramsInQuer
return rawURL
}
parsed, _ := url.Parse(rawURL)
parsed, err := url.Parse(rawURL)
if err != nil || parsed == nil {
logrus.Errorf("applySignature: 解析 URL 失败: %v", err)
return rawURL
}
q := parsed.Query()
// 收集所有参数并按 key 排序

View File

@@ -8,7 +8,7 @@ import (
dao "dataengine/dao/copydata"
taskDto "dataengine/model/dto/copydata"
"gitea.com/red-future/common/beans"
"gitea.redpowerfuture.com/red-future/common/beans"
"github.com/gogf/gf/v2/frame/g"
"github.com/sirupsen/logrus"
)

View File

@@ -4,7 +4,7 @@ import (
"context"
"time"
"gitea.com/red-future/common/db/gfdb"
"gitea.redpowerfuture.com/red-future/common/db/gfdb"
"github.com/sirupsen/logrus"
)
@@ -19,12 +19,8 @@ func InsertRows(ctx context.Context, tableName string, conflictKeys []string, ro
if rows[i] == nil {
rows[i] = make(map[string]interface{})
}
if _, ok := rows[i]["created_at"]; !ok {
rows[i]["created_at"] = now
}
if _, ok := rows[i]["updated_at"]; !ok {
rows[i]["updated_at"] = now
}
// 始终覆盖 updated_at不设置 created_at 让数据库维护首次值upsert 时不会覆盖)
rows[i]["updated_at"] = now
}
batchSize := 100

View File

@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"strconv"
"strings"
"sync"
"time"
@@ -13,7 +14,7 @@ import (
taskDto "dataengine/model/dto/copydata"
entity "dataengine/model/entity/dict"
"gitea.com/red-future/common/db/gfdb"
"gitea.redpowerfuture.com/red-future/common/db/gfdb"
"github.com/sirupsen/logrus"
)
@@ -38,8 +39,20 @@ type PrefetchConfig struct {
ValueField string `json:"value_field"`
}
// RecursiveConfig 递归遍历配置(如钉钉部门树)
type RecursiveConfig struct {
KeyField string `json:"key_field"`
TargetParam string `json:"target_param"`
}
// SyncByConfig 执行同步
func SyncByConfig(ctx context.Context, platformCode, interfaceCode string, isFullSync bool) (*SyncResult, error) {
// 创建超时 context 防止单次同步卡死
timeoutMin := GetSyncTimeout(ctx)
timeoutCtx, cancel := context.WithTimeout(ctx, time.Duration(timeoutMin)*time.Minute)
defer cancel()
ctx = timeoutCtx
// 内存锁:防止同一个接口被并发执行(两个调度周期重叠时跳过)
lockKey := platformCode + "/" + interfaceCode
if _, loaded := syncRunningMap.LoadOrStore(lockKey, true); loaded {
@@ -93,11 +106,16 @@ func SyncByConfig(ctx context.Context, platformCode, interfaceCode string, isFul
markSyncRunning(ctx, platformCode, interfaceCode, lastSyncTime)
api := NewApiClient(platform)
defer api.Close()
prefetch := parsePrefetchConfig(iface.RequestConfig)
if prefetch != nil {
return syncWithPrefetch(ctx, api, platform, iface, ifaces, td, prefetch, isFullSync, lastSyncTime, start)
}
recursive := parseRecursiveConfig(iface.RequestConfig)
if recursive != nil {
return syncRecursive(ctx, api, platform, iface, td, recursive, start)
}
return syncSingleAPI(ctx, api, platform, iface, td, isFullSync, lastSyncTime, start)
}
@@ -119,6 +137,8 @@ func syncSingleAPI(ctx context.Context, api *ApiClient, platform *PlatformConfig
pageSize := GetSyncPageSize(ctx)
if ps, ok := iface.RequestConfig["page_size"].(float64); ok {
pageSize = int(ps)
} else if ps, ok := iface.RequestConfig["pageSize"].(float64); ok {
pageSize = int(ps)
}
taskType := "incremental"
@@ -129,14 +149,19 @@ func syncSingleAPI(ctx context.Context, api *ApiClient, platform *PlatformConfig
inQuery := paramsInQuery(iface)
method := string(iface.Method)
// 游标分页首次请求需要 cursor=""(通过 extraParams 覆盖 buildReqBody 的 page=1 赋值)
// 游标分页首次请求需要处理初始游标值
firstExtra := map[string]interface{}{}
if isCursorPagination(iface) {
cp := "cursor"
if p, ok := iface.RequestConfig["page_param"].(string); ok && p != "" {
cp = p
}
firstExtra[cp] = ""
// 支持 initial_cursor 配置如钉钉HRM首次传 0
if icv, ok := iface.RequestConfig["initial_cursor"]; ok {
firstExtra[cp] = icv
} else {
firstExtra[cp] = ""
}
}
body := buildReqBody(iface, 1, pageSize, lastSyncTime, firstExtra)
resp, err := api.Request(ctx, method, iface.Url, body, inQuery)
@@ -151,6 +176,8 @@ func syncSingleAPI(ctx context.Context, api *ApiClient, platform *PlatformConfig
return nil, err
}
injectRowFields(rows, body, iface.RequestConfig)
result := &SyncResult{TableName: td.TableName, TotalPages: totalPages}
inserted, _ := savePage(ctx, td, rows)
result.InsertedRows += inserted
@@ -185,6 +212,7 @@ func syncSingleAPI(ctx context.Context, api *ApiClient, platform *PlatformConfig
}
nextCursor = nc
injectRowFields(rows, body, iface.RequestConfig)
inserted, _ = savePage(ctx, td, rows)
result.InsertedRows += inserted
result.TotalRows += len(rows)
@@ -194,22 +222,72 @@ func syncSingleAPI(ctx context.Context, api *ApiClient, platform *PlatformConfig
result.TotalPages++
time.Sleep(100 * time.Millisecond)
}
} else if iface.ResponseConfig != nil {
// hasMore 分页(如钉钉 offset/size + hasMore
if hf, _ := iface.ResponseConfig["has_more_field"].(string); hf != "" {
for page := 2; hasMoreCheck(resp.Body, hf); page++ {
body := buildReqBody(iface, page, pageSize, lastSyncTime, nil)
resp2, e2 := api.Request(ctx, method, iface.Url, body, inQuery)
if e2 != nil {
logrus.Errorf("第 %d 页请求失败: %v", page, e2)
break
}
rows2, _, mt2, _, pe2 := parseRespExt(resp2.Body, iface.ResponseConfig)
if pe2 != nil {
logrus.Errorf("第 %d 页解析失败: %v", page, pe2)
break
}
injectRowFields(rows2, body, iface.RequestConfig)
inserted2, _ := savePage(ctx, td, rows2)
result.InsertedRows += inserted2
result.TotalRows += len(rows2)
if mt2 > maxTime {
maxTime = mt2
}
resp = resp2
time.Sleep(100 * time.Millisecond)
}
} else {
// 普通分页
for page := 2; page <= totalPages; page++ {
body := buildReqBody(iface, page, pageSize, lastSyncTime, nil)
resp, err = api.Request(ctx, method, iface.Url, body, inQuery)
if err != nil {
logrus.Errorf("第 %d 页请求失败: %v", page, err)
recordFailure(ctx, platform.PlatformCode, iface.Code, taskType, fmt.Sprintf("第 %d 页请求失败: %v", page, err))
continue
}
rows, _, mt, _, pe := parseRespExt(resp.Body, iface.ResponseConfig)
if pe != nil {
logrus.Errorf("第 %d 页解析失败: %v", page, pe)
recordFailure(ctx, platform.PlatformCode, iface.Code, taskType, fmt.Sprintf("第 %d 页解析失败: %v", page, pe))
continue
}
injectRowFields(rows, body, iface.RequestConfig)
inserted, _ = savePage(ctx, td, rows)
result.InsertedRows += inserted
result.TotalRows += len(rows)
if mt > maxTime {
maxTime = mt
}
time.Sleep(100 * time.Millisecond)
}
}
} else {
// 普通分页
// 普通分页(无 response_config
for page := 2; page <= totalPages; page++ {
body := buildReqBody(iface, page, pageSize, lastSyncTime, nil)
resp, err := api.Request(ctx, method, iface.Url, body, inQuery)
resp, err = api.Request(ctx, method, iface.Url, body, inQuery)
if err != nil {
logrus.Errorf("第 %d 页请求失败: %v", page, err)
recordFailure(ctx, platform.PlatformCode, iface.Code, taskType, fmt.Sprintf("第 %d 页请求失败: %v", page, err))
continue
}
rows, _, mt, _, pe := parseRespExt(resp.Body, iface.ResponseConfig)
if pe != nil {
logrus.Errorf("第 %d 页解析失败: %v", page, pe)
recordFailure(ctx, platform.PlatformCode, iface.Code, taskType, fmt.Sprintf("第 %d 页解析失败: %v", page, pe))
continue
}
injectRowFields(rows, body, iface.RequestConfig)
inserted, _ = savePage(ctx, td, rows)
result.InsertedRows += inserted
result.TotalRows += len(rows)
@@ -238,6 +316,33 @@ func isCursorPagination(iface *entity.ApiInterface) bool {
return cp
}
// hasMoreCheck 从响应体中提取 has_more_field 的值
func hasMoreCheck(raw []byte, hasMorePath string) bool {
var respMap map[string]interface{}
if err := json.Unmarshal(raw, &respMap); err != nil {
return false
}
parts := strings.Split(hasMorePath, ".")
cc := respMap
for i, p := range parts {
if i == len(parts)-1 {
if b, ok := cc[p].(bool); ok {
return b
}
if s, ok := cc[p].(string); ok {
return s == "true"
}
return false
}
if m, ok := cc[p].(map[string]interface{}); ok {
cc = m
} else {
return false
}
}
return false
}
// collectPrefetchEntities 从 rows 中收集实体和行数据
func collectPrefetchEntities(rows []map[string]interface{}, prefetch *PrefetchConfig, allEntities *[]interface{}, allRows *[]map[string]interface{}) {
for _, item := range rows {
@@ -266,6 +371,12 @@ func syncWithPrefetch(ctx context.Context, api *ApiClient, platform *PlatformCon
// ====== 1. 预取阶段:分页拉取全部实体列表 ======
prefetchIface := findInterfaceByURL(allIfaces, prefetch.URL)
// 判断预取来源是否有递归配置(如钉钉部门树)
var prefetchRecursiveCfg *RecursiveConfig
if prefetchIface != nil {
prefetchRecursiveCfg = parseRecursiveConfig(prefetchIface.RequestConfig)
}
// 判断预取来源是否游标分页,以及分页参数名
prefetchIsCursor := false
prefetchPageParam := "page"
@@ -303,69 +414,127 @@ func syncWithPrefetch(ctx context.Context, api *ApiClient, platform *PlatformCon
allEntities := make([]interface{}, 0)
allRows := make([]map[string]interface{}, 0)
// 第一页(游标分页首次 cursor=""
firstExtra := make(map[string]interface{})
if prefetchIsCursor {
firstExtra[prefetchPageParam] = ""
}
prefetchReqIface := prefetchIface
if prefetchReqIface == nil {
prefetchReqIface = iface
}
body := buildReqBody(prefetchReqIface, 1, prefetchPageSize, lastSyncTime, firstExtra)
resp, err := api.Request(ctx, prefetchMethod, prefetch.URL, body, prefetchInQuery)
if err != nil {
recordFailure(ctx, platform.PlatformCode, iface.Code, taskType, fmt.Sprintf("预取第一页请求失败: %v", err))
return nil, fmt.Errorf("预取第一页失败: %w", err)
}
rows, prefetchTotalPages, _, nextCursor, err := parseRespExt(resp.Body, prefetchRespCfg)
if err != nil {
recordFailure(ctx, platform.PlatformCode, iface.Code, taskType, fmt.Sprintf("解析预取响应失败: %v", err))
return nil, fmt.Errorf("解析预取响应失败: %w", err)
}
collectPrefetchEntities(rows, prefetch, &allEntities, &allRows)
if prefetchIface != nil && prefetchRecursiveCfg != nil {
// ----- 递归遍历预取(如钉钉部门树)-----
maxDepth := 20
if md, ok := prefetchIface.RequestConfig["max_recursive_depth"].(float64); ok {
maxDepth = int(md)
}
processedKeys := make(map[string]bool)
type rItem struct {
depth int
keyVal interface{}
}
queue := []rItem{{depth: 0, keyVal: nil}}
// 分页循环
if prefetchIsCursor {
// 游标分页
for nextCursor != "" && nextCursor != "nomore" {
body := buildReqBody(prefetchReqIface, 1, prefetchPageSize, lastSyncTime, map[string]interface{}{
prefetchPageParam: nextCursor,
})
resp, err := api.Request(ctx, prefetchMethod, prefetch.URL, body, prefetchInQuery)
for len(queue) > 0 {
item := queue[0]
queue = queue[1:]
if item.depth > maxDepth {
continue
}
if item.keyVal != nil {
keyStr := fmt.Sprintf("%v", item.keyVal)
if processedKeys[keyStr] {
continue
}
processedKeys[keyStr] = true
}
extra := make(map[string]interface{})
if item.keyVal != nil {
extra[prefetchRecursiveCfg.TargetParam] = item.keyVal
}
body := buildReqBody(prefetchReqIface, 1, prefetchPageSize, 0, extra)
r2, err := api.Request(ctx, prefetchMethod, prefetch.URL, body, prefetchInQuery)
if err != nil {
logrus.Errorf("预取游标 %s 请求失败: %v", nextCursor, err)
break
logrus.Errorf("预取递归 [depth=%d] 请求失败: %v", item.depth, err)
continue
}
rows, _, _, nc, pe := parseRespExt(resp.Body, prefetchRespCfg)
itemRows, _, _, _, pe := parseRespExt(r2.Body, prefetchRespCfg)
if pe != nil {
logrus.Errorf("预取游标 %s 解析失败: %v", nextCursor, pe)
break
logrus.Errorf("预取递归 [depth=%d] 解析失败: %v", item.depth, pe)
continue
}
if len(rows) == 0 {
break
for _, row := range itemRows {
allRows = append(allRows, row)
if prefetch.ValueField == "" {
allEntities = append(allEntities, row)
} else if v, ok := row[prefetch.ValueField]; ok {
if f, ok := v.(float64); ok {
allEntities = append(allEntities, int64(f))
} else {
allEntities = append(allEntities, v)
}
}
if v, ok := row[prefetchRecursiveCfg.KeyField]; ok {
queue = append(queue, rItem{depth: item.depth + 1, keyVal: v})
}
}
nextCursor = nc
collectPrefetchEntities(rows, prefetch, &allEntities, &allRows)
time.Sleep(100 * time.Millisecond)
}
} else {
// 普通分页
for page := 2; page <= prefetchTotalPages; page++ {
body := buildReqBody(prefetchReqIface, page, prefetchPageSize, lastSyncTime, nil)
resp, err := api.Request(ctx, prefetchMethod, prefetch.URL, body, prefetchInQuery)
if err != nil {
logrus.Errorf("预取第 %d 页请求失败: %v", page, err)
continue
// ----- 常规分页预取 -----
firstExtra := make(map[string]interface{})
if prefetchIsCursor {
firstExtra[prefetchPageParam] = ""
}
body := buildReqBody(prefetchReqIface, 1, prefetchPageSize, lastSyncTime, firstExtra)
resp, err := api.Request(ctx, prefetchMethod, prefetch.URL, body, prefetchInQuery)
if err != nil {
recordFailure(ctx, platform.PlatformCode, iface.Code, taskType, fmt.Sprintf("预取第一页请求失败: %v", err))
return nil, fmt.Errorf("预取第一页失败: %w", err)
}
rows, prefetchTotalPages, _, nextCursor, err := parseRespExt(resp.Body, prefetchRespCfg)
if err != nil {
recordFailure(ctx, platform.PlatformCode, iface.Code, taskType, fmt.Sprintf("解析预取响应失败: %v", err))
return nil, fmt.Errorf("解析预取响应失败: %w", err)
}
collectPrefetchEntities(rows, prefetch, &allEntities, &allRows)
if prefetchIsCursor {
for nextCursor != "" && nextCursor != "nomore" {
body := buildReqBody(prefetchReqIface, 1, prefetchPageSize, lastSyncTime, map[string]interface{}{
prefetchPageParam: nextCursor,
})
resp, err := api.Request(ctx, prefetchMethod, prefetch.URL, body, prefetchInQuery)
if err != nil {
logrus.Errorf("预取游标 %s 请求失败: %v", nextCursor, err)
break
}
rows, _, _, nc, pe := parseRespExt(resp.Body, prefetchRespCfg)
if pe != nil {
logrus.Errorf("预取游标 %s 解析失败: %v", nextCursor, pe)
break
}
if len(rows) == 0 {
break
}
nextCursor = nc
collectPrefetchEntities(rows, prefetch, &allEntities, &allRows)
time.Sleep(100 * time.Millisecond)
}
rows, _, _, _, pe := parseRespExt(resp.Body, prefetchRespCfg)
if pe != nil {
logrus.Errorf("预取第 %d 页解析失败: %v", page, pe)
continue
} else {
for page := 2; page <= prefetchTotalPages; page++ {
body := buildReqBody(prefetchReqIface, page, prefetchPageSize, lastSyncTime, nil)
resp, err := api.Request(ctx, prefetchMethod, prefetch.URL, body, prefetchInQuery)
if err != nil {
logrus.Errorf("预取第 %d 页请求失败: %v", page, err)
continue
}
rows, _, _, _, pe := parseRespExt(resp.Body, prefetchRespCfg)
if pe != nil {
logrus.Errorf("预取第 %d 页解析失败: %v", page, pe)
continue
}
collectPrefetchEntities(rows, prefetch, &allEntities, &allRows)
time.Sleep(100 * time.Millisecond)
}
collectPrefetchEntities(rows, prefetch, &allEntities, &allRows)
time.Sleep(100 * time.Millisecond)
}
}
@@ -375,7 +544,7 @@ func syncWithPrefetch(ctx context.Context, api *ApiClient, platform *PlatformCon
}
logrus.Infof("预取到 %d 个实体", len(allEntities))
// 2. 将预取的数据也存入库(如账户列表存入 tencent_account_relation
// 将预取的数据也存入库(如账户列表存入 tencent_account_relation
if prefetchIface != nil && prefetchIface.TableDefinition != nil {
prefetchTd, err := ParseTableDefinition(prefetchIface.TableDefinition)
if err == nil {
@@ -386,11 +555,13 @@ func syncWithPrefetch(ctx context.Context, api *ApiClient, platform *PlatformCon
}
}
// 2. 并发处理每个实体的数据
// 并发处理每个实体的数据
result := &SyncResult{TableName: td.TableName}
pageSize := GetSyncPageSize(ctx)
if ps, ok := iface.RequestConfig["page_size"].(float64); ok {
pageSize = int(ps)
} else if ps, ok := iface.RequestConfig["pageSize"].(float64); ok {
pageSize = int(ps)
}
dataMethod := string(iface.Method)
@@ -411,52 +582,118 @@ func syncWithPrefetch(ctx context.Context, api *ApiClient, platform *PlatformCon
defer func() { <-sem }()
logrus.Infof(" 处理实体 [%d/%d]: %v", idx+1, len(allEntities), val)
page := 1
totalPages := 1
entityMaxTime := int64(0)
for page <= totalPages {
body := buildReqBody(iface, page, pageSize, lastSyncTime, map[string]interface{}{
if isCursorPagination(iface) {
// ----- 游标分页(如钉钉 user_list-----
cp := "cursor"
if p, ok := iface.RequestConfig["page_param"].(string); ok && p != "" {
cp = p
}
firstExtra := map[string]interface{}{
prefetch.TargetParam: val,
})
}
if icv, ok := iface.RequestConfig["initial_cursor"]; ok {
firstExtra[cp] = icv
} else {
firstExtra[cp] = ""
}
body := buildReqBody(iface, 1, pageSize, lastSyncTime, firstExtra)
resp, err := api.Request(ctx, dataMethod, iface.Url, body, inQuery)
if err != nil {
logrus.Errorf(" 实体 %v 第 %d 页失败: %v", val, page, err)
page++
time.Sleep(200 * time.Millisecond)
continue
logrus.Errorf(" 实体 %v 首次请求失败: %v", val, err)
return
}
rows, tp, mt, parseErr := parseResp(resp.Body, iface.ResponseConfig)
if parseErr != nil {
logrus.Errorf(" 解析响应失败: %v", parseErr)
page++
continue
rows, _, mt, nc, pe := parseRespExt(resp.Body, iface.ResponseConfig)
if pe != nil {
logrus.Errorf(" 实体 %v 解析首页失败: %v", val, pe)
return
}
if page == 1 {
totalPages = tp
}
for i := range rows {
rows[i][prefetch.TargetParam] = val
}
injectRowFields(rows, body, iface.RequestConfig)
inserted, _ := savePage(ctx, td, rows)
mu.Lock()
result.InsertedRows += inserted
result.TotalRows += len(rows)
mu.Unlock()
if mt > entityMaxTime {
entityMaxTime = mt
}
page++
time.Sleep(100 * time.Millisecond)
nextCursor := nc
for nextCursor != "" && nextCursor != "nomore" {
body := buildReqBody(iface, 1, pageSize, lastSyncTime, map[string]interface{}{
cp: nextCursor,
prefetch.TargetParam: val,
})
resp, err := api.Request(ctx, dataMethod, iface.Url, body, inQuery)
if err != nil {
logrus.Errorf(" 实体 %v 游标 %s 失败: %v", val, nextCursor, err)
break
}
rows, _, mt, nc, pe := parseRespExt(resp.Body, iface.ResponseConfig)
if pe != nil {
logrus.Errorf(" 实体 %v 游标 %s 解析失败: %v", val, nextCursor, pe)
break
}
if len(rows) == 0 {
break
}
nextCursor = nc
for i := range rows {
rows[i][prefetch.TargetParam] = val
}
injectRowFields(rows, body, iface.RequestConfig)
inserted, _ := savePage(ctx, td, rows)
mu.Lock()
result.InsertedRows += inserted
result.TotalRows += len(rows)
mu.Unlock()
if mt > entityMaxTime {
entityMaxTime = mt
}
time.Sleep(100 * time.Millisecond)
}
} else {
// ----- 普通分页 -----
page := 1
totalPages := 1
for page <= totalPages {
body := buildReqBody(iface, page, pageSize, lastSyncTime, map[string]interface{}{
prefetch.TargetParam: val,
})
resp, err := api.Request(ctx, dataMethod, iface.Url, body, inQuery)
if err != nil {
logrus.Errorf(" 实体 %v 第 %d 页失败: %v", val, page, err)
page++
time.Sleep(200 * time.Millisecond)
continue
}
rows, tp, mt, parseErr := parseResp(resp.Body, iface.ResponseConfig)
if parseErr != nil {
logrus.Errorf(" 解析响应失败: %v", parseErr)
page++
continue
}
if page == 1 {
totalPages = tp
}
for i := range rows {
rows[i][prefetch.TargetParam] = val
}
injectRowFields(rows, body, iface.RequestConfig)
inserted, _ := savePage(ctx, td, rows)
mu.Lock()
result.InsertedRows += inserted
result.TotalRows += len(rows)
mu.Unlock()
if mt > entityMaxTime {
entityMaxTime = mt
}
page++
time.Sleep(100 * time.Millisecond)
}
}
if entityMaxTime > 0 {
@@ -481,6 +718,90 @@ func syncWithPrefetch(ctx context.Context, api *ApiClient, platform *PlatformCon
return result, nil
}
// syncRecursive 递归遍历同步(如钉钉部门树:先查根级 → 对每个子部门递归查下级)
func syncRecursive(ctx context.Context, api *ApiClient, platform *PlatformConfig, iface *entity.ApiInterface, td *TableDefinition, recursive *RecursiveConfig, start time.Time) (*SyncResult, error) {
maxDepth := 20
if md, ok := iface.RequestConfig["max_recursive_depth"].(float64); ok {
maxDepth = int(md)
}
inQuery := paramsInQuery(iface)
method := string(iface.Method)
allRows := make([]map[string]interface{}, 0)
processedKeys := make(map[string]bool)
type queueItem struct {
depth int
keyVal interface{} // nil 表示根级
}
queue := []queueItem{{depth: 0, keyVal: nil}}
for len(queue) > 0 {
item := queue[0]
queue = queue[1:]
if item.depth > maxDepth {
logrus.Warnf("递归已达最大深度 %d终止该分支", maxDepth)
continue
}
// 防重复处理
if item.keyVal != nil {
keyStr := fmt.Sprintf("%v", item.keyVal)
if processedKeys[keyStr] {
continue
}
processedKeys[keyStr] = true
}
extraParams := make(map[string]interface{})
if item.keyVal != nil {
extraParams[recursive.TargetParam] = item.keyVal
}
body := buildReqBody(iface, 1, 100, 0, extraParams)
resp, err := api.Request(ctx, method, iface.Url, body, inQuery)
if err != nil {
logrus.Errorf("递归 [depth=%d] 请求失败: %v", item.depth, err)
recordFailure(ctx, platform.PlatformCode, iface.Code, "full", fmt.Sprintf("递归深度 %d 请求失败: %v", item.depth, err))
continue
}
rows, _, _, _, err := parseRespExt(resp.Body, iface.ResponseConfig)
if err != nil {
logrus.Errorf("递归 [depth=%d] 解析失败: %v", item.depth, err)
continue
}
for _, row := range rows {
allRows = append(allRows, row)
if v, ok := row[recursive.KeyField]; ok {
queue = append(queue, queueItem{depth: item.depth + 1, keyVal: v})
}
}
time.Sleep(100 * time.Millisecond)
}
if len(allRows) == 0 {
logrus.Warn("递归结果为空,跳过入库")
return &SyncResult{TableName: td.TableName, Duration: fmt.Sprintf("%.1fs", time.Since(start).Seconds())}, nil
}
inserted, _ := savePage(ctx, td, allRows)
updateSyncTime(ctx, platform.PlatformCode, iface.Code, time.Now().Unix())
result := &SyncResult{
TableName: td.TableName,
TotalRows: len(allRows),
InsertedRows: inserted,
Duration: fmt.Sprintf("%.1fs", time.Since(start).Seconds()),
}
logrus.Infof("递归同步完成 - 表:%s, %d条, 写入%d条, 耗时%s", td.TableName, result.TotalRows, result.InsertedRows, result.Duration)
return result, nil
}
// getTotalPages 从响应中提取总页数
func getTotalPages(raw []byte) int {
rows, tp, _, _, err := parseRespExt(raw, nil)
@@ -498,6 +819,12 @@ func toFloat64(v interface{}) (float64, bool) {
return float64(val), true
case int64:
return float64(val), true
case string:
// 支持字符串类型的成功值(如钉钉智能薪酬返回 code: "200"
if f, err := strconv.ParseFloat(val, 64); err == nil {
return f, true
}
return 0, false
default:
return 0, false
}
@@ -524,7 +851,10 @@ func buildPrefetchParams(iface *entity.ApiInterface) map[string]interface{} {
k == "page_size_param" || k == "time_field" || k == "parameters_location" ||
k == "filtering" || k == "group_by" || k == "date_range" ||
k == "body_wrapper_field" || k == "exclude_from_wrapper" ||
k == "cursor_pagination" || k == "time_field_mode" {
k == "cursor_pagination" || k == "time_field_mode" ||
k == "recursive" || k == "max_recursive_depth" ||
k == "initial_cursor" || k == "pagination_mode" ||
k == "full_sync_start_time" || k == "row_inject" {
continue
}
if k == pageParam || k == psParam {
@@ -567,6 +897,33 @@ func parsePrefetchConfig(requestConfig map[string]interface{}) *PrefetchConfig {
return pc
}
// parseRecursiveConfig 解析递归遍历配置
func parseRecursiveConfig(requestConfig map[string]interface{}) *RecursiveConfig {
if requestConfig == nil {
return nil
}
raw, ok := requestConfig["recursive"]
if !ok || raw == nil {
return nil
}
m, ok := raw.(map[string]interface{})
if !ok {
return nil
}
rc := &RecursiveConfig{}
if kf, _ := m["key_field"].(string); kf != "" {
rc.KeyField = kf
} else {
return nil
}
if tp, _ := m["target_param"].(string); tp != "" {
rc.TargetParam = tp
} else {
return nil
}
return rc
}
// extractValues 从 JSON 响应中提取值列表
func extractValues(raw []byte, path, valueField string) ([]interface{}, error) {
var resp map[string]interface{}
@@ -612,7 +969,10 @@ func buildReqBody(iface *entity.ApiInterface, page, pageSize int, lastSyncTime i
k == "page_param" || k == "page_size_param" || k == "parameters_location" ||
k == "cursor_pagination" || k == "time_field_mode" ||
k == "body_wrapper_field" || k == "exclude_from_wrapper" ||
k == "top_level_params" {
k == "top_level_params" || k == "recursive" ||
k == "max_recursive_depth" || k == "initial_cursor" ||
k == "pagination_mode" || k == "full_sync_start_time" ||
k == "row_inject" {
continue
}
body[k] = v
@@ -628,39 +988,68 @@ func buildReqBody(iface *entity.ApiInterface, page, pageSize int, lastSyncTime i
psParam = p
}
}
body[pageParam] = page
// 偏移量分页(如钉钉 offsetoffset = (page-1) * pageSize
paginationMode := ""
if iface.RequestConfig != nil {
if pm, ok := iface.RequestConfig["pagination_mode"].(string); ok {
paginationMode = pm
}
}
if paginationMode == "offset" {
body[pageParam] = (page - 1) * pageSize
} else {
body[pageParam] = page
}
body[psParam] = pageSize
// 时间过滤处理:支持两种模式
// 1. "filtering" 模式(默认):生成 filtering=[{"field":"...","operator":"GREATER_EQUALS","values":["..."]}](腾讯)
// 2. "range" 模式:生成 beginTime/endTime + queryType快手
if tf, ok := iface.RequestConfig["time_field"].(string); ok && tf != "" {
timeMode := "filtering"
if tm, ok := iface.RequestConfig["time_field_mode"].(string); ok && tm != "" {
timeMode = tm
}
if iface.RequestConfig != nil {
if tf, ok := iface.RequestConfig["time_field"].(string); ok && tf != "" {
timeMode := "filtering"
if tm, ok := iface.RequestConfig["time_field_mode"].(string); ok && tm != "" {
timeMode = tm
}
if timeMode == "range" {
// 快手模式beginTime/endTime毫秒时间戳
timeMs := lastSyncTime
if timeMs <= 0 {
// 全量默认90天前
timeMs = time.Now().Add(-90 * 24 * time.Hour).UnixMilli()
}
body["queryType"] = 2
body["beginTime"] = timeMs
body["endTime"] = time.Now().UnixMilli()
} else if lastSyncTime > 0 {
// 腾讯 filtering 模式(仅增量时)
timeFilter := map[string]interface{}{
"field": tf,
"operator": "GREATER_EQUALS",
"values": []interface{}{fmt.Sprintf("%d", lastSyncTime)},
}
if existing, ok := body["filtering"].([]interface{}); ok {
body["filtering"] = append(existing, timeFilter)
} else {
body["filtering"] = []interface{}{timeFilter}
if timeMode == "range" {
// 快手模式beginTime/endTime毫秒时间戳
timeMs := lastSyncTime
if timeMs <= 0 {
// 全量:优先使用配置的 full_sync_start_time否则默认90天前
if fst, ok := iface.RequestConfig["full_sync_start_time"].(float64); ok && fst > 0 {
timeMs = int64(fst)
} else {
timeMs = time.Now().Add(-90 * 24 * time.Hour).UnixMilli()
}
}
body["queryType"] = 2
body["beginTime"] = timeMs
body["endTime"] = time.Now().UnixMilli()
} else if lastSyncTime > 0 {
// 腾讯 filtering 模式(仅增量时)
timeFilter := map[string]interface{}{
"field": tf,
"operator": "GREATER_EQUALS",
"values": []interface{}{fmt.Sprintf("%d", lastSyncTime)},
}
if existing, ok := body["filtering"].([]interface{}); ok {
body["filtering"] = append(existing, timeFilter)
} else {
body["filtering"] = []interface{}{timeFilter}
}
} else if fst, ok := iface.RequestConfig["full_sync_start_time"].(float64); ok && fst > 0 {
// 全量 filtering 模式:指定了 full_sync_start_time从该时间戳开始拉取
timeFilter := map[string]interface{}{
"field": tf,
"operator": "GREATER_EQUALS",
"values": []interface{}{fmt.Sprintf("%d", int64(fst))},
}
if existing, ok := body["filtering"].([]interface{}); ok {
body["filtering"] = append(existing, timeFilter)
} else {
body["filtering"] = []interface{}{timeFilter}
}
}
}
}
@@ -687,8 +1076,12 @@ func buildReqBody(iface *entity.ApiInterface, page, pageSize int, lastSyncTime i
delete(body, k)
}
}
b, _ := json.Marshal(wrapperObj)
body[wf] = string(b)
b, err := json.Marshal(wrapperObj)
if err != nil {
logrus.Errorf("JSON序列化 wrapper 失败: %v", err)
} else {
body[wf] = string(b)
}
}
}
@@ -703,6 +1096,7 @@ func parseRespExt(raw []byte, rc map[string]interface{}) ([]map[string]interface
}
successField, successVal := "code", float64(0)
msgField, listPath, cursorPath := "message", "data", ""
hasMorePath := ""
singleRecord := false
if rc != nil {
if sf, _ := rc["success_field"].(string); sf != "" {
@@ -725,6 +1119,9 @@ func parseRespExt(raw []byte, rc map[string]interface{}) ([]map[string]interface
if sr, _ := rc["single_record"].(bool); sr {
singleRecord = true
}
if hm, _ := rc["has_more_field"].(string); hm != "" {
hasMorePath = hm
}
}
if v, ok := respMap[successField]; ok {
actual, _ := toFloat64(v)
@@ -820,6 +1217,23 @@ func parseRespExt(raw []byte, rc map[string]interface{}) ([]map[string]interface
if i == len(cp)-1 {
if s, ok := cc[p].(string); ok {
nextCursor = s
} else if f, ok := cc[p].(float64); ok {
// 数字游标(如钉钉 next_cursor=10
nextCursor = fmt.Sprintf("%.0f", f)
}
} else if m, ok := cc[p].(map[string]interface{}); ok {
cc = m
}
}
}
// has_more 字段支持false 时标记游标结束
if hasMorePath != "" {
parts := strings.Split(hasMorePath, ".")
cc := respMap
for i, p := range parts {
if i == len(parts)-1 {
if b, ok := cc[p].(bool); ok && !b {
nextCursor = "nomore"
}
} else if m, ok := cc[p].(map[string]interface{}); ok {
cc = m
@@ -950,3 +1364,32 @@ func findInterfaceByURL(ifaces []entity.ApiInterface, url string) *entity.ApiInt
}
return nil
}
// injectRowFields 将请求参数中 row_inject 指定的字段注入到响应行中
// 用于需要将请求参数(如 statisticsMonth持久化到表中但响应不含该字段的场景
func injectRowFields(rows []map[string]interface{}, body map[string]interface{}, requestConfig map[string]interface{}) {
if requestConfig == nil || body == nil {
return
}
rawInject, ok := requestConfig["row_inject"]
if !ok {
return
}
injectList, ok := rawInject.([]interface{})
if !ok {
return
}
for _, item := range injectList {
fieldName, ok := item.(string)
if !ok {
continue
}
val, exists := body[fieldName]
if !exists {
continue
}
for i := range rows {
rows[i][fieldName] = val
}
}
}

View File

@@ -41,3 +41,12 @@ func GetRetryCount(ctx context.Context) int {
}
return r
}
// GetSyncTimeout 获取单次同步超时时间分钟默认120全量超大表可适当调大
func GetSyncTimeout(ctx context.Context) int {
t := g.Cfg().MustGet(ctx, "sync.sync_timeout_minutes", 120).Int()
if t < 1 {
return 120
}
return t
}

View File

@@ -63,6 +63,15 @@ func (m *PlatformManager) GetPlatform(ctx context.Context, platformCode string)
cfg.AppSecret = as
}
}
case "APP_SIGNATURE":
if platform.AuthConfig != nil {
if aid, _ := platform.AuthConfig["app_id"].(string); aid != "" {
cfg.AppKey = aid
}
if as, _ := platform.AuthConfig["app_secret"].(string); as != "" {
cfg.AppSecret = as
}
}
default:
logrus.Warnf("平台 %s 认证类型 %s 未处理", platformCode, platform.AuthType)
}

View File

@@ -7,7 +7,7 @@ import (
dao "dataengine/dao/dict"
dto "dataengine/model/dto/dict"
"gitea.com/red-future/common/beans"
"gitea.redpowerfuture.com/red-future/common/beans"
"github.com/gogf/gf/v2/frame/g"
"github.com/sirupsen/logrus"
)
@@ -71,7 +71,7 @@ func runAutoSync(ctx context.Context) {
// InitAndStartAutoSync 在 main 中调用:初始化配置后启动自动同步和补偿
func InitAndStartAutoSync(ctx context.Context) {
// 读取配置中的同步开关
enabled := g.Cfg().MustGet(ctx, "sync.auto_sync_enabled", true).Bool()
enabled := g.Cfg().MustGet(ctx, "sync.auto_sync_enabled", false).Bool()
if enabled {
go StartAutoSync(ctx)
} else {

View File

@@ -5,7 +5,7 @@ import (
"fmt"
"strings"
"gitea.com/red-future/common/db/gfdb"
"gitea.redpowerfuture.com/red-future/common/db/gfdb"
"github.com/sirupsen/logrus"
)