重构数据引擎

service/sync/data_writer.go

@@ -0,0 +1,70 @@
+package sync
+
+import (
+	"context"
+	"time"
+
+	"gitea.com/red-future/common/db/gfdb"
+	"github.com/sirupsen/logrus"
+)
+
+// InsertRows 批量写入数据
+func InsertRows(ctx context.Context, tableName string, conflictKeys []string, rows []map[string]interface{}) (int, error) {
+	if len(rows) == 0 {
+		return 0, nil
+	}
+
+	now := time.Now()
+	for i := range rows {
+		if rows[i] == nil {
+			rows[i] = make(map[string]interface{})
+		}
+		if _, ok := rows[i]["created_at"]; !ok {
+			rows[i]["created_at"] = now
+		}
+		if _, ok := rows[i]["updated_at"]; !ok {
+			rows[i]["updated_at"] = now
+		}
+	}
+
+	batchSize := 100
+	total := 0
+	for i := 0; i < len(rows); i += batchSize {
+		end := i + batchSize
+		if end > len(rows) {
+			end = len(rows)
+		}
+		batch := rows[i:end]
+
+		m := gfdb.DB(ctx).Model(ctx, tableName).Data(batch)
+		if len(conflictKeys) > 0 {
+			keys := make([]interface{}, len(conflictKeys))
+			for j, k := range conflictKeys {
+				keys[j] = k
+			}
+			m = m.OnConflict(keys...)
+		}
+		_, err := m.Save()
+		if err != nil {
+			logrus.Errorf("批量写入 %s 失败: %v", tableName, err)
+			for _, row := range batch {
+				mm := gfdb.DB(ctx).Model(ctx, tableName).Data(row)
+				if len(conflictKeys) > 0 {
+					keys := make([]interface{}, len(conflictKeys))
+					for j, k := range conflictKeys {
+						keys[j] = k
+					}
+					mm = mm.OnConflict(keys...)
+				}
+				if _, e := mm.Save(); e != nil {
+					logrus.Errorf("逐条写入失败: %v", e)
+				} else {
+					total++
+				}
+			}
+		} else {
+			total += len(batch)
+		}
+	}
+	return total, nil
+}