文件存储-增加文件上传记录接口,增加定时同步租户文件存储容量信息接口

This commit is contained in:
2025-12-26 13:59:02 +08:00
parent 5693dd4952
commit 36c9b61db0
14 changed files with 518 additions and 0 deletions

42
config.yml Normal file
View File

@@ -0,0 +1,42 @@
server:
address: ":3008"
name: "oss"
rate:
limit: 200
burst: 300
mongo:
logger:
level: "all"
stdout: true
address: "mongodb://192.168.3.200:27017/oss?retryWrites=true"
redis:
# 集群模式配置方法
default:
address: 192.168.3.200:6379
db: 0
idleTimeout: "60s" #连接最大空闲时间使用时间字符串例如30s/1m/1d
maxConnLifetime: "90s" #连接最长存活时间使用时间字符串例如30s/1m/1d
waitTimeout: "60s" #等待连接池连接的超时时间使用时间字符串例如30s/1m/1d
dialTimeout: "30s" #TCP连接的超时时间使用时间字符串例如30s/1m/1d
readTimeout: "30s" #TCP的Read操作超时时间使用时间字符串例如30s/1m/1d
writeTimeout: "30s" #TCP的Write操作超时时间使用时间字符串例如30s/1m/1d
maxActive: 100
consul:
address: 192.168.3.200:8500
# pass: jiahui8888
jaeger: #链路追踪
addr: 192.168.3.200:4318
# MinIO 连接配置
minio:
endpoint: "192.168.3.200:9000" # 核心:仅协议+主机+端口,无路径/末尾斜杠
accessKey: "rag_flow" # 访问密钥(本地默认)
secretKey: "infini_rag_flow" # 秘密密钥(本地默认)
secure: false # 本地 MinIO 关闭 SSL生产按需改为 true
region: "us-east-1" # 与 MinIO 服务端 REGION 一致(默认 us-east-1
bucketName: "my-dev-bucket" # 默认桶名(可选)
presignedExpire: "5m" # 预签名URL过期时间≥1秒支持m/h/d等单位
# 文件存储初始化容量大小配置
oss:
capacitySize: 500

7
consts/collections.go Normal file
View File

@@ -0,0 +1,7 @@
package consts
// MongoDB集合名称常量
const (
FileCollection = "file"
TenantOssTotalCollection = "tenant_oss_total"
)

5
consts/redis_key.go Normal file
View File

@@ -0,0 +1,5 @@
package consts
const TenantOssTotalKey = "oss:total:tenantId-%s"
const FileLockKey = "oss:lock:tenantId-%s"
const OssTotalKey = "oss:total:*"

View File

@@ -0,0 +1,20 @@
package controller
import (
"context"
"oss/model/dto"
"oss/service"
)
type file struct{}
var File = new(file)
// init 初始化表单配置
func init() {
}
// UploadFile 上传文件
func (c *file) UploadFile(ctx context.Context, req *dto.UploadFileReq) (res *dto.UploadFileRes, err error) {
return service.File.UploadFile(ctx, req)
}

18
dao/file_dao.go Normal file
View File

@@ -0,0 +1,18 @@
package dao
import (
"context"
"gitee.com/red-future---jilin-g/common/mongo"
"oss/consts"
"oss/model/entity"
)
var File = &file{}
type file struct{}
// Insert 插入
func (d *file) Insert(ctx context.Context, entity *entity.File) (err error) {
_, err = mongo.Insert(ctx, []interface{}{entity}, consts.FileCollection)
return
}

40
dao/tenant_oss_total.go Normal file
View File

@@ -0,0 +1,40 @@
package dao
import (
"context"
"gitee.com/red-future---jilin-g/common/mongo"
"github.com/gogf/gf/v2/frame/g"
"go.mongodb.org/mongo-driver/v2/bson"
"oss/consts"
"oss/model/entity"
)
var TenantOssTotal = &tenantOssTotal{}
type tenantOssTotal struct{}
// Insert 插入
func (d *tenantOssTotal) Insert(ctx context.Context, entity []interface{}) (err error) {
_, err = mongo.Insert(ctx, entity, consts.TenantOssTotalCollection)
return
}
// SaveOrUpdate 增加或更新
func (d *tenantOssTotal) SaveOrUpdate(ctx context.Context, filterData []*entity.TenantOssTotal, updateData []*entity.TenantOssTotal) (err error) {
if !g.IsEmpty(updateData) {
var filter, update []bson.M
for i, v := range filterData {
filter = append(filter, bson.M{"tenantId": v.TenantId})
update = append(update, bson.M{"$set": bson.M{"usedOssSize": updateData[i].UsedOssSize, "totalOssSize": updateData[i].TotalOssSize}})
}
_, err = mongo.SaveOrUpdate(ctx, filter, update, consts.TenantOssTotalCollection)
}
return
}
func (d *tenantOssTotal) GetOneByTenantId(ctx context.Context, tenantId string) (e *entity.TenantOssTotal, err error) {
filter := bson.M{"tenantId": tenantId}
e = &entity.TenantOssTotal{}
err = mongo.FindOne(ctx, filter, e, consts.TenantOssTotalCollection)
return
}

101
go.mod Normal file
View File

@@ -0,0 +1,101 @@
module oss
go 1.25.3
require (
gitee.com/red-future---jilin-g/common v0.2.5
github.com/gogf/gf/contrib/drivers/mysql/v2 v2.9.5
github.com/gogf/gf/contrib/nosql/redis/v2 v2.9.5
github.com/gogf/gf/v2 v2.9.5
go.mongodb.org/mongo-driver/v2 v2.4.1
)
//replace gitee.com/red-future---jilin-g/common v0.2.5 => ../common
require (
github.com/BurntSushi/toml v1.5.0 // indirect
github.com/armon/go-metrics v0.4.1 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/clbanning/mxj/v2 v2.7.0 // indirect
github.com/dgraph-io/badger/v4 v4.2.0 // indirect
github.com/dgraph-io/ristretto v0.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/fatih/color v1.18.0 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-sql-driver/mysql v1.7.1 // indirect
github.com/gogf/gf/contrib/registry/consul/v2 v2.9.5 // indirect
github.com/gogf/gf/contrib/trace/otlphttp/v2 v2.9.5 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
github.com/golang/glog v1.2.5 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/golang/snappy v1.0.0 // indirect
github.com/google/flatbuffers v1.12.1 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/websocket v1.5.3 // indirect
github.com/grokify/html-strip-tags-go v0.1.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect
github.com/hashicorp/consul/api v1.26.1 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-hclog v1.5.0 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/hashicorp/serf v0.10.1 // indirect
github.com/klauspost/compress v1.18.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
github.com/klauspost/crc32 v1.3.0 // indirect
github.com/magiconair/properties v1.8.10 // indirect
github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/minio/crc64nvme v1.1.0 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/minio/minio-go/v7 v7.0.97 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/olekukonko/errors v1.1.0 // indirect
github.com/olekukonko/ll v0.0.9 // indirect
github.com/olekukonko/tablewriter v1.1.0 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/redis/go-redis/v9 v9.12.1 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/rs/xid v1.6.0 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/tiger1103/gfast-token v1.0.10 // indirect
github.com/tinylib/msgp v1.3.0 // indirect
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
github.com/xdg-go/scram v1.1.2 // indirect
github.com/xdg-go/stringprep v1.0.4 // indirect
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
go.opencensus.io v0.23.0 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/otel v1.38.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 // indirect
go.opentelemetry.io/otel/metric v1.38.0 // indirect
go.opentelemetry.io/otel/sdk v1.38.0 // indirect
go.opentelemetry.io/otel/trace v1.38.0 // indirect
go.opentelemetry.io/proto/otlp v1.7.1 // indirect
golang.org/x/crypto v0.44.0 // indirect
golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect
golang.org/x/net v0.47.0 // indirect
golang.org/x/sync v0.18.0 // indirect
golang.org/x/sys v0.38.0 // indirect
golang.org/x/text v0.31.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect
google.golang.org/grpc v1.75.0 // indirect
google.golang.org/protobuf v1.36.8 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

35
main.go Normal file
View File

@@ -0,0 +1,35 @@
package main
import (
"context"
"github.com/gogf/gf/v2/os/glog"
"github.com/gogf/gf/v2/os/gtimer"
"oss/controller"
"oss/service"
"time"
"gitee.com/red-future---jilin-g/common/http"
"gitee.com/red-future---jilin-g/common/jaeger"
_ "gitee.com/red-future---jilin-g/common/mongo"
_ "github.com/gogf/gf/contrib/nosql/redis/v2"
)
func main() {
ctx := context.Background()
defer jaeger.ShutDown(ctx)
// 注册路由
http.RouteRegister([]interface{}{
controller.File,
})
gtimer.AddSingleton(ctx, time.Minute*5, func(ctx context.Context) {
err := service.TenantOssTotal.UpdateUsedOssSize(ctx)
if err != nil {
glog.Error(ctx, "UpdateUsedOssSize err: %v", err)
}
})
// 保持应用运行
select {}
}

18
model/dto/file_dto.go Normal file
View File

@@ -0,0 +1,18 @@
package dto
import (
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/net/ghttp"
)
// UploadFileReq 上传文件请求
type UploadFileReq struct {
g.Meta `path:"/uploadFile" method:"post" tags:"存储管理" summary:"上传文件" dc:"上传文件"`
File *ghttp.UploadFile `json:"file" type:"file"` // 文件URL
}
// UploadFileRes 上传文件响应
type UploadFileRes struct {
FileURL string `json:"fileURL" dc:"上传地址"`
FileAddressPrefix string `json:"fileAddressPrefix"`
}

View File

@@ -0,0 +1,17 @@
package dto
import (
"github.com/gogf/gf/v2/frame/g"
"oss/model/entity"
)
// GetByTenantIdReq 根据租户id获取存储总量请求
type GetByTenantIdReq struct {
g.Meta `path:"/GetOneByTenantId" method:"get" tags:"租户存储总量管理" summary:"获取存储总量" dc:"获取存储总量"`
TenantId string `json:"tenantId" v:"required#租户id不能为空"`
}
// GetByTenantIdRes 根据租户id获取存储总量响应
type GetByTenantIdRes struct {
*entity.TenantOssTotal
}

19
model/entity/file.go Normal file
View File

@@ -0,0 +1,19 @@
package entity
import (
"gitee.com/red-future---jilin-g/common/do"
"oss/consts"
)
// File 存储文件实体
type File struct {
do.MongoBaseDO `bson:",inline"` // 嵌入基础字段Id, Creator, CreatedAt, Updater, UpdatedAt, TenantId, IsDeleted
// 基础信息
FileURL string `bson:"fileURL" json:"fileURL"` // 图URL
FileSize int64 `bson:"fileSize" json:"fileSize"`
}
// CollectionName 存储集合名称
func (File) CollectionName() string {
return consts.FileCollection
}

View File

@@ -0,0 +1,19 @@
package entity
import (
"gitee.com/red-future---jilin-g/common/do"
"oss/consts"
)
// TenantOssTotal 租户储存服务总计实体
type TenantOssTotal struct {
do.MongoBaseDO `bson:",inline"` // 嵌入基础字段Id, Creator, CreatedAt, Updater, UpdatedAt, TenantId, IsDeleted
// 基础信息
UsedOssSize int64 `bson:"usedOssSize" json:"usedOssSize"`
TotalOssSize int64 `bson:"totalOssSize" json:"totalOssSize"`
}
// CollectionName 租户储存服务总计集合名称
func (TenantOssTotal) CollectionName() string {
return consts.TenantOssTotalCollection
}

119
service/file_service.go Normal file
View File

@@ -0,0 +1,119 @@
package service
import (
"context"
"fmt"
"gitee.com/red-future---jilin-g/common/minio"
"gitee.com/red-future---jilin-g/common/mongo"
"gitee.com/red-future---jilin-g/common/redis"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/util/gconv"
"oss/consts"
"oss/dao"
"oss/model/dto"
"oss/model/entity"
"time"
)
type file struct{}
// File 存储文件服务
var File = new(file)
func (f *file) UploadFile(ctx context.Context, req *dto.UploadFileReq) (res *dto.UploadFileRes, err error) {
tenantId := ""
fileSize := req.File.Size
totalFileSize := int64(0)
// 获取租户id
user, err := mongo.GetTenantInfo(ctx)
if err != nil {
return
}
// 获取redis-租户存储容量总数key
tenantOssTotalKey := fmt.Sprintf(consts.TenantOssTotalKey, gconv.String(user.TenantId))
// 获取redis-租户存储-锁key
fileLockKey := fmt.Sprintf(consts.FileLockKey, gconv.String(user.TenantId))
i := 0
LOCK:
if ok, err := redis.RedisClient.SetNX(ctx, fileLockKey, fileSize); !ok || err != nil {
// 获取锁失败,需要重试
if i < 5 {
i++
time.Sleep(5 * time.Millisecond)
goto LOCK
}
}
// 设置redis-租户存储-锁key超时时间1分钟
err = redis.RedisClient.SetEX(ctx, fileLockKey, fileSize, gconv.Int64(time.Minute*1))
if err != nil {
return nil, err
}
// 获取redis-租户存储容量总数
get, err := redis.RedisClient.Get(ctx, tenantOssTotalKey)
if err != nil {
return nil, err
}
tenantOssTotalEntity := &entity.TenantOssTotal{}
if g.IsEmpty(get) {
//查询数据库-获取租户存储容量总数
getByTenantIdReq := &dto.GetByTenantIdReq{
TenantId: gconv.String(user.TenantId),
}
tenantOssTotal, err := TenantOssTotal.GetOneByTenantId(ctx, getByTenantIdReq)
if err != nil {
return nil, err
}
if g.IsEmpty(tenantOssTotal) {
tenantOssTotalEntity.TenantId = user.TenantId
tenantOssTotalEntity.UsedOssSize = int64(0)
tenantOssTotalEntity.TotalOssSize = g.Cfg().MustGet(ctx, "oss.capacitySize").Int64()
} else {
tenantOssTotalEntity = tenantOssTotal.TenantOssTotal
}
} else {
// 反序列化-redis获取租户存储容量总数
err = gconv.Struct(get, tenantOssTotalEntity)
if err != nil {
return nil, err
}
}
tenantId = gconv.String(tenantOssTotalEntity.TenantId)
fileSize = tenantOssTotalEntity.UsedOssSize + fileSize
totalFileSize = tenantOssTotalEntity.TotalOssSize
// 设置redis-租户存储容量总数
tenantOssTotalKeyMap := map[string]interface{}{"tenantId": tenantId, "UsedOssSize": fileSize, "TotalOssSize": totalFileSize}
// 修改redis-租户存储容量总数 超时时间10分钟
err = redis.RedisClient.SetEX(ctx, tenantOssTotalKey, tenantOssTotalKeyMap, gconv.Int64(time.Minute*10))
if err != nil {
return nil, err
}
if fileSize < totalFileSize {
// 删除redis-租户存储-锁key
_, err = redis.RedisClient.Del(ctx, fileLockKey)
if err != nil {
return nil, err
}
} else {
return nil, gerror.New("存储服务内存不足")
}
// 上传图片
fileURL, err := minio.UploadImage(ctx, req.File)
if err != nil {
return nil, err
}
// 插入数据库
ossEntity := &entity.File{
FileURL: fileURL,
FileSize: fileSize,
}
err = dao.File.Insert(ctx, ossEntity)
if err != nil {
return nil, err
}
// 返回图片url
return &dto.UploadFileRes{
FileURL: fileURL,
FileAddressPrefix: minio.GetImgAddressPrefix(ctx),
}, err
}

View File

@@ -0,0 +1,58 @@
package service
import (
"context"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/util/gconv"
"oss/consts"
"oss/dao"
"oss/model/dto"
"oss/model/entity"
)
type tenantOssTotal struct{}
// TenantOssTotal 存储文件服务
var TenantOssTotal = new(tenantOssTotal)
func (s *tenantOssTotal) GetOneByTenantId(ctx context.Context, req *dto.GetByTenantIdReq) (res *dto.GetByTenantIdRes, err error) {
e, err := dao.TenantOssTotal.GetOneByTenantId(ctx, req.TenantId)
if err != nil {
return nil, err
}
return &dto.GetByTenantIdRes{
TenantOssTotal: e,
}, nil
}
func (s *tenantOssTotal) UpdateUsedOssSize(ctx context.Context) (err error) {
// 使用 Keys 取出所有key
keys, err := g.Redis().Keys(ctx, consts.OssTotalKey)
if err != nil {
return
}
updateData := make([]*entity.TenantOssTotal, 0)
filterData := make([]*entity.TenantOssTotal, 0)
for _, key := range keys {
get, err := g.Redis().Get(ctx, key)
if err != nil {
return err
}
e := &entity.TenantOssTotal{}
err = gconv.Struct(get, e)
if err != nil {
return err
}
updateData = append(updateData, e)
totalOssSize := &entity.TenantOssTotal{}
totalOssSize.TenantId = e.TenantId
filterData = append(filterData, totalOssSize)
}
// 更新数据库
err = dao.TenantOssTotal.SaveOrUpdate(ctx, filterData, updateData)
if err != nil {
return err
}
return err
}