diff --git a/config.yml b/config.yml new file mode 100644 index 0000000..56198f0 --- /dev/null +++ b/config.yml @@ -0,0 +1,42 @@ +server: + address: ":3008" + name: "oss" +rate: + limit: 200 + burst: 300 +mongo: + logger: + level: "all" + stdout: true + address: "mongodb://192.168.3.200:27017/oss?retryWrites=true" +redis: + # 集群模式配置方法 + default: + address: 192.168.3.200:6379 + db: 0 + idleTimeout: "60s" #连接最大空闲时间,使用时间字符串例如30s/1m/1d + maxConnLifetime: "90s" #连接最长存活时间,使用时间字符串例如30s/1m/1d + waitTimeout: "60s" #等待连接池连接的超时时间,使用时间字符串例如30s/1m/1d + dialTimeout: "30s" #TCP连接的超时时间,使用时间字符串例如30s/1m/1d + readTimeout: "30s" #TCP的Read操作超时时间,使用时间字符串例如30s/1m/1d + writeTimeout: "30s" #TCP的Write操作超时时间,使用时间字符串例如30s/1m/1d + maxActive: 100 +consul: + address: 192.168.3.200:8500 +# pass: jiahui8888 +jaeger: #链路追踪 + addr: 192.168.3.200:4318 + +# MinIO 连接配置 +minio: + endpoint: "192.168.3.200:9000" # 核心:仅协议+主机+端口,无路径/末尾斜杠 + accessKey: "rag_flow" # 访问密钥(本地默认) + secretKey: "infini_rag_flow" # 秘密密钥(本地默认) + secure: false # 本地 MinIO 关闭 SSL(生产按需改为 true) + region: "us-east-1" # 与 MinIO 服务端 REGION 一致(默认 us-east-1) + bucketName: "my-dev-bucket" # 默认桶名(可选) + presignedExpire: "5m" # 预签名URL过期时间(≥1秒,支持m/h/d等单位) + +# 文件存储初始化容量大小配置 +oss: + capacitySize: 500 \ No newline at end of file diff --git a/consts/collections.go b/consts/collections.go new file mode 100644 index 0000000..11b3457 --- /dev/null +++ b/consts/collections.go @@ -0,0 +1,7 @@ +package consts + +// MongoDB集合名称常量 +const ( + FileCollection = "file" + TenantOssTotalCollection = "tenant_oss_total" +) diff --git a/consts/redis_key.go b/consts/redis_key.go new file mode 100644 index 0000000..ee8c17d --- /dev/null +++ b/consts/redis_key.go @@ -0,0 +1,5 @@ +package consts + +const TenantOssTotalKey = "oss:total:tenantId-%s" +const FileLockKey = "oss:lock:tenantId-%s" +const OssTotalKey = "oss:total:*" diff --git a/controller/file_controller.go b/controller/file_controller.go new file mode 100644 index 0000000..bb7c881 --- /dev/null +++ b/controller/file_controller.go @@ -0,0 +1,20 @@ +package controller + +import ( + "context" + "oss/model/dto" + "oss/service" +) + +type file struct{} + +var File = new(file) + +// init 初始化表单配置 +func init() { +} + +// UploadFile 上传文件 +func (c *file) UploadFile(ctx context.Context, req *dto.UploadFileReq) (res *dto.UploadFileRes, err error) { + return service.File.UploadFile(ctx, req) +} diff --git a/dao/file_dao.go b/dao/file_dao.go new file mode 100644 index 0000000..fa40c35 --- /dev/null +++ b/dao/file_dao.go @@ -0,0 +1,18 @@ +package dao + +import ( + "context" + "gitee.com/red-future---jilin-g/common/mongo" + "oss/consts" + "oss/model/entity" +) + +var File = &file{} + +type file struct{} + +// Insert 插入 +func (d *file) Insert(ctx context.Context, entity *entity.File) (err error) { + _, err = mongo.Insert(ctx, []interface{}{entity}, consts.FileCollection) + return +} diff --git a/dao/tenant_oss_total.go b/dao/tenant_oss_total.go new file mode 100644 index 0000000..1d04c0d --- /dev/null +++ b/dao/tenant_oss_total.go @@ -0,0 +1,40 @@ +package dao + +import ( + "context" + "gitee.com/red-future---jilin-g/common/mongo" + "github.com/gogf/gf/v2/frame/g" + "go.mongodb.org/mongo-driver/v2/bson" + "oss/consts" + "oss/model/entity" +) + +var TenantOssTotal = &tenantOssTotal{} + +type tenantOssTotal struct{} + +// Insert 插入 +func (d *tenantOssTotal) Insert(ctx context.Context, entity []interface{}) (err error) { + _, err = mongo.Insert(ctx, entity, consts.TenantOssTotalCollection) + return +} + +// SaveOrUpdate 增加或更新 +func (d *tenantOssTotal) SaveOrUpdate(ctx context.Context, filterData []*entity.TenantOssTotal, updateData []*entity.TenantOssTotal) (err error) { + if !g.IsEmpty(updateData) { + var filter, update []bson.M + for i, v := range filterData { + filter = append(filter, bson.M{"tenantId": v.TenantId}) + update = append(update, bson.M{"$set": bson.M{"usedOssSize": updateData[i].UsedOssSize, "totalOssSize": updateData[i].TotalOssSize}}) + } + _, err = mongo.SaveOrUpdate(ctx, filter, update, consts.TenantOssTotalCollection) + } + return +} + +func (d *tenantOssTotal) GetOneByTenantId(ctx context.Context, tenantId string) (e *entity.TenantOssTotal, err error) { + filter := bson.M{"tenantId": tenantId} + e = &entity.TenantOssTotal{} + err = mongo.FindOne(ctx, filter, e, consts.TenantOssTotalCollection) + return +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..452e9e7 --- /dev/null +++ b/go.mod @@ -0,0 +1,101 @@ +module oss + +go 1.25.3 + +require ( + gitee.com/red-future---jilin-g/common v0.2.5 + github.com/gogf/gf/contrib/drivers/mysql/v2 v2.9.5 + github.com/gogf/gf/contrib/nosql/redis/v2 v2.9.5 + github.com/gogf/gf/v2 v2.9.5 + go.mongodb.org/mongo-driver/v2 v2.4.1 +) + +//replace gitee.com/red-future---jilin-g/common v0.2.5 => ../common + +require ( + github.com/BurntSushi/toml v1.5.0 // indirect + github.com/armon/go-metrics v0.4.1 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/clbanning/mxj/v2 v2.7.0 // indirect + github.com/dgraph-io/badger/v4 v4.2.0 // indirect + github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/fatih/color v1.18.0 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-sql-driver/mysql v1.7.1 // indirect + github.com/gogf/gf/contrib/registry/consul/v2 v2.9.5 // indirect + github.com/gogf/gf/contrib/trace/otlphttp/v2 v2.9.5 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v5 v5.0.0 // indirect + github.com/golang/glog v1.2.5 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/golang/snappy v1.0.0 // indirect + github.com/google/flatbuffers v1.12.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/websocket v1.5.3 // indirect + github.com/grokify/html-strip-tags-go v0.1.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect + github.com/hashicorp/consul/api v1.26.1 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-hclog v1.5.0 // indirect + github.com/hashicorp/go-immutable-radix v1.3.1 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-rootcerts v1.0.2 // indirect + github.com/hashicorp/golang-lru v0.5.4 // indirect + github.com/hashicorp/serf v0.10.1 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/magiconair/properties v1.8.10 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/minio/crc64nvme v1.1.0 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/minio-go/v7 v7.0.97 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/olekukonko/errors v1.1.0 // indirect + github.com/olekukonko/ll v0.0.9 // indirect + github.com/olekukonko/tablewriter v1.1.0 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/redis/go-redis/v9 v9.12.1 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/stretchr/objx v0.5.2 // indirect + github.com/tiger1103/gfast-token v1.0.10 // indirect + github.com/tinylib/msgp v1.3.0 // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect + github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect + go.opencensus.io v0.23.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel v1.38.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 // indirect + go.opentelemetry.io/otel/metric v1.38.0 // indirect + go.opentelemetry.io/otel/sdk v1.38.0 // indirect + go.opentelemetry.io/otel/trace v1.38.0 // indirect + go.opentelemetry.io/proto/otlp v1.7.1 // indirect + golang.org/x/crypto v0.44.0 // indirect + golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/text v0.31.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/grpc v1.75.0 // indirect + google.golang.org/protobuf v1.36.8 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/main.go b/main.go new file mode 100644 index 0000000..3e7e14e --- /dev/null +++ b/main.go @@ -0,0 +1,35 @@ +package main + +import ( + "context" + "github.com/gogf/gf/v2/os/glog" + "github.com/gogf/gf/v2/os/gtimer" + "oss/controller" + "oss/service" + "time" + + "gitee.com/red-future---jilin-g/common/http" + "gitee.com/red-future---jilin-g/common/jaeger" + _ "gitee.com/red-future---jilin-g/common/mongo" + _ "github.com/gogf/gf/contrib/nosql/redis/v2" +) + +func main() { + ctx := context.Background() + defer jaeger.ShutDown(ctx) + + // 注册路由 + http.RouteRegister([]interface{}{ + controller.File, + }) + + gtimer.AddSingleton(ctx, time.Minute*5, func(ctx context.Context) { + err := service.TenantOssTotal.UpdateUsedOssSize(ctx) + if err != nil { + glog.Error(ctx, "UpdateUsedOssSize err: %v", err) + } + }) + + // 保持应用运行 + select {} +} diff --git a/model/dto/file_dto.go b/model/dto/file_dto.go new file mode 100644 index 0000000..a367fde --- /dev/null +++ b/model/dto/file_dto.go @@ -0,0 +1,18 @@ +package dto + +import ( + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/net/ghttp" +) + +// UploadFileReq 上传文件请求 +type UploadFileReq struct { + g.Meta `path:"/uploadFile" method:"post" tags:"存储管理" summary:"上传文件" dc:"上传文件"` + File *ghttp.UploadFile `json:"file" type:"file"` // 文件URL +} + +// UploadFileRes 上传文件响应 +type UploadFileRes struct { + FileURL string `json:"fileURL" dc:"上传地址"` + FileAddressPrefix string `json:"fileAddressPrefix"` +} diff --git a/model/dto/tenant_oss_total.go b/model/dto/tenant_oss_total.go new file mode 100644 index 0000000..c355acc --- /dev/null +++ b/model/dto/tenant_oss_total.go @@ -0,0 +1,17 @@ +package dto + +import ( + "github.com/gogf/gf/v2/frame/g" + "oss/model/entity" +) + +// GetByTenantIdReq 根据租户id获取存储总量请求 +type GetByTenantIdReq struct { + g.Meta `path:"/GetOneByTenantId" method:"get" tags:"租户存储总量管理" summary:"获取存储总量" dc:"获取存储总量"` + TenantId string `json:"tenantId" v:"required#租户id不能为空"` +} + +// GetByTenantIdRes 根据租户id获取存储总量响应 +type GetByTenantIdRes struct { + *entity.TenantOssTotal +} diff --git a/model/entity/file.go b/model/entity/file.go new file mode 100644 index 0000000..df6c9cc --- /dev/null +++ b/model/entity/file.go @@ -0,0 +1,19 @@ +package entity + +import ( + "gitee.com/red-future---jilin-g/common/do" + "oss/consts" +) + +// File 存储文件实体 +type File struct { + do.MongoBaseDO `bson:",inline"` // 嵌入基础字段:Id, Creator, CreatedAt, Updater, UpdatedAt, TenantId, IsDeleted + // 基础信息 + FileURL string `bson:"fileURL" json:"fileURL"` // 图URL + FileSize int64 `bson:"fileSize" json:"fileSize"` +} + +// CollectionName 存储集合名称 +func (File) CollectionName() string { + return consts.FileCollection +} diff --git a/model/entity/tenant_oss_total.go b/model/entity/tenant_oss_total.go new file mode 100644 index 0000000..c7fc67e --- /dev/null +++ b/model/entity/tenant_oss_total.go @@ -0,0 +1,19 @@ +package entity + +import ( + "gitee.com/red-future---jilin-g/common/do" + "oss/consts" +) + +// TenantOssTotal 租户储存服务总计实体 +type TenantOssTotal struct { + do.MongoBaseDO `bson:",inline"` // 嵌入基础字段:Id, Creator, CreatedAt, Updater, UpdatedAt, TenantId, IsDeleted + // 基础信息 + UsedOssSize int64 `bson:"usedOssSize" json:"usedOssSize"` + TotalOssSize int64 `bson:"totalOssSize" json:"totalOssSize"` +} + +// CollectionName 租户储存服务总计集合名称 +func (TenantOssTotal) CollectionName() string { + return consts.TenantOssTotalCollection +} diff --git a/service/file_service.go b/service/file_service.go new file mode 100644 index 0000000..eb94f5b --- /dev/null +++ b/service/file_service.go @@ -0,0 +1,119 @@ +package service + +import ( + "context" + "fmt" + "gitee.com/red-future---jilin-g/common/minio" + "gitee.com/red-future---jilin-g/common/mongo" + "gitee.com/red-future---jilin-g/common/redis" + "github.com/gogf/gf/v2/errors/gerror" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/util/gconv" + "oss/consts" + "oss/dao" + "oss/model/dto" + "oss/model/entity" + "time" +) + +type file struct{} + +// File 存储文件服务 +var File = new(file) + +func (f *file) UploadFile(ctx context.Context, req *dto.UploadFileReq) (res *dto.UploadFileRes, err error) { + tenantId := "" + fileSize := req.File.Size + totalFileSize := int64(0) + // 获取租户id + user, err := mongo.GetTenantInfo(ctx) + if err != nil { + return + } + // 获取redis-租户存储容量总数key + tenantOssTotalKey := fmt.Sprintf(consts.TenantOssTotalKey, gconv.String(user.TenantId)) + // 获取redis-租户存储-锁key + fileLockKey := fmt.Sprintf(consts.FileLockKey, gconv.String(user.TenantId)) + i := 0 +LOCK: + if ok, err := redis.RedisClient.SetNX(ctx, fileLockKey, fileSize); !ok || err != nil { + // 获取锁失败,需要重试 + if i < 5 { + i++ + time.Sleep(5 * time.Millisecond) + goto LOCK + } + } + // 设置redis-租户存储-锁key超时时间1分钟 + err = redis.RedisClient.SetEX(ctx, fileLockKey, fileSize, gconv.Int64(time.Minute*1)) + if err != nil { + return nil, err + } + // 获取redis-租户存储容量总数 + get, err := redis.RedisClient.Get(ctx, tenantOssTotalKey) + if err != nil { + return nil, err + } + tenantOssTotalEntity := &entity.TenantOssTotal{} + if g.IsEmpty(get) { + //查询数据库-获取租户存储容量总数 + getByTenantIdReq := &dto.GetByTenantIdReq{ + TenantId: gconv.String(user.TenantId), + } + tenantOssTotal, err := TenantOssTotal.GetOneByTenantId(ctx, getByTenantIdReq) + if err != nil { + return nil, err + } + if g.IsEmpty(tenantOssTotal) { + tenantOssTotalEntity.TenantId = user.TenantId + tenantOssTotalEntity.UsedOssSize = int64(0) + tenantOssTotalEntity.TotalOssSize = g.Cfg().MustGet(ctx, "oss.capacitySize").Int64() + } else { + tenantOssTotalEntity = tenantOssTotal.TenantOssTotal + } + } else { + // 反序列化-redis获取租户存储容量总数 + err = gconv.Struct(get, tenantOssTotalEntity) + if err != nil { + return nil, err + } + } + tenantId = gconv.String(tenantOssTotalEntity.TenantId) + fileSize = tenantOssTotalEntity.UsedOssSize + fileSize + totalFileSize = tenantOssTotalEntity.TotalOssSize + // 设置redis-租户存储容量总数 + tenantOssTotalKeyMap := map[string]interface{}{"tenantId": tenantId, "UsedOssSize": fileSize, "TotalOssSize": totalFileSize} + // 修改redis-租户存储容量总数 超时时间10分钟 + err = redis.RedisClient.SetEX(ctx, tenantOssTotalKey, tenantOssTotalKeyMap, gconv.Int64(time.Minute*10)) + if err != nil { + return nil, err + } + if fileSize < totalFileSize { + // 删除redis-租户存储-锁key + _, err = redis.RedisClient.Del(ctx, fileLockKey) + if err != nil { + return nil, err + } + } else { + return nil, gerror.New("存储服务内存不足") + } + // 上传图片 + fileURL, err := minio.UploadImage(ctx, req.File) + if err != nil { + return nil, err + } + // 插入数据库 + ossEntity := &entity.File{ + FileURL: fileURL, + FileSize: fileSize, + } + err = dao.File.Insert(ctx, ossEntity) + if err != nil { + return nil, err + } + // 返回图片url + return &dto.UploadFileRes{ + FileURL: fileURL, + FileAddressPrefix: minio.GetImgAddressPrefix(ctx), + }, err +} diff --git a/service/tenant_oss_total_service.go b/service/tenant_oss_total_service.go new file mode 100644 index 0000000..7bd7fec --- /dev/null +++ b/service/tenant_oss_total_service.go @@ -0,0 +1,58 @@ +package service + +import ( + "context" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/util/gconv" + "oss/consts" + "oss/dao" + "oss/model/dto" + "oss/model/entity" +) + +type tenantOssTotal struct{} + +// TenantOssTotal 存储文件服务 +var TenantOssTotal = new(tenantOssTotal) + +func (s *tenantOssTotal) GetOneByTenantId(ctx context.Context, req *dto.GetByTenantIdReq) (res *dto.GetByTenantIdRes, err error) { + e, err := dao.TenantOssTotal.GetOneByTenantId(ctx, req.TenantId) + if err != nil { + return nil, err + } + return &dto.GetByTenantIdRes{ + TenantOssTotal: e, + }, nil +} + +func (s *tenantOssTotal) UpdateUsedOssSize(ctx context.Context) (err error) { + // 使用 Keys 取出所有key + keys, err := g.Redis().Keys(ctx, consts.OssTotalKey) + if err != nil { + return + } + updateData := make([]*entity.TenantOssTotal, 0) + filterData := make([]*entity.TenantOssTotal, 0) + for _, key := range keys { + get, err := g.Redis().Get(ctx, key) + if err != nil { + return err + } + e := &entity.TenantOssTotal{} + err = gconv.Struct(get, e) + if err != nil { + return err + } + updateData = append(updateData, e) + totalOssSize := &entity.TenantOssTotal{} + totalOssSize.TenantId = e.TenantId + filterData = append(filterData, totalOssSize) + + } + // 更新数据库 + err = dao.TenantOssTotal.SaveOrUpdate(ctx, filterData, updateData) + if err != nil { + return err + } + return err +}