red-future/common
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 19 Wiki Activity

重构消息队列模块，新增NATS和RabbitMQ连接实现，移除旧版消息队列代码

Browse Source
This commit is contained in:
曲皓东
2026-01-31 05:17:14 +08:00
committed by 张斌
parent 6acdbb6e88
commit ee6d3c9033
17 changed files with 966 additions and 2391 deletions
Download Patch File Download Diff File Expand all files Collapse all files

280
message/redis_msg.go
View File

@@ -2,7 +2,6 @@ package message
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
@@ -12,151 +11,137 @@ import (
"github.com/gogf/gf/v2/util/gconv"
)
// redisMessageClient Redis 实现
type redisMessageClient struct {
clientType messageClientType
type RedisPublishMsgConfig struct {
QueueName string
Data any
}
type RedisSubscribeMsgConfig struct {
QueueName string
ConsumerName string
AutoAck bool
PrefetchCount int
HandleFunc func(ctx context.Context, message map[string]interface{}) error
}
func (*RedisPublishMsgConfig) GetPublishMsgType() {
}
func (*RedisSubscribeMsgConfig) GetSubscribeMsgType() {
}
func init() {
// 注册 Redis 插件（连接由 RegisterPlugin 异步处理）
registerPlugin(MessageRedis, func() messageUtil {
return &redis{}
})
}
type redis struct{}
// RedisStreamMessage Redis Stream 消息结构
type RedisStreamMessage struct {
type redisStreamMessage struct {
ID string
Values map[string]interface{}
}
// StreamGroup 创建消费组（支持单个或批量）
func (q *redisMessageClient) streamGroup(ctx context.Context, configs ...interface{}) error {
if len(configs) == 0 {
return fmt.Errorf("配置不能为空")
// Ping 检测 Redis 连接状态
func (c *redis) ping(ctx context.Context) bool {
conn, err := getDefaultDataSource()
if err != nil {
return false
}
for _, config := range configs {
cfg, ok := config.(*RedisConfig)
if !ok {
return fmt.Errorf("无效的 Redis 配置类型")
}
if err := q.createStreamGroup(ctx, cfg); err != nil {
return err
}
return conn.redisPing(ctx)
}
// Reconnect 重连 Redis
func (c *redis) reconnect(ctx context.Context) error {
conn, err := getDefaultDataSource()
if err != nil {
return fmt.Errorf("获取默认连接失败: %w", err)
}
if err := conn.redisReconnect(ctx); err != nil {
return fmt.Errorf("redis重连失败: %w", err)
}
return nil
}
// streamGroup 内部单个创建消费组
func (q *redisMessageClient) createStreamGroup(ctx context.Context, cfg *RedisConfig) error {
// 获取默认数据源
ds, err := GetManager().GetDefaultDataSource()
// Close 关闭 Redis 连接
func (c *redis) close(ctx context.Context) error {
conn, err := getDefaultDataSource()
if err != nil {
return fmt.Errorf("获取默认数据源失败: %w", err)
return fmt.Errorf("获取默认连接失败: %w", err)
}
// 检查连接状态,未连接则自动重连
if !ds.IsConnected() {
if err := ds.Reconnect(ctx); err != nil {
return fmt.Errorf("redis重连失败: %w", err)
}
if err := conn.redisClose(ctx); err != nil {
return fmt.Errorf("关闭redis连接失败: %w", err)
}
_, err = ds.Redis().Do(ctx, "XGROUP", "CREATE", cfg.Stream, cfg.Group, "0", "MKSTREAM")
if err != nil {
errStr := err.Error()
if strings.Contains(errStr, "BUSYGROUP") && strings.Contains(errStr, "already exists") {
glog.Infof(ctx, "✅ Redis 消费者组已存在: %s", cfg.Group)
return nil
}
return fmt.Errorf("初始化消费者组失败: %w", err)
}
glog.Infof(ctx, "✅ Redis 消费者组创建成功: %s", cfg.Group)
return nil
}
// Publish 内部单个发布消息
func (q *redisMessageClient) publish(ctx context.Context, config interface{}, data interface{}) error {
ds, err := GetManager().GetDefaultDataSource()
if err != nil {
return fmt.Errorf("获取默认数据源失败: %w", err)
}
if !ds.IsConnected() {
if err := ds.Reconnect(ctx); err != nil {
return fmt.Errorf("redis重连失败: %w", err)
}
}
cfg, ok := config.(*RedisConfig)
// Publish 发布消息
func (c *redis) Publish(ctx context.Context, msgConfig messagePublishConfig) error {
cfg, ok := msgConfig.(*RedisPublishMsgConfig)
if !ok {
return fmt.Errorf("无效的redis配置类型")
return fmt.Errorf("无效的 Redis 配置类型")
}
values := gconv.Map(data)
if g.IsEmpty(cfg.QueueName) {
return fmt.Errorf("队列名称不能为空")
}
if g.IsEmpty(cfg.Data) {
return fmt.Errorf("数据不能为空")
}
conn, err := getDefaultDataSource()
if err != nil {
return fmt.Errorf("获取默认连接失败: %w", err)
}
if !conn.getIsConnected() {
if err := conn.redisReconnect(ctx); err != nil {
return fmt.Errorf("redis重连失败: %w", err)
}
}
values := gconv.Map(cfg.Data)
args := make([]interface{}, 0, len(values)*2+2)
args = append(args, cfg.Stream, "*")
args = append(args, cfg.QueueName, "*")
for key, val := range values {
args = append(args, key, val)
}
result, err := ds.Redis().Do(ctx, "XADD", args...)
result, err := conn.getClient().Do(ctx, "XADD", args...)
if err != nil {
g.Log().Errorf(ctx, "❌ Redis 发布消息失败: topic=%s, err=%v", cfg.Stream, err)
g.Log().Errorf(ctx, "❌ Redis 发布消息失败: key=%s, err=%v", cfg.QueueName, err)
return err
}
g.Log().Infof(ctx, "✅ Redis 发布消息成功: topic=%s, messageID=%s", cfg.Stream, gconv.String(result))
g.Log().Infof(ctx, "✅ Redis 发布消息成功: key=%s, messageID=%s", cfg.QueueName, gconv.String(result))
return nil
}
// PublishDelayed 发布延迟消息（使用 ZSET）
func (q *redisMessageClient) publishDelayed(ctx context.Context, config interface{}, data interface{}, delay int) error {
ds, err := GetManager().GetDefaultDataSource()
if err != nil {
return fmt.Errorf("获取默认数据源失败: %w", err)
}
if !ds.IsConnected() {
if err := ds.Reconnect(ctx); err != nil {
return fmt.Errorf("redis重连失败: %w", err)
}
}
cfg, ok := config.(*RedisConfig)
// Subscribe 订阅消息
func (c *redis) Subscribe(ctx context.Context, msgConfig messageSubscribeConfig) error {
cfg, ok := msgConfig.(*RedisSubscribeMsgConfig)
if !ok {
return fmt.Errorf("无效的redis配置类型")
return fmt.Errorf("无效的 Redis 配置类型")
}
payload, err := json.Marshal(data)
if err != nil {
return fmt.Errorf("序列化数据失败: %w", err)
if g.IsEmpty(cfg.QueueName) {
return fmt.Errorf("队列名称不能为空")
}
score := float64(time.Now().Add(time.Duration(delay)).UnixMilli())
delayedKey := fmt.Sprintf("delayed:%s", cfg.Stream)
// ZADD delayedKey score payload
_, err = ds.Redis().Do(ctx, "ZADD", delayedKey, score, string(payload))
if err != nil {
return err
if g.IsEmpty(cfg.ConsumerName) {
return fmt.Errorf("消费者名称不能为空")
}
g.Log().Infof(ctx, "✅ Redis 延迟消息已发布: topic=%s, delay=%v", cfg.Stream, delay)
return nil
if g.IsEmpty(cfg.HandleFunc) {
return fmt.Errorf("处理函数不能为空")
}
return c.createSubscribe(ctx, cfg.QueueName, cfg.ConsumerName, cfg.PrefetchCount, cfg.AutoAck, cfg.HandleFunc)
}
// Subscribe 订阅消息（支持单个或批量）
func (q *redisMessageClient) subscribe(ctx context.Context, configs ...interface{}) error {
if len(configs) == 0 {
return fmt.Errorf("配置不能为空")
}
for _, config := range configs {
cfg, ok := config.(*RedisConfig)
if !ok {
return fmt.Errorf("无效的 Redis 配置类型")
}
handler := cfg.HandleFunc
if handler == nil {
return fmt.Errorf("必须提供处理函数")
}
if err := q.createSubscribe(ctx, cfg, handler); err != nil {
return err
}
}
return nil
}
// subscribe 内部单个订阅消息
func (q *redisMessageClient) createSubscribe(ctx context.Context, cfg *RedisConfig, handler func(ctx context.Context, message map[string]interface{}) error) error {
// createSubscribe 内部订阅消息
func (c *redis) createSubscribe(ctx context.Context, key, consumerName string, prefetchCount int, autoAck bool, handler func(ctx context.Context, message map[string]interface{}) error) error {
go func() {
defer func() {
if r := recover(); r != nil {
@@ -174,10 +159,10 @@ func (q *redisMessageClient) createSubscribe(ctx context.Context, cfg *RedisConf
for {
select {
case <-ctx.Done():
g.Log().Infof(ctx, "🔕 Redis 消费者停止: topic=%s", cfg.Stream)
g.Log().Infof(ctx, "🔕 Redis 消费者停止: topic=%s", key)
return
case <-retryTicker.C:
err := q.consumeMessages(ctx, cfg, handler)
err := c.consumeMessages(ctx, key, consumerName, prefetchCount, autoAck, handler)
if err != nil {
// 对于超时错误,返回nil继续循环,而不是返回错误
if strings.Contains(err.Error(), "i/o timeout") || strings.Contains(err.Error(), "timeout") ||
@@ -216,25 +201,25 @@ func (q *redisMessageClient) createSubscribe(ctx context.Context, cfg *RedisConf
}
// consumeMessages 消费消息
func (q *redisMessageClient) consumeMessages(ctx context.Context, cfg *RedisConfig, handler func(ctx context.Context, message map[string]interface{}) error) error {
ds, err := GetManager().GetDefaultDataSource()
func (c *redis) consumeMessages(ctx context.Context, key, consumerName string, prefetchCount int, autoAck bool, handler func(ctx context.Context, message map[string]interface{}) error) error {
conn, err := getDefaultDataSource()
if err != nil {
return fmt.Errorf("获取默认数据源失败: %w", err)
return fmt.Errorf("获取默认连接失败: %w", err)
}
if !ds.IsConnected() {
if err := ds.Reconnect(ctx); err != nil {
if !conn.getIsConnected() {
if err := conn.redisReconnect(ctx); err != nil {
return fmt.Errorf("redis重连失败: %w", err)
}
}
// 检查消费者组是否存在
if err := q.createStreamGroup(ctx, cfg); err != nil {
if err := c.createStreamGroup(ctx, key); err != nil {
return fmt.Errorf("create stream group failed: %w", err)
}
// 使用带重试的命令执行
result, err := ds.Redis().Do(ctx, "XREADGROUP", "GROUP", cfg.Group, cfg.Consumer, "COUNT", cfg.Count, "BLOCK", 0, "STREAMS", cfg.Stream, ">")
result, err := conn.getClient().Do(ctx, "XREADGROUP", "GROUP", "default", consumerName, "COUNT", prefetchCount, "BLOCK", 0, "STREAMS", key, ">")
if err != nil {
if strings.Contains(err.Error(), "i/o timeout") || strings.Contains(err.Error(), "timeout") ||
strings.Contains(err.Error(), "context deadline exceeded") || strings.Contains(err.Error(), "context canceled") {
@@ -242,7 +227,7 @@ func (q *redisMessageClient) consumeMessages(ctx context.Context, cfg *RedisConf
}
return err
}
messages, err := q.parseStreamResult(result)
messages, err := c.parseStreamResult(result)
if err != nil {
return err
}
@@ -254,8 +239,8 @@ func (q *redisMessageClient) consumeMessages(ctx context.Context, cfg *RedisConf
}
// ACK 消息
if cfg.AutoAck {
if err := q.ackMessage(ctx, cfg.Stream, cfg.Group, msg.ID); err != nil {
if autoAck {
if err := c.ackMessage(ctx, key, "default", msg.ID); err != nil {
g.Log().Errorf(ctx, "❌ ACK 消息失败: messageID=%s, err=%v", msg.ID, err)
}
}
@@ -264,15 +249,42 @@ func (q *redisMessageClient) consumeMessages(ctx context.Context, cfg *RedisConf
return nil
}
// ackMessage ACK 消息
func (q *redisMessageClient) ackMessage(ctx context.Context, streamKey, groupName string, messageIDs ...string) error {
ds, err := GetManager().GetDefaultDataSource()
// createStreamGroup 内部单个创建消费组
func (c *redis) createStreamGroup(ctx context.Context, key string) error {
conn, err := getDefaultDataSource()
if err != nil {
return fmt.Errorf("获取默认数据源失败: %w", err)
return fmt.Errorf("获取默认连接失败: %w", err)
}
if !ds.IsConnected() {
if err := ds.Reconnect(ctx); err != nil {
if !conn.getIsConnected() {
if err := conn.redisReconnect(ctx); err != nil {
return fmt.Errorf("redis重连失败: %w", err)
}
}
groupName := "default"
_, err = conn.getClient().Do(ctx, "XGROUP", "CREATE", key, groupName, "0", "MKSTREAM")
if err != nil {
errStr := err.Error()
if strings.Contains(errStr, "BUSYGROUP") && strings.Contains(errStr, "already exists") {
glog.Infof(ctx, "✅ Redis 消费者组已存在: %s", groupName)
return nil
}
return fmt.Errorf("初始化消费者组失败: %w", err)
}
glog.Infof(ctx, "✅ Redis 消费者组创建成功: %s", groupName)
return nil
}
// ackMessage ACK 消息
func (c *redis) ackMessage(ctx context.Context, streamKey, groupName string, messageIDs ...string) error {
conn, err := getDefaultDataSource()
if err != nil {
return fmt.Errorf("获取默认连接失败: %w", err)
}
if !conn.getIsConnected() {
if err := conn.redisReconnect(ctx); err != nil {
return fmt.Errorf("redis重连失败: %w", err)
}
}
@@ -282,14 +294,14 @@ func (q *redisMessageClient) ackMessage(ctx context.Context, streamKey, groupNam
for _, id := range messageIDs {
args = append(args, id)
}
_, err = ds.Redis().Do(ctx, "XACK", args...)
_, err = conn.getClient().Do(ctx, "XACK", args...)
return err
}
// parseStreamResult 解析 Stream 结果
func (q *redisMessageClient) parseStreamResult(result interface{}) ([]RedisStreamMessage, error) {
func (c *redis) parseStreamResult(result interface{}) ([]redisStreamMessage, error) {
if result == nil {
return []RedisStreamMessage{}, nil
return []redisStreamMessage{}, nil
}
var resultVal interface{}
@@ -303,15 +315,15 @@ func (q *redisMessageClient) parseStreamResult(result interface{}) ([]RedisStrea
// 检查是否为空
if resultVal == nil {
return []RedisStreamMessage{}, nil
return []redisStreamMessage{}, nil
}
// 预分配切片容量,避免多次扩容
messages := make([]RedisStreamMessage, 0)
messages := make([]redisStreamMessage, 0)
if streamsMap, ok := resultVal.(map[interface{}]interface{}); ok {
for _, streamMsg := range streamsMap {
msgArray, ok := streamMsg.([]interface{})
for _, streamData := range streamsMap {
msgArray, ok := streamData.([]interface{})
if !ok {
continue
}
@@ -332,7 +344,7 @@ func (q *redisMessageClient) parseStreamResult(result interface{}) ([]RedisStrea
values[key] = fieldsArray[i+1]
}
}
messages = append(messages, RedisStreamMessage{
messages = append(messages, redisStreamMessage{
ID: msgID,
Values: values,
})