feat: optimize code and support running in single process mode (#3142)

* pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * monolithic * fix: DeleteDoc crash * fix: DeleteDoc crash * fix: monolithic * fix: monolithic * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group * fix: mq * fix: mq * fix: user msg timestamp * fix: mq * 1 * 1 * 1 * 1 * 1 * 1 * 1 * seq read config * seq read config * 1 * 1 * fix: the source message of the reference is withdrawn, and the referenced message is deleted * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1
2026-05-07 10:35:59 +08:00 · 2025-02-14 16:18:27 +08:00
parent 4b3a2b7483
commit f322ddca77
100 changed files with 3044 additions and 1857 deletions
@@ -18,14 +18,13 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
-	"github.com/openimsdk/open-im-server/v3/pkg/rpcli"
-	"github.com/openimsdk/tools/discovery"
-	"strconv"
-	"strings"
+
 	"sync"
 	"time"

-	"github.com/IBM/sarama"
+	"github.com/openimsdk/open-im-server/v3/pkg/rpcli"
+	"github.com/openimsdk/tools/discovery"
+
 	"github.com/go-redis/redis"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller"
@@ -64,9 +63,7 @@ type userHasReadSeq struct {
 }

 type OnlineHistoryRedisConsumerHandler struct {
-	historyConsumerGroup *kafka.MConsumerGroup
-
-	redisMessageBatches *batcher.Batcher[sarama.ConsumerMessage]
+	redisMessageBatches *batcher.Batcher[ConsumerMessage]

 	msgTransferDatabase         controller.MsgTransferDatabase
 	conversationUserHasReadChan chan *userHasReadSeq
@@ -76,12 +73,13 @@ type OnlineHistoryRedisConsumerHandler struct {
 	conversationClient *rpcli.ConversationClient
 }

-func NewOnlineHistoryRedisConsumerHandler(ctx context.Context, client discovery.SvcDiscoveryRegistry, config *Config, database controller.MsgTransferDatabase) (*OnlineHistoryRedisConsumerHandler, error) {
-	kafkaConf := config.KafkaConfig
-	historyConsumerGroup, err := kafka.NewMConsumerGroup(kafkaConf.Build(), kafkaConf.ToRedisGroupID, []string{kafkaConf.ToRedisTopic}, false)
-	if err != nil {
-		return nil, err
-	}
+type ConsumerMessage struct {
+	Ctx   context.Context
+	Key   string
+	Value []byte
+}
+
+func NewOnlineHistoryRedisConsumerHandler(ctx context.Context, client discovery.Conn, config *Config, database controller.MsgTransferDatabase) (*OnlineHistoryRedisConsumerHandler, error) {
 	groupConn, err := client.GetConn(ctx, config.Discovery.RpcService.Group)
 	if err != nil {
 		return nil, err
@@ -97,7 +95,7 @@ func NewOnlineHistoryRedisConsumerHandler(ctx context.Context, client discovery.
 	och.conversationClient = rpcli.NewConversationClient(conversationConn)
 	och.wg.Add(1)

-	b := batcher.New[sarama.ConsumerMessage](
+	b := batcher.New[ConsumerMessage](
 		batcher.WithSize(size),
 		batcher.WithWorker(worker),
 		batcher.WithInterval(interval),
@@ -109,16 +107,15 @@ func NewOnlineHistoryRedisConsumerHandler(ctx context.Context, client discovery.
 		hashCode := stringutil.GetHashCode(key)
 		return int(hashCode) % och.redisMessageBatches.Worker()
 	}
-	b.Key = func(consumerMessage *sarama.ConsumerMessage) string {
-		return string(consumerMessage.Key)
+	b.Key = func(consumerMessage *ConsumerMessage) string {
+		return consumerMessage.Key
 	}
 	b.Do = och.do
 	och.redisMessageBatches = b
-	och.historyConsumerGroup = historyConsumerGroup

 	return &och, nil
 }
-func (och *OnlineHistoryRedisConsumerHandler) do(ctx context.Context, channelID int, val *batcher.Msg[sarama.ConsumerMessage]) {
+func (och *OnlineHistoryRedisConsumerHandler) do(ctx context.Context, channelID int, val *batcher.Msg[ConsumerMessage]) {
 	ctx = mcontext.WithTriggerIDContext(ctx, val.TriggerID())
 	ctxMessages := och.parseConsumerMessages(ctx, val.Val())
 	ctx = withAggregationCtx(ctx, ctxMessages)
@@ -189,7 +186,7 @@ func (och *OnlineHistoryRedisConsumerHandler) doSetReadSeq(ctx context.Context,

 }

-func (och *OnlineHistoryRedisConsumerHandler) parseConsumerMessages(ctx context.Context, consumerMessages []*sarama.ConsumerMessage) []*ContextMsg {
+func (och *OnlineHistoryRedisConsumerHandler) parseConsumerMessages(ctx context.Context, consumerMessages []*ConsumerMessage) []*ContextMsg {
 	var ctxMessages []*ContextMsg
 	for i := 0; i < len(consumerMessages); i++ {
 		ctxMsg := &ContextMsg{}
@@ -199,16 +196,9 @@ func (och *OnlineHistoryRedisConsumerHandler) parseConsumerMessages(ctx context.
 			log.ZWarn(ctx, "msg_transfer Unmarshal msg err", err, string(consumerMessages[i].Value))
 			continue
 		}
-		var arr []string
-		for i, header := range consumerMessages[i].Headers {
-			arr = append(arr, strconv.Itoa(i), string(header.Key), string(header.Value))
-		}
-		log.ZDebug(ctx, "consumer.kafka.GetContextWithMQHeader", "len", len(consumerMessages[i].Headers),
-			"header", strings.Join(arr, ", "))
-		ctxMsg.ctx = kafka.GetContextWithMQHeader(consumerMessages[i].Headers)
+		ctxMsg.ctx = consumerMessages[i].Ctx
 		ctxMsg.message = msgFromMQ
-		log.ZDebug(ctx, "message parse finish", "message", msgFromMQ, "key",
-			string(consumerMessages[i].Key))
+		log.ZDebug(ctx, "message parse finish", "message", msgFromMQ, "key", consumerMessages[i].Key)
 		ctxMessages = append(ctxMessages, ctxMsg)
 	}
 	return ctxMessages
@@ -383,7 +373,9 @@ func (och *OnlineHistoryRedisConsumerHandler) Close() {
 func (och *OnlineHistoryRedisConsumerHandler) toPushTopic(ctx context.Context, key, conversationID string, msgs []*ContextMsg) {
 	for _, v := range msgs {
 		log.ZDebug(ctx, "push msg to topic", "msg", v.message.String())
-		_, _, _ = och.msgTransferDatabase.MsgToPushMQ(v.ctx, key, conversationID, v.message)
+		if err := och.msgTransferDatabase.MsgToPushMQ(v.ctx, key, conversationID, v.message); err != nil {
+			log.ZError(ctx, "msg to push topic error", err, "msg", v.message.String())
+		}
 	}
 }

@@ -401,35 +393,10 @@ func withAggregationCtx(ctx context.Context, values []*ContextMsg) context.Conte
 	return mcontext.SetOperationID(ctx, allMessageOperationID)
 }

-func (och *OnlineHistoryRedisConsumerHandler) Setup(_ sarama.ConsumerGroupSession) error { return nil }
-func (och *OnlineHistoryRedisConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) error {
+func (och *OnlineHistoryRedisConsumerHandler) HandlerRedisMessage(ctx context.Context, key string, value []byte) error { // a instance in the consumer group
+	err := och.redisMessageBatches.Put(ctx, &ConsumerMessage{Ctx: ctx, Key: key, Value: value})
+	if err != nil {
+		log.ZWarn(ctx, "put msg to  error", err, "key", key, "value", value)
+	}
 	return nil
 }
-
-func (och *OnlineHistoryRedisConsumerHandler) ConsumeClaim(session sarama.ConsumerGroupSession,
-	claim sarama.ConsumerGroupClaim) error { // a instance in the consumer group
-	log.ZDebug(context.Background(), "online new session msg come", "highWaterMarkOffset",
-		claim.HighWaterMarkOffset(), "topic", claim.Topic(), "partition", claim.Partition())
-	och.redisMessageBatches.OnComplete = func(lastMessage *sarama.ConsumerMessage, totalCount int) {
-		session.MarkMessage(lastMessage, "")
-		session.Commit()
-	}
-	for {
-		select {
-		case msg, ok := <-claim.Messages():
-			if !ok {
-				return nil
-			}
-
-			if len(msg.Value) == 0 {
-				continue
-			}
-			err := och.redisMessageBatches.Put(context.Background(), msg)
-			if err != nil {
-				log.ZWarn(context.Background(), "put msg to  error", err, "msg", msg)
-			}
-		case <-session.Context().Done():
-			return nil
-		}
-	}
-}