Merge branch 'errcode' of github.com:OpenIMSDK/Open-IM-Server into errcode

internal/msgtransfer/init.go

@@ -2,8 +2,6 @@ package msgtransfer
 
 import (
 	"fmt"
-	"sync"
-
 	"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
 	"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/cache"
 	"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/controller"
@@ -43,25 +41,28 @@ func StartTransfer(prometheusPort int) error {
 	if err != nil {
 		return err
 	}
-	cacheModel := cache.NewCacheModel(rdb)
+	msgModel := cache.NewMsgCacheModel(rdb)
+	notificationModel := cache.NewNotificationCacheModel(rdb)
 	msgDocModel := unrelation.NewMsgMongoDriver(mongo.GetDatabase())
+	notificationDocModel := unrelation.NewNotificationMongoDriver(mongo.GetDatabase())
 	extendMsgModel := unrelation.NewExtendMsgSetMongoDriver(mongo.GetDatabase())
 	extendMsgCache := cache.NewExtendMsgSetCacheRedis(rdb, extendMsgModel, cache.GetDefaultOpt())
 	chatLogDatabase := controller.NewChatLogDatabase(relation.NewChatLogGorm(db))
 	extendMsgDatabase := controller.NewExtendMsgDatabase(extendMsgModel, extendMsgCache, tx.NewMongo(mongo.GetClient()))
-	msgDatabase := controller.NewMsgDatabase(msgDocModel, cacheModel)
+	msgDatabase := controller.NewMsgDatabase(msgDocModel, msgModel)
+	notificationDatabase := controller.NewNotificationDatabase(notificationDocModel, notificationModel)
 	conversationRpcClient := rpcclient.NewConversationClient(client)
 
-	msgTransfer := NewMsgTransfer(chatLogDatabase, extendMsgDatabase, msgDatabase, conversationRpcClient)
+	msgTransfer := NewMsgTransfer(chatLogDatabase, extendMsgDatabase, msgDatabase, notificationDatabase, conversationRpcClient)
 	msgTransfer.initPrometheus()
 	return msgTransfer.Start(prometheusPort)
 }
 
 func NewMsgTransfer(chatLogDatabase controller.ChatLogDatabase,
-	extendMsgDatabase controller.ExtendMsgDatabase, msgDatabase controller.MsgDatabase,
+	extendMsgDatabase controller.ExtendMsgDatabase, msgDatabase controller.MsgDatabase, notificationDatabase controller.MsgDatabase,
 	conversationRpcClient *rpcclient.ConversationClient) *MsgTransfer {
 	return &MsgTransfer{persistentCH: NewPersistentConsumerHandler(chatLogDatabase), historyCH: NewOnlineHistoryRedisConsumerHandler(msgDatabase, conversationRpcClient),
-		historyMongoCH: NewOnlineHistoryMongoConsumerHandler(msgDatabase), modifyCH: NewModifyMsgConsumerHandler(extendMsgDatabase)}
+		historyMongoCH: NewOnlineHistoryMongoConsumerHandler(msgDatabase, notificationDatabase), modifyCH: NewModifyMsgConsumerHandler(extendMsgDatabase)}
 }
 
 func (m *MsgTransfer) initPrometheus() {
@@ -76,8 +77,8 @@ func (m *MsgTransfer) initPrometheus() {
 }
 
 func (m *MsgTransfer) Start(prometheusPort int) error {
-	var wg sync.WaitGroup
-	wg.Add(4)
+	//var wg sync.WaitGroup
+	//wg.Add(4)
 	fmt.Println("start msg transfer", "prometheusPort:", prometheusPort)
 	if config.Config.ChatPersistenceMysql {
 		go m.persistentCH.persistentConsumerGroup.RegisterHandleAndConsumer(m.persistentCH)
@@ -91,6 +92,6 @@ func (m *MsgTransfer) Start(prometheusPort int) error {
 	if err != nil {
 		return err
 	}
-	wg.Wait()
+	//wg.Wait()
 	return nil
 }

internal/msgtransfer/online_history_msg_handler.go

@@ -5,6 +5,9 @@ import (
 	"sync"
 	"time"
 
+	"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/sdkws"
+	"github.com/go-redis/redis"
+
 	"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
 	"github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant"
 	"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/controller"
@@ -18,8 +21,7 @@ import (
 	"github.com/OpenIMSDK/Open-IM-Server/pkg/rpcclient"
 	"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
 	"github.com/Shopify/sarama"
-	"github.com/go-redis/redis/v8"
-	"google.golang.org/protobuf/proto"
+	"github.com/golang/protobuf/proto"
 )
 
 const ConsumerMsgs = 3
@@ -43,7 +45,7 @@ type Cmd2Value struct {
 	Value interface{}
 }
 type ContextMsg struct {
-	message *pbMsg.MsgDataToMQ
+	message *sdkws.MsgData
 	ctx     context.Context
 }
 
@@ -101,9 +103,9 @@ func (och *OnlineHistoryRedisConsumerHandler) Run(channelID int) {
 }
 
 // 获取消息/通知 存储的消息列表， 不存储并且推送的消息列表，
-func (och *OnlineHistoryRedisConsumerHandler) getPushStorageMsgList(conversationID string, totalMsgs []*ContextMsg) (storageMsgList, notStorageMsgList, storageNotificatoinList, notStorageNotificationList, modifyMsgList []*pbMsg.MsgDataToMQ) {
-	isStorage := func(msg *pbMsg.MsgDataToMQ) bool {
-		options2 := utils.Options(msg.MsgData.Options)
+func (och *OnlineHistoryRedisConsumerHandler) getPushStorageMsgList(sourceID string, totalMsgs []*ContextMsg) (storageMsgList, notStorageMsgList, storageNotificatoinList, notStorageNotificationList, modifyMsgList []*pbMsg.MsgDataToMQ) {
+	isStorage := func(msg *sdkws.MsgData) bool {
+		options2 := utils.Options(msg.Options)
 		if options2.IsHistory() {
 			return true
 		} else {
@@ -115,13 +117,13 @@ func (och *OnlineHistoryRedisConsumerHandler) getPushStorageMsgList(conversation
 		return false
 	}
 	for _, v := range totalMsgs {
-		options := utils.Options(v.message.MsgData.Options)
+		options := utils.Options(v.message.Options)
 		if options.IsNotification() {
 			// 原通知
 			notificationMsg := proto.Clone(v.message).(*pbMsg.MsgDataToMQ)
 			if options.IsSendMsg() {
 				// 消息
-				v.message.MsgData.Options = utils.WithOptions(utils.Options(v.message.MsgData.Options), utils.WithNotification(false), utils.WithSendMsg(false))
+				v.message.Options = utils.WithOptions(utils.Options(v.message.MsgData.Options), utils.WithNotification(false), utils.WithSendMsg(false))
 				storageMsgList = append(storageMsgList, v.message)
 			}
 			if isStorage(notificationMsg) {
@@ -136,7 +138,7 @@ func (och *OnlineHistoryRedisConsumerHandler) getPushStorageMsgList(conversation
 				notStorageMsgList = append(notStorageMsgList, v.message)
 			}
 		}
-		if v.message.MsgData.ContentType == constant.ReactionMessageModifier || v.message.MsgData.ContentType == constant.ReactionMessageDeleter {
+		if v.message.ContentType == constant.ReactionMessageModifier || v.message.ContentType == constant.ReactionMessageDeleter {
 			modifyMsgList = append(modifyMsgList, v.message)
 		}
 	}
@@ -251,7 +253,7 @@ func (och *OnlineHistoryRedisConsumerHandler) MessagesDistributionHandle() {
 				log.ZDebug(ctx, "batch messages come to distribution center", "length", len(consumerMessages))
 				for i := 0; i < len(consumerMessages); i++ {
 					ctxMsg := &ContextMsg{}
-					msgFromMQ := pbMsg.MsgDataToMQ{}
+					var msgFromMQ sdkws.MsgData
 					err := proto.Unmarshal(consumerMessages[i].Value, &msgFromMQ)
 					if err != nil {
 						log.ZError(ctx, "msg_transfer Unmarshal msg err", err, string(consumerMessages[i].Value))

internal/msgtransfer/online_msg_to_mongo_handler.go

@@ -2,6 +2,10 @@ package msgtransfer
 
 import (
 	"context"
+	"errors"
+
+	"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/sdkws"
+	"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
 
 	"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
 	"github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant"
@@ -10,8 +14,6 @@ import (
 	"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
 	"github.com/OpenIMSDK/Open-IM-Server/pkg/common/mcontext"
 	pbMsg "github.com/OpenIMSDK/Open-IM-Server/pkg/proto/msg"
-	"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/sdkws"
-	"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
 	"github.com/Shopify/sarama"
 	"google.golang.org/protobuf/proto"
 )
@@ -19,14 +21,16 @@ import (
 type OnlineHistoryMongoConsumerHandler struct {
 	historyConsumerGroup *kfk.MConsumerGroup
 	msgDatabase          controller.MsgDatabase
+	notificationDatabase controller.NotificationDatabase
 }
 
-func NewOnlineHistoryMongoConsumerHandler(database controller.MsgDatabase) *OnlineHistoryMongoConsumerHandler {
+func NewOnlineHistoryMongoConsumerHandler(database controller.MsgDatabase, notificationDatabase controller.NotificationDatabase) *OnlineHistoryMongoConsumerHandler {
 	mc := &OnlineHistoryMongoConsumerHandler{
 		historyConsumerGroup: kfk.NewMConsumerGroup(&kfk.MConsumerGroupConfig{KafkaVersion: sarama.V2_0_0_0,
 			OffsetsInitial: sarama.OffsetNewest, IsReturnErr: false}, []string{config.Config.Kafka.MsgToMongo.Topic},
 			config.Config.Kafka.MsgToMongo.Addr, config.Config.Kafka.ConsumerGroupID.MsgToMongo),
-		msgDatabase: database,
+		msgDatabase:          database,
+		notificationDatabase: notificationDatabase,
 	}
 	return mc
 }
@@ -40,31 +44,54 @@ func (mc *OnlineHistoryMongoConsumerHandler) handleChatWs2Mongo(ctx context.Cont
 		log.Error("msg_transfer Unmarshal msg err", "", "msg", string(msg), "err", err.Error())
 		return
 	}
-	log.Info(operationID, "BatchInsertChat2DB userID: ", msgFromMQ.ConversationID, "msgFromMQ.LastSeq: ", msgFromMQ.LastSeq)
-	err = mc.msgDatabase.BatchInsertChat2DB(ctx, msgFromMQ.ConversationID, msgFromMQ.Messages, msgFromMQ.LastSeq)
-	if err != nil {
-		log.NewError(operationID, "single data insert to mongo err", err.Error(), msgFromMQ.Messages, msgFromMQ.ConversationID, msgFromMQ.TriggerID)
+	log.Info(operationID, "BatchInsertChat2DB userID: ", msgFromMQ.AggregationID, "msgFromMQ.LastSeq: ", msgFromMQ.LastSeq)
+	if len(msgFromMQ.MsgData) == 0 {
+		log.ZError(ctx, "msgFromMQ.MsgData is empty", errors.New("msgFromMQ.MsgData is empty"), "cMsg", cMsg)
+		return
 	}
-	err = mc.msgDatabase.DeleteMessageFromCache(ctx, msgFromMQ.ConversationID, msgFromMQ.Messages)
-	if err != nil {
-		log.NewError(operationID, "remove cache msg from redis err", err.Error(), msgFromMQ.Messages, msgFromMQ.ConversationID, msgFromMQ.TriggerID)
-	}
-	for _, v := range msgFromMQ.Messages {
-		if v.MsgData.ContentType == constant.DeleteMessageNotification {
-			tips := sdkws.TipsComm{}
-			DeleteMessageTips := sdkws.DeleteMessageTips{}
-			err := proto.Unmarshal(v.MsgData.Content, &tips)
-			if err != nil {
-				log.NewError(operationID, "tips unmarshal err:", err.Error(), v.String())
-				continue
+	isNotification := msgFromMQ.MsgData[0].Options[constant.IsNotification]
+	if isNotification {
+		err = mc.notificationDatabase.BatchInsertChat2DB(ctx, msgFromMQ.AggregationID, msgFromMQ.MsgData, msgFromMQ.LastSeq)
+		if err != nil {
+			log.NewError(operationID, "single data insert to mongo err", err.Error(), msgFromMQ.MsgData, msgFromMQ.AggregationID, msgFromMQ.TriggerID)
+		}
+		err = mc.notificationDatabase.DeleteMessageFromCache(ctx, msgFromMQ.AggregationID, msgFromMQ.MsgData)
+		if err != nil {
+			log.NewError(operationID, "remove cache msg from redis err", err.Error(), msgFromMQ.MsgData, msgFromMQ.AggregationID, msgFromMQ.TriggerID)
+		}
+		for _, v := range msgFromMQ.MsgData {
+			if v.ContentType == constant.DeleteMessageNotification {
+				deleteMessageTips := sdkws.DeleteMessageTips{}
+				err := proto.Unmarshal(v.Content, &deleteMessageTips)
+				if err != nil {
+					log.NewError(operationID, "tips unmarshal err:", err.Error(), v.String())
+					continue
+				}
+				if totalUnExistSeqs, err := mc.notificationDatabase.DelMsgBySeqs(ctx, deleteMessageTips.UserID, deleteMessageTips.Seqs); err != nil {
+					log.NewError(operationID, utils.GetSelfFuncName(), "DelMsgBySeqs args: ", deleteMessageTips.UserID, deleteMessageTips.Seqs, "error:", err.Error(), "totalUnExistSeqs: ", totalUnExistSeqs)
+				}
 			}
-			err = proto.Unmarshal(tips.Detail, &DeleteMessageTips)
-			if err != nil {
-				log.NewError(operationID, "deleteMessageTips unmarshal err:", err.Error(), v.String())
-				continue
-			}
-			if totalUnExistSeqs, err := mc.msgDatabase.DelMsgBySeqs(ctx, DeleteMessageTips.UserID, DeleteMessageTips.Seqs); err != nil {
-				log.NewError(operationID, utils.GetSelfFuncName(), "DelMsgBySeqs args: ", DeleteMessageTips.UserID, DeleteMessageTips.Seqs, "error:", err.Error(), "totalUnExistSeqs: ", totalUnExistSeqs)
+		}
+	} else {
+		err = mc.msgDatabase.BatchInsertChat2DB(ctx, msgFromMQ.AggregationID, msgFromMQ.MsgData, msgFromMQ.LastSeq)
+		if err != nil {
+			log.NewError(operationID, "single data insert to mongo err", err.Error(), msgFromMQ.MsgData, msgFromMQ.AggregationID, msgFromMQ.TriggerID)
+		}
+		err = mc.msgDatabase.DeleteMessageFromCache(ctx, msgFromMQ.AggregationID, msgFromMQ.MsgData)
+		if err != nil {
+			log.NewError(operationID, "remove cache msg from redis err", err.Error(), msgFromMQ.MsgData, msgFromMQ.AggregationID, msgFromMQ.TriggerID)
+		}
+		for _, v := range msgFromMQ.MsgData {
+			if v.ContentType == constant.DeleteMessageNotification {
+				deleteMessageTips := sdkws.DeleteMessageTips{}
+				err := proto.Unmarshal(v.Content, &deleteMessageTips)
+				if err != nil {
+					log.NewError(operationID, "tips unmarshal err:", err.Error(), v.String())
+					continue
+				}
+				if totalUnExistSeqs, err := mc.msgDatabase.DelMsgBySeqs(ctx, deleteMessageTips.UserID, deleteMessageTips.Seqs); err != nil {
+					log.NewError(operationID, utils.GetSelfFuncName(), "DelMsgBySeqs args: ", deleteMessageTips.UserID, deleteMessageTips.Seqs, "error:", err.Error(), "totalUnExistSeqs: ", totalUnExistSeqs)
+				}
 			}
 		}
 	}

internal/push/offlinepush/fcm/push.go

@@ -20,10 +20,10 @@ var Terminal = []int{constant.IOSPlatformID, constant.AndroidPlatformID, constan
 
 type Fcm struct {
 	fcmMsgCli *messaging.Client
-	cache     cache.Model
+	cache     cache.MsgModel
 }
 
-func NewClient(cache cache.Model) *Fcm {
+func NewClient(cache cache.MsgModel) *Fcm {
 	opt := option.WithCredentialsFile(filepath.Join(config.Root, "config", config.Config.Push.Fcm.ServiceAccount))
 	fcmApp, err := firebase.NewApp(context.Background(), nil, opt)
 	if err != nil {

internal/push/offlinepush/fcm/push_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 func Test_Push(t *testing.T) {
-	var redis cache.Model
+	var redis cache.MsgModel
 	offlinePusher := NewClient(redis)
 	err := offlinePusher.Push(context.Background(), []string{"userID1"}, "test", "test", &offlinepush.Opts{})
 	assert.Nil(t, err)

internal/push/offlinepush/getui/push.go

@@ -38,12 +38,12 @@ const (
 )
 
 type Client struct {
-	cache           cache.Model
+	cache           cache.MsgModel
 	tokenExpireTime int64
 	taskIDTTL       int64
 }
 
-func NewClient(cache cache.Model) *Client {
+func NewClient(cache cache.MsgModel) *Client {
 	return &Client{cache: cache, tokenExpireTime: tokenExpireTime, taskIDTTL: taskIDTTL}
 }
 

internal/push/push_rpc_server.go

@@ -22,7 +22,7 @@ func Start(client discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) e
 	if err != nil {
 		return err
 	}
-	cacheModel := cache.NewCacheModel(rdb)
+	cacheModel := cache.NewMsgCacheModel(rdb)
 	offlinePusher := NewOfflinePusher(cacheModel)
 	database := controller.NewPushDatabase(cacheModel)
 	pusher := NewPusher(client, offlinePusher, database, localcache.NewGroupLocalCache(client), localcache.NewConversationLocalCache(client))

internal/push/push_to_client.go

@@ -44,7 +44,7 @@ func NewPusher(client discoveryregistry.SvcDiscoveryRegistry, offlinePusher offl
 	}
 }
 
-func NewOfflinePusher(cache cache.Model) offlinepush.OfflinePusher {
+func NewOfflinePusher(cache cache.MsgModel) offlinepush.OfflinePusher {
 	var offlinePusher offlinepush.OfflinePusher
 	if config.Config.Push.Getui.Enable {
 		offlinePusher = getui.NewClient(cache)

internal/rpc/auth/auth.go

@@ -32,7 +32,7 @@ func Start(client discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) e
 	pbAuth.RegisterAuthServer(server, &authServer{
 		userRpcClient:  rpcclient.NewUserClient(client),
 		RegisterCenter: client,
-		authDatabase:   controller.NewAuthDatabase(cache.NewCacheModel(rdb), config.Config.TokenPolicy.AccessSecret, config.Config.TokenPolicy.AccessExpire),
+		authDatabase:   controller.NewAuthDatabase(cache.NewMsgCacheModel(rdb), config.Config.TokenPolicy.AccessSecret, config.Config.TokenPolicy.AccessExpire),
 	})
 	return nil
 }

internal/rpc/msg/lock.go

@@ -15,10 +15,10 @@ type MessageLocker interface {
 	UnLockGlobalMessage(ctx context.Context, clientMsgID string) (err error)
 }
 type LockerMessage struct {
-	cache cache.Model
+	cache cache.MsgModel
 }
 
-func NewLockerMessage(cache cache.Model) *LockerMessage {
+func NewLockerMessage(cache cache.MsgModel) *LockerMessage {
 	return &LockerMessage{cache: cache}
 }
 func (l *LockerMessage) LockMessageTypeKey(ctx context.Context, clientMsgID, typeKey string) (err error) {

internal/rpc/msg/server.go

@@ -57,7 +57,7 @@ func Start(client discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) e
 	if err != nil {
 		return err
 	}
-	cacheModel := cache.NewCacheModel(rdb)
+	cacheModel := cache.NewMsgCacheModel(rdb)
 	msgDocModel := unrelation.NewMsgMongoDriver(mongo.GetDatabase())
 	extendMsgModel := unrelation.NewExtendMsgSetMongoDriver(mongo.GetDatabase())
 	extendMsgCacheModel := cache.NewExtendMsgSetCacheRedis(rdb, extendMsgModel, cache.GetDefaultOpt())

internal/rpc/third/third.go

@@ -37,7 +37,7 @@ func Start(client discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) e
 		return err
 	}
 	third.RegisterThirdServer(server, &thirdServer{
-		thirdDatabase: controller.NewThirdDatabase(cache.NewCacheModel(rdb)),
+		thirdDatabase: controller.NewThirdDatabase(cache.NewMsgCacheModel(rdb)),
 		userRpcClient: rpcclient.NewUserClient(client),
 		s3dataBase:    controller.NewS3Database(o, relation.NewObjectHash(db), relation.NewObjectInfo(db), relation.NewObjectPut(db), u),
 	})

internal/tools/msg_test.go

@@ -63,7 +63,7 @@ func TestDeleteMongoMsgAndResetRedisSeq(t *testing.T) {
 	if err != nil {
 		return
 	}
-	cacheModel := cache.NewCacheModel(rdb)
+	cacheModel := cache.NewMsgCacheModel(rdb)
 	mongoClient := mgo.GetDatabase().Collection(unRelationTb.MsgDocModel{}.TableName())
 
 	ctx := context.Background()