mirror of
https://github.com/openimsdk/open-im-server.git
synced 2026-05-02 16:15:59 +08:00
style: add format
Signed-off-by: Xinwei Xiong(cubxxw-openim) <3293172751nss@gmail.com>
This commit is contained in:
@@ -5,6 +5,9 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/cache"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/controller"
|
||||
@@ -17,8 +20,6 @@ import (
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/prome"
|
||||
openKeeper "github.com/OpenIMSDK/Open-IM-Server/pkg/discoveryregistry/zookeeper"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/rpcclient"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
type MsgTransfer struct {
|
||||
@@ -47,9 +48,18 @@ func StartTransfer(prometheusPort int) error {
|
||||
if err := mongo.CreateMsgIndex(); err != nil {
|
||||
return err
|
||||
}
|
||||
client, err := openKeeper.NewClient(config.Config.Zookeeper.ZkAddr, config.Config.Zookeeper.Schema,
|
||||
openKeeper.WithFreq(time.Hour), openKeeper.WithRoundRobin(), openKeeper.WithUserNameAndPassword(config.Config.Zookeeper.Username,
|
||||
config.Config.Zookeeper.Password), openKeeper.WithTimeout(10), openKeeper.WithLogger(log.NewZkLogger()))
|
||||
client, err := openKeeper.NewClient(
|
||||
config.Config.Zookeeper.ZkAddr,
|
||||
config.Config.Zookeeper.Schema,
|
||||
openKeeper.WithFreq(
|
||||
time.Hour,
|
||||
),
|
||||
openKeeper.WithRoundRobin(),
|
||||
openKeeper.WithUserNameAndPassword(config.Config.Zookeeper.Username,
|
||||
config.Config.Zookeeper.Password),
|
||||
openKeeper.WithTimeout(10),
|
||||
openKeeper.WithLogger(log.NewZkLogger()),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -66,7 +76,13 @@ func StartTransfer(prometheusPort int) error {
|
||||
msgDatabase := controller.NewCommonMsgDatabase(msgDocModel, msgModel)
|
||||
conversationRpcClient := rpcclient.NewConversationRpcClient(client)
|
||||
groupRpcClient := rpcclient.NewGroupRpcClient(client)
|
||||
msgTransfer := NewMsgTransfer(chatLogDatabase, extendMsgDatabase, msgDatabase, &conversationRpcClient, &groupRpcClient)
|
||||
msgTransfer := NewMsgTransfer(
|
||||
chatLogDatabase,
|
||||
extendMsgDatabase,
|
||||
msgDatabase,
|
||||
&conversationRpcClient,
|
||||
&groupRpcClient,
|
||||
)
|
||||
msgTransfer.initPrometheus()
|
||||
return msgTransfer.Start(prometheusPort)
|
||||
}
|
||||
@@ -74,8 +90,14 @@ func StartTransfer(prometheusPort int) error {
|
||||
func NewMsgTransfer(chatLogDatabase controller.ChatLogDatabase,
|
||||
extendMsgDatabase controller.ExtendMsgDatabase, msgDatabase controller.CommonMsgDatabase,
|
||||
conversationRpcClient *rpcclient.ConversationRpcClient, groupRpcClient *rpcclient.GroupRpcClient) *MsgTransfer {
|
||||
return &MsgTransfer{persistentCH: NewPersistentConsumerHandler(chatLogDatabase), historyCH: NewOnlineHistoryRedisConsumerHandler(msgDatabase, conversationRpcClient, groupRpcClient),
|
||||
historyMongoCH: NewOnlineHistoryMongoConsumerHandler(msgDatabase), modifyCH: NewModifyMsgConsumerHandler(extendMsgDatabase)}
|
||||
return &MsgTransfer{
|
||||
persistentCH: NewPersistentConsumerHandler(chatLogDatabase),
|
||||
historyCH: NewOnlineHistoryRedisConsumerHandler(msgDatabase, conversationRpcClient, groupRpcClient),
|
||||
historyMongoCH: NewOnlineHistoryMongoConsumerHandler(
|
||||
msgDatabase,
|
||||
),
|
||||
modifyCH: NewModifyMsgConsumerHandler(extendMsgDatabase),
|
||||
}
|
||||
}
|
||||
|
||||
func (m *MsgTransfer) initPrometheus() {
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
"github.com/Shopify/sarama"
|
||||
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/controller"
|
||||
@@ -14,7 +16,6 @@ import (
|
||||
pbMsg "github.com/OpenIMSDK/Open-IM-Server/pkg/proto/msg"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/sdkws"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
|
||||
"github.com/Shopify/sarama"
|
||||
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
@@ -41,7 +42,18 @@ func (mmc *ModifyMsgConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSessi
|
||||
claim sarama.ConsumerGroupClaim) error {
|
||||
for msg := range claim.Messages() {
|
||||
ctx := mmc.modifyMsgConsumerGroup.GetContextFromMsg(msg)
|
||||
log.ZDebug(ctx, "kafka get info to mysql", "ModifyMsgConsumerHandler", msg.Topic, "msgPartition", msg.Partition, "msg", string(msg.Value), "key", string(msg.Key))
|
||||
log.ZDebug(
|
||||
ctx,
|
||||
"kafka get info to mysql",
|
||||
"ModifyMsgConsumerHandler",
|
||||
msg.Topic,
|
||||
"msgPartition",
|
||||
msg.Partition,
|
||||
"msg",
|
||||
string(msg.Value),
|
||||
"key",
|
||||
string(msg.Key),
|
||||
)
|
||||
if len(msg.Value) != 0 {
|
||||
mmc.ModifyMsg(ctx, msg, string(msg.Key), sess)
|
||||
} else {
|
||||
@@ -52,7 +64,12 @@ func (mmc *ModifyMsgConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSessi
|
||||
return nil
|
||||
}
|
||||
|
||||
func (mmc *ModifyMsgConsumerHandler) ModifyMsg(ctx context.Context, cMsg *sarama.ConsumerMessage, msgKey string, _ sarama.ConsumerGroupSession) {
|
||||
func (mmc *ModifyMsgConsumerHandler) ModifyMsg(
|
||||
ctx context.Context,
|
||||
cMsg *sarama.ConsumerMessage,
|
||||
msgKey string,
|
||||
_ sarama.ConsumerGroupSession,
|
||||
) {
|
||||
msgFromMQ := pbMsg.MsgDataToModifyByMQ{}
|
||||
operationID := mcontext.GetOperationID(ctx)
|
||||
err := proto.Unmarshal(cMsg.Value, &msgFromMQ)
|
||||
@@ -92,7 +109,8 @@ func (mmc *ModifyMsgConsumerHandler) ModifyMsg(ctx context.Context, cMsg *sarama
|
||||
}
|
||||
|
||||
if err := mmc.extendMsgDatabase.InsertExtendMsg(ctx, notification.ConversationID, notification.SessionType, &extendMsg); err != nil {
|
||||
// log.ZError(ctx, "MsgFirstModify InsertExtendMsg failed", notification.ConversationID, notification.SessionType, extendMsg, err.Error())
|
||||
// log.ZError(ctx, "MsgFirstModify InsertExtendMsg failed", notification.ConversationID,
|
||||
// notification.SessionType, extendMsg, err.Error())
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
|
||||
@@ -9,6 +9,10 @@ import (
|
||||
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
|
||||
|
||||
"github.com/Shopify/sarama"
|
||||
"github.com/go-redis/redis"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/controller"
|
||||
@@ -18,9 +22,6 @@ import (
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/sdkws"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/rpcclient"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
|
||||
"github.com/Shopify/sarama"
|
||||
"github.com/go-redis/redis"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
const ConsumerMsgs = 3
|
||||
@@ -63,7 +64,11 @@ type OnlineHistoryRedisConsumerHandler struct {
|
||||
groupRpcClient *rpcclient.GroupRpcClient
|
||||
}
|
||||
|
||||
func NewOnlineHistoryRedisConsumerHandler(database controller.CommonMsgDatabase, conversationRpcClient *rpcclient.ConversationRpcClient, groupRpcClient *rpcclient.GroupRpcClient) *OnlineHistoryRedisConsumerHandler {
|
||||
func NewOnlineHistoryRedisConsumerHandler(
|
||||
database controller.CommonMsgDatabase,
|
||||
conversationRpcClient *rpcclient.ConversationRpcClient,
|
||||
groupRpcClient *rpcclient.GroupRpcClient,
|
||||
) *OnlineHistoryRedisConsumerHandler {
|
||||
var och OnlineHistoryRedisConsumerHandler
|
||||
och.msgDatabase = database
|
||||
och.msgDistributionCh = make(chan Cmd2Value) //no buffer channel
|
||||
@@ -77,7 +82,8 @@ func NewOnlineHistoryRedisConsumerHandler(database controller.CommonMsgDatabase,
|
||||
och.historyConsumerGroup = kafka.NewMConsumerGroup(&kafka.MConsumerGroupConfig{KafkaVersion: sarama.V2_0_0_0,
|
||||
OffsetsInitial: sarama.OffsetNewest, IsReturnErr: false}, []string{config.Config.Kafka.LatestMsgToRedis.Topic},
|
||||
config.Config.Kafka.Addr, config.Config.Kafka.ConsumerGroupID.MsgToRedis)
|
||||
//statistics.NewStatistics(&och.singleMsgSuccessCount, config.Config.ModuleName.MsgTransferName, fmt.Sprintf("%d second singleMsgCount insert to mongo", constant.StatisticsTimeInterval), constant.StatisticsTimeInterval)
|
||||
// statistics.NewStatistics(&och.singleMsgSuccessCount, config.Config.ModuleName.MsgTransferName, fmt.Sprintf("%d
|
||||
// second singleMsgCount insert to mongo", constant.StatisticsTimeInterval), constant.StatisticsTimeInterval)
|
||||
return &och
|
||||
}
|
||||
|
||||
@@ -90,16 +96,53 @@ func (och *OnlineHistoryRedisConsumerHandler) Run(channelID int) {
|
||||
msgChannelValue := cmd.Value.(MsgChannelValue)
|
||||
ctxMsgList := msgChannelValue.ctxMsgList
|
||||
ctx := msgChannelValue.ctx
|
||||
log.ZDebug(ctx, "msg arrived channel", "channel id", channelID, "msgList length", len(ctxMsgList), "uniqueKey", msgChannelValue.uniqueKey)
|
||||
storageMsgList, notStorageMsgList, storageNotificationList, notStorageNotificationList, modifyMsgList := och.getPushStorageMsgList(ctxMsgList)
|
||||
log.ZDebug(ctx, "msg lens", "storageMsgList", len(storageMsgList), "notStorageMsgList", len(notStorageMsgList),
|
||||
"storageNotificationList", len(storageNotificationList), "notStorageNotificationList", len(notStorageNotificationList), "modifyMsgList", len(modifyMsgList))
|
||||
log.ZDebug(
|
||||
ctx,
|
||||
"msg arrived channel",
|
||||
"channel id",
|
||||
channelID,
|
||||
"msgList length",
|
||||
len(ctxMsgList),
|
||||
"uniqueKey",
|
||||
msgChannelValue.uniqueKey,
|
||||
)
|
||||
storageMsgList, notStorageMsgList, storageNotificationList, notStorageNotificationList, modifyMsgList := och.getPushStorageMsgList(
|
||||
ctxMsgList,
|
||||
)
|
||||
log.ZDebug(
|
||||
ctx,
|
||||
"msg lens",
|
||||
"storageMsgList",
|
||||
len(storageMsgList),
|
||||
"notStorageMsgList",
|
||||
len(notStorageMsgList),
|
||||
"storageNotificationList",
|
||||
len(storageNotificationList),
|
||||
"notStorageNotificationList",
|
||||
len(notStorageNotificationList),
|
||||
"modifyMsgList",
|
||||
len(modifyMsgList),
|
||||
)
|
||||
conversationIDMsg := utils.GetChatConversationIDByMsg(ctxMsgList[0].message)
|
||||
conversationIDNotification := utils.GetNotificationConversationID(ctxMsgList[0].message)
|
||||
och.handleMsg(ctx, msgChannelValue.uniqueKey, conversationIDMsg, storageMsgList, notStorageMsgList)
|
||||
och.handleNotification(ctx, msgChannelValue.uniqueKey, conversationIDNotification, storageNotificationList, notStorageNotificationList)
|
||||
och.handleNotification(
|
||||
ctx,
|
||||
msgChannelValue.uniqueKey,
|
||||
conversationIDNotification,
|
||||
storageNotificationList,
|
||||
notStorageNotificationList,
|
||||
)
|
||||
if err := och.msgDatabase.MsgToModifyMQ(ctx, msgChannelValue.uniqueKey, conversationIDNotification, modifyMsgList); err != nil {
|
||||
log.ZError(ctx, "msg to modify mq error", err, "uniqueKey", msgChannelValue.uniqueKey, "modifyMsgList", modifyMsgList)
|
||||
log.ZError(
|
||||
ctx,
|
||||
"msg to modify mq error",
|
||||
err,
|
||||
"uniqueKey",
|
||||
msgChannelValue.uniqueKey,
|
||||
"modifyMsgList",
|
||||
modifyMsgList,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -107,7 +150,9 @@ func (och *OnlineHistoryRedisConsumerHandler) Run(channelID int) {
|
||||
}
|
||||
|
||||
// 获取消息/通知 存储的消息列表, 不存储并且推送的消息列表,
|
||||
func (och *OnlineHistoryRedisConsumerHandler) getPushStorageMsgList(totalMsgs []*ContextMsg) (storageMsgList, notStorageMsgList, storageNotificatoinList, notStorageNotificationList, modifyMsgList []*sdkws.MsgData) {
|
||||
func (och *OnlineHistoryRedisConsumerHandler) getPushStorageMsgList(
|
||||
totalMsgs []*ContextMsg,
|
||||
) (storageMsgList, notStorageMsgList, storageNotificatoinList, notStorageNotificationList, modifyMsgList []*sdkws.MsgData) {
|
||||
isStorage := func(msg *sdkws.MsgData) bool {
|
||||
options2 := utils.Options(msg.Options)
|
||||
if options2.IsHistory() {
|
||||
@@ -130,11 +175,17 @@ func (och *OnlineHistoryRedisConsumerHandler) getPushStorageMsgList(totalMsgs []
|
||||
msg.Options = utils.NewMsgOptions()
|
||||
}
|
||||
if options.IsOfflinePush() {
|
||||
v.message.Options = utils.WithOptions(utils.Options(v.message.Options), utils.WithOfflinePush(false))
|
||||
v.message.Options = utils.WithOptions(
|
||||
utils.Options(v.message.Options),
|
||||
utils.WithOfflinePush(false),
|
||||
)
|
||||
msg.Options = utils.WithOptions(utils.Options(msg.Options), utils.WithOfflinePush(true))
|
||||
}
|
||||
if options.IsUnreadCount() {
|
||||
v.message.Options = utils.WithOptions(utils.Options(v.message.Options), utils.WithUnreadCount(false))
|
||||
v.message.Options = utils.WithOptions(
|
||||
utils.Options(v.message.Options),
|
||||
utils.WithUnreadCount(false),
|
||||
)
|
||||
msg.Options = utils.WithOptions(utils.Options(msg.Options), utils.WithUnreadCount(true))
|
||||
}
|
||||
storageMsgList = append(storageMsgList, msg)
|
||||
@@ -151,19 +202,32 @@ func (och *OnlineHistoryRedisConsumerHandler) getPushStorageMsgList(totalMsgs []
|
||||
notStorageMsgList = append(notStorageMsgList, v.message)
|
||||
}
|
||||
}
|
||||
if v.message.ContentType == constant.ReactionMessageModifier || v.message.ContentType == constant.ReactionMessageDeleter {
|
||||
if v.message.ContentType == constant.ReactionMessageModifier ||
|
||||
v.message.ContentType == constant.ReactionMessageDeleter {
|
||||
modifyMsgList = append(modifyMsgList, v.message)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (och *OnlineHistoryRedisConsumerHandler) handleNotification(ctx context.Context, key, conversationID string, storageList, notStorageList []*sdkws.MsgData) {
|
||||
func (och *OnlineHistoryRedisConsumerHandler) handleNotification(
|
||||
ctx context.Context,
|
||||
key, conversationID string,
|
||||
storageList, notStorageList []*sdkws.MsgData,
|
||||
) {
|
||||
och.toPushTopic(ctx, key, conversationID, notStorageList)
|
||||
if len(storageList) > 0 {
|
||||
lastSeq, _, err := och.msgDatabase.BatchInsertChat2Cache(ctx, conversationID, storageList)
|
||||
if err != nil {
|
||||
log.ZError(ctx, "notification batch insert to redis error", err, "conversationID", conversationID, "storageList", storageList)
|
||||
log.ZError(
|
||||
ctx,
|
||||
"notification batch insert to redis error",
|
||||
err,
|
||||
"conversationID",
|
||||
conversationID,
|
||||
"storageList",
|
||||
storageList,
|
||||
)
|
||||
return
|
||||
}
|
||||
log.ZDebug(ctx, "success to next topic", "conversationID", conversationID)
|
||||
@@ -172,13 +236,21 @@ func (och *OnlineHistoryRedisConsumerHandler) handleNotification(ctx context.Con
|
||||
}
|
||||
}
|
||||
|
||||
func (och *OnlineHistoryRedisConsumerHandler) toPushTopic(ctx context.Context, key, conversationID string, msgs []*sdkws.MsgData) {
|
||||
func (och *OnlineHistoryRedisConsumerHandler) toPushTopic(
|
||||
ctx context.Context,
|
||||
key, conversationID string,
|
||||
msgs []*sdkws.MsgData,
|
||||
) {
|
||||
for _, v := range msgs {
|
||||
och.msgDatabase.MsgToPushMQ(ctx, key, conversationID, v)
|
||||
}
|
||||
}
|
||||
|
||||
func (och *OnlineHistoryRedisConsumerHandler) handleMsg(ctx context.Context, key, conversationID string, storageList, notStorageList []*sdkws.MsgData) {
|
||||
func (och *OnlineHistoryRedisConsumerHandler) handleMsg(
|
||||
ctx context.Context,
|
||||
key, conversationID string,
|
||||
storageList, notStorageList []*sdkws.MsgData,
|
||||
) {
|
||||
och.toPushTopic(ctx, key, conversationID, notStorageList)
|
||||
if len(storageList) > 0 {
|
||||
lastSeq, isNewConversation, err := och.msgDatabase.BatchInsertChat2Cache(ctx, conversationID, storageList)
|
||||
@@ -240,11 +312,26 @@ func (och *OnlineHistoryRedisConsumerHandler) MessagesDistributionHandle() {
|
||||
for i, header := range consumerMessages[i].Headers {
|
||||
arr = append(arr, strconv.Itoa(i), string(header.Key), string(header.Value))
|
||||
}
|
||||
log.ZInfo(ctx, "consumer.kafka.GetContextWithMQHeader", "len", len(consumerMessages[i].Headers), "header", strings.Join(arr, ", "))
|
||||
log.ZInfo(
|
||||
ctx,
|
||||
"consumer.kafka.GetContextWithMQHeader",
|
||||
"len",
|
||||
len(consumerMessages[i].Headers),
|
||||
"header",
|
||||
strings.Join(arr, ", "),
|
||||
)
|
||||
ctxMsg.ctx = kafka.GetContextWithMQHeader(consumerMessages[i].Headers)
|
||||
ctxMsg.message = msgFromMQ
|
||||
log.ZDebug(ctx, "single msg come to distribution center", "message", msgFromMQ, "key", string(consumerMessages[i].Key))
|
||||
//aggregationMsgs[string(consumerMessages[i].Key)] = append(aggregationMsgs[string(consumerMessages[i].Key)], ctxMsg)
|
||||
log.ZDebug(
|
||||
ctx,
|
||||
"single msg come to distribution center",
|
||||
"message",
|
||||
msgFromMQ,
|
||||
"key",
|
||||
string(consumerMessages[i].Key),
|
||||
)
|
||||
// aggregationMsgs[string(consumerMessages[i].Key)] =
|
||||
// append(aggregationMsgs[string(consumerMessages[i].Key)], ctxMsg)
|
||||
if oldM, ok := aggregationMsgs[string(consumerMessages[i].Key)]; ok {
|
||||
oldM = append(oldM, ctxMsg)
|
||||
aggregationMsgs[string(consumerMessages[i].Key)] = oldM
|
||||
@@ -260,7 +347,16 @@ func (och *OnlineHistoryRedisConsumerHandler) MessagesDistributionHandle() {
|
||||
hashCode := utils.GetHashCode(uniqueKey)
|
||||
channelID := hashCode % ChannelNum
|
||||
newCtx := withAggregationCtx(ctx, v)
|
||||
log.ZDebug(newCtx, "generate channelID", "hashCode", hashCode, "channelID", channelID, "uniqueKey", uniqueKey)
|
||||
log.ZDebug(
|
||||
newCtx,
|
||||
"generate channelID",
|
||||
"hashCode",
|
||||
hashCode,
|
||||
"channelID",
|
||||
channelID,
|
||||
"uniqueKey",
|
||||
uniqueKey,
|
||||
)
|
||||
och.chArrays[channelID] <- Cmd2Value{Cmd: SourceMessages, Value: MsgChannelValue{uniqueKey: uniqueKey, ctxMsgList: v, ctx: newCtx}}
|
||||
}
|
||||
}
|
||||
@@ -288,7 +384,10 @@ func (och *OnlineHistoryRedisConsumerHandler) Cleanup(_ sarama.ConsumerGroupSess
|
||||
return nil
|
||||
}
|
||||
|
||||
func (och *OnlineHistoryRedisConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { // a instance in the consumer group
|
||||
func (och *OnlineHistoryRedisConsumerHandler) ConsumeClaim(
|
||||
sess sarama.ConsumerGroupSession,
|
||||
claim sarama.ConsumerGroupClaim,
|
||||
) error { // a instance in the consumer group
|
||||
for {
|
||||
if sess == nil {
|
||||
log.ZWarn(context.Background(), "sess == nil, waiting", nil)
|
||||
|
||||
@@ -3,13 +3,14 @@ package msgtransfer
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/Shopify/sarama"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/controller"
|
||||
kfk "github.com/OpenIMSDK/Open-IM-Server/pkg/common/kafka"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
|
||||
pbMsg "github.com/OpenIMSDK/Open-IM-Server/pkg/proto/msg"
|
||||
"github.com/Shopify/sarama"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
type OnlineHistoryMongoConsumerHandler struct {
|
||||
@@ -27,7 +28,12 @@ func NewOnlineHistoryMongoConsumerHandler(database controller.CommonMsgDatabase)
|
||||
return mc
|
||||
}
|
||||
|
||||
func (mc *OnlineHistoryMongoConsumerHandler) handleChatWs2Mongo(ctx context.Context, cMsg *sarama.ConsumerMessage, key string, session sarama.ConsumerGroupSession) {
|
||||
func (mc *OnlineHistoryMongoConsumerHandler) handleChatWs2Mongo(
|
||||
ctx context.Context,
|
||||
cMsg *sarama.ConsumerMessage,
|
||||
key string,
|
||||
session sarama.ConsumerGroupSession,
|
||||
) {
|
||||
msg := cMsg.Value
|
||||
msgFromMQ := pbMsg.MsgDataToMongoByMQ{}
|
||||
err := proto.Unmarshal(msg, &msgFromMQ)
|
||||
@@ -42,7 +48,15 @@ func (mc *OnlineHistoryMongoConsumerHandler) handleChatWs2Mongo(ctx context.Cont
|
||||
log.ZInfo(ctx, "mongo consumer recv msg", "msgs", msgFromMQ.MsgData)
|
||||
err = mc.msgDatabase.BatchInsertChat2DB(ctx, msgFromMQ.ConversationID, msgFromMQ.MsgData, msgFromMQ.LastSeq)
|
||||
if err != nil {
|
||||
log.ZError(ctx, "single data insert to mongo err", err, "msg", msgFromMQ.MsgData, "conversationID", msgFromMQ.ConversationID)
|
||||
log.ZError(
|
||||
ctx,
|
||||
"single data insert to mongo err",
|
||||
err,
|
||||
"msg",
|
||||
msgFromMQ.MsgData,
|
||||
"conversationID",
|
||||
msgFromMQ.ConversationID,
|
||||
)
|
||||
}
|
||||
var seqs []int64
|
||||
for _, msg := range msgFromMQ.MsgData {
|
||||
@@ -50,14 +64,26 @@ func (mc *OnlineHistoryMongoConsumerHandler) handleChatWs2Mongo(ctx context.Cont
|
||||
}
|
||||
err = mc.msgDatabase.DeleteMessagesFromCache(ctx, msgFromMQ.ConversationID, seqs)
|
||||
if err != nil {
|
||||
log.ZError(ctx, "remove cache msg from redis err", err, "msg", msgFromMQ.MsgData, "conversationID", msgFromMQ.ConversationID)
|
||||
log.ZError(
|
||||
ctx,
|
||||
"remove cache msg from redis err",
|
||||
err,
|
||||
"msg",
|
||||
msgFromMQ.MsgData,
|
||||
"conversationID",
|
||||
msgFromMQ.ConversationID,
|
||||
)
|
||||
}
|
||||
mc.msgDatabase.DelUserDeleteMsgsList(ctx, msgFromMQ.ConversationID, seqs)
|
||||
}
|
||||
|
||||
func (OnlineHistoryMongoConsumerHandler) Setup(_ sarama.ConsumerGroupSession) error { return nil }
|
||||
func (OnlineHistoryMongoConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) error { return nil }
|
||||
func (mc *OnlineHistoryMongoConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { // a instance in the consumer group
|
||||
|
||||
func (mc *OnlineHistoryMongoConsumerHandler) ConsumeClaim(
|
||||
sess sarama.ConsumerGroupSession,
|
||||
claim sarama.ConsumerGroupClaim,
|
||||
) error { // a instance in the consumer group
|
||||
log.ZDebug(context.Background(), "online new session msg come", "highWaterMarkOffset",
|
||||
claim.HighWaterMarkOffset(), "topic", claim.Topic(), "partition", claim.Partition())
|
||||
for msg := range claim.Messages() {
|
||||
|
||||
@@ -3,8 +3,7 @@
|
||||
** copyright('tuoyun,www.tuoyun.net').
|
||||
** author("fg,Gordon@tuoyun.net").
|
||||
** time(2021/5/11 15:37).
|
||||
*/
|
||||
package msgtransfer
|
||||
*/package msgtransfer
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -35,7 +34,12 @@ func NewPersistentConsumerHandler(database controller.ChatLogDatabase) *Persiste
|
||||
}
|
||||
}
|
||||
|
||||
func (pc *PersistentConsumerHandler) handleChatWs2Mysql(ctx context.Context, cMsg *sarama.ConsumerMessage, msgKey string, _ sarama.ConsumerGroupSession) {
|
||||
func (pc *PersistentConsumerHandler) handleChatWs2Mysql(
|
||||
ctx context.Context,
|
||||
cMsg *sarama.ConsumerMessage,
|
||||
msgKey string,
|
||||
_ sarama.ConsumerGroupSession,
|
||||
) {
|
||||
msg := cMsg.Value
|
||||
var tag bool
|
||||
msgFromMQ := pbMsg.MsgDataToMQ{}
|
||||
@@ -73,10 +77,25 @@ func (pc *PersistentConsumerHandler) handleChatWs2Mysql(ctx context.Context, cMs
|
||||
}
|
||||
func (PersistentConsumerHandler) Setup(_ sarama.ConsumerGroupSession) error { return nil }
|
||||
func (PersistentConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) error { return nil }
|
||||
func (pc *PersistentConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error {
|
||||
|
||||
func (pc *PersistentConsumerHandler) ConsumeClaim(
|
||||
sess sarama.ConsumerGroupSession,
|
||||
claim sarama.ConsumerGroupClaim,
|
||||
) error {
|
||||
for msg := range claim.Messages() {
|
||||
ctx := pc.persistentConsumerGroup.GetContextFromMsg(msg)
|
||||
log.ZDebug(ctx, "kafka get info to mysql", "msgTopic", msg.Topic, "msgPartition", msg.Partition, "msg", string(msg.Value), "key", string(msg.Key))
|
||||
log.ZDebug(
|
||||
ctx,
|
||||
"kafka get info to mysql",
|
||||
"msgTopic",
|
||||
msg.Topic,
|
||||
"msgPartition",
|
||||
msg.Partition,
|
||||
"msg",
|
||||
string(msg.Value),
|
||||
"key",
|
||||
string(msg.Key),
|
||||
)
|
||||
if len(msg.Value) != 0 {
|
||||
pc.handleChatWs2Mysql(ctx, msg, string(msg.Key), sess)
|
||||
} else {
|
||||
|
||||
Reference in New Issue
Block a user