Merge remote-tracking branch 'origin/modify' into v2.3.0release

# Conflicts:
#	config/config.yaml
#	pkg/common/constant/constant.go
#	pkg/proto/sdk_ws/ws.pb.go
#	pkg/proto/sdk_ws/ws.proto

pkg/common/config/config.go

@@ -248,11 +248,16 @@ type config struct {
 			Addr  []string `yaml:"addr"`
 			Topic string   `yaml:"topic"`
 		}
+		MsgToModify struct {
+			Addr  []string `yaml:"addr"`
+			Topic string   `yaml:"topic"`
+		}
 		ConsumerGroupID struct {
-			MsgToRedis string `yaml:"msgToTransfer"`
-			MsgToMongo string `yaml:"msgToMongo"`
-			MsgToMySql string `yaml:"msgToMySql"`
-			MsgToPush  string `yaml:"msgToPush"`
+			MsgToRedis  string `yaml:"msgToTransfer"`
+			MsgToMongo  string `yaml:"msgToMongo"`
+			MsgToMySql  string `yaml:"msgToMySql"`
+			MsgToPush   string `yaml:"msgToPush"`
+			MsgToModify string `yaml:"msgToModify"`
 		}
 	}
 	Secret                            string `yaml:"secret"`

pkg/common/constant/constant.go

@@ -48,6 +48,8 @@ const (
 	AdvancedRevoke               = 118 //影响前者消息
 	CustomNotTriggerConversation = 119
 	CustomOnlineOnly             = 120
+	ReactionMessageModifier      = 121
+	ReactionMessageDeleter       = 122
 
 	Common             = 200
 	GroupMsg           = 201
@@ -169,6 +171,7 @@ const (
 	IsNotPrivate               = "notPrivate"
 	IsSenderConversationUpdate = "senderConversationUpdate"
 	IsSenderNotificationPush   = "senderNotificationPush"
+	IsReactionFromCache        = "reactionFromCache"
 
 	//GroupStatus
 	GroupOk              = 0
@@ -217,6 +220,8 @@ const (
 	CallbackBeforeCreateGroupCommand        = "callbackBeforeCreateGroupCommand"
 	CallbackBeforeMemberJoinGroupCommand    = "callbackBeforeMemberJoinGroupCommand"
 	CallbackBeforeSetGroupMemberInfoCommand = "CallbackBeforeSetGroupMemberInfoCommand"
+	CallbackBeforeSetMessageReactionExtensionCommand     = "callbackBeforeSetMessageReactionExtensionCommand"
+	CallbackBeforeDeleteMessageReactionExtensionsCommand = "callbackBeforeDeleteMessageReactionExtensionsCommand"
 
 	//callback actionCode
 	ActionAllow     = 0

pkg/common/db/RedisModel.go

@@ -39,6 +39,7 @@ const (
 	groupMinSeq                   = "GROUP_MIN_SEQ:"
 	sendMsgFailedFlag             = "SEND_MSG_FAILED_FLAG:"
 	userBadgeUnreadCountSum       = "USER_BADGE_UNREAD_COUNT_SUM:"
+	exTypeKeyLocker               = "EX_LOCK:"
 )
 
 func (d *DataBases) JudgeAccountEXISTS(account string) (bool, error) {
@@ -447,3 +448,61 @@ func (d *DataBases) GetUserBadgeUnreadCountSum(uid string) (int, error) {
 	seq, err := d.RDB.Get(context.Background(), key).Result()
 	return utils.StringToInt(seq), err
 }
+func (d *DataBases) JudgeMessageReactionEXISTS(clientMsgID string, sessionType int32) (bool, error) {
+	key := getMessageReactionExPrefix(clientMsgID, sessionType)
+	n, err := d.RDB.Exists(context.Background(), key).Result()
+	if n > 0 {
+		return true, err
+	} else {
+		return false, err
+	}
+}
+
+func (d *DataBases) GetOneMessageAllReactionList(clientMsgID string, sessionType int32) (map[string]string, error) {
+	key := getMessageReactionExPrefix(clientMsgID, sessionType)
+	return d.RDB.HGetAll(context.Background(), key).Result()
+
+}
+func (d *DataBases) DeleteOneMessageKey(clientMsgID string, sessionType int32, subKey string) error {
+	key := getMessageReactionExPrefix(clientMsgID, sessionType)
+	return d.RDB.HDel(context.Background(), key, subKey).Err()
+
+}
+func (d *DataBases) SetMessageReactionExpire(clientMsgID string, sessionType int32, expiration time.Duration) (bool, error) {
+	key := getMessageReactionExPrefix(clientMsgID, sessionType)
+	return d.RDB.Expire(context.Background(), key, expiration).Result()
+}
+func (d *DataBases) GetMessageTypeKeyValue(clientMsgID string, sessionType int32, typeKey string) (string, error) {
+	key := getMessageReactionExPrefix(clientMsgID, sessionType)
+	result, err := d.RDB.HGet(context.Background(), key, typeKey).Result()
+	return result, err
+
+}
+func (d *DataBases) SetMessageTypeKeyValue(clientMsgID string, sessionType int32, typeKey, value string) error {
+	key := getMessageReactionExPrefix(clientMsgID, sessionType)
+	return d.RDB.HSet(context.Background(), key, typeKey, value).Err()
+
+}
+func (d *DataBases) LockMessageTypeKey(clientMsgID string, TypeKey string) error {
+	key := exTypeKeyLocker + clientMsgID + "_" + TypeKey
+	return d.RDB.SetNX(context.Background(), key, 1, time.Minute).Err()
+}
+func (d *DataBases) UnLockMessageTypeKey(clientMsgID string, TypeKey string) error {
+	key := exTypeKeyLocker + clientMsgID + "_" + TypeKey
+	return d.RDB.Del(context.Background(), key).Err()
+
+}
+
+func getMessageReactionExPrefix(clientMsgID string, sessionType int32) string {
+	switch sessionType {
+	case constant.SingleChatType:
+		return "EX_SINGLE_" + clientMsgID
+	case constant.GroupChatType:
+		return "EX_GROUP_" + clientMsgID
+	case constant.SuperGroupChatType:
+		return "EX_SUPER_GROUP_" + clientMsgID
+	case constant.NotificationChatType:
+		return "EX_NOTIFICATION" + clientMsgID
+	}
+	return ""
+}

pkg/common/db/extend_msg_mongo_model.go

@@ -2,46 +2,61 @@ package db
 
 import (
 	"Open_IM/pkg/common/config"
+	server_api_params "Open_IM/pkg/proto/sdk_ws"
 	"Open_IM/pkg/utils"
 	"context"
+	"errors"
 	"fmt"
 	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
 	"strconv"
+	"strings"
 	"time"
 
 	"go.mongodb.org/mongo-driver/bson"
 )
 
 const cExtendMsgSet = "extend_msgs"
+const MaxNum = 100
 
 type ExtendMsgSet struct {
-	ID               string               `bson:"id" json:"ID"`
+	SourceID         string               `bson:"source_id" json:"sourceID"`
+	SessionType      int32                `bson:"session_type" json:"sessionType"`
 	ExtendMsgs       map[string]ExtendMsg `bson:"extend_msgs" json:"extendMsgs"`
-	LatestUpdateTime int32                `bson:"latest_update_time" json:"latestUpdateTime"`
-	AttachedInfo     *string              `bson:"attached_info" json:"attachedInfo"`
-	Ex               *string              `bson:"ex" json:"ex"`
 	ExtendMsgNum     int32                `bson:"extend_msg_num" json:"extendMsgNum"`
-	CreateTime       int32                `bson:"create_time" json:"createTime"`
+	CreateTime       int64                `bson:"create_time" json:"createTime"`               // this block's create time
+	MaxMsgUpdateTime int64                `bson:"max_msg_update_time" json:"maxMsgUpdateTime"` // index find msg
 }
 
-type ReactionExtendMsgSet struct {
-	UserKey          string `bson:"user_key" json:"userKey"`
+type KeyValue struct {
+	TypeKey          string `bson:"type_key" json:"typeKey"`
 	Value            string `bson:"value" json:"value"`
-	LatestUpdateTime int32  `bson:"latest_update_time" json:"latestUpdateTime"`
+	LatestUpdateTime int64  `bson:"latest_update_time" json:"latestUpdateTime"`
 }
 
 type ExtendMsg struct {
-	Content          map[string]ReactionExtendMsgSet `bson:"content" json:"content"`
-	ClientMsgID      string                          `bson:"client_msg_id" json:"clientMsgID"`
-	CreateTime       int32                           `bson:"create_time" json:"createTime"`
-	LatestUpdateTime int32                           `bson:"latest_update_time" json:"latestUpdateTime"`
+	ReactionExtensionList map[string]KeyValue `bson:"reaction_extension_list" json:"reactionExtensionList"`
+	ClientMsgID           string              `bson:"client_msg_id" json:"clientMsgID"`
+	MsgFirstModifyTime    int64               `bson:"msg_first_modify_time" json:"msgFirstModifyTime"` // this extendMsg create time
+	AttachedInfo          string              `bson:"attached_info" json:"attachedInfo"`
+	Ex                    string              `bson:"ex" json:"ex"`
 }
 
-func GetExtendMsgSetID(ID string, index int32) string {
+func GetExtendMsgMaxNum() int32 {
+	return MaxNum
+}
+
+func GetExtendMsgSourceID(ID string, index int32) string {
 	return ID + ":" + strconv.Itoa(int(index))
 }
 
+func SplitSourceIDAndGetIndex(sourceID string) int32 {
+	l := strings.Split(sourceID, ":")
+	index, _ := strconv.Atoi(l[len(l)-1])
+	return int32(index)
+}
+
 func (d *DataBases) CreateExtendMsgSet(set *ExtendMsgSet) error {
 	ctx, _ := context.WithTimeout(context.Background(), time.Duration(config.Config.Mongo.DBTimeout)*time.Second)
 	c := d.mongoClient.Database(config.Config.Mongo.DBDatabase).Collection(cExtendMsgSet)
@@ -75,61 +90,116 @@ func (d *DataBases) GetAllExtendMsgSet(ID string, opts *GetAllExtendMsgSetOpts)
 	return sets, nil
 }
 
-type GetExtendMsgSetOpts struct {
-	ExcludeExtendMsgs bool
-}
-
-func (d *DataBases) GetExtendMsgSet(ID string, index int32, opts *GetExtendMsgSetOpts) (*ExtendMsgSet, error) {
-	ctx, _ := context.WithTimeout(context.Background(), time.Duration(config.Config.Mongo.DBTimeout)*time.Second)
-	c := d.mongoClient.Database(config.Config.Mongo.DBDatabase).Collection(cExtendMsgSet)
-	var set ExtendMsgSet
-	var findOneOpt *options.FindOneOptions
-	if opts != nil {
-		if opts.ExcludeExtendMsgs {
-			findOneOpt = &options.FindOneOptions{}
-			findOneOpt.SetProjection(bson.M{"extend_msgs": 0})
-		}
+func (d *DataBases) GetExtendMsgSet(ctx context.Context, sourceID string, sessionType int32, maxMsgUpdateTime int64, c *mongo.Collection) (*ExtendMsgSet, error) {
+	regex := fmt.Sprintf("^%s", sourceID)
+	var err error
+	findOpts := options.Find().SetLimit(1).SetSkip(0).SetSort(bson.M{"source_id": -1}).SetProjection(bson.M{"extend_msgs": 0})
+	// update newest
+	find := bson.M{"source_id": primitive.Regex{Pattern: regex}, "session_type": sessionType}
+	if maxMsgUpdateTime > 0 {
+		find["max_msg_update_time"] = maxMsgUpdateTime
 	}
-	err := c.FindOne(ctx, bson.M{"uid": GetExtendMsgSetID(ID, index)}, findOneOpt).Decode(&set)
-	return &set, err
+	result, err := c.Find(ctx, find, findOpts)
+	if err != nil {
+		return nil, utils.Wrap(err, "")
+	}
+	var setList []ExtendMsgSet
+	if err := result.All(ctx, &setList); err != nil {
+		return nil, utils.Wrap(err, "")
+	}
+	if len(setList) == 0 {
+		return nil, nil
+	}
+	return &setList[0], nil
 }
 
 // first modify msg
-func (d *DataBases) InsertExtendMsgAndGetIndex(ID string, index int32, msg *ExtendMsg) error {
+func (d *DataBases) InsertExtendMsg(sourceID string, sessionType int32, msg *ExtendMsg) error {
 	ctx, _ := context.WithTimeout(context.Background(), time.Duration(config.Config.Mongo.DBTimeout)*time.Second)
 	c := d.mongoClient.Database(config.Config.Mongo.DBDatabase).Collection(cExtendMsgSet)
-	_, err := c.UpdateOne(ctx, bson.M{"uid": GetExtendMsgSetID(ID, index)}, bson.M{"$set": bson.M{"latest_update_time": utils.GetCurrentTimestampBySecond(), "$inc": bson.M{"extend_msg_num": 1}, fmt.Sprintf("extend_msgs.%s", msg.ClientMsgID): msg}})
-	return err
+	set, err := d.GetExtendMsgSet(ctx, sourceID, sessionType, 0, c)
+	if err != nil {
+		return utils.Wrap(err, "")
+	}
+	if set == nil || set.ExtendMsgNum >= GetExtendMsgMaxNum() {
+		var index int32
+		if set != nil {
+			index = SplitSourceIDAndGetIndex(set.SourceID)
+		}
+		err = d.CreateExtendMsgSet(&ExtendMsgSet{
+			SourceID:         GetExtendMsgSourceID(sourceID, index),
+			SessionType:      sessionType,
+			ExtendMsgs:       map[string]ExtendMsg{msg.ClientMsgID: *msg},
+			ExtendMsgNum:     1,
+			CreateTime:       msg.MsgFirstModifyTime,
+			MaxMsgUpdateTime: msg.MsgFirstModifyTime,
+		})
+	} else {
+		_, err = c.UpdateOne(ctx, bson.M{"source_id": set.SourceID, "session_type": sessionType}, bson.M{"$set": bson.M{"max_msg_update_time": msg.MsgFirstModifyTime, "$inc": bson.M{"extend_msg_num": 1}, fmt.Sprintf("extend_msgs.%s", msg.ClientMsgID): msg}})
+	}
+	return utils.Wrap(err, "")
 }
 
 // insert or update
-func (d *DataBases) InsertOrUpdateReactionExtendMsgSet(ID string, index int32, clientMsgID, userID, value string) error {
+func (d *DataBases) InsertOrUpdateReactionExtendMsgSet(sourceID string, sessionType int32, clientMsgID string, msgFirstModifyTime int64, reactionExtensionList map[string]*server_api_params.KeyValue) error {
 	ctx, _ := context.WithTimeout(context.Background(), time.Duration(config.Config.Mongo.DBTimeout)*time.Second)
 	c := d.mongoClient.Database(config.Config.Mongo.DBDatabase).Collection(cExtendMsgSet)
-	reactionExtendMsgSet := ReactionExtendMsgSet{
-		UserKey:          userID,
-		Value:            value,
-		LatestUpdateTime: int32(utils.GetCurrentTimestampBySecond()),
+	var updateBson = bson.M{}
+	for _, v := range reactionExtensionList {
+		updateBson[fmt.Sprintf("extend_msgs.%s.%s", clientMsgID, v.TypeKey)] = v
 	}
 	upsert := true
 	opt := &options.UpdateOptions{
 		Upsert: &upsert,
 	}
-	_, err := c.UpdateOne(ctx, bson.M{"uid": GetExtendMsgSetID(ID, index)}, bson.M{"$set": bson.M{"latest_update_time": utils.GetCurrentTimestampBySecond()}, fmt.Sprintf("extend_msgs.%s.%s", clientMsgID, userID): reactionExtendMsgSet}, opt)
+	set, err := d.GetExtendMsgSet(ctx, sourceID, sessionType, msgFirstModifyTime, c)
+	if err != nil {
+		return utils.Wrap(err, "")
+	}
+	if set == nil {
+		return errors.New(fmt.Sprintf("sourceID %s has no set", sourceID))
+	}
+	_, err = c.UpdateOne(ctx, bson.M{"source_id": set.SourceID, "session_type": sessionType}, bson.M{"$set": updateBson}, opt)
+	return utils.Wrap(err, "")
+}
+
+// delete TypeKey
+func (d *DataBases) DeleteReactionExtendMsgSet(sourceID string, sessionType int32, clientMsgID string, msgFirstModifyTime int64, reactionExtensionList map[string]*server_api_params.KeyValue) error {
+	ctx, _ := context.WithTimeout(context.Background(), time.Duration(config.Config.Mongo.DBTimeout)*time.Second)
+	c := d.mongoClient.Database(config.Config.Mongo.DBDatabase).Collection(cExtendMsgSet)
+	var updateBson = bson.M{}
+	for _, v := range reactionExtensionList {
+		updateBson[fmt.Sprintf("extend_msgs.%s.%s", clientMsgID, v.TypeKey)] = ""
+	}
+	set, err := d.GetExtendMsgSet(ctx, sourceID, sessionType, msgFirstModifyTime, c)
+	if err != nil {
+		return utils.Wrap(err, "")
+	}
+	if set == nil {
+		return errors.New(fmt.Sprintf("sourceID %s has no set", sourceID))
+	}
+	_, err = c.UpdateOne(ctx, bson.M{"source_id": set.SourceID, "session_type": sessionType}, bson.M{"$unset": updateBson})
 	return err
 }
 
-func (d *DataBases) DeleteReactionExtendMsgSet(ID string, index int32, clientMsgID, userID string) error {
+func (d *DataBases) GetExtendMsg(sourceID string, sessionType int32, clientMsgID string, maxMsgUpdateTime int64) (extendMsg *ExtendMsg, err error) {
 	ctx, _ := context.WithTimeout(context.Background(), time.Duration(config.Config.Mongo.DBTimeout)*time.Second)
 	c := d.mongoClient.Database(config.Config.Mongo.DBDatabase).Collection(cExtendMsgSet)
-	_, err := c.UpdateOne(ctx, bson.M{"uid": GetExtendMsgSetID(ID, index)}, bson.M{"$unset": bson.M{}})
-	return err
-}
-
-// by index start end
-func (d *DataBases) GetExtendMsgList(ID string, index int32, clientMsgID string) (extendMsg *ExtendMsg, err error) {
-	ctx, _ := context.WithTimeout(context.Background(), time.Duration(config.Config.Mongo.DBTimeout)*time.Second)
-	c := d.mongoClient.Database(config.Config.Mongo.DBDatabase).Collection(cExtendMsgSet)
-	err = c.FindOne(ctx, bson.M{"uid": GetExtendMsgSetID(ID, index), "extend_msgs": bson.M{}}).Decode(&extendMsg)
-	return extendMsg, err
+	findOpts := options.Find().SetLimit(1).SetSkip(0).SetSort(bson.M{"source_id": -1}).SetProjection(bson.M{fmt.Sprintf("extend_msgs.%s", clientMsgID): 1})
+	regex := fmt.Sprintf("^%s", sourceID)
+	result, err := c.Find(ctx, bson.M{"source_id": primitive.Regex{Pattern: regex}, "session_type": sessionType, "max_msg_update_time": bson.M{"$lte": maxMsgUpdateTime}}, findOpts)
+	if err != nil {
+		return nil, utils.Wrap(err, "")
+	}
+	var setList []ExtendMsgSet
+	if err := result.All(ctx, &setList); err != nil {
+		return nil, utils.Wrap(err, "")
+	}
+	if len(setList) == 0 {
+		return nil, utils.Wrap(errors.New("GetExtendMsg failed, len(setList) == 0"), "")
+	}
+	if v, ok := setList[0].ExtendMsgs[clientMsgID]; ok {
+		return &v, nil
+	}
+	return nil, errors.New(fmt.Sprintf("cant find client msg id: %s", clientMsgID))
 }

pkg/common/db/rocks_cache/rocks_cache.go

@@ -567,37 +567,9 @@ func DelConversationFromCache(ownerUserID, conversationID string) error {
 	return utils.Wrap(db.DB.Rc.TagAsDeleted(conversationCache+ownerUserID+":"+conversationID), "DelConversationFromCache err")
 }
 
-func GetExtendMsgSetFromCache(ID string, index int32) (*db.ExtendMsgSet, error) {
-	getExtendMsgSet := func() (string, error) {
-		extendMsgSet, err := db.DB.GetExtendMsgSet(ID, index, &db.GetExtendMsgSetOpts{ExcludeExtendMsgs: false})
-		if err != nil {
-			return "", utils.Wrap(err, "GetExtendMsgSet failed")
-		}
-		bytes, err := json.Marshal(extendMsgSet)
-		if err != nil {
-			return "", utils.Wrap(err, "Marshal failed")
-		}
-		return string(bytes), nil
-	}
-	extendMsgSetStr, err := db.DB.Rc.Fetch(extendMsgSetCache+db.GetExtendMsgSetID(ID, index), time.Second*30*60, getExtendMsgSet)
-	if err != nil {
-		return nil, utils.Wrap(err, "Fetch failed")
-	}
-	extendMsgSet := &db.ExtendMsgSet{}
-	err = json.Unmarshal([]byte(extendMsgSetStr), extendMsgSet)
-	if err != nil {
-		return nil, utils.Wrap(err, "Unmarshal failed")
-	}
-	return extendMsgSet, nil
-}
-
-func DelExtendMsgSetFromCache(ID string, index int32) error {
-	return utils.Wrap(db.DB.Rc.TagAsDeleted(extendMsgSetCache+db.GetExtendMsgSetID(ID, index)), "DelExtendMsgSetFromCache err")
-}
-
-func GetExtendMsg(ID string, index int32, clientMsgID string) (*db.ExtendMsg, error) {
+func GetExtendMsg(sourceID string, sessionType int32, clientMsgID string, firstModifyTime int64) (*db.ExtendMsg, error) {
 	getExtendMsg := func() (string, error) {
-		extendMsg, err := db.DB.GetExtendMsgList(ID, index, clientMsgID)
+		extendMsg, err := db.DB.GetExtendMsg(sourceID, sessionType, clientMsgID, firstModifyTime)
 		if err != nil {
 			return "", utils.Wrap(err, "GetExtendMsgList failed")
 		}
@@ -608,7 +580,7 @@ func GetExtendMsg(ID string, index int32, clientMsgID string) (*db.ExtendMsg, er
 		return string(bytes), nil
 	}
 
-	extendMsgStr, err := db.DB.Rc.Fetch(extendMsgCache+db.GetExtendMsgSetID(ID, index)+":"+clientMsgID, time.Second*30*60, getExtendMsg)
+	extendMsgStr, err := db.DB.Rc.Fetch(extendMsgCache+clientMsgID, time.Second*30*60, getExtendMsg)
 	if err != nil {
 		return nil, utils.Wrap(err, "Fetch failed")
 	}
@@ -621,5 +593,5 @@ func GetExtendMsg(ID string, index int32, clientMsgID string) (*db.ExtendMsg, er
 }
 
 func DelExtendMsg(ID string, index int32, clientMsgID string) error {
-	return utils.Wrap(db.DB.Rc.TagAsDeleted(extendMsgCache+db.GetExtendMsgSetID(ID, index)+":"+clientMsgID), "DelExtendMsg err")
+	return utils.Wrap(db.DB.Rc.TagAsDeleted(extendMsgCache+clientMsgID), "DelExtendMsg err")
 }