feat: add local cache for high frequency reads (#2036)

* feat: msg local cache

* feat: msg local cache

* feat: msg local cache

* feat: msg local cache

* feat: msg local cache

* feat: msg local cache

* fix: mongo

* fix: mongo

* fix: mongo

* openim.yaml

* localcache

* localcache

* localcache

* localcache

* localcache

* localcache

* localcache

* localcache

* localcache

* local cache

* local cache

* local cache

* local cache

* fix: GroupApplicationAcceptedNotification

* fix: GroupApplicationAcceptedNotification

* fix: NotificationUserInfoUpdate

* feat: cache add single-flight and timing-wheel.

* feat: local cache

* feat: local cache

* feat: local cache

* feat: cache add single-flight and timing-wheel.

* feat: local cache

* feat: local cache

* feat: local cache

* feat: local cache

* feat: local cache

* feat: local cache

* feat: local cache

* feat: local cache

* feat: local cache

* feat: local cache

* feat: local cache

* feat: local cache

* feat: local cache

* feat: local cache

* feat: local cache

* feat: local cache

* feat: local cache

* feat: msg rpc local cache

* feat: msg rpc local cache

* feat: msg rpc local cache

* feat: msg rpc local cache

* feat: msg rpc local cache

* feat: msg rpc local cache

* refactor: refactor the code of push and optimization.

* cicd: robot automated Change

* refactor: rename cache.

* merge

* fix: refactor project dir avoid import cycle.

* update tools

* merge

* feat: conversation FindRecvMsgNotNotifyUserIDs

* feat: conversation FindRecvMsgNotNotifyUserIDs

* feat: conversation FindRecvMsgNotNotifyUserIDs

* merge

* merge the latest main

---------

Co-authored-by: Gordon <46924906+FGadvancer@users.noreply.github.com>
Co-authored-by: withchao <withchao@users.noreply.github.com>

pkg/common/cachekey/black.go

@@ -0,0 +1,15 @@
+package cachekey
+
+const (
+	BlackIDsKey = "BLACK_IDS:"
+	IsBlackKey  = "IS_BLACK:" // local cache
+)
+
+func GetBlackIDsKey(ownerUserID string) string {
+	return BlackIDsKey + ownerUserID
+
+}
+
+func GetIsBlackIDsKey(possibleBlackUserID, userID string) string {
+	return IsBlackKey + userID + "-" + possibleBlackUserID
+}

pkg/common/cachekey/conversation.go

@@ -0,0 +1,44 @@
+package cachekey
+
+const (
+	ConversationKey                          = "CONVERSATION:"
+	ConversationIDsKey                       = "CONVERSATION_IDS:"
+	ConversationIDsHashKey                   = "CONVERSATION_IDS_HASH:"
+	ConversationHasReadSeqKey                = "CONVERSATION_HAS_READ_SEQ:"
+	RecvMsgOptKey                            = "RECV_MSG_OPT:"
+	SuperGroupRecvMsgNotNotifyUserIDsKey     = "SUPER_GROUP_RECV_MSG_NOT_NOTIFY_USER_IDS:"
+	SuperGroupRecvMsgNotNotifyUserIDsHashKey = "SUPER_GROUP_RECV_MSG_NOT_NOTIFY_USER_IDS_HASH:"
+	ConversationNotReceiveMessageUserIDsKey  = "CONVERSATION_NOT_RECEIVE_MESSAGE_USER_IDS:"
+)
+
+func GetConversationKey(ownerUserID, conversationID string) string {
+	return ConversationKey + ownerUserID + ":" + conversationID
+}
+
+func GetConversationIDsKey(ownerUserID string) string {
+	return ConversationIDsKey + ownerUserID
+}
+
+func GetSuperGroupRecvNotNotifyUserIDsKey(groupID string) string {
+	return SuperGroupRecvMsgNotNotifyUserIDsKey + groupID
+}
+
+func GetRecvMsgOptKey(ownerUserID, conversationID string) string {
+	return RecvMsgOptKey + ownerUserID + ":" + conversationID
+}
+
+func GetSuperGroupRecvNotNotifyUserIDsHashKey(groupID string) string {
+	return SuperGroupRecvMsgNotNotifyUserIDsHashKey + groupID
+}
+
+func GetConversationHasReadSeqKey(ownerUserID, conversationID string) string {
+	return ConversationHasReadSeqKey + ownerUserID + ":" + conversationID
+}
+
+func GetConversationNotReceiveMessageUserIDsKey(conversationID string) string {
+	return ConversationNotReceiveMessageUserIDsKey + conversationID
+}
+
+func GetUserConversationIDsHashKey(ownerUserID string) string {
+	return ConversationIDsHashKey + ownerUserID
+}

pkg/common/cachekey/friend.go

@@ -0,0 +1,24 @@
+package cachekey
+
+const (
+	FriendIDsKey        = "FRIEND_IDS:"
+	TwoWayFriendsIDsKey = "COMMON_FRIENDS_IDS:"
+	FriendKey           = "FRIEND_INFO:"
+	IsFriendKey         = "IS_FRIEND:" // local cache key
+)
+
+func GetFriendIDsKey(ownerUserID string) string {
+	return FriendIDsKey + ownerUserID
+}
+
+func GetTwoWayFriendsIDsKey(ownerUserID string) string {
+	return TwoWayFriendsIDsKey + ownerUserID
+}
+
+func GetFriendKey(ownerUserID, friendUserID string) string {
+	return FriendKey + ownerUserID + "-" + friendUserID
+}
+
+func GetIsFriendKey(possibleFriendUserID, userID string) string {
+	return IsFriendKey + possibleFriendUserID + "-" + userID
+}

pkg/common/cachekey/group.go

@@ -0,0 +1,45 @@
+package cachekey
+
+import (
+	"strconv"
+	"time"
+)
+
+const (
+	groupExpireTime            = time.Second * 60 * 60 * 12
+	GroupInfoKey               = "GROUP_INFO:"
+	GroupMemberIDsKey          = "GROUP_MEMBER_IDS:"
+	GroupMembersHashKey        = "GROUP_MEMBERS_HASH2:"
+	GroupMemberInfoKey         = "GROUP_MEMBER_INFO:"
+	JoinedGroupsKey            = "JOIN_GROUPS_KEY:"
+	GroupMemberNumKey          = "GROUP_MEMBER_NUM_CACHE:"
+	GroupRoleLevelMemberIDsKey = "GROUP_ROLE_LEVEL_MEMBER_IDS:"
+)
+
+func GetGroupInfoKey(groupID string) string {
+	return GroupInfoKey + groupID
+}
+
+func GetJoinedGroupsKey(userID string) string {
+	return JoinedGroupsKey + userID
+}
+
+func GetGroupMembersHashKey(groupID string) string {
+	return GroupMembersHashKey + groupID
+}
+
+func GetGroupMemberIDsKey(groupID string) string {
+	return GroupMemberIDsKey + groupID
+}
+
+func GetGroupMemberInfoKey(groupID, userID string) string {
+	return GroupMemberInfoKey + groupID + "-" + userID
+}
+
+func GetGroupMemberNumKey(groupID string) string {
+	return GroupMemberNumKey + groupID
+}
+
+func GetGroupRoleLevelMemberIDsKey(groupID string, roleLevel int32) string {
+	return GroupRoleLevelMemberIDsKey + groupID + "-" + strconv.Itoa(int(roleLevel))
+}

pkg/common/cachekey/user.go

@@ -0,0 +1,14 @@
+package cachekey
+
+const (
+	UserInfoKey             = "USER_INFO:"
+	UserGlobalRecvMsgOptKey = "USER_GLOBAL_RECV_MSG_OPT_KEY:"
+)
+
+func GetUserInfoKey(userID string) string {
+	return UserInfoKey + userID
+}
+
+func GetUserGlobalRecvMsgOptKey(userID string) string {
+	return UserGlobalRecvMsgOptKey + userID
+}

pkg/common/config/config.go

@@ -16,6 +16,7 @@ package config
 
 import (
 	"bytes"
+	"time"
 
 	"github.com/OpenIMSDK/tools/discoveryregistry"
 	"gopkg.in/yaml.v3"
@@ -266,6 +267,8 @@ type GlobalConfig struct {
 		FriendVerify *bool `yaml:"friendVerify"`
 	} `yaml:"messageVerify"`
 
+	LocalCache localCache `yaml:"localCache"`
+
 	IOSPush struct {
 		PushSound  string `yaml:"pushSound"`
 		BadgeCount bool   `yaml:"badgeCount"`
@@ -382,6 +385,33 @@ type notification struct {
 	ConversationSetPrivate NotificationConf `yaml:"conversationSetPrivate"`
 }
 
+type LocalCache struct {
+	Topic         string `yaml:"topic"`
+	SlotNum       int    `yaml:"slotNum"`
+	SlotSize      int    `yaml:"slotSize"`
+	SuccessExpire int    `yaml:"successExpire"` // second
+	FailedExpire  int    `yaml:"failedExpire"`  // second
+}
+
+func (l LocalCache) Failed() time.Duration {
+	return time.Second * time.Duration(l.FailedExpire)
+}
+
+func (l LocalCache) Success() time.Duration {
+	return time.Second * time.Duration(l.SuccessExpire)
+}
+
+func (l LocalCache) Enable() bool {
+	return l.Topic != "" && l.SlotNum > 0 && l.SlotSize > 0
+}
+
+type localCache struct {
+	User         LocalCache `yaml:"user"`
+	Group        LocalCache `yaml:"group"`
+	Friend       LocalCache `yaml:"friend"`
+	Conversation LocalCache `yaml:"conversation"`
+}
+
 func (c *GlobalConfig) GetServiceNames() []string {
 	return []string{
 		c.RpcRegisterName.OpenImUserName,

pkg/common/config/parse_test.go

@@ -16,6 +16,8 @@ package config
 
 import (
 	_ "embed"
+	"fmt"
+	"gopkg.in/yaml.v3"
 	"reflect"
 	"testing"
 
@@ -116,3 +118,13 @@ func TestInitConfig(t *testing.T) {
 		})
 	}
 }
+
+func TestName(t *testing.T) {
+	Config.LocalCache.Friend.Topic = "friend"
+	Config.LocalCache.Friend.SlotNum = 500
+	Config.LocalCache.Friend.SlotSize = 20000
+
+	data, _ := yaml.Marshal(&Config)
+
+	fmt.Println(string(data))
+}

pkg/common/db/cache/black.go

@@ -16,6 +16,9 @@ package cache
 
 import (
 	"context"
+	"github.com/OpenIMSDK/tools/log"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/cachekey"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
 	"time"
 
 	"github.com/dtm-labs/rockscache"
@@ -47,11 +50,15 @@ type BlackCacheRedis struct {
 
 func NewBlackCacheRedis(rdb redis.UniversalClient, blackDB relationtb.BlackModelInterface, options rockscache.Options) BlackCache {
 	rcClient := rockscache.NewClient(rdb, options)
-
+	mc := NewMetaCacheRedis(rcClient)
+	b := config.Config.LocalCache.Friend
+	log.ZDebug(context.Background(), "black local cache init", "Topic", b.Topic, "SlotNum", b.SlotNum, "SlotSize", b.SlotSize, "enable", b.Enable())
+	mc.SetTopic(b.Topic)
+	mc.SetRawRedisClient(rdb)
 	return &BlackCacheRedis{
 		expireTime: blackExpireTime,
 		rcClient:   rcClient,
-		metaCache:  NewMetaCacheRedis(rcClient),
+		metaCache:  mc,
 		blackDB:    blackDB,
 	}
 }
@@ -61,12 +68,12 @@ func (b *BlackCacheRedis) NewCache() BlackCache {
 		expireTime: b.expireTime,
 		rcClient:   b.rcClient,
 		blackDB:    b.blackDB,
-		metaCache:  NewMetaCacheRedis(b.rcClient, b.metaCache.GetPreDelKeys()...),
+		metaCache:  b.Copy(),
 	}
 }
 
 func (b *BlackCacheRedis) getBlackIDsKey(ownerUserID string) string {
-	return blackIDsKey + ownerUserID
+	return cachekey.GetBlackIDsKey(ownerUserID)
 }
 
 func (b *BlackCacheRedis) GetBlackIDs(ctx context.Context, userID string) (blackIDs []string, err error) {

pkg/common/db/cache/config.go

@@ -0,0 +1,66 @@
+package cache
+
+import (
+	"github.com/openimsdk/open-im-server/v3/pkg/common/cachekey"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
+	"strings"
+	"sync"
+)
+
+var (
+	once      sync.Once
+	subscribe map[string][]string
+)
+
+func getPublishKey(topic string, key []string) []string {
+	if topic == "" || len(key) == 0 {
+		return nil
+	}
+	once.Do(func() {
+		list := []struct {
+			Local config.LocalCache
+			Keys  []string
+		}{
+			{
+				Local: config.Config.LocalCache.User,
+				Keys:  []string{cachekey.UserInfoKey, cachekey.UserGlobalRecvMsgOptKey},
+			},
+			{
+				Local: config.Config.LocalCache.Group,
+				Keys:  []string{cachekey.GroupMemberIDsKey, cachekey.GroupInfoKey, cachekey.GroupMemberInfoKey},
+			},
+			{
+				Local: config.Config.LocalCache.Friend,
+				Keys:  []string{cachekey.FriendIDsKey, cachekey.BlackIDsKey},
+			},
+			{
+				Local: config.Config.LocalCache.Conversation,
+				Keys:  []string{cachekey.ConversationKey, cachekey.ConversationIDsKey, cachekey.ConversationNotReceiveMessageUserIDsKey},
+			},
+		}
+		subscribe = make(map[string][]string)
+		for _, v := range list {
+			if v.Local.Enable() {
+				subscribe[v.Local.Topic] = v.Keys
+			}
+		}
+	})
+	prefix, ok := subscribe[topic]
+	if !ok {
+		return nil
+	}
+	res := make([]string, 0, len(key))
+	for _, k := range key {
+		var exist bool
+		for _, p := range prefix {
+			if strings.HasPrefix(k, p) {
+				exist = true
+				break
+			}
+		}
+		if exist {
+			res = append(res, k)
+		}
+	}
+	return res
+}

pkg/common/db/cache/conversation.go

@@ -16,6 +16,9 @@ package cache
 
 import (
 	"context"
+	"github.com/OpenIMSDK/tools/log"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/cachekey"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
 	"math/big"
 	"strings"
 	"time"
@@ -27,14 +30,14 @@ import (
 )
 
 const (
-	conversationKey                          = "CONVERSATION:"
-	conversationIDsKey                       = "CONVERSATION_IDS:"
-	conversationIDsHashKey                   = "CONVERSATION_IDS_HASH:"
-	conversationHasReadSeqKey                = "CONVERSATION_HAS_READ_SEQ:"
-	recvMsgOptKey                            = "RECV_MSG_OPT:"
-	superGroupRecvMsgNotNotifyUserIDsKey     = "SUPER_GROUP_RECV_MSG_NOT_NOTIFY_USER_IDS:"
-	superGroupRecvMsgNotNotifyUserIDsHashKey = "SUPER_GROUP_RECV_MSG_NOT_NOTIFY_USER_IDS_HASH:"
-	conversationNotReceiveMessageUserIDsKey  = "CONVERSATION_NOT_RECEIVE_MESSAGE_USER_IDS:"
+	//conversationKey                          = "CONVERSATION:"
+	//conversationIDsKey                       = "CONVERSATION_IDS:"
+	//conversationIDsHashKey                   = "CONVERSATION_IDS_HASH:"
+	//conversationHasReadSeqKey                = "CONVERSATION_HAS_READ_SEQ:"
+	//recvMsgOptKey                            = "RECV_MSG_OPT:"
+	//superGroupRecvMsgNotNotifyUserIDsKey     = "SUPER_GROUP_RECV_MSG_NOT_NOTIFY_USER_IDS:"
+	//superGroupRecvMsgNotNotifyUserIDsHashKey = "SUPER_GROUP_RECV_MSG_NOT_NOTIFY_USER_IDS_HASH:"
+	//conversationNotReceiveMessageUserIDsKey  = "CONVERSATION_NOT_RECEIVE_MESSAGE_USER_IDS:"
 
 	conversationExpireTime = time.Second * 60 * 60 * 12
 )
@@ -81,10 +84,14 @@ type ConversationCache interface {
 
 func NewConversationRedis(rdb redis.UniversalClient, opts rockscache.Options, db relationtb.ConversationModelInterface) ConversationCache {
 	rcClient := rockscache.NewClient(rdb, opts)
-
+	mc := NewMetaCacheRedis(rcClient)
+	c := config.Config.LocalCache.Conversation
+	log.ZDebug(context.Background(), "black local cache init", "Topic", c.Topic, "SlotNum", c.SlotNum, "SlotSize", c.SlotSize, "enable", c.Enable())
+	mc.SetTopic(c.Topic)
+	mc.SetRawRedisClient(rdb)
 	return &ConversationRedisCache{
 		rcClient:       rcClient,
-		metaCache:      NewMetaCacheRedis(rcClient),
+		metaCache:      mc,
 		conversationDB: db,
 		expireTime:     conversationExpireTime,
 	}
@@ -115,38 +122,42 @@ type ConversationRedisCache struct {
 func (c *ConversationRedisCache) NewCache() ConversationCache {
 	return &ConversationRedisCache{
 		rcClient:       c.rcClient,
-		metaCache:      NewMetaCacheRedis(c.rcClient, c.metaCache.GetPreDelKeys()...),
+		metaCache:      c.Copy(),
 		conversationDB: c.conversationDB,
 		expireTime:     c.expireTime,
 	}
 }
 
 func (c *ConversationRedisCache) getConversationKey(ownerUserID, conversationID string) string {
-	return conversationKey + ownerUserID + ":" + conversationID
+	return cachekey.GetConversationKey(ownerUserID, conversationID)
 }
 
 func (c *ConversationRedisCache) getConversationIDsKey(ownerUserID string) string {
-	return conversationIDsKey + ownerUserID
+	return cachekey.GetConversationIDsKey(ownerUserID)
 }
 
 func (c *ConversationRedisCache) getSuperGroupRecvNotNotifyUserIDsKey(groupID string) string {
-	return superGroupRecvMsgNotNotifyUserIDsKey + groupID
+	return cachekey.GetSuperGroupRecvNotNotifyUserIDsKey(groupID)
 }
 
 func (c *ConversationRedisCache) getRecvMsgOptKey(ownerUserID, conversationID string) string {
-	return recvMsgOptKey + ownerUserID + ":" + conversationID
+	return cachekey.GetRecvMsgOptKey(ownerUserID, conversationID)
 }
 
 func (c *ConversationRedisCache) getSuperGroupRecvNotNotifyUserIDsHashKey(groupID string) string {
-	return superGroupRecvMsgNotNotifyUserIDsHashKey + groupID
+	return cachekey.GetSuperGroupRecvNotNotifyUserIDsHashKey(groupID)
 }
 
 func (c *ConversationRedisCache) getConversationHasReadSeqKey(ownerUserID, conversationID string) string {
-	return conversationHasReadSeqKey + ownerUserID + ":" + conversationID
+	return cachekey.GetConversationHasReadSeqKey(ownerUserID, conversationID)
 }
 
 func (c *ConversationRedisCache) getConversationNotReceiveMessageUserIDsKey(conversationID string) string {
-	return conversationNotReceiveMessageUserIDsKey + conversationID
+	return cachekey.GetConversationNotReceiveMessageUserIDsKey(conversationID)
+}
+
+func (c *ConversationRedisCache) getUserConversationIDsHashKey(ownerUserID string) string {
+	return cachekey.GetUserConversationIDsHashKey(ownerUserID)
 }
 
 func (c *ConversationRedisCache) GetUserConversationIDs(ctx context.Context, ownerUserID string) ([]string, error) {
@@ -166,10 +177,6 @@ func (c *ConversationRedisCache) DelConversationIDs(userIDs ...string) Conversat
 	return cache
 }
 
-func (c *ConversationRedisCache) getUserConversationIDsHashKey(ownerUserID string) string {
-	return conversationIDsHashKey + ownerUserID
-}
-
 func (c *ConversationRedisCache) GetUserConversationIDsHash(ctx context.Context, ownerUserID string) (hash uint64, err error) {
 	return getCache(
 		ctx,

pkg/common/db/cache/friend.go

@@ -16,6 +16,9 @@ package cache
 
 import (
 	"context"
+	"github.com/OpenIMSDK/tools/log"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/cachekey"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
 	"time"
 
 	"github.com/OpenIMSDK/tools/utils"
@@ -25,10 +28,10 @@ import (
 )
 
 const (
-	friendExpireTime    = time.Second * 60 * 60 * 12
-	friendIDsKey        = "FRIEND_IDS:"
-	TwoWayFriendsIDsKey = "COMMON_FRIENDS_IDS:"
-	friendKey           = "FRIEND_INFO:"
+	friendExpireTime = time.Second * 60 * 60 * 12
+	//friendIDsKey        = "FRIEND_IDS:"
+	//TwoWayFriendsIDsKey = "COMMON_FRIENDS_IDS:"
+	//friendKey           = "FRIEND_INFO:"
 )
 
 // FriendCache is an interface for caching friend-related data.
@@ -58,8 +61,13 @@ type FriendCacheRedis struct {
 func NewFriendCacheRedis(rdb redis.UniversalClient, friendDB relationtb.FriendModelInterface,
 	options rockscache.Options) FriendCache {
 	rcClient := rockscache.NewClient(rdb, options)
+	mc := NewMetaCacheRedis(rcClient)
+	f := config.Config.LocalCache.Friend
+	log.ZDebug(context.Background(), "friend local cache init", "Topic", f.Topic, "SlotNum", f.SlotNum, "SlotSize", f.SlotSize, "enable", f.Enable())
+	mc.SetTopic(f.Topic)
+	mc.SetRawRedisClient(rdb)
 	return &FriendCacheRedis{
-		metaCache:  NewMetaCacheRedis(rcClient),
+		metaCache:  mc,
 		friendDB:   friendDB,
 		expireTime: friendExpireTime,
 		rcClient:   rcClient,
@@ -70,7 +78,7 @@ func NewFriendCacheRedis(rdb redis.UniversalClient, friendDB relationtb.FriendMo
 func (f *FriendCacheRedis) NewCache() FriendCache {
 	return &FriendCacheRedis{
 		rcClient:   f.rcClient,
-		metaCache:  NewMetaCacheRedis(f.rcClient, f.metaCache.GetPreDelKeys()...),
+		metaCache:  f.Copy(),
 		friendDB:   f.friendDB,
 		expireTime: f.expireTime,
 	}
@@ -78,17 +86,17 @@ func (f *FriendCacheRedis) NewCache() FriendCache {
 
 // getFriendIDsKey returns the key for storing friend IDs in the cache.
 func (f *FriendCacheRedis) getFriendIDsKey(ownerUserID string) string {
-	return friendIDsKey + ownerUserID
+	return cachekey.GetFriendIDsKey(ownerUserID)
 }
 
 // getTwoWayFriendsIDsKey returns the key for storing two-way friend IDs in the cache.
 func (f *FriendCacheRedis) getTwoWayFriendsIDsKey(ownerUserID string) string {
-	return TwoWayFriendsIDsKey + ownerUserID
+	return cachekey.GetTwoWayFriendsIDsKey(ownerUserID)
 }
 
 // getFriendKey returns the key for storing friend info in the cache.
 func (f *FriendCacheRedis) getFriendKey(ownerUserID, friendUserID string) string {
-	return friendKey + ownerUserID + "-" + friendUserID
+	return cachekey.GetFriendKey(ownerUserID, friendUserID)
 }
 
 // GetFriendIDs retrieves friend IDs from the cache or the database if not found.

pkg/common/db/cache/group.go

@@ -17,7 +17,8 @@ package cache
 import (
 	"context"
 	"fmt"
-	"strconv"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/cachekey"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
 	"time"
 
 	"github.com/OpenIMSDK/protocol/constant"
@@ -30,15 +31,14 @@ import (
 )
 
 const (
-	groupExpireTime     = time.Second * 60 * 60 * 12
-	groupInfoKey        = "GROUP_INFO:"
-	groupMemberIDsKey   = "GROUP_MEMBER_IDS:"
-	groupMembersHashKey = "GROUP_MEMBERS_HASH2:"
-	groupMemberInfoKey  = "GROUP_MEMBER_INFO:"
-	//groupOwnerInfoKey   = "GROUP_OWNER_INFO:".
-	joinedGroupsKey            = "JOIN_GROUPS_KEY:"
-	groupMemberNumKey          = "GROUP_MEMBER_NUM_CACHE:"
-	groupRoleLevelMemberIDsKey = "GROUP_ROLE_LEVEL_MEMBER_IDS:"
+	groupExpireTime = time.Second * 60 * 60 * 12
+	//groupInfoKey        = "GROUP_INFO:"
+	//groupMemberIDsKey   = "GROUP_MEMBER_IDS:"
+	//groupMembersHashKey = "GROUP_MEMBERS_HASH2:"
+	//groupMemberInfoKey  = "GROUP_MEMBER_INFO:"
+	//joinedGroupsKey            = "JOIN_GROUPS_KEY:"
+	//groupMemberNumKey          = "GROUP_MEMBER_NUM_CACHE:"
+	//groupRoleLevelMemberIDsKey = "GROUP_ROLE_LEVEL_MEMBER_IDS:"
 )
 
 type GroupHash interface {
@@ -101,12 +101,16 @@ func NewGroupCacheRedis(
 	opts rockscache.Options,
 ) GroupCache {
 	rcClient := rockscache.NewClient(rdb, opts)
-
+	mc := NewMetaCacheRedis(rcClient)
+	g := config.Config.LocalCache.Group
+	mc.SetTopic(g.Topic)
+	log.ZDebug(context.Background(), "group local cache init", "Topic", g.Topic, "SlotNum", g.SlotNum, "SlotSize", g.SlotSize, "enable", g.Enable())
+	mc.SetRawRedisClient(rdb)
 	return &GroupCacheRedis{
 		rcClient: rcClient, expireTime: groupExpireTime,
 		groupDB: groupDB, groupMemberDB: groupMemberDB, groupRequestDB: groupRequestDB,
 		groupHash: hashCode,
-		metaCache: NewMetaCacheRedis(rcClient),
+		metaCache: mc,
 	}
 }
 
@@ -117,36 +121,36 @@ func (g *GroupCacheRedis) NewCache() GroupCache {
 		groupDB:        g.groupDB,
 		groupMemberDB:  g.groupMemberDB,
 		groupRequestDB: g.groupRequestDB,
-		metaCache:      NewMetaCacheRedis(g.rcClient, g.metaCache.GetPreDelKeys()...),
+		metaCache:      g.Copy(),
 	}
 }
 
 func (g *GroupCacheRedis) getGroupInfoKey(groupID string) string {
-	return groupInfoKey + groupID
+	return cachekey.GetGroupInfoKey(groupID)
 }
 
 func (g *GroupCacheRedis) getJoinedGroupsKey(userID string) string {
-	return joinedGroupsKey + userID
+	return cachekey.GetJoinedGroupsKey(userID)
 }
 
 func (g *GroupCacheRedis) getGroupMembersHashKey(groupID string) string {
-	return groupMembersHashKey + groupID
+	return cachekey.GetGroupMembersHashKey(groupID)
 }
 
 func (g *GroupCacheRedis) getGroupMemberIDsKey(groupID string) string {
-	return groupMemberIDsKey + groupID
+	return cachekey.GetGroupMemberIDsKey(groupID)
 }
 
 func (g *GroupCacheRedis) getGroupMemberInfoKey(groupID, userID string) string {
-	return groupMemberInfoKey + groupID + "-" + userID
+	return cachekey.GetGroupMemberInfoKey(groupID, userID)
 }
 
 func (g *GroupCacheRedis) getGroupMemberNumKey(groupID string) string {
-	return groupMemberNumKey + groupID
+	return cachekey.GetGroupMemberNumKey(groupID)
 }
 
 func (g *GroupCacheRedis) getGroupRoleLevelMemberIDsKey(groupID string, roleLevel int32) string {
-	return groupRoleLevelMemberIDsKey + groupID + "-" + strconv.Itoa(int(roleLevel))
+	return cachekey.GetGroupRoleLevelMemberIDsKey(groupID, roleLevel)
 }
 
 func (g *GroupCacheRedis) GetGroupIndex(group *relationtb.GroupModel, keys []string) (int, error) {

pkg/common/db/cache/meta_cache.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"github.com/redis/go-redis/v9"
 	"fmt"
 	"time"
 
@@ -43,6 +44,9 @@ type metaCache interface {
 	AddKeys(keys ...string)
 	ClearKeys()
 	GetPreDelKeys() []string
+	SetTopic(topic string)
+	SetRawRedisClient(cli redis.UniversalClient)
+	Copy() metaCache
 }
 
 func NewMetaCacheRedis(rcClient *rockscache.Client, keys ...string) metaCache {
@@ -50,10 +54,36 @@ func NewMetaCacheRedis(rcClient *rockscache.Client, keys ...string) metaCache {
 }
 
 type metaCacheRedis struct {
+	topic         string
 	rcClient      *rockscache.Client
 	keys          []string
 	maxRetryTimes int
 	retryInterval time.Duration
+	redisClient   redis.UniversalClient
+}
+
+func (m *metaCacheRedis) Copy() metaCache {
+	var keys []string
+	if len(m.keys) > 0 {
+		keys = make([]string, 0, len(m.keys)*2)
+		keys = append(keys, m.keys...)
+	}
+	return &metaCacheRedis{
+		topic:         m.topic,
+		rcClient:      m.rcClient,
+		keys:          keys,
+		maxRetryTimes: m.maxRetryTimes,
+		retryInterval: m.retryInterval,
+		redisClient:   redisClient,
+	}
+}
+
+func (m *metaCacheRedis) SetTopic(topic string) {
+	m.topic = topic
+}
+
+func (m *metaCacheRedis) SetRawRedisClient(cli redis.UniversalClient) {
+	m.redisClient = cli
 }
 
 func (m *metaCacheRedis) ExecDel(ctx context.Context, distinct ...bool) error {
@@ -61,7 +91,7 @@ func (m *metaCacheRedis) ExecDel(ctx context.Context, distinct ...bool) error {
 		m.keys = utils.Distinct(m.keys)
 	}
 	if len(m.keys) > 0 {
-		log.ZDebug(ctx, "delete cache", "keys", m.keys)
+		log.ZDebug(ctx, "delete cache", "topic", m.topic, "keys", m.keys)
 		for _, key := range m.keys {
 			for i := 0; i < m.maxRetryTimes; i++ {
 				if err := m.rcClient.TagAsDeleted(key); err != nil {
@@ -71,31 +101,18 @@ func (m *metaCacheRedis) ExecDel(ctx context.Context, distinct ...bool) error {
 				}
 				break
 			}
-
-			//retryTimes := 0
-			//for {
-			//	m.rcClient.TagAsDeleted()
-			//	if err := m.rcClient.TagAsDeletedBatch2(ctx, []string{key}); err != nil {
-			//		if retryTimes >= m.maxRetryTimes {
-			//			err = errs.ErrInternalServer.Wrap(
-			//				fmt.Sprintf(
-			//					"delete cache error: %v, keys: %v, retry times %d, please check redis server",
-			//					err,
-			//					key,
-			//					retryTimes,
-			//				),
-			//			)
-			//			log.ZWarn(ctx, "delete cache failed, please handle keys", err, "keys", key)
-			//			return err
-			//		}
-			//		retryTimes++
-			//	} else {
-			//		break
-			//	}
-			//}
+		}
+		if pk := getPublishKey(m.topic, m.keys); len(pk) > 0 {
+			data, err := json.Marshal(pk)
+			if err != nil {
+				log.ZError(ctx, "keys json marshal failed", err, "topic", m.topic, "keys", pk)
+			} else {
+				if err := m.redisClient.Publish(ctx, m.topic, string(data)).Err(); err != nil {
+					log.ZError(ctx, "redis publish cache delete error", err, "topic", m.topic, "keys", pk)
+				}
+			}
 		}
 	}
-
 	return nil
 }
 

pkg/common/db/cache/msg.go

@@ -38,12 +38,12 @@ const (
 	conversationUserMinSeq = "CON_USER_MIN_SEQ:"
 	hasReadSeq             = "HAS_READ_SEQ:"
 
-	appleDeviceToken = "DEVICE_TOKEN"
-	getuiToken       = "GETUI_TOKEN"
-	getuiTaskID      = "GETUI_TASK_ID"
-	signalCache      = "SIGNAL_CACHE:"
-	signalListCache  = "SIGNAL_LIST_CACHE:"
-	FCM_TOKEN        = "FCM_TOKEN:"
+	//appleDeviceToken = "DEVICE_TOKEN"
+	getuiToken  = "GETUI_TOKEN"
+	getuiTaskID = "GETUI_TASK_ID"
+	//signalCache      = "SIGNAL_CACHE:"
+	//signalListCache  = "SIGNAL_LIST_CACHE:"
+	FCM_TOKEN = "FCM_TOKEN:"
 
 	messageCache            = "MESSAGE_CACHE:"
 	messageDelUserList      = "MESSAGE_DEL_USER_LIST:"
@@ -143,6 +143,10 @@ func (c *msgCache) getHasReadSeqKey(conversationID string, userID string) string
 	return hasReadSeq + userID + ":" + conversationID
 }
 
+func (c *msgCache) getConversationUserMinSeqKey(conversationID, userID string) string {
+	return conversationUserMinSeq + conversationID + "u:" + userID
+}
+
 func (c *msgCache) setSeq(ctx context.Context, conversationID string, seq int64, getkey func(conversationID string) string) error {
 	return errs.Wrap(c.rdb.Set(ctx, getkey(conversationID), seq, 0).Err())
 }
@@ -208,10 +212,6 @@ func (c *msgCache) GetMinSeq(ctx context.Context, conversationID string) (int64,
 	return c.getSeq(ctx, conversationID, c.getMinSeqKey)
 }
 
-func (c *msgCache) getConversationUserMinSeqKey(conversationID, userID string) string {
-	return conversationUserMinSeq + conversationID + "u:" + userID
-}
-
 func (c *msgCache) GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) {
 	val, err := c.rdb.Get(ctx, c.getConversationUserMinSeqKey(conversationID, userID)).Int64()
 	if err != nil {

pkg/common/db/cache/user.go

@@ -18,6 +18,8 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/cachekey"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
 	"hash/crc32"
 	"strconv"
 	"time"
@@ -32,8 +34,8 @@ import (
 )
 
 const (
-	userExpireTime            = time.Second * 60 * 60 * 12
-	userInfoKey               = "USER_INFO:"
+	userExpireTime = time.Second * 60 * 60 * 12
+	//userInfoKey               = "USER_INFO:"
 	userGlobalRecvMsgOptKey   = "USER_GLOBAL_RECV_MSG_OPT_KEY:"
 	olineStatusKey            = "ONLINE_STATUS:"
 	userOlineStatusExpireTime = time.Second * 60 * 60 * 24
@@ -68,7 +70,11 @@ func NewUserCacheRedis(
 	options rockscache.Options,
 ) UserCache {
 	rcClient := rockscache.NewClient(rdb, options)
-
+	mc := NewMetaCacheRedis(rcClient)
+	u := config.Config.LocalCache.User
+	log.ZDebug(context.Background(), "user local cache init", "Topic", u.Topic, "SlotNum", u.SlotNum, "SlotSize", u.SlotSize, "enable", u.Enable())
+	mc.SetTopic(u.Topic)
+	mc.SetRawRedisClient(rdb)
 	return &UserCacheRedis{
 		rdb:        rdb,
 		metaCache:  NewMetaCacheRedis(rcClient),
@@ -81,7 +87,7 @@ func NewUserCacheRedis(
 func (u *UserCacheRedis) NewCache() UserCache {
 	return &UserCacheRedis{
 		rdb:        u.rdb,
-		metaCache:  NewMetaCacheRedis(u.rcClient, u.metaCache.GetPreDelKeys()...),
+		metaCache:  u.Copy(),
 		userDB:     u.userDB,
 		expireTime: u.expireTime,
 		rcClient:   u.rcClient,
@@ -89,18 +95,17 @@ func (u *UserCacheRedis) NewCache() UserCache {
 }
 
 func (u *UserCacheRedis) getUserInfoKey(userID string) string {
-	return userInfoKey + userID
+	return cachekey.GetUserInfoKey(userID)
 }
 
 func (u *UserCacheRedis) getUserGlobalRecvMsgOptKey(userID string) string {
-	return userGlobalRecvMsgOptKey + userID
+	return cachekey.GetUserGlobalRecvMsgOptKey(userID)
 }
 
 func (u *UserCacheRedis) GetUserInfo(ctx context.Context, userID string) (userInfo *relationtb.UserModel, err error) {
 	return getCache(ctx, u.rcClient, u.getUserInfoKey(userID), u.expireTime, func(ctx context.Context) (*relationtb.UserModel, error) {
 		return u.userDB.Take(ctx, userID)
-	},
-	)
+	})
 }
 
 func (u *UserCacheRedis) GetUsersInfo(ctx context.Context, userIDs []string) ([]*relationtb.UserModel, error) {

pkg/common/db/localcache/conversation.go

@@ -1,86 +0,0 @@
-// Copyright © 2023 OpenIM. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package localcache
-
-import (
-	"context"
-	"sync"
-
-	"github.com/OpenIMSDK/protocol/conversation"
-	"github.com/openimsdk/open-im-server/v3/pkg/rpcclient"
-)
-
-type ConversationLocalCache struct {
-	lock                              sync.Mutex
-	superGroupRecvMsgNotNotifyUserIDs map[string]Hash
-	conversationIDs                   map[string]Hash
-	client                            *rpcclient.ConversationRpcClient
-}
-
-type Hash struct {
-	hash uint64
-	ids  []string
-}
-
-func NewConversationLocalCache(client *rpcclient.ConversationRpcClient) *ConversationLocalCache {
-	return &ConversationLocalCache{
-		superGroupRecvMsgNotNotifyUserIDs: make(map[string]Hash),
-		conversationIDs:                   make(map[string]Hash),
-		client:                            client,
-	}
-}
-
-func (g *ConversationLocalCache) GetRecvMsgNotNotifyUserIDs(ctx context.Context, groupID string) ([]string, error) {
-	resp, err := g.client.Client.GetRecvMsgNotNotifyUserIDs(ctx, &conversation.GetRecvMsgNotNotifyUserIDsReq{
-		GroupID: groupID,
-	})
-	if err != nil {
-		return nil, err
-	}
-	return resp.UserIDs, nil
-}
-
-func (g *ConversationLocalCache) GetConversationIDs(ctx context.Context, userID string) ([]string, error) {
-	resp, err := g.client.Client.GetUserConversationIDsHash(ctx, &conversation.GetUserConversationIDsHashReq{
-		OwnerUserID: userID,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	g.lock.Lock()
-	hash, ok := g.conversationIDs[userID]
-	g.lock.Unlock()
-
-	if !ok || hash.hash != resp.Hash {
-		conversationIDsResp, err := g.client.Client.GetConversationIDs(ctx, &conversation.GetConversationIDsReq{
-			UserID: userID,
-		})
-		if err != nil {
-			return nil, err
-		}
-
-		g.lock.Lock()
-		defer g.lock.Unlock()
-		g.conversationIDs[userID] = Hash{
-			hash: resp.Hash,
-			ids:  conversationIDsResp.ConversationIDs,
-		}
-
-		return conversationIDsResp.ConversationIDs, nil
-	}
-
-	return hash.ids, nil
-}

pkg/common/db/localcache/doc.go

@@ -1,15 +0,0 @@
-// Copyright © 2023 OpenIM. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package localcache // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/localcache"

pkg/common/db/localcache/group.go

@@ -1,77 +0,0 @@
-// Copyright © 2023 OpenIM. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package localcache
-
-import (
-	"context"
-	"sync"
-
-	"github.com/OpenIMSDK/protocol/group"
-	"github.com/OpenIMSDK/tools/errs"
-	"github.com/openimsdk/open-im-server/v3/pkg/rpcclient"
-)
-
-type GroupLocalCache struct {
-	lock   sync.Mutex
-	cache  map[string]GroupMemberIDsHash
-	client *rpcclient.GroupRpcClient
-}
-
-type GroupMemberIDsHash struct {
-	memberListHash uint64
-	userIDs        []string
-}
-
-func NewGroupLocalCache(client *rpcclient.GroupRpcClient) *GroupLocalCache {
-	return &GroupLocalCache{
-		cache:  make(map[string]GroupMemberIDsHash, 0),
-		client: client,
-	}
-}
-
-func (g *GroupLocalCache) GetGroupMemberIDs(ctx context.Context, groupID string) ([]string, error) {
-	resp, err := g.client.Client.GetGroupAbstractInfo(ctx, &group.GetGroupAbstractInfoReq{
-		GroupIDs: []string{groupID},
-	})
-	if err != nil {
-		return nil, err
-	}
-	if len(resp.GroupAbstractInfos) < 1 {
-		return nil, errs.ErrGroupIDNotFound
-	}
-
-	g.lock.Lock()
-	localHashInfo, ok := g.cache[groupID]
-	if ok && localHashInfo.memberListHash == resp.GroupAbstractInfos[0].GroupMemberListHash {
-		g.lock.Unlock()
-		return localHashInfo.userIDs, nil
-	}
-	g.lock.Unlock()
-
-	groupMembersResp, err := g.client.Client.GetGroupMemberUserIDs(ctx, &group.GetGroupMemberUserIDsReq{
-		GroupID: groupID,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	g.lock.Lock()
-	defer g.lock.Unlock()
-	g.cache[groupID] = GroupMemberIDsHash{
-		memberListHash: resp.GroupAbstractInfos[0].GroupMemberListHash,
-		userIDs:        groupMembersResp.UserIDs,
-	}
-	return g.cache[groupID].userIDs, nil
-}

pkg/common/db/localcache/meta_local_cache.go

@@ -1,15 +0,0 @@
-// Copyright © 2023 OpenIM. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package localcache

pkg/common/db/mgo/conversation.go

@@ -96,8 +96,14 @@ func (c *ConversationMgo) FindUserIDAllConversations(ctx context.Context, userID
 	return mgoutil.Find[*relation.ConversationModel](ctx, c.coll, bson.M{"owner_user_id": userID})
 }
 
-func (c *ConversationMgo) FindRecvMsgNotNotifyUserIDs(ctx context.Context, groupID string) ([]string, error) {
-	return mgoutil.Find[string](ctx, c.coll, bson.M{"group_id": groupID, "recv_msg_opt": constant.ReceiveNotNotifyMessage}, options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1}))
+func (c *ConversationMgo) FindRecvMsgUserIDs(ctx context.Context, conversationID string, recvOpts []int) ([]string, error) {
+	var filter any
+	if len(recvOpts) == 0 {
+		filter = bson.M{"conversation_id": conversationID}
+	} else {
+		filter = bson.M{"conversation_id": conversationID, "recv_msg_opt": bson.M{"$in": recvOpts}}
+	}
+	return mgoutil.Find[string](ctx, c.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1}))
 }
 
 func (c *ConversationMgo) GetUserRecvMsgOpt(ctx context.Context, ownerUserID, conversationID string) (opt int, err error) {

pkg/common/db/table/relation/conversation.go

@@ -53,7 +53,7 @@ type ConversationModelInterface interface {
 	Take(ctx context.Context, userID, conversationID string) (conversation *ConversationModel, err error)
 	FindConversationID(ctx context.Context, userID string, conversationIDs []string) (existConversationID []string, err error)
 	FindUserIDAllConversations(ctx context.Context, userID string) (conversations []*ConversationModel, err error)
-	FindRecvMsgNotNotifyUserIDs(ctx context.Context, groupID string) ([]string, error)
+	FindRecvMsgUserIDs(ctx context.Context, conversationID string, recvOpts []int) ([]string, error)
 	GetUserRecvMsgOpt(ctx context.Context, ownerUserID, conversationID string) (opt int, err error)
 	GetAllConversationIDs(ctx context.Context) ([]string, error)
 	GetAllConversationIDsNumber(ctx context.Context) (int64, error)

pkg/common/redispubsub/redispubliser.go

@@ -0,0 +1,16 @@
+package redispubsub
+
+import "github.com/redis/go-redis/v9"
+
+type Publisher struct {
+	client  redis.UniversalClient
+	channel string
+}
+
+func NewPublisher(client redis.UniversalClient, channel string) *Publisher {
+	return &Publisher{client: client, channel: channel}
+}
+
+func (p *Publisher) Publish(message string) error {
+	return p.client.Publish(ctx, p.channel, message).Err()
+}

pkg/common/redispubsub/redissubscriber.go

@@ -0,0 +1,34 @@
+package redispubsub
+
+import (
+	"context"
+	"github.com/redis/go-redis/v9"
+)
+
+var ctx = context.Background()
+
+type Subscriber struct {
+	client  redis.UniversalClient
+	channel string
+}
+
+func NewSubscriber(client redis.UniversalClient, channel string) *Subscriber {
+	return &Subscriber{client: client, channel: channel}
+}
+
+func (s *Subscriber) OnMessage(ctx context.Context, callback func(string)) error {
+	messageChannel := s.client.Subscribe(ctx, s.channel).Channel()
+
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case msg := <-messageChannel:
+				callback(msg.Payload)
+			}
+		}
+	}()
+
+	return nil
+}