feat: support incremental synchronization (#2379)

* fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * friend incr sync * friend incr sync * friend incr sync * friend incr sync * friend incr sync * mage * optimization version log * optimization version log * sync * sync * sync * group sync * sync option * sync option * refactor: replace `friend` package with `realtion`. * refactor: update lastest commit to relation. * sync option * sync option * sync option * sync * sync * go.mod * update: go mod * refactor: change incremental to full * feat: get full friend user ids * feat: api and config * group version * merge * fix: sort by id avoid unstable sort friends. * group * group * group * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * user version * fix: sort by id avoid unstable sort friends. * test: test log add. * test: debug log remove. * fix: transfer group owner incr version more than 1. * fix: add condition to kick owner. * feat: replace resp nil * feat: replace nil * fix: delete cache of max group joined version avoid sync joined group failed. * fix: nil * fix: delete cache of max group joined version avoid sync joined group failed. * fix: delete cache of max group joined version avoid sync joined group failed. * return group information for any changes * online cache --------- Co-authored-by: withchao <withchao@users.noreply.github.com> Co-authored-by: Monet Lee <monet_lee@163.com> Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <1186114839@qq.com>
2026-05-08 19:16:35 +08:00 · 2024-07-01 17:38:14 +08:00
parent fe7c029c2a
commit 88c0d5f5ad
56 changed files with 2023 additions and 325 deletions
@@ -19,6 +19,8 @@ const (
 	TwoWayFriendsIDsKey = "COMMON_FRIENDS_IDS:"
 	FriendKey           = "FRIEND_INFO:"
 	IsFriendKey         = "IS_FRIEND:" // local cache key
+	//FriendSyncSortUserIDsKey = "FRIEND_SYNC_SORT_USER_IDS:"
+	FriendMaxVersionKey = "FRIEND_MAX_VERSION:"
 )

 func GetFriendIDsKey(ownerUserID string) string {
@@ -33,6 +35,14 @@ func GetFriendKey(ownerUserID, friendUserID string) string {
 	return FriendKey + ownerUserID + "-" + friendUserID
 }

+func GetFriendMaxVersionKey(ownerUserID string) string {
+	return FriendMaxVersionKey + ownerUserID
+}
+
 func GetIsFriendKey(possibleFriendUserID, userID string) string {
 	return IsFriendKey + possibleFriendUserID + "-" + userID
 }
+
+//func GetFriendSyncSortUserIDsKey(ownerUserID string, count int) string {
+//	return FriendSyncSortUserIDsKey + strconv.Itoa(count) + ":" + ownerUserID
+//}
@@ -28,6 +28,8 @@ const (
 	JoinedGroupsKey            = "JOIN_GROUPS_KEY:"
 	GroupMemberNumKey          = "GROUP_MEMBER_NUM_CACHE:"
 	GroupRoleLevelMemberIDsKey = "GROUP_ROLE_LEVEL_MEMBER_IDS:"
+	GroupMemberMaxVersionKey   = "GROUP_MEMBER_MAX_VERSION:"
+	GroupJoinMaxVersionKey     = "GROUP_JOIN_MAX_VERSION:"
 )

 func GetGroupInfoKey(groupID string) string {
@@ -57,3 +59,11 @@ func GetGroupMemberNumKey(groupID string) string {
 func GetGroupRoleLevelMemberIDsKey(groupID string, roleLevel int32) string {
 	return GroupRoleLevelMemberIDsKey + groupID + "-" + strconv.Itoa(int(roleLevel))
 }
+
+func GetGroupMemberMaxVersionKey(groupID string) string {
+	return GroupMemberMaxVersionKey + groupID
+}
+
+func GetJoinGroupMaxVersionKey(userID string) string {
+	return GroupJoinMaxVersionKey + userID
+}
@@ -32,4 +32,16 @@ type FriendCache interface {
 	DelFriend(ownerUserID, friendUserID string) FriendCache
 	// Delete friends when friends' info changed
 	DelFriends(ownerUserID string, friendUserIDs []string) FriendCache
+
+	DelOwner(friendUserID string, ownerUserIDs []string) FriendCache
+
+	DelMaxFriendVersion(ownerUserIDs ...string) FriendCache
+
+	//DelSortFriendUserIDs(ownerUserIDs ...string) FriendCache
+
+	//FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error)
+
+	//FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*relationtb.VersionLog, error)
+
+	FindMaxFriendVersion(ctx context.Context, ownerUserID string) (*relationtb.VersionLog, error)
 }
@@ -46,7 +46,6 @@ type GroupCache interface {
 	GetGroupMemberInfo(ctx context.Context, groupID, userID string) (groupMember *model.GroupMember, err error)
 	GetGroupMembersInfo(ctx context.Context, groupID string, userID []string) (groupMembers []*model.GroupMember, err error)
 	GetAllGroupMembersInfo(ctx context.Context, groupID string) (groupMembers []*model.GroupMember, err error)
-	GetGroupMembersPage(ctx context.Context, groupID string, userID []string, showNumber, pageNumber int32) (total uint32, groupMembers []*model.GroupMember, err error)
 	FindGroupMemberUser(ctx context.Context, groupIDs []string, userID string) ([]*model.GroupMember, error)

 	GetGroupRoleLevelMemberIDs(ctx context.Context, groupID string, roleLevel int32) ([]string, error)
@@ -59,4 +58,12 @@ type GroupCache interface {
 	GetGroupRolesLevelMemberInfo(ctx context.Context, groupID string, roleLevels []int32) ([]*model.GroupMember, error)
 	GetGroupMemberNum(ctx context.Context, groupID string) (memberNum int64, err error)
 	DelGroupsMemberNum(groupID ...string) GroupCache
+
+	//FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error)
+	//FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error)
+
+	DelMaxGroupMemberVersion(groupIDs ...string) GroupCache
+	DelMaxJoinGroupVersion(userIDs ...string) GroupCache
+	FindMaxGroupMemberVersion(ctx context.Context, groupID string) (*model.VersionLog, error)
+	FindMaxJoinGroupVersion(ctx context.Context, userID string) (*model.VersionLog, error)
 }
@@ -16,6 +16,8 @@ package redis

 import (
 	"context"
+	"time"
+
 	"github.com/dtm-labs/rockscache"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache"
@@ -25,7 +27,6 @@ import (
 	"github.com/openimsdk/tools/log"
 	"github.com/openimsdk/tools/utils/datautil"
 	"github.com/redis/go-redis/v9"
-	"time"
 )

 const (
@@ -38,6 +39,7 @@ type FriendCacheRedis struct {
 	friendDB   database.Friend
 	expireTime time.Duration
 	rcClient   *rockscache.Client
+	syncCount  int
 }

 // NewFriendCacheRedis creates a new instance of FriendCacheRedis.
@@ -68,6 +70,14 @@ func (f *FriendCacheRedis) getFriendIDsKey(ownerUserID string) string {
 	return cachekey.GetFriendIDsKey(ownerUserID)
 }

+//func (f *FriendCacheRedis) getFriendSyncSortUserIDsKey(ownerUserID string) string {
+//	return cachekey.GetFriendSyncSortUserIDsKey(ownerUserID, f.syncCount)
+//}
+
+func (f *FriendCacheRedis) getFriendMaxVersionKey(ownerUserID string) string {
+	return cachekey.GetFriendMaxVersionKey(ownerUserID)
+}
+
 // getTwoWayFriendsIDsKey returns the key for storing two-way friend IDs in the cache.
 func (f *FriendCacheRedis) getTwoWayFriendsIDsKey(ownerUserID string) string {
 	return cachekey.GetTwoWayFriendsIDsKey(ownerUserID)
@@ -97,6 +107,16 @@ func (f *FriendCacheRedis) DelFriendIDs(ownerUserIDs ...string) cache.FriendCach
 	return newFriendCache
 }

+//func (f *FriendCacheRedis) DelSortFriendUserIDs(ownerUserIDs ...string) cache.FriendCache {
+//	newGroupCache := f.CloneFriendCache()
+//	keys := make([]string, 0, len(ownerUserIDs))
+//	for _, userID := range ownerUserIDs {
+//		keys = append(keys, f.getFriendSyncSortUserIDsKey(userID))
+//	}
+//	newGroupCache.AddKeys(keys...)
+//	return newGroupCache
+//}
+
 // GetTwoWayFriendIDs retrieves two-way friend IDs from the cache.
 func (f *FriendCacheRedis) GetTwoWayFriendIDs(ctx context.Context, ownerUserID string) (twoWayFriendIDs []string, err error) {
 	friendIDs, err := f.GetFriendIDs(ctx, ownerUserID)
@@ -151,3 +171,41 @@ func (f *FriendCacheRedis) DelFriends(ownerUserID string, friendUserIDs []string

 	return newFriendCache
 }
+
+func (f *FriendCacheRedis) DelOwner(friendUserID string, ownerUserIDs []string) cache.FriendCache {
+	newFriendCache := f.CloneFriendCache()
+
+	for _, ownerUserID := range ownerUserIDs {
+		key := f.getFriendKey(ownerUserID, friendUserID)
+		newFriendCache.AddKeys(key) // Assuming AddKeys marks the keys for deletion
+	}
+
+	return newFriendCache
+}
+
+func (f *FriendCacheRedis) DelMaxFriendVersion(ownerUserIDs ...string) cache.FriendCache {
+	newFriendCache := f.CloneFriendCache()
+	for _, ownerUserID := range ownerUserIDs {
+		key := f.getFriendMaxVersionKey(ownerUserID)
+		newFriendCache.AddKeys(key) // Assuming AddKeys marks the keys for deletion
+	}
+
+	return newFriendCache
+}
+
+//func (f *FriendCacheRedis) FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) {
+//	userIDs, err := f.GetFriendIDs(ctx, ownerUserID)
+//	if err != nil {
+//		return nil, err
+//	}
+//	if len(userIDs) > f.syncCount {
+//		userIDs = userIDs[:f.syncCount]
+//	}
+//	return userIDs, nil
+//}
+
+func (f *FriendCacheRedis) FindMaxFriendVersion(ctx context.Context, ownerUserID string) (*model.VersionLog, error) {
+	return getCache(ctx, f.rcClient, f.getFriendMaxVersionKey(ownerUserID), f.expireTime, func(ctx context.Context) (*model.VersionLog, error) {
+		return f.friendDB.FindIncrVersion(ctx, ownerUserID, 0, 0)
+	})
+}
@@ -27,7 +27,6 @@ import (
 	"github.com/openimsdk/protocol/constant"
 	"github.com/openimsdk/tools/errs"
 	"github.com/openimsdk/tools/log"
-	"github.com/openimsdk/tools/utils/datautil"
 	"github.com/redis/go-redis/v9"
 	"time"
 )
@@ -111,6 +110,14 @@ func (g *GroupCacheRedis) getGroupRoleLevelMemberIDsKey(groupID string, roleLeve
 	return cachekey.GetGroupRoleLevelMemberIDsKey(groupID, roleLevel)
 }

+func (g *GroupCacheRedis) getGroupMemberMaxVersionKey(groupID string) string {
+	return cachekey.GetGroupMemberMaxVersionKey(groupID)
+}
+
+func (g *GroupCacheRedis) getJoinGroupMaxVersionKey(userID string) string {
+	return cachekey.GetJoinGroupMaxVersionKey(userID)
+}
+
 func (g *GroupCacheRedis) GetGroupIndex(group *model.Group, keys []string) (int, error) {
 	key := g.getGroupInfoKey(group.GroupID)
 	for i, _key := range keys {
@@ -246,9 +253,17 @@ func (g *GroupCacheRedis) DelGroupMemberIDs(groupID string) cache.GroupCache {
 	return cache
 }

+func (g *GroupCacheRedis) findUserJoinedGroupID(ctx context.Context, userID string) ([]string, error) {
+	groupIDs, err := g.groupMemberDB.FindUserJoinedGroupID(ctx, userID)
+	if err != nil {
+		return nil, err
+	}
+	return g.groupDB.FindJoinSortGroupID(ctx, groupIDs)
+}
+
 func (g *GroupCacheRedis) GetJoinedGroupIDs(ctx context.Context, userID string) (joinedGroupIDs []string, err error) {
 	return getCache(ctx, g.rcClient, g.getJoinedGroupsKey(userID), g.expireTime, func(ctx context.Context) ([]string, error) {
-		return g.groupMemberDB.FindUserJoinedGroupID(ctx, userID)
+		return g.findUserJoinedGroupID(ctx, userID)
 	})
 }

@@ -277,26 +292,6 @@ func (g *GroupCacheRedis) GetGroupMembersInfo(ctx context.Context, groupID strin
 	})
 }

-func (g *GroupCacheRedis) GetGroupMembersPage(
-	ctx context.Context,
-	groupID string,
-	userIDs []string,
-	showNumber, pageNumber int32,
-) (total uint32, groupMembers []*model.GroupMember, err error) {
-	groupMemberIDs, err := g.GetGroupMemberIDs(ctx, groupID)
-	if err != nil {
-		return 0, nil, err
-	}
-	if userIDs != nil {
-		userIDs = datautil.BothExist(userIDs, groupMemberIDs)
-	} else {
-		userIDs = groupMemberIDs
-	}
-	groupMembers, err = g.GetGroupMembersInfo(ctx, groupID, datautil.Paginate(userIDs, int(showNumber), int(showNumber)))
-
-	return uint32(len(userIDs)), groupMembers, err
-}
-
 func (g *GroupCacheRedis) GetAllGroupMembersInfo(ctx context.Context, groupID string) (groupMembers []*model.GroupMember, err error) {
 	groupMemberIDs, err := g.GetGroupMemberIDs(ctx, groupID)
 	if err != nil {
@@ -406,3 +401,57 @@ func (g *GroupCacheRedis) FindGroupMemberUser(ctx context.Context, groupIDs []st
 		return g.groupMemberDB.Take(ctx, groupID, userID)
 	})
 }
+
+//func (g *GroupCacheRedis) FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) {
+//	userIDs, err := g.GetGroupMemberIDs(ctx, groupID)
+//	if err != nil {
+//		return nil, err
+//	}
+//	if len(userIDs) > g.syncCount {
+//		userIDs = userIDs[:g.syncCount]
+//	}
+//	return userIDs, nil
+//}
+//
+//func (g *GroupCacheRedis) FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) {
+//	groupIDs, err := g.GetJoinedGroupIDs(ctx, userID)
+//	if err != nil {
+//		return nil, err
+//	}
+//	if len(groupIDs) > g.syncCount {
+//		groupIDs = groupIDs[:g.syncCount]
+//	}
+//	return groupIDs, nil
+//}
+
+func (g *GroupCacheRedis) DelMaxGroupMemberVersion(groupIDs ...string) cache.GroupCache {
+	keys := make([]string, 0, len(groupIDs))
+	for _, groupID := range groupIDs {
+		keys = append(keys, g.getGroupMemberMaxVersionKey(groupID))
+	}
+	cache := g.CloneGroupCache()
+	cache.AddKeys(keys...)
+	return cache
+}
+
+func (g *GroupCacheRedis) DelMaxJoinGroupVersion(userIDs ...string) cache.GroupCache {
+	keys := make([]string, 0, len(userIDs))
+	for _, userID := range userIDs {
+		keys = append(keys, g.getJoinGroupMaxVersionKey(userID))
+	}
+	cache := g.CloneGroupCache()
+	cache.AddKeys(keys...)
+	return cache
+}
+
+func (g *GroupCacheRedis) FindMaxGroupMemberVersion(ctx context.Context, groupID string) (*model.VersionLog, error) {
+	return getCache(ctx, g.rcClient, g.getGroupMemberMaxVersionKey(groupID), g.expireTime, func(ctx context.Context) (*model.VersionLog, error) {
+		return g.groupMemberDB.FindMemberIncrVersion(ctx, groupID, 0, 0)
+	})
+}
+
+func (g *GroupCacheRedis) FindMaxJoinGroupVersion(ctx context.Context, userID string) (*model.VersionLog, error) {
+	return getCache(ctx, g.rcClient, g.getJoinGroupMaxVersionKey(userID), g.expireTime, func(ctx context.Context) (*model.VersionLog, error) {
+		return g.groupMemberDB.FindJoinIncrVersion(ctx, userID, 0, 0)
+	})
+}