feat: support incremental synchronization (#2379)

* fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * friend incr sync * friend incr sync * friend incr sync * friend incr sync * friend incr sync * mage * optimization version log * optimization version log * sync * sync * sync * group sync * sync option * sync option * refactor: replace `friend` package with `realtion`. * refactor: update lastest commit to relation. * sync option * sync option * sync option * sync * sync * go.mod * update: go mod * refactor: change incremental to full * feat: get full friend user ids * feat: api and config * group version * merge * fix: sort by id avoid unstable sort friends. * group * group * group * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * user version * fix: sort by id avoid unstable sort friends. * test: test log add. * test: debug log remove. * fix: transfer group owner incr version more than 1. * fix: add condition to kick owner. * feat: replace resp nil * feat: replace nil * fix: delete cache of max group joined version avoid sync joined group failed. * fix: nil * fix: delete cache of max group joined version avoid sync joined group failed. * fix: delete cache of max group joined version avoid sync joined group failed. * return group information for any changes * online cache --------- Co-authored-by: withchao <withchao@users.noreply.github.com> Co-authored-by: Monet Lee <monet_lee@163.com> Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <1186114839@qq.com>
2026-05-06 10:05:58 +08:00 · 2024-07-01 17:38:14 +08:00
parent fe7c029c2a
commit 88c0d5f5ad
56 changed files with 2023 additions and 325 deletions
@@ -77,6 +77,16 @@ type FriendDatabase interface {

 	// UpdateFriends updates fields for friends
 	UpdateFriends(ctx context.Context, ownerUserID string, friendUserIDs []string, val map[string]any) (err error)
+
+	//FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error)
+
+	FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error)
+
+	FindMaxFriendVersionCache(ctx context.Context, ownerUserID string) (*model.VersionLog, error)
+
+	FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error)
+
+	OwnerIncrVersion(ctx context.Context, ownerUserID string, friendUserIDs []string, state int32) error
 }

 type friendDatabase struct {
@@ -175,7 +185,7 @@ func (f *friendDatabase) BecomeFriends(ctx context.Context, ownerUserID string,
 			return err
 		}
 		newFriendIDs = append(newFriendIDs, ownerUserID)
-		cache = cache.DelFriendIDs(newFriendIDs...)
+		cache = cache.DelFriendIDs(newFriendIDs...).DelMaxFriendVersion(newFriendIDs...)
 		return cache.ChainExecDel(ctx)

 	})
@@ -278,7 +288,7 @@ func (f *friendDatabase) AgreeFriendRequest(ctx context.Context, friendRequest *
 				return err
 			}
 		}
-		return f.cache.DelFriendIDs(friendRequest.ToUserID, friendRequest.FromUserID).ChainExecDel(ctx)
+		return f.cache.DelFriendIDs(friendRequest.ToUserID, friendRequest.FromUserID).DelMaxFriendVersion(friendRequest.ToUserID, friendRequest.FromUserID).ChainExecDel(ctx)
 	})
 }

@@ -287,7 +297,8 @@ func (f *friendDatabase) Delete(ctx context.Context, ownerUserID string, friendU
 	if err := f.friend.Delete(ctx, ownerUserID, friendUserIDs); err != nil {
 		return err
 	}
-	return f.cache.DelFriendIDs(append(friendUserIDs, ownerUserID)...).ChainExecDel(ctx)
+	userIds := append(friendUserIDs, ownerUserID)
+	return f.cache.DelFriendIDs(userIds...).DelMaxFriendVersion(userIds...).ChainExecDel(ctx)
 }

 // UpdateRemark updates the remark for a friend. Zero value for remark is also supported.
@@ -295,7 +306,7 @@ func (f *friendDatabase) UpdateRemark(ctx context.Context, ownerUserID, friendUs
 	if err := f.friend.UpdateRemark(ctx, ownerUserID, friendUserID, remark); err != nil {
 		return err
 	}
-	return f.cache.DelFriend(ownerUserID, friendUserID).ChainExecDel(ctx)
+	return f.cache.DelFriend(ownerUserID, friendUserID).DelMaxFriendVersion(ownerUserID).ChainExecDel(ctx)
 }

 // PageOwnerFriends retrieves the list of friends for the ownerUserID. It does not return an error if the result is empty.
@@ -324,9 +335,6 @@ func (f *friendDatabase) FindFriendsWithError(ctx context.Context, ownerUserID s
 	if err != nil {
 		return
 	}
-	if len(friends) != len(friendUserIDs) {
-		err = errs.ErrRecordNotFound.Wrap()
-	}
 	return
 }

@@ -341,8 +349,37 @@ func (f *friendDatabase) UpdateFriends(ctx context.Context, ownerUserID string,
 	if len(val) == 0 {
 		return nil
 	}
-	if err := f.friend.UpdateFriends(ctx, ownerUserID, friendUserIDs, val); err != nil {
+	return f.tx.Transaction(ctx, func(ctx context.Context) error {
+		if err := f.friend.UpdateFriends(ctx, ownerUserID, friendUserIDs, val); err != nil {
+			return err
+		}
+		return f.cache.DelFriends(ownerUserID, friendUserIDs).DelMaxFriendVersion(ownerUserID).ChainExecDel(ctx)
+	})
+}
+
+//func (f *friendDatabase) FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) {
+//	return f.cache.FindSortFriendUserIDs(ctx, ownerUserID)
+//}
+
+func (f *friendDatabase) FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) {
+	return f.friend.FindIncrVersion(ctx, ownerUserID, version, limit)
+}
+
+func (f *friendDatabase) FindMaxFriendVersionCache(ctx context.Context, ownerUserID string) (*model.VersionLog, error) {
+	return f.cache.FindMaxFriendVersion(ctx, ownerUserID)
+}
+
+func (f *friendDatabase) FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) {
+	return f.friend.FindFriendUserID(ctx, friendUserID)
+}
+
+//func (f *friendDatabase) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) {
+//	return f.friend.SearchFriend(ctx, ownerUserID, keyword, pagination)
+//}
+
+func (f *friendDatabase) OwnerIncrVersion(ctx context.Context, ownerUserID string, friendUserIDs []string, state int32) error {
+	if err := f.friend.IncrVersion(ctx, ownerUserID, friendUserIDs, state); err != nil {
 		return err
 	}
-	return f.cache.DelFriends(ownerUserID, friendUserIDs).ChainExecDel(ctx)
+	return f.cache.DelMaxFriendVersion(ownerUserID).ChainExecDel(ctx)
 }
@@ -106,6 +106,20 @@ type GroupDatabase interface {
 	CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error)
 	// DeleteGroupMemberHash deletes the hash entries for group members in specified groups.
 	DeleteGroupMemberHash(ctx context.Context, groupIDs []string) error
+
+	FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error)
+	FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error)
+	MemberGroupIncrVersion(ctx context.Context, groupID string, userIDs []string, state int32) error
+
+	//FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error)
+	//FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error)
+
+	FindMaxGroupMemberVersionCache(ctx context.Context, groupID string) (*model.VersionLog, error)
+	FindMaxJoinGroupVersionCache(ctx context.Context, userID string) (*model.VersionLog, error)
+
+	SearchJoinGroup(ctx context.Context, userID string, keyword string, pagination pagination.Pagination) (int64, []*model.Group, error)
+
+	FindJoinGroupID(ctx context.Context, userID string) ([]string, error)
 }

 func NewGroupDatabase(
@@ -134,6 +148,10 @@ type groupDatabase struct {
 	cache          cache.GroupCache
 }

+func (g *groupDatabase) FindJoinGroupID(ctx context.Context, userID string) ([]string, error) {
+	return g.cache.GetJoinedGroupIDs(ctx, userID)
+}
+
 func (g *groupDatabase) FindGroupMembers(ctx context.Context, groupID string, userIDs []string) ([]*model.GroupMember, error) {
 	return g.cache.GetGroupMembersInfo(ctx, groupID, userIDs)
 }
@@ -174,7 +192,8 @@ func (g *groupDatabase) CreateGroup(ctx context.Context, groups []*model.Group,
 					DelGroupMembersHash(group.GroupID).
 					DelGroupsMemberNum(group.GroupID).
 					DelGroupMemberIDs(group.GroupID).
-					DelGroupAllRoleLevel(group.GroupID)
+					DelGroupAllRoleLevel(group.GroupID).
+					DelMaxGroupMemberVersion(group.GroupID)
 			}
 		}
 		if len(groupMembers) > 0 {
@@ -187,7 +206,9 @@ func (g *groupDatabase) CreateGroup(ctx context.Context, groups []*model.Group,
 					DelGroupMemberIDs(groupMember.GroupID).
 					DelJoinedGroupID(groupMember.UserID).
 					DelGroupMembersInfo(groupMember.GroupID, groupMember.UserID).
-					DelGroupAllRoleLevel(groupMember.GroupID)
+					DelGroupAllRoleLevel(groupMember.GroupID).
+					DelMaxJoinGroupVersion(groupMember.UserID).
+					DelMaxGroupMemberVersion(groupMember.GroupID)
 			}
 		}
 		return c.ChainExecDel(ctx)
@@ -219,10 +240,15 @@ func (g *groupDatabase) SearchGroup(ctx context.Context, keyword string, paginat
 }

 func (g *groupDatabase) UpdateGroup(ctx context.Context, groupID string, data map[string]any) error {
-	if err := g.groupDB.UpdateMap(ctx, groupID, data); err != nil {
-		return err
-	}
-	return g.cache.DelGroupsInfo(groupID).ChainExecDel(ctx)
+	return g.ctxTx.Transaction(ctx, func(ctx context.Context) error {
+		if err := g.groupDB.UpdateMap(ctx, groupID, data); err != nil {
+			return err
+		}
+		if err := g.groupMemberDB.MemberGroupIncrVersion(ctx, groupID, []string{""}, model.VersionStateUpdate); err != nil {
+			return err
+		}
+		return g.cache.CloneGroupCache().DelGroupsInfo(groupID).DelMaxGroupMemberVersion(groupID).ChainExecDel(ctx)
+	})
 }

 func (g *groupDatabase) DismissGroup(ctx context.Context, groupID string, deleteMember bool) error {
@@ -244,7 +270,19 @@ func (g *groupDatabase) DismissGroup(ctx context.Context, groupID string, delete
 				DelGroupsMemberNum(groupID).
 				DelGroupMembersHash(groupID).
 				DelGroupAllRoleLevel(groupID).
-				DelGroupMembersInfo(groupID, userIDs...)
+				DelGroupMembersInfo(groupID, userIDs...).
+				DelMaxGroupMemberVersion(groupID).
+				DelMaxJoinGroupVersion(userIDs...)
+			for _, userID := range userIDs {
+				if err := g.groupMemberDB.JoinGroupIncrVersion(ctx, userID, []string{groupID}, model.VersionStateDelete); err != nil {
+					return err
+				}
+			}
+		} else {
+			if err := g.groupMemberDB.MemberGroupIncrVersion(ctx, groupID, []string{""}, model.VersionStateUpdate); err != nil {
+				return err
+			}
+			c = c.DelMaxGroupMemberVersion(groupID)
 		}
 		return c.DelGroupsInfo(groupID).ChainExecDel(ctx)
 	})
@@ -316,7 +354,9 @@ func (g *groupDatabase) HandlerGroupRequest(ctx context.Context, groupID string,
 				DelGroupMemberIDs(groupID).
 				DelGroupsMemberNum(groupID).
 				DelJoinedGroupID(member.UserID).
-				DelGroupRoleLevel(groupID, []int32{member.RoleLevel})
+				DelGroupRoleLevel(groupID, []int32{member.RoleLevel}).
+				DelMaxJoinGroupVersion(userID).
+				DelMaxGroupMemberVersion(groupID)
 			if err := c.ChainExecDel(ctx); err != nil {
 				return err
 			}
@@ -326,17 +366,21 @@ func (g *groupDatabase) HandlerGroupRequest(ctx context.Context, groupID string,
 }

 func (g *groupDatabase) DeleteGroupMember(ctx context.Context, groupID string, userIDs []string) error {
-	if err := g.groupMemberDB.Delete(ctx, groupID, userIDs); err != nil {
-		return err
-	}
-	c := g.cache.CloneGroupCache()
-	return c.DelGroupMembersHash(groupID).
-		DelGroupMemberIDs(groupID).
-		DelGroupsMemberNum(groupID).
-		DelJoinedGroupID(userIDs...).
-		DelGroupMembersInfo(groupID, userIDs...).
-		DelGroupAllRoleLevel(groupID).
-		ChainExecDel(ctx)
+	return g.ctxTx.Transaction(ctx, func(ctx context.Context) error {
+		if err := g.groupMemberDB.Delete(ctx, groupID, userIDs); err != nil {
+			return err
+		}
+		c := g.cache.CloneGroupCache()
+		return c.DelGroupMembersHash(groupID).
+			DelGroupMemberIDs(groupID).
+			DelGroupsMemberNum(groupID).
+			DelJoinedGroupID(userIDs...).
+			DelGroupMembersInfo(groupID, userIDs...).
+			DelGroupAllRoleLevel(groupID).
+			DelMaxGroupMemberVersion(groupID).
+			DelMaxJoinGroupVersion(userIDs...).
+			ChainExecDel(ctx)
+	})
 }

 func (g *groupDatabase) MapGroupMemberUserID(ctx context.Context, groupIDs []string) (map[string]*common.GroupSimpleUserID, error) {
@@ -357,29 +401,35 @@ func (g *groupDatabase) MapGroupMemberNum(ctx context.Context, groupIDs []string

 func (g *groupDatabase) TransferGroupOwner(ctx context.Context, groupID string, oldOwnerUserID, newOwnerUserID string, roleLevel int32) error {
 	return g.ctxTx.Transaction(ctx, func(ctx context.Context) error {
-		if err := g.groupMemberDB.UpdateRoleLevel(ctx, groupID, oldOwnerUserID, roleLevel); err != nil {
-			return err
-		}
-		if err := g.groupMemberDB.UpdateRoleLevel(ctx, groupID, newOwnerUserID, constant.GroupOwner); err != nil {
+		if err := g.groupMemberDB.UpdateUserRoleLevels(ctx, groupID, oldOwnerUserID, roleLevel, newOwnerUserID, constant.GroupOwner); err != nil {
 			return err
 		}
 		c := g.cache.CloneGroupCache()
 		return c.DelGroupMembersInfo(groupID, oldOwnerUserID, newOwnerUserID).
 			DelGroupAllRoleLevel(groupID).
-			DelGroupMembersHash(groupID).ChainExecDel(ctx)
+			DelGroupMembersHash(groupID).
+			DelMaxGroupMemberVersion(groupID).
+			DelGroupMemberIDs(groupID).
+			ChainExecDel(ctx)
 	})
 }

 func (g *groupDatabase) UpdateGroupMember(ctx context.Context, groupID string, userID string, data map[string]any) error {
-	if err := g.groupMemberDB.Update(ctx, groupID, userID, data); err != nil {
-		return err
+	if len(data) == 0 {
+		return nil
 	}
-	c := g.cache.CloneGroupCache()
-	c = c.DelGroupMembersInfo(groupID, userID)
-	if g.groupMemberDB.IsUpdateRoleLevel(data) {
-		c = c.DelGroupAllRoleLevel(groupID)
-	}
-	return c.ChainExecDel(ctx)
+	return g.ctxTx.Transaction(ctx, func(ctx context.Context) error {
+		if err := g.groupMemberDB.Update(ctx, groupID, userID, data); err != nil {
+			return err
+		}
+		c := g.cache.CloneGroupCache()
+		c = c.DelGroupMembersInfo(groupID, userID)
+		if g.groupMemberDB.IsUpdateRoleLevel(data) {
+			c = c.DelGroupAllRoleLevel(groupID).DelGroupMemberIDs(groupID)
+		}
+		c = c.DelMaxGroupMemberVersion(groupID)
+		return c.ChainExecDel(ctx)
+	})
 }

 func (g *groupDatabase) UpdateGroupMembers(ctx context.Context, data []*common.BatchUpdateGroupMember) error {
@@ -390,9 +440,9 @@ func (g *groupDatabase) UpdateGroupMembers(ctx context.Context, data []*common.B
 				return err
 			}
 			if g.groupMemberDB.IsUpdateRoleLevel(item.Map) {
-				c = c.DelGroupAllRoleLevel(item.GroupID)
+				c = c.DelGroupAllRoleLevel(item.GroupID).DelGroupMemberIDs(item.GroupID)
 			}
-			c = c.DelGroupMembersInfo(item.GroupID, item.UserID).DelGroupMembersHash(item.GroupID)
+			c = c.DelGroupMembersInfo(item.GroupID, item.UserID).DelMaxGroupMemberVersion(item.GroupID).DelGroupMembersHash(item.GroupID)
 		}
 		return c.ChainExecDel(ctx)
 	})
@@ -443,3 +493,34 @@ func (g *groupDatabase) DeleteGroupMemberHash(ctx context.Context, groupIDs []st
 	}
 	return c.ChainExecDel(ctx)
 }
+
+func (g *groupDatabase) FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) {
+	return g.groupMemberDB.FindMemberIncrVersion(ctx, groupID, version, limit)
+}
+
+func (g *groupDatabase) FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) {
+	return g.groupMemberDB.FindJoinIncrVersion(ctx, userID, version, limit)
+}
+
+func (g *groupDatabase) FindMaxGroupMemberVersionCache(ctx context.Context, groupID string) (*model.VersionLog, error) {
+	return g.cache.FindMaxGroupMemberVersion(ctx, groupID)
+}
+
+func (g *groupDatabase) FindMaxJoinGroupVersionCache(ctx context.Context, userID string) (*model.VersionLog, error) {
+	return g.cache.FindMaxJoinGroupVersion(ctx, userID)
+}
+
+func (g *groupDatabase) SearchJoinGroup(ctx context.Context, userID string, keyword string, pagination pagination.Pagination) (int64, []*model.Group, error) {
+	groupIDs, err := g.cache.GetJoinedGroupIDs(ctx, userID)
+	if err != nil {
+		return 0, nil, err
+	}
+	return g.groupDB.SearchJoin(ctx, groupIDs, keyword, pagination)
+}
+
+func (g *groupDatabase) MemberGroupIncrVersion(ctx context.Context, groupID string, userIDs []string, state int32) error {
+	if err := g.groupMemberDB.MemberGroupIncrVersion(ctx, groupID, userIDs, state); err != nil {
+		return err
+	}
+	return g.cache.DelMaxGroupMemberVersion(groupID).ChainExecDel(ctx)
+}
@@ -60,6 +60,8 @@ type UserDatabase interface {
 	CountTotal(ctx context.Context, before *time.Time) (int64, error)
 	// CountRangeEverydayTotal Get the user increment in the range
 	CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error)
+
+	SortQuery(ctx context.Context, userIDName map[string]string, asc bool) ([]*model.User, error)
 	// SubscribeUsersStatus Subscribe a user's presence status
 	SubscribeUsersStatus(ctx context.Context, userID string, userIDs []string) error
 	// UnsubscribeUsersStatus unsubscribe a user's presence status
@@ -210,6 +212,10 @@ func (u *userDatabase) CountRangeEverydayTotal(ctx context.Context, start time.T
 	return u.userDB.CountRangeEverydayTotal(ctx, start, end)
 }

+func (u *userDatabase) SortQuery(ctx context.Context, userIDName map[string]string, asc bool) ([]*model.User, error) {
+	return u.userDB.SortQuery(ctx, userIDName, asc)
+}
+
 // SubscribeUsersStatus Subscribe or unsubscribe a user's presence status.
 func (u *userDatabase) SubscribeUsersStatus(ctx context.Context, userID string, userIDs []string) error {
 	err := u.mongoDB.AddSubscriptionList(ctx, userID, userIDs)