feat: support incremental synchronization (#2379)

* fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * friend incr sync * friend incr sync * friend incr sync * friend incr sync * friend incr sync * mage * optimization version log * optimization version log * sync * sync * sync * group sync * sync option * sync option * refactor: replace `friend` package with `realtion`. * refactor: update lastest commit to relation. * sync option * sync option * sync option * sync * sync * go.mod * update: go mod * refactor: change incremental to full * feat: get full friend user ids * feat: api and config * group version * merge * fix: sort by id avoid unstable sort friends. * group * group * group * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * user version * fix: sort by id avoid unstable sort friends. * test: test log add. * test: debug log remove. * fix: transfer group owner incr version more than 1. * fix: add condition to kick owner. * feat: replace resp nil * feat: replace nil * fix: delete cache of max group joined version avoid sync joined group failed. * fix: nil * fix: delete cache of max group joined version avoid sync joined group failed. * fix: delete cache of max group joined version avoid sync joined group failed. * return group information for any changes * online cache --------- Co-authored-by: withchao <withchao@users.noreply.github.com> Co-authored-by: Monet Lee <monet_lee@163.com> Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <1186114839@qq.com>
2026-05-12 13:05:58 +08:00 · 2024-07-01 17:38:14 +08:00
parent fe7c029c2a
commit 88c0d5f5ad
56 changed files with 2023 additions and 325 deletions
@@ -17,7 +17,7 @@ package api
 import (
 	"github.com/gin-gonic/gin"
 	"github.com/openimsdk/open-im-server/v3/pkg/rpcclient"
-	"github.com/openimsdk/protocol/friend"
+	"github.com/openimsdk/protocol/relation"
 	"github.com/openimsdk/tools/a2r"
 )

@@ -28,68 +28,82 @@ func NewFriendApi(client rpcclient.Friend) FriendApi {
 }

 func (o *FriendApi) ApplyToAddFriend(c *gin.Context) {
-	a2r.Call(friend.FriendClient.ApplyToAddFriend, o.Client, c)
+	a2r.Call(relation.FriendClient.ApplyToAddFriend, o.Client, c)
 }

 func (o *FriendApi) RespondFriendApply(c *gin.Context) {
-	a2r.Call(friend.FriendClient.RespondFriendApply, o.Client, c)
+	a2r.Call(relation.FriendClient.RespondFriendApply, o.Client, c)
 }

 func (o *FriendApi) DeleteFriend(c *gin.Context) {
-	a2r.Call(friend.FriendClient.DeleteFriend, o.Client, c)
+	a2r.Call(relation.FriendClient.DeleteFriend, o.Client, c)
 }

 func (o *FriendApi) GetFriendApplyList(c *gin.Context) {
-	a2r.Call(friend.FriendClient.GetPaginationFriendsApplyTo, o.Client, c)
+	a2r.Call(relation.FriendClient.GetPaginationFriendsApplyTo, o.Client, c)
 }

 func (o *FriendApi) GetDesignatedFriendsApply(c *gin.Context) {
-	a2r.Call(friend.FriendClient.GetDesignatedFriendsApply, o.Client, c)
+	a2r.Call(relation.FriendClient.GetDesignatedFriendsApply, o.Client, c)
 }

 func (o *FriendApi) GetSelfApplyList(c *gin.Context) {
-	a2r.Call(friend.FriendClient.GetPaginationFriendsApplyFrom, o.Client, c)
+	a2r.Call(relation.FriendClient.GetPaginationFriendsApplyFrom, o.Client, c)
 }

 func (o *FriendApi) GetFriendList(c *gin.Context) {
-	a2r.Call(friend.FriendClient.GetPaginationFriends, o.Client, c)
+	a2r.Call(relation.FriendClient.GetPaginationFriends, o.Client, c)
 }

 func (o *FriendApi) GetDesignatedFriends(c *gin.Context) {
-	a2r.Call(friend.FriendClient.GetDesignatedFriends, o.Client, c)
+	a2r.Call(relation.FriendClient.GetDesignatedFriends, o.Client, c)
+	//a2r.Call(relation.FriendClient.GetDesignatedFriends, o.Client, c, a2r.NewNilReplaceOption(relation.FriendClient.GetDesignatedFriends))
 }

 func (o *FriendApi) SetFriendRemark(c *gin.Context) {
-	a2r.Call(friend.FriendClient.SetFriendRemark, o.Client, c)
+	a2r.Call(relation.FriendClient.SetFriendRemark, o.Client, c)
 }

 func (o *FriendApi) AddBlack(c *gin.Context) {
-	a2r.Call(friend.FriendClient.AddBlack, o.Client, c)
+	a2r.Call(relation.FriendClient.AddBlack, o.Client, c)
 }

 func (o *FriendApi) GetPaginationBlacks(c *gin.Context) {
-	a2r.Call(friend.FriendClient.GetPaginationBlacks, o.Client, c)
+	a2r.Call(relation.FriendClient.GetPaginationBlacks, o.Client, c)
 }

 func (o *FriendApi) RemoveBlack(c *gin.Context) {
-	a2r.Call(friend.FriendClient.RemoveBlack, o.Client, c)
+	a2r.Call(relation.FriendClient.RemoveBlack, o.Client, c)
 }

 func (o *FriendApi) ImportFriends(c *gin.Context) {
-	a2r.Call(friend.FriendClient.ImportFriends, o.Client, c)
+	a2r.Call(relation.FriendClient.ImportFriends, o.Client, c)
 }

 func (o *FriendApi) IsFriend(c *gin.Context) {
-	a2r.Call(friend.FriendClient.IsFriend, o.Client, c)
+	a2r.Call(relation.FriendClient.IsFriend, o.Client, c)
 }

 func (o *FriendApi) GetFriendIDs(c *gin.Context) {
-	a2r.Call(friend.FriendClient.GetFriendIDs, o.Client, c)
+	a2r.Call(relation.FriendClient.GetFriendIDs, o.Client, c)
 }

 func (o *FriendApi) GetSpecifiedFriendsInfo(c *gin.Context) {
-	a2r.Call(friend.FriendClient.GetSpecifiedFriendsInfo, o.Client, c)
+	a2r.Call(relation.FriendClient.GetSpecifiedFriendsInfo, o.Client, c)
 }
+
 func (o *FriendApi) UpdateFriends(c *gin.Context) {
-	a2r.Call(friend.FriendClient.UpdateFriends, o.Client, c)
+	a2r.Call(relation.FriendClient.UpdateFriends, o.Client, c)
+}
+
+func (o *FriendApi) GetIncrementalFriends(c *gin.Context) {
+	a2r.Call(relation.FriendClient.GetIncrementalFriends, o.Client, c)
+}
+
+func (o *FriendApi) GetIncrementalBlacks(c *gin.Context) {
+	a2r.Call(relation.FriendClient.GetIncrementalBlacks, o.Client, c)
+}
+
+func (o *FriendApi) GetFullFriendUserIDs(c *gin.Context) {
+	a2r.Call(relation.FriendClient.GetFullFriendUserIDs, o.Client, c)
 }
@@ -19,6 +19,8 @@ import (
 	"github.com/openimsdk/open-im-server/v3/pkg/rpcclient"
 	"github.com/openimsdk/protocol/group"
 	"github.com/openimsdk/tools/a2r"
+	"github.com/openimsdk/tools/apiresp"
+	"github.com/openimsdk/tools/log"
 )

 type GroupApi rpcclient.Group
@@ -65,6 +67,7 @@ func (o *GroupApi) GetGroupUsersReqApplicationList(c *gin.Context) {

 func (o *GroupApi) GetGroupsInfo(c *gin.Context) {
 	a2r.Call(group.GroupClient.GetGroupsInfo, o.Client, c)
+	//a2r.Call(group.GroupClient.GetGroupsInfo, o.Client, c, a2r.NewNilReplaceOption(group.GroupClient.GetGroupsInfo))
 }

 func (o *GroupApi) KickGroupMember(c *gin.Context) {
@@ -73,6 +76,7 @@ func (o *GroupApi) KickGroupMember(c *gin.Context) {

 func (o *GroupApi) GetGroupMembersInfo(c *gin.Context) {
 	a2r.Call(group.GroupClient.GetGroupMembersInfo, o.Client, c)
+	//a2r.Call(group.GroupClient.GetGroupMembersInfo, o.Client, c, a2r.NewNilReplaceOption(group.GroupClient.GetGroupMembersInfo))
 }

 func (o *GroupApi) GetGroupMemberList(c *gin.Context) {
@@ -134,3 +138,61 @@ func (o *GroupApi) GetGroups(c *gin.Context) {
 func (o *GroupApi) GetGroupMemberUserIDs(c *gin.Context) {
 	a2r.Call(group.GroupClient.GetGroupMemberUserIDs, o.Client, c)
 }
+
+func (o *GroupApi) GetIncrementalJoinGroup(c *gin.Context) {
+	a2r.Call(group.GroupClient.GetIncrementalJoinGroup, o.Client, c)
+}
+
+func (o *GroupApi) GetIncrementalGroupMember(c *gin.Context) {
+	a2r.Call(group.GroupClient.GetIncrementalGroupMember, o.Client, c)
+}
+
+func (o *GroupApi) GetIncrementalGroupMemberBatch(c *gin.Context) {
+	type BatchIncrementalReq struct {
+		UserID string                                `json:"user_id"`
+		List   []*group.GetIncrementalGroupMemberReq `json:"list"`
+	}
+	type BatchIncrementalResp struct {
+		List map[string]*group.GetIncrementalGroupMemberResp `json:"list"`
+	}
+	req, err := a2r.ParseRequestNotCheck[BatchIncrementalReq](c)
+	if err != nil {
+		apiresp.GinError(c, err)
+		return
+	}
+	resp := &BatchIncrementalResp{
+		List: make(map[string]*group.GetIncrementalGroupMemberResp),
+	}
+	var (
+		changeCount int
+	)
+	for _, req := range req.List {
+		if _, ok := resp.List[req.GroupID]; ok {
+			continue
+		}
+		res, err := o.Client.GetIncrementalGroupMember(c, req)
+		if err != nil {
+			if len(resp.List) == 0 {
+				apiresp.GinError(c, err)
+			} else {
+				log.ZError(c, "group incr sync versopn", err, "groupID", req.GroupID, "success", len(resp.List))
+				apiresp.GinSuccess(c, resp)
+			}
+			return
+		}
+		resp.List[req.GroupID] = res
+		changeCount += len(res.Insert) + len(res.Delete) + len(res.Update)
+		if changeCount >= 200 {
+			break
+		}
+	}
+	apiresp.GinSuccess(c, resp)
+}
+
+func (o *GroupApi) GetFullGroupMemberUserIDs(c *gin.Context) {
+	a2r.Call(group.GroupClient.GetFullGroupMemberUserIDs, o.Client, c)
+}
+
+func (o *GroupApi) GetFullJoinGroupIDs(c *gin.Context) {
+	a2r.Call(group.GroupClient.GetFullJoinGroupIDs, o.Client, c)
+}
@@ -2,6 +2,9 @@ package api

 import (
 	"fmt"
+	"net/http"
+	"strings"
+
 	"github.com/gin-gonic/gin"
 	"github.com/gin-gonic/gin/binding"
 	"github.com/go-playground/validator/v10"
@@ -14,8 +17,6 @@ import (
 	"github.com/openimsdk/tools/mw"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
-	"net/http"
-	"strings"
 )

 func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.Engine {
@@ -81,11 +82,14 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En
 		friendRouterGroup.POST("/add_black", f.AddBlack)
 		friendRouterGroup.POST("/get_black_list", f.GetPaginationBlacks)
 		friendRouterGroup.POST("/remove_black", f.RemoveBlack)
+		friendRouterGroup.POST("/get_incremental_blacks", f.GetIncrementalBlacks)
 		friendRouterGroup.POST("/import_friend", f.ImportFriends)
 		friendRouterGroup.POST("/is_friend", f.IsFriend)
 		friendRouterGroup.POST("/get_friend_id", f.GetFriendIDs)
 		friendRouterGroup.POST("/get_specified_friends_info", f.GetSpecifiedFriendsInfo)
 		friendRouterGroup.POST("/update_friends", f.UpdateFriends)
+		friendRouterGroup.POST("/get_incremental_friends", f.GetIncrementalFriends)
+		friendRouterGroup.POST("/get_full_friend_user_ids", f.GetFullFriendUserIDs)
 	}
 	g := NewGroupApi(*groupRpc)
 	groupRouterGroup := r.Group("/group")
@@ -114,6 +118,11 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En
 		groupRouterGroup.POST("/get_group_abstract_info", g.GetGroupAbstractInfo)
 		groupRouterGroup.POST("/get_groups", g.GetGroups)
 		groupRouterGroup.POST("/get_group_member_user_id", g.GetGroupMemberUserIDs)
+		groupRouterGroup.POST("/get_incremental_join_group", g.GetIncrementalJoinGroup)
+		groupRouterGroup.POST("/get_incremental_group_member", g.GetIncrementalGroupMember)
+		groupRouterGroup.POST("/get_incremental_group_member_batch", g.GetIncrementalGroupMemberBatch)
+		groupRouterGroup.POST("/get_full_group_member_user_ids", g.GetFullGroupMemberUserIDs)
+		groupRouterGroup.POST("/get_full_join_group_ids", g.GetFullJoinGroupIDs)
 	}
 	// certificate
 	authRouterGroup := r.Group("/auth")