feat: support incremental synchronization (#2379)

* fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * friend incr sync * friend incr sync * friend incr sync * friend incr sync * friend incr sync * mage * optimization version log * optimization version log * sync * sync * sync * group sync * sync option * sync option * refactor: replace `friend` package with `realtion`. * refactor: update lastest commit to relation. * sync option * sync option * sync option * sync * sync * go.mod * update: go mod * refactor: change incremental to full * feat: get full friend user ids * feat: api and config * group version * merge * fix: sort by id avoid unstable sort friends. * group * group * group * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * user version * fix: sort by id avoid unstable sort friends. * test: test log add. * test: debug log remove. * fix: transfer group owner incr version more than 1. * fix: add condition to kick owner. * feat: replace resp nil * feat: replace nil * fix: delete cache of max group joined version avoid sync joined group failed. * fix: nil * fix: delete cache of max group joined version avoid sync joined group failed. * fix: delete cache of max group joined version avoid sync joined group failed. * return group information for any changes * online cache --------- Co-authored-by: withchao <withchao@users.noreply.github.com> Co-authored-by: Monet Lee <monet_lee@163.com> Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <1186114839@qq.com>
2026-05-12 13:05:58 +08:00 · 2024-07-01 17:38:14 +08:00
parent fe7c029c2a
commit 88c0d5f5ad
56 changed files with 2023 additions and 325 deletions
@@ -0,0 +1,149 @@
+package group
+
+import (
+	"context"
+	"github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion"
+	"github.com/openimsdk/open-im-server/v3/pkg/authverify"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/servererrs"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
+	"github.com/openimsdk/open-im-server/v3/pkg/util/hashutil"
+	"github.com/openimsdk/protocol/constant"
+	pbgroup "github.com/openimsdk/protocol/group"
+	"github.com/openimsdk/protocol/sdkws"
+	"slices"
+)
+
+func (s *groupServer) GetFullGroupMemberUserIDs(ctx context.Context, req *pbgroup.GetFullGroupMemberUserIDsReq) (*pbgroup.GetFullGroupMemberUserIDsResp, error) {
+	vl, err := s.db.FindMaxGroupMemberVersionCache(ctx, req.GroupID)
+	if err != nil {
+		return nil, err
+	}
+	userIDs, err := s.db.FindGroupMemberUserID(ctx, req.GroupID)
+	if err != nil {
+		return nil, err
+	}
+	idHash := hashutil.IdHash(userIDs)
+	if req.IdHash == idHash {
+		userIDs = nil
+	}
+	return &pbgroup.GetFullGroupMemberUserIDsResp{
+		Version:   idHash,
+		VersionID: vl.ID.Hex(),
+		Equal:     req.IdHash == idHash,
+		UserIDs:   userIDs,
+	}, nil
+}
+
+func (s *groupServer) GetFullJoinGroupIDs(ctx context.Context, req *pbgroup.GetFullJoinGroupIDsReq) (*pbgroup.GetFullJoinGroupIDsResp, error) {
+	vl, err := s.db.FindMaxJoinGroupVersionCache(ctx, req.UserID)
+	if err != nil {
+		return nil, err
+	}
+	groupIDs, err := s.db.FindJoinGroupID(ctx, req.UserID)
+	if err != nil {
+		return nil, err
+	}
+	idHash := hashutil.IdHash(groupIDs)
+	if req.IdHash == idHash {
+		groupIDs = nil
+	}
+	return &pbgroup.GetFullJoinGroupIDsResp{
+		Version:   idHash,
+		VersionID: vl.ID.Hex(),
+		Equal:     req.IdHash == idHash,
+		GroupIDs:  groupIDs,
+	}, nil
+}
+
+func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgroup.GetIncrementalGroupMemberReq) (*pbgroup.GetIncrementalGroupMemberResp, error) {
+	group, err := s.db.TakeGroup(ctx, req.GroupID)
+	if err != nil {
+		return nil, err
+	}
+	if group.Status == constant.GroupStatusDismissed {
+		return nil, servererrs.ErrDismissedAlready.Wrap()
+	}
+	var hasGroupUpdate bool
+	opt := incrversion.Option[*sdkws.GroupMemberFullInfo, pbgroup.GetIncrementalGroupMemberResp]{
+		Ctx:           ctx,
+		VersionKey:    req.GroupID,
+		VersionID:     req.VersionID,
+		VersionNumber: req.Version,
+		Version: func(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) {
+			vl, err := s.db.FindMemberIncrVersion(ctx, groupID, version, limit)
+			if err != nil {
+				return nil, err
+			}
+			vl.Logs = slices.DeleteFunc(vl.Logs, func(elem model.VersionLogElem) bool {
+				if elem.EID == "" {
+					vl.LogLen--
+					hasGroupUpdate = true
+					return true
+				}
+				return false
+			})
+			if vl.LogLen > 0 {
+				hasGroupUpdate = true
+			}
+			return vl, nil
+		},
+		CacheMaxVersion: s.db.FindMaxGroupMemberVersionCache,
+		Find: func(ctx context.Context, ids []string) ([]*sdkws.GroupMemberFullInfo, error) {
+			return s.getGroupMembersInfo(ctx, req.GroupID, ids)
+		},
+		ID: func(elem *sdkws.GroupMemberFullInfo) string { return elem.UserID },
+		Resp: func(version *model.VersionLog, delIDs []string, insertList, updateList []*sdkws.GroupMemberFullInfo, full bool) *pbgroup.GetIncrementalGroupMemberResp {
+			return &pbgroup.GetIncrementalGroupMemberResp{
+				VersionID: version.ID.Hex(),
+				Version:   uint64(version.Version),
+				Full:      full,
+				Delete:    delIDs,
+				Insert:    insertList,
+				Update:    updateList,
+			}
+		},
+	}
+	resp, err := opt.Build()
+	if err != nil {
+		return nil, err
+	}
+	if resp.Full || hasGroupUpdate {
+		count, err := s.db.FindGroupMemberNum(ctx, group.GroupID)
+		if err != nil {
+			return nil, err
+		}
+		owner, err := s.db.TakeGroupOwner(ctx, group.GroupID)
+		if err != nil {
+			return nil, err
+		}
+		resp.Group = s.groupDB2PB(group, owner.UserID, count)
+	}
+	return resp, nil
+}
+
+func (s *groupServer) GetIncrementalJoinGroup(ctx context.Context, req *pbgroup.GetIncrementalJoinGroupReq) (*pbgroup.GetIncrementalJoinGroupResp, error) {
+	if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil {
+		return nil, err
+	}
+	opt := incrversion.Option[*sdkws.GroupInfo, pbgroup.GetIncrementalJoinGroupResp]{
+		Ctx:             ctx,
+		VersionKey:      req.UserID,
+		VersionID:       req.VersionID,
+		VersionNumber:   req.Version,
+		Version:         s.db.FindJoinIncrVersion,
+		CacheMaxVersion: s.db.FindMaxJoinGroupVersionCache,
+		Find:            s.getGroupsInfo,
+		ID:              func(elem *sdkws.GroupInfo) string { return elem.GroupID },
+		Resp: func(version *model.VersionLog, delIDs []string, insertList, updateList []*sdkws.GroupInfo, full bool) *pbgroup.GetIncrementalJoinGroupResp {
+			return &pbgroup.GetIncrementalJoinGroupResp{
+				VersionID: version.ID.Hex(),
+				Version:   uint64(version.Version),
+				Full:      full,
+				Delete:    delIDs,
+				Insert:    insertList,
+				Update:    updateList,
+			}
+		},
+	}
+	return opt.Build()
+}