feat: support incremental synchronization (#2379)

* fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * friend incr sync * friend incr sync * friend incr sync * friend incr sync * friend incr sync * mage * optimization version log * optimization version log * sync * sync * sync * group sync * sync option * sync option * refactor: replace `friend` package with `realtion`. * refactor: update lastest commit to relation. * sync option * sync option * sync option * sync * sync * go.mod * update: go mod * refactor: change incremental to full * feat: get full friend user ids * feat: api and config * group version * merge * fix: sort by id avoid unstable sort friends. * group * group * group * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * user version * fix: sort by id avoid unstable sort friends. * test: test log add. * test: debug log remove. * fix: transfer group owner incr version more than 1. * fix: add condition to kick owner. * feat: replace resp nil * feat: replace nil * fix: delete cache of max group joined version avoid sync joined group failed. * fix: nil * fix: delete cache of max group joined version avoid sync joined group failed. * fix: delete cache of max group joined version avoid sync joined group failed. * return group information for any changes * online cache --------- Co-authored-by: withchao <withchao@users.noreply.github.com> Co-authored-by: Monet Lee <monet_lee@163.com> Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <1186114839@qq.com>
2026-05-07 10:35:59 +08:00 · 2024-07-01 17:38:14 +08:00
parent fe7c029c2a
commit 88c0d5f5ad
56 changed files with 2023 additions and 325 deletions
@@ -16,6 +16,7 @@ package database

 import (
 	"context"
+
 	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
 	"github.com/openimsdk/tools/db/pagination"
 )
@@ -46,4 +47,14 @@ type Friend interface {
 	FindFriendUserIDs(ctx context.Context, ownerUserID string) (friendUserIDs []string, err error)
 	// UpdateFriends update friends' fields
 	UpdateFriends(ctx context.Context, ownerUserID string, friendUserIDs []string, val map[string]any) (err error)
+
+	FindIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error)
+
+	FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error)
+
+	//SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error)
+
+	FindOwnerFriendUserIds(ctx context.Context, ownerUserID string, limit int) ([]string, error)
+
+	IncrVersion(ctx context.Context, ownerUserID string, friendUserIDs []string, state int32) error
 }
@@ -32,4 +32,8 @@ type Group interface {
 	CountTotal(ctx context.Context, before *time.Time) (count int64, err error)
 	// Get Group total quantity every day
 	CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error)
+
+	FindJoinSortGroupID(ctx context.Context, groupIDs []string) ([]string, error)
+
+	SearchJoin(ctx context.Context, groupIDs []string, keyword string, pagination pagination.Pagination) (int64, []*model.Group, error)
 }
@@ -25,6 +25,7 @@ type GroupMember interface {
 	Delete(ctx context.Context, groupID string, userIDs []string) (err error)
 	Update(ctx context.Context, groupID string, userID string, data map[string]any) (err error)
 	UpdateRoleLevel(ctx context.Context, groupID string, userID string, roleLevel int32) error
+	UpdateUserRoleLevels(ctx context.Context, groupID string, firstUserID string, firstUserRoleLevel int32, secondUserID string, secondUserRoleLevel int32) error
 	FindMemberUserID(ctx context.Context, groupID string) (userIDs []string, err error)
 	Take(ctx context.Context, groupID string, userID string) (groupMember *model.GroupMember, err error)
 	TakeOwner(ctx context.Context, groupID string) (groupMember *model.GroupMember, err error)
@@ -34,4 +35,8 @@ type GroupMember interface {
 	TakeGroupMemberNum(ctx context.Context, groupID string) (count int64, err error)
 	FindUserManagedGroupID(ctx context.Context, userID string) (groupIDs []string, err error)
 	IsUpdateRoleLevel(data map[string]any) bool
+	JoinGroupIncrVersion(ctx context.Context, userID string, groupIDs []string, state int32) error
+	MemberGroupIncrVersion(ctx context.Context, groupID string, userIDs []string, state int32) error
+	FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error)
+	FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error)
 }
@@ -27,7 +27,7 @@ import (
 )

 func NewBlackMongo(db *mongo.Database) (database.Black, error) {
-	coll := db.Collection("black")
+	coll := db.Collection(database.BlackName)
 	_, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{
 		Keys: bson.D{
 			{Key: "owner_user_id", Value: 1},
@@ -16,6 +16,7 @@ package mgo

 import (
 	"context"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/database"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
 	"time"

@@ -29,7 +30,7 @@ import (
 )

 func NewConversationMongo(db *mongo.Database) (*ConversationMgo, error) {
-	coll := db.Collection("conversation")
+	coll := db.Collection(database.ConversationName)
 	_, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{
 		Keys: bson.D{
 			{Key: "owner_user_id", Value: 1},
@@ -18,6 +18,8 @@ import (
 	"context"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/database"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"

 	"github.com/openimsdk/tools/db/mongoutil"
 	"github.com/openimsdk/tools/db/pagination"
@@ -28,12 +30,13 @@ import (

 // FriendMgo implements Friend using MongoDB as the storage backend.
 type FriendMgo struct {
-	coll *mongo.Collection
+	coll  *mongo.Collection
+	owner database.VersionLog
 }

 // NewFriendMongo creates a new instance of FriendMgo with the provided MongoDB database.
 func NewFriendMongo(db *mongo.Database) (database.Friend, error) {
-	coll := db.Collection("friend")
+	coll := db.Collection(database.FriendName)
 	_, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{
 		Keys: bson.D{
 			{Key: "owner_user_id", Value: 1},
@@ -44,12 +47,41 @@ func NewFriendMongo(db *mongo.Database) (database.Friend, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &FriendMgo{coll: coll}, nil
+	owner, err := NewVersionLog(db.Collection(database.FriendVersionName))
+	if err != nil {
+		return nil, err
+	}
+	return &FriendMgo{coll: coll, owner: owner}, nil
+}
+
+func (f *FriendMgo) friendSort() any {
+	return bson.D{{"is_pinned", -1}, {"_id", 1}}
 }

 // Create inserts multiple friend records.
 func (f *FriendMgo) Create(ctx context.Context, friends []*model.Friend) error {
-	return mongoutil.InsertMany(ctx, f.coll, friends)
+	for i, friend := range friends {
+		if friend.ID.IsZero() {
+			friends[i].ID = primitive.NewObjectID()
+		}
+		if friend.CreateTime.IsZero() {
+			friends[i].CreateTime = time.Now()
+		}
+	}
+	return mongoutil.IncrVersion(func() error {
+		return mongoutil.InsertMany(ctx, f.coll, friends)
+	}, func() error {
+		mp := make(map[string][]string)
+		for _, friend := range friends {
+			mp[friend.OwnerUserID] = append(mp[friend.OwnerUserID], friend.FriendUserID)
+		}
+		for ownerUserID, friendUserIDs := range mp {
+			if err := f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, model.VersionStateInsert); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
 }

 // Delete removes specified friends of the owner user.
@@ -58,11 +90,15 @@ func (f *FriendMgo) Delete(ctx context.Context, ownerUserID string, friendUserID
 		"owner_user_id":  ownerUserID,
 		"friend_user_id": bson.M{"$in": friendUserIDs},
 	}
-	return mongoutil.DeleteOne(ctx, f.coll, filter)
+	return mongoutil.IncrVersion(func() error {
+		return mongoutil.DeleteOne(ctx, f.coll, filter)
+	}, func() error {
+		return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, model.VersionStateDelete)
+	})
 }

 // UpdateByMap updates specific fields of a friend document using a map.
-func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendUserID string, args map[string]interface{}) error {
+func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendUserID string, args map[string]any) error {
 	if len(args) == 0 {
 		return nil
 	}
@@ -70,30 +106,55 @@ func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendU
 		"owner_user_id":  ownerUserID,
 		"friend_user_id": friendUserID,
 	}
-	return mongoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": args}, true)
+	return mongoutil.IncrVersion(func() error {
+		return mongoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": args}, true)
+	}, func() error {
+		return f.owner.IncrVersion(ctx, ownerUserID, []string{friendUserID}, model.VersionStateUpdate)
+	})
 }

-// Update modifies multiple friend documents.
-// func (f *FriendMgo) Update(ctx context.Context, friends []*relation.Friend) error {
-// 	filter := bson.M{
-// 		"owner_user_id":  ownerUserID,
-// 		"friend_user_id": friendUserID,
-// 	}
-// 	return mgotool.UpdateMany(ctx, f.coll, filter, friends)
-// }
-
 // UpdateRemark updates the remark for a specific friend.
 func (f *FriendMgo) UpdateRemark(ctx context.Context, ownerUserID, friendUserID, remark string) error {
 	return f.UpdateByMap(ctx, ownerUserID, friendUserID, map[string]any{"remark": remark})
 }

+func (f *FriendMgo) fillTime(friends ...*model.Friend) {
+	for i, friend := range friends {
+		if friend.CreateTime.IsZero() {
+			friends[i].CreateTime = friend.ID.Timestamp()
+		}
+	}
+}
+
+func (f *FriendMgo) findOne(ctx context.Context, filter any) (*model.Friend, error) {
+	friend, err := mongoutil.FindOne[*model.Friend](ctx, f.coll, filter)
+	if err != nil {
+		return nil, err
+	}
+	f.fillTime(friend)
+	return friend, nil
+}
+
+func (f *FriendMgo) find(ctx context.Context, filter any) ([]*model.Friend, error) {
+	friends, err := mongoutil.Find[*model.Friend](ctx, f.coll, filter)
+	if err != nil {
+		return nil, err
+	}
+	f.fillTime(friends...)
+	return friends, nil
+}
+
+func (f *FriendMgo) findPage(ctx context.Context, filter any, pagination pagination.Pagination, opts ...*options.FindOptions) (int64, []*model.Friend, error) {
+	return mongoutil.FindPage[*model.Friend](ctx, f.coll, filter, pagination, opts...)
+}
+
 // Take retrieves a single friend document. Returns an error if not found.
 func (f *FriendMgo) Take(ctx context.Context, ownerUserID, friendUserID string) (*model.Friend, error) {
 	filter := bson.M{
 		"owner_user_id":  ownerUserID,
 		"friend_user_id": friendUserID,
 	}
-	return mongoutil.FindOne[*model.Friend](ctx, f.coll, filter)
+	return f.findOne(ctx, filter)
 }

 // FindUserState finds the friendship status between two users.
@@ -104,7 +165,7 @@ func (f *FriendMgo) FindUserState(ctx context.Context, userID1, userID2 string)
 			{"owner_user_id": userID2, "friend_user_id": userID1},
 		},
 	}
-	return mongoutil.Find[*model.Friend](ctx, f.coll, filter)
+	return f.find(ctx, filter)
 }

 // FindFriends retrieves a list of friends for a given owner. Missing friends do not cause an error.
@@ -113,7 +174,7 @@ func (f *FriendMgo) FindFriends(ctx context.Context, ownerUserID string, friendU
 		"owner_user_id":  ownerUserID,
 		"friend_user_id": bson.M{"$in": friendUserIDs},
 	}
-	return mongoutil.Find[*model.Friend](ctx, f.coll, filter)
+	return f.find(ctx, filter)
 }

 // FindReversalFriends finds users who have added the specified user as a friend.
@@ -122,25 +183,33 @@ func (f *FriendMgo) FindReversalFriends(ctx context.Context, friendUserID string
 		"owner_user_id":  bson.M{"$in": ownerUserIDs},
 		"friend_user_id": friendUserID,
 	}
-	return mongoutil.Find[*model.Friend](ctx, f.coll, filter)
+	return f.find(ctx, filter)
 }

 // FindOwnerFriends retrieves a paginated list of friends for a given owner.
 func (f *FriendMgo) FindOwnerFriends(ctx context.Context, ownerUserID string, pagination pagination.Pagination) (int64, []*model.Friend, error) {
 	filter := bson.M{"owner_user_id": ownerUserID}
-	return mongoutil.FindPage[*model.Friend](ctx, f.coll, filter, pagination)
+	opt := options.Find().SetSort(f.friendSort())
+	return f.findPage(ctx, filter, pagination, opt)
+}
+
+func (f *FriendMgo) FindOwnerFriendUserIds(ctx context.Context, ownerUserID string, limit int) ([]string, error) {
+	filter := bson.M{"owner_user_id": ownerUserID}
+	opt := options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1}).SetSort(f.friendSort()).SetLimit(int64(limit))
+	return mongoutil.Find[string](ctx, f.coll, filter, opt)
 }

 // FindInWhoseFriends finds users who have added the specified user as a friend, with pagination.
 func (f *FriendMgo) FindInWhoseFriends(ctx context.Context, friendUserID string, pagination pagination.Pagination) (int64, []*model.Friend, error) {
 	filter := bson.M{"friend_user_id": friendUserID}
-	return mongoutil.FindPage[*model.Friend](ctx, f.coll, filter, pagination)
+	opt := options.Find().SetSort(f.friendSort())
+	return f.findPage(ctx, filter, pagination, opt)
 }

 // FindFriendUserIDs retrieves a list of friend user IDs for a given owner.
 func (f *FriendMgo) FindFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) {
 	filter := bson.M{"owner_user_id": ownerUserID}
-	return mongoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1}))
+	return mongoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1}).SetSort(f.friendSort()))
 }

 func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, friendUserIDs []string, val map[string]any) error {
@@ -158,7 +227,24 @@ func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, frien
 	// Create an update document
 	update := bson.M{"$set": val}

-	// Perform the update operation for all matching documents
-	_, err := mongoutil.UpdateMany(ctx, f.coll, filter, update)
-	return err
+	return mongoutil.IncrVersion(func() error {
+		return mongoutil.Ignore(mongoutil.UpdateMany(ctx, f.coll, filter, update))
+	}, func() error {
+		return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, model.VersionStateUpdate)
+	})
+}
+
+func (f *FriendMgo) FindIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) {
+	return f.owner.FindChangeLog(ctx, ownerUserID, version, limit)
+}
+
+func (f *FriendMgo) FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) {
+	filter := bson.M{
+		"friend_user_id": friendUserID,
+	}
+	return mongoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1}).SetSort(f.friendSort()))
+}
+
+func (f *FriendMgo) IncrVersion(ctx context.Context, ownerUserID string, friendUserIDs []string, state int32) error {
+	return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, state)
 }
@@ -27,7 +27,7 @@ import (
 )

 func NewFriendRequestMongo(db *mongo.Database) (database.FriendRequest, error) {
-	coll := db.Collection("friend_request")
+	coll := db.Collection(database.FriendRequestName)
 	_, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{
 		Keys: bson.D{
 			{Key: "from_user_id", Value: 1},
@@ -30,7 +30,7 @@ import (
 )

 func NewGroupMongo(db *mongo.Database) (database.Group, error) {
-	coll := db.Collection("group")
+	coll := db.Collection(database.GroupName)
 	_, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{
 		Keys: bson.D{
 			{Key: "group_id", Value: 1},
@@ -47,6 +47,10 @@ type GroupMgo struct {
 	coll *mongo.Collection
 }

+func (g *GroupMgo) sortGroup() any {
+	return bson.D{{"group_name", 1}, {"create_time", 1}}
+}
+
 func (g *GroupMgo) Create(ctx context.Context, groups []*model.Group) (err error) {
 	return mongoutil.InsertMany(ctx, g.coll, groups)
 }
@@ -126,3 +130,32 @@ func (g *GroupMgo) CountRangeEverydayTotal(ctx context.Context, start time.Time,
 	}
 	return res, nil
 }
+
+func (g *GroupMgo) FindJoinSortGroupID(ctx context.Context, groupIDs []string) ([]string, error) {
+	if len(groupIDs) < 2 {
+		return groupIDs, nil
+	}
+	filter := bson.M{
+		"group_id": bson.M{"$in": groupIDs},
+		"status":   bson.M{"$ne": constant.GroupStatusDismissed},
+	}
+	opt := options.Find().SetSort(g.sortGroup()).SetProjection(bson.M{"_id": 0, "group_id": 1})
+	return mongoutil.Find[string](ctx, g.coll, filter, opt)
+}
+
+func (g *GroupMgo) SearchJoin(ctx context.Context, groupIDs []string, keyword string, pagination pagination.Pagination) (int64, []*model.Group, error) {
+	if len(groupIDs) == 0 {
+		return 0, nil, nil
+	}
+	filter := bson.M{
+		"group_id": bson.M{"$in": groupIDs},
+		"status":   bson.M{"$ne": constant.GroupStatusDismissed},
+	}
+	if keyword != "" {
+		filter["group_name"] = bson.M{"$regex": keyword}
+	}
+	// Define the sorting options
+	opts := options.Find().SetSort(g.sortGroup())
+	// Perform the search with pagination and sorting
+	return mongoutil.FindPage[*model.Group](ctx, g.coll, filter, pagination, opts)
+}
@@ -18,6 +18,7 @@ import (
 	"context"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/database"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
+	"github.com/openimsdk/tools/log"

 	"github.com/openimsdk/protocol/constant"
 	"github.com/openimsdk/tools/db/mongoutil"
@@ -29,7 +30,7 @@ import (
 )

 func NewGroupMember(db *mongo.Database) (database.GroupMember, error) {
-	coll := db.Collection("group_member")
+	coll := db.Collection(database.GroupMemberName)
 	_, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{
 		Keys: bson.D{
 			{Key: "group_id", Value: 1},
@@ -40,15 +41,53 @@ func NewGroupMember(db *mongo.Database) (database.GroupMember, error) {
 	if err != nil {
 		return nil, errs.Wrap(err)
 	}
-	return &GroupMemberMgo{coll: coll}, nil
+	member, err := NewVersionLog(db.Collection(database.GroupMemberVersionName))
+	if err != nil {
+		return nil, err
+	}
+	join, err := NewVersionLog(db.Collection(database.GroupJoinVersionName))
+	if err != nil {
+		return nil, err
+	}
+	return &GroupMemberMgo{coll: coll, member: member, join: join}, nil
 }

 type GroupMemberMgo struct {
-	coll *mongo.Collection
+	coll   *mongo.Collection
+	member database.VersionLog
+	join   database.VersionLog
+}
+
+func (g *GroupMemberMgo) memberSort() any {
+	return bson.D{{"role_level", -1}, {"create_time", -1}}
 }

 func (g *GroupMemberMgo) Create(ctx context.Context, groupMembers []*model.GroupMember) (err error) {
-	return mongoutil.InsertMany(ctx, g.coll, groupMembers)
+	return mongoutil.IncrVersion(func() error {
+		return mongoutil.InsertMany(ctx, g.coll, groupMembers)
+	}, func() error {
+		gms := make(map[string][]string)
+		for _, member := range groupMembers {
+			gms[member.GroupID] = append(gms[member.GroupID], member.UserID)
+		}
+		for groupID, userIDs := range gms {
+			if err := g.member.IncrVersion(ctx, groupID, userIDs, model.VersionStateInsert); err != nil {
+				return err
+			}
+		}
+		return nil
+	}, func() error {
+		gms := make(map[string][]string)
+		for _, member := range groupMembers {
+			gms[member.UserID] = append(gms[member.UserID], member.GroupID)
+		}
+		for userID, groupIDs := range gms {
+			if err := g.join.IncrVersion(ctx, userID, groupIDs, model.VersionStateInsert); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
 }

 func (g *GroupMemberMgo) Delete(ctx context.Context, groupID string, userIDs []string) (err error) {
@@ -56,24 +95,62 @@ func (g *GroupMemberMgo) Delete(ctx context.Context, groupID string, userIDs []s
 	if len(userIDs) > 0 {
 		filter["user_id"] = bson.M{"$in": userIDs}
 	}
-	return mongoutil.DeleteMany(ctx, g.coll, filter)
+	return mongoutil.IncrVersion(func() error {
+		return mongoutil.DeleteMany(ctx, g.coll, filter)
+	}, func() error {
+		if len(userIDs) == 0 {
+			return g.member.Delete(ctx, groupID)
+		} else {
+			return g.member.IncrVersion(ctx, groupID, userIDs, model.VersionStateDelete)
+		}
+	}, func() error {
+		for _, userID := range userIDs {
+			if err := g.join.IncrVersion(ctx, userID, []string{groupID}, model.VersionStateDelete); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
 }

 func (g *GroupMemberMgo) UpdateRoleLevel(ctx context.Context, groupID string, userID string, roleLevel int32) error {
-	return g.Update(ctx, groupID, userID, bson.M{"role_level": roleLevel})
+	return mongoutil.IncrVersion(func() error {
+		return mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID},
+			bson.M{"$set": bson.M{"role_level": roleLevel}}, true)
+	}, func() error {
+		return g.member.IncrVersion(ctx, groupID, []string{userID}, model.VersionStateUpdate)
+	})
+}
+func (g *GroupMemberMgo) UpdateUserRoleLevels(ctx context.Context, groupID string, firstUserID string, firstUserRoleLevel int32, secondUserID string, secondUserRoleLevel int32) error {
+	return mongoutil.IncrVersion(func() error {
+		if err := mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": firstUserID},
+			bson.M{"$set": bson.M{"role_level": firstUserRoleLevel}}, true); err != nil {
+			return err
+		}
+		if err := mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": secondUserID},
+			bson.M{"$set": bson.M{"role_level": secondUserRoleLevel}}, true); err != nil {
+			return err
+		}
+
+		return nil
+	}, func() error {
+		return g.member.IncrVersion(ctx, groupID, []string{firstUserID, secondUserID}, model.VersionStateUpdate)
+	})
 }

 func (g *GroupMemberMgo) Update(ctx context.Context, groupID string, userID string, data map[string]any) (err error) {
-	return mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID}, bson.M{"$set": data}, true)
-}
-
-func (g *GroupMemberMgo) Find(ctx context.Context, groupIDs []string, userIDs []string, roleLevels []int32) (groupMembers []*model.GroupMember, err error) {
-	// TODO implement me
-	panic("implement me")
+	if len(data) == 0 {
+		return nil
+	}
+	return mongoutil.IncrVersion(func() error {
+		return mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID}, bson.M{"$set": data}, true)
+	}, func() error {
+		return g.member.IncrVersion(ctx, groupID, []string{userID}, model.VersionStateUpdate)
+	})
 }

 func (g *GroupMemberMgo) FindMemberUserID(ctx context.Context, groupID string) (userIDs []string, err error) {
-	return mongoutil.Find[string](ctx, g.coll, bson.M{"group_id": groupID}, options.Find().SetProjection(bson.M{"_id": 0, "user_id": 1}))
+	return mongoutil.Find[string](ctx, g.coll, bson.M{"group_id": groupID}, options.Find().SetProjection(bson.M{"_id": 0, "user_id": 1}).SetSort(g.memberSort()))
 }

 func (g *GroupMemberMgo) Take(ctx context.Context, groupID string, userID string) (groupMember *model.GroupMember, err error) {
@@ -88,13 +165,13 @@ func (g *GroupMemberMgo) FindRoleLevelUserIDs(ctx context.Context, groupID strin
 	return mongoutil.Find[string](ctx, g.coll, bson.M{"group_id": groupID, "role_level": roleLevel}, options.Find().SetProjection(bson.M{"_id": 0, "user_id": 1}))
 }

-func (g *GroupMemberMgo) SearchMember(ctx context.Context, keyword string, groupID string, pagination pagination.Pagination) (total int64, groupList []*model.GroupMember, err error) {
+func (g *GroupMemberMgo) SearchMember(ctx context.Context, keyword string, groupID string, pagination pagination.Pagination) (int64, []*model.GroupMember, error) {
 	filter := bson.M{"group_id": groupID, "nickname": bson.M{"$regex": keyword}}
-	return mongoutil.FindPage[*model.GroupMember](ctx, g.coll, filter, pagination)
+	return mongoutil.FindPage[*model.GroupMember](ctx, g.coll, filter, pagination, options.Find().SetSort(g.memberSort()))
 }

 func (g *GroupMemberMgo) FindUserJoinedGroupID(ctx context.Context, userID string) (groupIDs []string, err error) {
-	return mongoutil.Find[string](ctx, g.coll, bson.M{"user_id": userID}, options.Find().SetProjection(bson.M{"_id": 0, "group_id": 1}))
+	return mongoutil.Find[string](ctx, g.coll, bson.M{"user_id": userID}, options.Find().SetProjection(bson.M{"_id": 0, "group_id": 1}).SetSort(g.memberSort()))
 }

 func (g *GroupMemberMgo) TakeGroupMemberNum(ctx context.Context, groupID string) (count int64, err error) {
@@ -118,3 +195,21 @@ func (g *GroupMemberMgo) IsUpdateRoleLevel(data map[string]any) bool {
 	_, ok := data["role_level"]
 	return ok
 }
+
+func (g *GroupMemberMgo) JoinGroupIncrVersion(ctx context.Context, userID string, groupIDs []string, state int32) error {
+	return g.join.IncrVersion(ctx, userID, groupIDs, state)
+}
+
+func (g *GroupMemberMgo) MemberGroupIncrVersion(ctx context.Context, groupID string, userIDs []string, state int32) error {
+	return g.member.IncrVersion(ctx, groupID, userIDs, state)
+}
+
+func (g *GroupMemberMgo) FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) {
+	log.ZDebug(ctx, "find member incr version", "groupID", groupID, "version", version)
+	return g.member.FindChangeLog(ctx, groupID, version, limit)
+}
+
+func (g *GroupMemberMgo) FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) {
+	log.ZDebug(ctx, "find join incr version", "userID", userID, "version", version)
+	return g.join.FindChangeLog(ctx, userID, version, limit)
+}
@@ -28,7 +28,7 @@ import (
 )

 func NewGroupRequestMgo(db *mongo.Database) (database.GroupRequest, error) {
-	coll := db.Collection("group_request")
+	coll := db.Collection(database.GroupRequestName)
 	_, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{
 		Keys: bson.D{
 			{Key: "group_id", Value: 1},
@@ -28,7 +28,7 @@ import (
 )

 func NewLogMongo(db *mongo.Database) (database.Log, error) {
-	coll := db.Collection("log")
+	coll := db.Collection(database.LogName)
 	_, err := coll.Indexes().CreateMany(context.Background(), []mongo.IndexModel{
 		{
 			Keys: bson.D{
@@ -27,7 +27,7 @@ import (
 )

 func NewS3Mongo(db *mongo.Database) (database.ObjectInfo, error) {
-	coll := db.Collection("s3")
+	coll := db.Collection(database.ObjectName)
 	_, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{
 		Keys: bson.D{
 			{Key: "name", Value: 1},
@@ -31,7 +31,7 @@ import (
 )

 func NewUserMongo(db *mongo.Database) (database.User, error) {
-	coll := db.Collection("user")
+	coll := db.Collection(database.UserName)
 	_, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{
 		Keys: bson.D{
 			{Key: "user_id", Value: 1},
@@ -319,3 +319,69 @@ func (u *UserMgo) CountRangeEverydayTotal(ctx context.Context, start time.Time,
 	}
 	return res, nil
 }
+
+func (u *UserMgo) SortQuery(ctx context.Context, userIDName map[string]string, asc bool) ([]*model.User, error) {
+	if len(userIDName) == 0 {
+		return nil, nil
+	}
+	userIDs := make([]string, 0, len(userIDName))
+	attached := make(map[string]string)
+	for userID, name := range userIDName {
+		userIDs = append(userIDs, userID)
+		if name == "" {
+			continue
+		}
+		attached[userID] = name
+	}
+	var sortValue int
+	if asc {
+		sortValue = 1
+	} else {
+		sortValue = -1
+	}
+	if len(attached) == 0 {
+		filter := bson.M{"user_id": bson.M{"$in": userIDs}}
+		opt := options.Find().SetSort(bson.M{"nickname": sortValue})
+		return mongoutil.Find[*model.User](ctx, u.coll, filter, opt)
+	}
+	pipeline := []bson.M{
+		{
+			"$match": bson.M{
+				"user_id": bson.M{"$in": userIDs},
+			},
+		},
+		{
+			"$addFields": bson.M{
+				"_query_sort_name": bson.M{
+					"$arrayElemAt": []any{
+						bson.M{
+							"$filter": bson.M{
+								"input": bson.M{
+									"$objectToArray": attached,
+								},
+								"as": "item",
+								"cond": bson.M{
+									"$eq": []any{"$$item.k", "$user_id"},
+								},
+							},
+						},
+						0,
+					},
+				},
+			},
+		},
+		{
+			"$addFields": bson.M{
+				"_query_sort_name": bson.M{
+					"$ifNull": []any{"$_query_sort_name.v", "$nickname"},
+				},
+			},
+		},
+		{
+			"$sort": bson.M{
+				"_query_sort_name": sortValue,
+			},
+		},
+	}
+	return mongoutil.Aggregate[*model.User](ctx, u.coll, pipeline)
+}
@@ -0,0 +1,265 @@
+package mgo
+
+import (
+	"context"
+	"errors"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/database"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx"
+	"github.com/openimsdk/tools/db/mongoutil"
+	"github.com/openimsdk/tools/errs"
+	"github.com/openimsdk/tools/log"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+	"time"
+)
+
+func NewVersionLog(coll *mongo.Collection) (database.VersionLog, error) {
+	lm := &VersionLogMgo{coll: coll}
+	if lm.initIndex(context.Background()) != nil {
+		return nil, errs.ErrInternalServer.WrapMsg("init index failed", "coll", coll.Name())
+	}
+	return lm, nil
+}
+
+type VersionLogMgo struct {
+	coll *mongo.Collection
+}
+
+func (l *VersionLogMgo) initIndex(ctx context.Context) error {
+	_, err := l.coll.Indexes().CreateOne(ctx, mongo.IndexModel{
+		Keys: bson.M{
+			"d_id": 1,
+		},
+	})
+	return err
+}
+
+func (l *VersionLogMgo) IncrVersion(ctx context.Context, dId string, eIds []string, state int32) error {
+	_, err := l.IncrVersionResult(ctx, dId, eIds, state)
+	return err
+}
+
+func (l *VersionLogMgo) IncrVersionResult(ctx context.Context, dId string, eIds []string, state int32) (*model.VersionLog, error) {
+	vl, err := l.incrVersionResult(ctx, dId, eIds, state)
+	if err != nil {
+		return nil, err
+	}
+	versionctx.GetVersionLog(ctx).Append(versionctx.Collection{
+		Name: l.coll.Name(),
+		Doc:  vl,
+	})
+	return vl, nil
+}
+
+func (l *VersionLogMgo) incrVersionResult(ctx context.Context, dId string, eIds []string, state int32) (*model.VersionLog, error) {
+	if len(eIds) == 0 {
+		return nil, errs.ErrArgs.WrapMsg("elem id is empty", "dId", dId)
+	}
+	now := time.Now()
+	if res, err := l.writeLogBatch2(ctx, dId, eIds, state, now); err == nil {
+		return res, nil
+	} else if !errors.Is(err, mongo.ErrNoDocuments) {
+		return nil, err
+	}
+	if res, err := l.initDoc(ctx, dId, eIds, state, now); err == nil {
+		return res, nil
+	} else if !mongo.IsDuplicateKeyError(err) {
+		return nil, err
+	}
+	return l.writeLogBatch2(ctx, dId, eIds, state, now)
+}
+
+func (l *VersionLogMgo) initDoc(ctx context.Context, dId string, eIds []string, state int32, now time.Time) (*model.VersionLog, error) {
+	wl := model.VersionLogTable{
+		ID:         primitive.NewObjectID(),
+		DID:        dId,
+		Logs:       make([]model.VersionLogElem, 0, len(eIds)),
+		Version:    database.FirstVersion,
+		Deleted:    database.DefaultDeleteVersion,
+		LastUpdate: now,
+	}
+	for _, eId := range eIds {
+		wl.Logs = append(wl.Logs, model.VersionLogElem{
+			EID:        eId,
+			State:      state,
+			Version:    database.FirstVersion,
+			LastUpdate: now,
+		})
+	}
+	if _, err := l.coll.InsertOne(ctx, &wl); err != nil {
+		return nil, err
+	}
+	return wl.VersionLog(), nil
+}
+
+func (l *VersionLogMgo) writeLogBatch2(ctx context.Context, dId string, eIds []string, state int32, now time.Time) (*model.VersionLog, error) {
+	if eIds == nil {
+		eIds = []string{}
+	}
+	filter := bson.M{
+		"d_id": dId,
+	}
+	elems := make([]bson.M, 0, len(eIds))
+	for _, eId := range eIds {
+		elems = append(elems, bson.M{
+			"e_id":        eId,
+			"version":     "$version",
+			"state":       state,
+			"last_update": now,
+		})
+	}
+	pipeline := []bson.M{
+		{
+			"$addFields": bson.M{
+				"delete_e_ids": eIds,
+			},
+		},
+		{
+			"$set": bson.M{
+				"version":     bson.M{"$add": []any{"$version", 1}},
+				"last_update": now,
+			},
+		},
+		{
+			"$set": bson.M{
+				"logs": bson.M{
+					"$filter": bson.M{
+						"input": "$logs",
+						"as":    "log",
+						"cond": bson.M{
+							"$not": bson.M{
+								"$in": []any{"$$log.e_id", "$delete_e_ids"},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			"$set": bson.M{
+				"logs": bson.M{
+					"$concatArrays": []any{
+						"$logs",
+						elems,
+					},
+				},
+			},
+		},
+		{
+			"$unset": "delete_e_ids",
+		},
+	}
+	opt := options.FindOneAndUpdate().SetUpsert(false).SetReturnDocument(options.After).SetProjection(bson.M{"logs": 0})
+	return mongoutil.FindOneAndUpdate[*model.VersionLog](ctx, l.coll, filter, pipeline, opt)
+}
+
+func (l *VersionLogMgo) findDoc(ctx context.Context, dId string) (*model.VersionLog, error) {
+	vl, err := mongoutil.FindOne[*model.VersionLogTable](ctx, l.coll, bson.M{"d_id": dId}, options.FindOne().SetProjection(bson.M{"logs": 0}))
+	if err != nil {
+		return nil, err
+	}
+	return vl.VersionLog(), nil
+}
+
+func (l *VersionLogMgo) FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error) {
+	if wl, err := l.findChangeLog(ctx, dId, version, limit); err == nil {
+		return wl, nil
+	} else if !errors.Is(err, mongo.ErrNoDocuments) {
+		return nil, err
+	}
+	log.ZDebug(ctx, "init doc", "dId", dId)
+	if res, err := l.initDoc(ctx, dId, nil, 0, time.Now()); err == nil {
+		log.ZDebug(ctx, "init doc success", "dId", dId)
+		return res, nil
+	} else if mongo.IsDuplicateKeyError(err) {
+		return l.findChangeLog(ctx, dId, version, limit)
+	} else {
+		return nil, err
+	}
+}
+
+func (l *VersionLogMgo) findChangeLog(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error) {
+	if version == 0 && limit == 0 {
+		return l.findDoc(ctx, dId)
+	}
+	pipeline := []bson.M{
+		{
+			"$match": bson.M{
+				"d_id": dId,
+			},
+		},
+		{
+			"$addFields": bson.M{
+				"logs": bson.M{
+					"$cond": bson.M{
+						"if": bson.M{
+							"$or": []bson.M{
+								{"$lt": []any{"$version", version}},
+								{"$gte": []any{"$deleted", version}},
+							},
+						},
+						"then": []any{},
+						"else": "$logs",
+					},
+				},
+			},
+		},
+		{
+			"$addFields": bson.M{
+				"logs": bson.M{
+					"$filter": bson.M{
+						"input": "$logs",
+						"as":    "l",
+						"cond": bson.M{
+							"$gt": []any{"$$l.version", version},
+						},
+					},
+				},
+			},
+		},
+		{
+			"$addFields": bson.M{
+				"log_len": bson.M{"$size": "$logs"},
+			},
+		},
+		{
+			"$addFields": bson.M{
+				"logs": bson.M{
+					"$cond": bson.M{
+						"if": bson.M{
+							"$gt": []any{"$log_len", limit},
+						},
+						"then": []any{},
+						"else": "$logs",
+					},
+				},
+			},
+		},
+	}
+	if limit <= 0 {
+		pipeline = pipeline[:len(pipeline)-1]
+	}
+	vl, err := mongoutil.Aggregate[*model.VersionLog](ctx, l.coll, pipeline)
+	if err != nil {
+		return nil, err
+	}
+	if len(vl) == 0 {
+		return nil, mongo.ErrNoDocuments
+	}
+	return vl[0], nil
+}
+
+func (l *VersionLogMgo) DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error {
+	return mongoutil.DeleteMany(ctx, l.coll, bson.M{
+		"last_update": bson.M{
+			"$lt": deadline,
+		},
+	})
+}
+
+func (l *VersionLogMgo) Delete(ctx context.Context, dId string) error {
+	return mongoutil.DeleteOne(ctx, l.coll, bson.M{"d_id": dId})
+}
@@ -0,0 +1,39 @@
+package mgo
+
+import (
+	"context"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+	"testing"
+	"time"
+)
+
+func Result[V any](val V, err error) V {
+	if err != nil {
+		panic(err)
+	}
+	return val
+}
+
+func Check(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func TestName(t *testing.T) {
+	cli := Result(mongo.Connect(context.Background(), options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second)))
+	coll := cli.Database("openim_v3").Collection("version_test")
+	tmp, err := NewVersionLog(coll)
+	if err != nil {
+		panic(err)
+	}
+	vl := tmp.(*VersionLogMgo)
+	res, err := vl.writeLogBatch2(context.Background(), "100", []string{"1000", "1001", "1003"}, model.VersionStateInsert, time.Now())
+	if err != nil {
+		t.Log(err)
+		return
+	}
+	t.Logf("%+v", res)
+}
@@ -0,0 +1,17 @@
+package database
+
+const (
+	BlackName              = "black"
+	ConversationName       = "conversation"
+	FriendName             = "friend"
+	FriendVersionName      = "friend_version"
+	FriendRequestName      = "friend_request"
+	GroupName              = "group"
+	GroupMemberName        = "group_member"
+	GroupMemberVersionName = "group_member_version"
+	GroupJoinVersionName   = "group_join_version"
+	GroupRequestName       = "group_request"
+	LogName                = "log"
+	ObjectName             = "s3"
+	UserName               = "user"
+)
@@ -39,6 +39,9 @@ type User interface {
 	CountTotal(ctx context.Context, before *time.Time) (count int64, err error)
 	// Get user total quantity every day
 	CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error)
+
+	SortQuery(ctx context.Context, userIDName map[string]string, asc bool) ([]*model.User, error)
+
 	// CRUD user command
 	AddUserCommand(ctx context.Context, userID string, Type int32, UUID string, value string, ex string) error
 	DeleteUserCommand(ctx context.Context, userID string, Type int32, UUID string) error
@@ -0,0 +1,19 @@
+package database
+
+import (
+	"context"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
+	"time"
+)
+
+const (
+	FirstVersion         = 1
+	DefaultDeleteVersion = 0
+)
+
+type VersionLog interface {
+	IncrVersion(ctx context.Context, dId string, eIds []string, state int32) error
+	FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error)
+	DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error
+	Delete(ctx context.Context, dId string) error
+}