feat: support incremental synchronization (#2379)

* fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * friend incr sync * friend incr sync * friend incr sync * friend incr sync * friend incr sync * mage * optimization version log * optimization version log * sync * sync * sync * group sync * sync option * sync option * refactor: replace `friend` package with `realtion`. * refactor: update lastest commit to relation. * sync option * sync option * sync option * sync * sync * go.mod * update: go mod * refactor: change incremental to full * feat: get full friend user ids * feat: api and config * group version * merge * fix: sort by id avoid unstable sort friends. * group * group * group * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * user version * fix: sort by id avoid unstable sort friends. * test: test log add. * test: debug log remove. * fix: transfer group owner incr version more than 1. * fix: add condition to kick owner. * feat: replace resp nil * feat: replace nil * fix: delete cache of max group joined version avoid sync joined group failed. * fix: nil * fix: delete cache of max group joined version avoid sync joined group failed. * fix: delete cache of max group joined version avoid sync joined group failed. * return group information for any changes * online cache --------- Co-authored-by: withchao <withchao@users.noreply.github.com> Co-authored-by: Monet Lee <monet_lee@163.com> Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <1186114839@qq.com>
2026-05-16 23:09:01 +08:00 · 2024-07-01 17:38:14 +08:00
parent fe7c029c2a
commit 88c0d5f5ad
56 changed files with 2023 additions and 325 deletions
@@ -18,6 +18,8 @@ import (
 	"context"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/database"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"

 	"github.com/openimsdk/tools/db/mongoutil"
 	"github.com/openimsdk/tools/db/pagination"
@@ -28,12 +30,13 @@ import (

 // FriendMgo implements Friend using MongoDB as the storage backend.
 type FriendMgo struct {
-	coll *mongo.Collection
+	coll  *mongo.Collection
+	owner database.VersionLog
 }

 // NewFriendMongo creates a new instance of FriendMgo with the provided MongoDB database.
 func NewFriendMongo(db *mongo.Database) (database.Friend, error) {
-	coll := db.Collection("friend")
+	coll := db.Collection(database.FriendName)
 	_, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{
 		Keys: bson.D{
 			{Key: "owner_user_id", Value: 1},
@@ -44,12 +47,41 @@ func NewFriendMongo(db *mongo.Database) (database.Friend, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &FriendMgo{coll: coll}, nil
+	owner, err := NewVersionLog(db.Collection(database.FriendVersionName))
+	if err != nil {
+		return nil, err
+	}
+	return &FriendMgo{coll: coll, owner: owner}, nil
+}
+
+func (f *FriendMgo) friendSort() any {
+	return bson.D{{"is_pinned", -1}, {"_id", 1}}
 }

 // Create inserts multiple friend records.
 func (f *FriendMgo) Create(ctx context.Context, friends []*model.Friend) error {
-	return mongoutil.InsertMany(ctx, f.coll, friends)
+	for i, friend := range friends {
+		if friend.ID.IsZero() {
+			friends[i].ID = primitive.NewObjectID()
+		}
+		if friend.CreateTime.IsZero() {
+			friends[i].CreateTime = time.Now()
+		}
+	}
+	return mongoutil.IncrVersion(func() error {
+		return mongoutil.InsertMany(ctx, f.coll, friends)
+	}, func() error {
+		mp := make(map[string][]string)
+		for _, friend := range friends {
+			mp[friend.OwnerUserID] = append(mp[friend.OwnerUserID], friend.FriendUserID)
+		}
+		for ownerUserID, friendUserIDs := range mp {
+			if err := f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, model.VersionStateInsert); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
 }

 // Delete removes specified friends of the owner user.
@@ -58,11 +90,15 @@ func (f *FriendMgo) Delete(ctx context.Context, ownerUserID string, friendUserID
 		"owner_user_id":  ownerUserID,
 		"friend_user_id": bson.M{"$in": friendUserIDs},
 	}
-	return mongoutil.DeleteOne(ctx, f.coll, filter)
+	return mongoutil.IncrVersion(func() error {
+		return mongoutil.DeleteOne(ctx, f.coll, filter)
+	}, func() error {
+		return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, model.VersionStateDelete)
+	})
 }

 // UpdateByMap updates specific fields of a friend document using a map.
-func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendUserID string, args map[string]interface{}) error {
+func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendUserID string, args map[string]any) error {
 	if len(args) == 0 {
 		return nil
 	}
@@ -70,30 +106,55 @@ func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendU
 		"owner_user_id":  ownerUserID,
 		"friend_user_id": friendUserID,
 	}
-	return mongoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": args}, true)
+	return mongoutil.IncrVersion(func() error {
+		return mongoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": args}, true)
+	}, func() error {
+		return f.owner.IncrVersion(ctx, ownerUserID, []string{friendUserID}, model.VersionStateUpdate)
+	})
 }

-// Update modifies multiple friend documents.
-// func (f *FriendMgo) Update(ctx context.Context, friends []*relation.Friend) error {
-// 	filter := bson.M{
-// 		"owner_user_id":  ownerUserID,
-// 		"friend_user_id": friendUserID,
-// 	}
-// 	return mgotool.UpdateMany(ctx, f.coll, filter, friends)
-// }
-
 // UpdateRemark updates the remark for a specific friend.
 func (f *FriendMgo) UpdateRemark(ctx context.Context, ownerUserID, friendUserID, remark string) error {
 	return f.UpdateByMap(ctx, ownerUserID, friendUserID, map[string]any{"remark": remark})
 }

+func (f *FriendMgo) fillTime(friends ...*model.Friend) {
+	for i, friend := range friends {
+		if friend.CreateTime.IsZero() {
+			friends[i].CreateTime = friend.ID.Timestamp()
+		}
+	}
+}
+
+func (f *FriendMgo) findOne(ctx context.Context, filter any) (*model.Friend, error) {
+	friend, err := mongoutil.FindOne[*model.Friend](ctx, f.coll, filter)
+	if err != nil {
+		return nil, err
+	}
+	f.fillTime(friend)
+	return friend, nil
+}
+
+func (f *FriendMgo) find(ctx context.Context, filter any) ([]*model.Friend, error) {
+	friends, err := mongoutil.Find[*model.Friend](ctx, f.coll, filter)
+	if err != nil {
+		return nil, err
+	}
+	f.fillTime(friends...)
+	return friends, nil
+}
+
+func (f *FriendMgo) findPage(ctx context.Context, filter any, pagination pagination.Pagination, opts ...*options.FindOptions) (int64, []*model.Friend, error) {
+	return mongoutil.FindPage[*model.Friend](ctx, f.coll, filter, pagination, opts...)
+}
+
 // Take retrieves a single friend document. Returns an error if not found.
 func (f *FriendMgo) Take(ctx context.Context, ownerUserID, friendUserID string) (*model.Friend, error) {
 	filter := bson.M{
 		"owner_user_id":  ownerUserID,
 		"friend_user_id": friendUserID,
 	}
-	return mongoutil.FindOne[*model.Friend](ctx, f.coll, filter)
+	return f.findOne(ctx, filter)
 }

 // FindUserState finds the friendship status between two users.
@@ -104,7 +165,7 @@ func (f *FriendMgo) FindUserState(ctx context.Context, userID1, userID2 string)
 			{"owner_user_id": userID2, "friend_user_id": userID1},
 		},
 	}
-	return mongoutil.Find[*model.Friend](ctx, f.coll, filter)
+	return f.find(ctx, filter)
 }

 // FindFriends retrieves a list of friends for a given owner. Missing friends do not cause an error.
@@ -113,7 +174,7 @@ func (f *FriendMgo) FindFriends(ctx context.Context, ownerUserID string, friendU
 		"owner_user_id":  ownerUserID,
 		"friend_user_id": bson.M{"$in": friendUserIDs},
 	}
-	return mongoutil.Find[*model.Friend](ctx, f.coll, filter)
+	return f.find(ctx, filter)
 }

 // FindReversalFriends finds users who have added the specified user as a friend.
@@ -122,25 +183,33 @@ func (f *FriendMgo) FindReversalFriends(ctx context.Context, friendUserID string
 		"owner_user_id":  bson.M{"$in": ownerUserIDs},
 		"friend_user_id": friendUserID,
 	}
-	return mongoutil.Find[*model.Friend](ctx, f.coll, filter)
+	return f.find(ctx, filter)
 }

 // FindOwnerFriends retrieves a paginated list of friends for a given owner.
 func (f *FriendMgo) FindOwnerFriends(ctx context.Context, ownerUserID string, pagination pagination.Pagination) (int64, []*model.Friend, error) {
 	filter := bson.M{"owner_user_id": ownerUserID}
-	return mongoutil.FindPage[*model.Friend](ctx, f.coll, filter, pagination)
+	opt := options.Find().SetSort(f.friendSort())
+	return f.findPage(ctx, filter, pagination, opt)
+}
+
+func (f *FriendMgo) FindOwnerFriendUserIds(ctx context.Context, ownerUserID string, limit int) ([]string, error) {
+	filter := bson.M{"owner_user_id": ownerUserID}
+	opt := options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1}).SetSort(f.friendSort()).SetLimit(int64(limit))
+	return mongoutil.Find[string](ctx, f.coll, filter, opt)
 }

 // FindInWhoseFriends finds users who have added the specified user as a friend, with pagination.
 func (f *FriendMgo) FindInWhoseFriends(ctx context.Context, friendUserID string, pagination pagination.Pagination) (int64, []*model.Friend, error) {
 	filter := bson.M{"friend_user_id": friendUserID}
-	return mongoutil.FindPage[*model.Friend](ctx, f.coll, filter, pagination)
+	opt := options.Find().SetSort(f.friendSort())
+	return f.findPage(ctx, filter, pagination, opt)
 }

 // FindFriendUserIDs retrieves a list of friend user IDs for a given owner.
 func (f *FriendMgo) FindFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) {
 	filter := bson.M{"owner_user_id": ownerUserID}
-	return mongoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1}))
+	return mongoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1}).SetSort(f.friendSort()))
 }

 func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, friendUserIDs []string, val map[string]any) error {
@@ -158,7 +227,24 @@ func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, frien
 	// Create an update document
 	update := bson.M{"$set": val}

-	// Perform the update operation for all matching documents
-	_, err := mongoutil.UpdateMany(ctx, f.coll, filter, update)
-	return err
+	return mongoutil.IncrVersion(func() error {
+		return mongoutil.Ignore(mongoutil.UpdateMany(ctx, f.coll, filter, update))
+	}, func() error {
+		return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, model.VersionStateUpdate)
+	})
+}
+
+func (f *FriendMgo) FindIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) {
+	return f.owner.FindChangeLog(ctx, ownerUserID, version, limit)
+}
+
+func (f *FriendMgo) FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) {
+	filter := bson.M{
+		"friend_user_id": friendUserID,
+	}
+	return mongoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1}).SetSort(f.friendSort()))
+}
+
+func (f *FriendMgo) IncrVersion(ctx context.Context, ownerUserID string, friendUserIDs []string, state int32) error {
+	return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, state)
 }