feat: group members, friends sorting version, client online subscription (#2427)

* fix: GroupApplicationAcceptedNotification

* fix: GroupApplicationAcceptedNotification

* fix: NotificationUserInfoUpdate

* cicd: robot automated Change

* fix: component

* fix: getConversationInfo

* feat: cron task

* feat: cron task

* feat: cron task

* feat: cron task

* feat: cron task

* fix: minio config url recognition error

* new mongo

* new mongo

* new mongo

* new mongo

* new mongo

* new mongo

* new mongo

* new mongo

* friend incr sync

* friend incr sync

* friend incr sync

* friend incr sync

* friend incr sync

* mage

* optimization version log

* optimization version log

* sync

* sync

* sync

* group sync

* sync option

* sync option

* refactor: replace `friend` package with `realtion`.

* refactor: update lastest commit to relation.

* sync option

* sync option

* sync option

* sync

* sync

* go.mod

* seq

* update: go mod

* refactor: change incremental to full

* feat: get full friend user ids

* feat: api and config

* seq

* group version

* merge

* seq

* seq

* seq

* fix: sort by id avoid unstable sort friends.

* group

* group

* group

* fix: sort by id avoid unstable sort friends.

* fix: sort by id avoid unstable sort friends.

* fix: sort by id avoid unstable sort friends.

* user version

* seq

* seq

* seq user

* user online

* implement minio expire delete.

* user online

* config

* fix

* fix

* implement minio expire delete logic.

* online cache

* online cache

* online cache

* online cache

* online cache

* online cache

* online cache

* online cache

* online cache

* online cache

* online cache

* online cache

* feat: implement scheduled delete outdated object in minio.

* update gomake version

* update gomake version

* implement FindExpires pagination.

* remove unnesseary incr.

* fix uncorrect args call.

* online push

* online push

* online push

* resolving conflicts

* resolving conflicts

* test

* api prommetrics

* api prommetrics

* api prommetrics

* api prommetrics

* api prommetrics

* rpc prommetrics

* rpc prommetrics

* online status

* online status

* online status

* online status

* sub

* conversation version incremental

* merge seq

* merge online

* merge online

* merge online

* merge seq

* GetOwnerConversation

* fix: change incremental syncer router name.

* rockscache batch get

* rockscache seq batch get

* fix: GetMsgDocModelByIndex bug

* update go.mod

* update go.mod

* merge

* feat: prometheus

* feat: prometheus

* group member sort

* sub

* sub

* fix: seq conversion bug

* fix: redis pipe exec

* sort version

* sort version

* sort version

* remove old version online subscription

* remove old version online subscription

* version log index

---------

Co-authored-by: withchao <withchao@users.noreply.github.com>
Co-authored-by: Monet Lee <monet_lee@163.com>
Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com>
Co-authored-by: icey-yu <1186114839@qq.com>

pkg/common/storage/controller/user.go

@@ -62,14 +62,6 @@ type UserDatabase interface {
 	CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error)
 
 	SortQuery(ctx context.Context, userIDName map[string]string, asc bool) ([]*model.User, error)
-	// SubscribeUsersStatus Subscribe a user's presence status
-	SubscribeUsersStatus(ctx context.Context, userID string, userIDs []string) error
-	// UnsubscribeUsersStatus unsubscribe a user's presence status
-	UnsubscribeUsersStatus(ctx context.Context, userID string, userIDs []string) error
-	// GetAllSubscribeList Get a list of all subscriptions
-	GetAllSubscribeList(ctx context.Context, userID string) ([]string, error)
-	// GetSubscribedList Get all subscribed lists
-	GetSubscribedList(ctx context.Context, userID string) ([]string, error)
 
 	// CRUD user command
 	AddUserCommand(ctx context.Context, userID string, Type int32, UUID string, value string, ex string) error
@@ -80,14 +72,13 @@ type UserDatabase interface {
 }
 
 type userDatabase struct {
-	tx      tx.Tx
-	userDB  database.User
-	cache   cache.UserCache
-	mongoDB database.SubscribeUser
+	tx     tx.Tx
+	userDB database.User
+	cache  cache.UserCache
 }
 
-func NewUserDatabase(userDB database.User, cache cache.UserCache, tx tx.Tx, mongoDB database.SubscribeUser) UserDatabase {
-	return &userDatabase{userDB: userDB, cache: cache, tx: tx, mongoDB: mongoDB}
+func NewUserDatabase(userDB database.User, cache cache.UserCache, tx tx.Tx) UserDatabase {
+	return &userDatabase{userDB: userDB, cache: cache, tx: tx}
 }
 
 func (u *userDatabase) InitOnce(ctx context.Context, users []*model.User) error {
@@ -212,36 +203,6 @@ func (u *userDatabase) SortQuery(ctx context.Context, userIDName map[string]stri
 	return u.userDB.SortQuery(ctx, userIDName, asc)
 }
 
-// SubscribeUsersStatus Subscribe or unsubscribe a user's presence status.
-func (u *userDatabase) SubscribeUsersStatus(ctx context.Context, userID string, userIDs []string) error {
-	err := u.mongoDB.AddSubscriptionList(ctx, userID, userIDs)
-	return err
-}
-
-// UnsubscribeUsersStatus unsubscribe a user's presence status.
-func (u *userDatabase) UnsubscribeUsersStatus(ctx context.Context, userID string, userIDs []string) error {
-	err := u.mongoDB.UnsubscriptionList(ctx, userID, userIDs)
-	return err
-}
-
-// GetAllSubscribeList Get a list of all subscriptions.
-func (u *userDatabase) GetAllSubscribeList(ctx context.Context, userID string) ([]string, error) {
-	list, err := u.mongoDB.GetAllSubscribeList(ctx, userID)
-	if err != nil {
-		return nil, err
-	}
-	return list, nil
-}
-
-// GetSubscribedList Get all subscribed lists.
-func (u *userDatabase) GetSubscribedList(ctx context.Context, userID string) ([]string, error) {
-	list, err := u.mongoDB.GetSubscribedList(ctx, userID)
-	if err != nil {
-		return nil, err
-	}
-	return list, nil
-}
-
 func (u *userDatabase) AddUserCommand(ctx context.Context, userID string, Type int32, UUID string, value string, ex string) error {
 	return u.userDB.AddUserCommand(ctx, userID, Type, UUID, value, ex)
 }

pkg/common/storage/database/mgo/friend.go

@@ -109,7 +109,13 @@ func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendU
 	return mongoutil.IncrVersion(func() error {
 		return mongoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": args}, true)
 	}, func() error {
-		return f.owner.IncrVersion(ctx, ownerUserID, []string{friendUserID}, model.VersionStateUpdate)
+		var friendUserIDs []string
+		if f.IsUpdateIsPinned(args) {
+			friendUserIDs = []string{model.VersionSortChangeID, friendUserID}
+		} else {
+			friendUserIDs = []string{friendUserID}
+		}
+		return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, model.VersionStateUpdate)
 	})
 }
 
@@ -214,7 +220,7 @@ func (f *FriendMgo) FindFriendUserIDs(ctx context.Context, ownerUserID string) (
 
 func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, friendUserIDs []string, val map[string]any) error {
 	// Ensure there are IDs to update
-	if len(friendUserIDs) == 0 {
+	if len(friendUserIDs) == 0 || len(val) == 0 {
 		return nil // Or return an error if you expect there to always be IDs
 	}
 
@@ -230,7 +236,13 @@ func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, frien
 	return mongoutil.IncrVersion(func() error {
 		return mongoutil.Ignore(mongoutil.UpdateMany(ctx, f.coll, filter, update))
 	}, func() error {
-		return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, model.VersionStateUpdate)
+		var userIDs []string
+		if f.IsUpdateIsPinned(val) {
+			userIDs = append([]string{model.VersionSortChangeID}, friendUserIDs...)
+		} else {
+			userIDs = friendUserIDs
+		}
+		return f.owner.IncrVersion(ctx, ownerUserID, userIDs, model.VersionStateUpdate)
 	})
 }
 
@@ -248,3 +260,11 @@ func (f *FriendMgo) FindFriendUserID(ctx context.Context, friendUserID string) (
 func (f *FriendMgo) IncrVersion(ctx context.Context, ownerUserID string, friendUserIDs []string, state int32) error {
 	return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, state)
 }
+
+func (f *FriendMgo) IsUpdateIsPinned(data map[string]any) bool {
+	if data == nil {
+		return false
+	}
+	_, ok := data["is_pinned"]
+	return ok
+}

pkg/common/storage/database/mgo/group_member.go

@@ -59,7 +59,7 @@ type GroupMemberMgo struct {
 }
 
 func (g *GroupMemberMgo) memberSort() any {
-	return bson.D{{"role_level", -1}, {"create_time", -1}}
+	return bson.D{{"role_level", -1}, {"create_time", 1}}
 }
 
 func (g *GroupMemberMgo) Create(ctx context.Context, groupMembers []*model.GroupMember) (err error) {
@@ -118,7 +118,7 @@ func (g *GroupMemberMgo) UpdateRoleLevel(ctx context.Context, groupID string, us
 		return mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID},
 			bson.M{"$set": bson.M{"role_level": roleLevel}}, true)
 	}, func() error {
-		return g.member.IncrVersion(ctx, groupID, []string{userID}, model.VersionStateUpdate)
+		return g.member.IncrVersion(ctx, groupID, []string{model.VersionSortChangeID, userID}, model.VersionStateUpdate)
 	})
 }
 func (g *GroupMemberMgo) UpdateUserRoleLevels(ctx context.Context, groupID string, firstUserID string, firstUserRoleLevel int32, secondUserID string, secondUserRoleLevel int32) error {
@@ -131,10 +131,9 @@ func (g *GroupMemberMgo) UpdateUserRoleLevels(ctx context.Context, groupID strin
 			bson.M{"$set": bson.M{"role_level": secondUserRoleLevel}}, true); err != nil {
 			return err
 		}
-
 		return nil
 	}, func() error {
-		return g.member.IncrVersion(ctx, groupID, []string{firstUserID, secondUserID}, model.VersionStateUpdate)
+		return g.member.IncrVersion(ctx, groupID, []string{model.VersionSortChangeID, firstUserID, secondUserID}, model.VersionStateUpdate)
 	})
 }
 
@@ -145,7 +144,13 @@ func (g *GroupMemberMgo) Update(ctx context.Context, groupID string, userID stri
 	return mongoutil.IncrVersion(func() error {
 		return mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID}, bson.M{"$set": data}, true)
 	}, func() error {
-		return g.member.IncrVersion(ctx, groupID, []string{userID}, model.VersionStateUpdate)
+		var userIDs []string
+		if g.IsUpdateRoleLevel(data) {
+			userIDs = []string{model.VersionSortChangeID, userID}
+		} else {
+			userIDs = []string{userID}
+		}
+		return g.member.IncrVersion(ctx, groupID, userIDs, model.VersionStateUpdate)
 	})
 }
 

pkg/common/storage/database/mgo/subscribe.go

@@ -1,189 +0,0 @@
-// Copyright © 2023 OpenIM. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package mgo
-
-import (
-	"context"
-	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/database"
-	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
-
-	"github.com/openimsdk/tools/errs"
-	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/mongo"
-	"go.mongodb.org/mongo-driver/mongo/options"
-)
-
-// prefixes and suffixes.
-const (
-	SubscriptionPrefix = "subscription_prefix"
-	SubscribedPrefix   = "subscribed_prefix"
-)
-
-// MaximumSubscription Maximum number of subscriptions.
-const (
-	MaximumSubscription = 3000
-)
-
-func NewUserMongoDriver(database *mongo.Database) database.SubscribeUser {
-	return &UserMongoDriver{
-		userCollection: database.Collection(model.SubscribeUserTableName),
-	}
-}
-
-type UserMongoDriver struct {
-	userCollection *mongo.Collection
-}
-
-// AddSubscriptionList Subscriber's handling of thresholds.
-func (u *UserMongoDriver) AddSubscriptionList(ctx context.Context, userID string, userIDList []string) error {
-	// Check the number of lists in the key.
-	pipeline := mongo.Pipeline{
-		{{"$match", bson.D{{"user_id", SubscriptionPrefix + userID}}}},
-		{{"$project", bson.D{{"count", bson.D{{"$size", "$user_id_list"}}}}}},
-	}
-	// perform aggregate operations
-	cursor, err := u.userCollection.Aggregate(ctx, pipeline)
-	if err != nil {
-		return errs.Wrap(err)
-	}
-	defer cursor.Close(ctx)
-	var cnt struct {
-		Count int `bson:"count"`
-	}
-	// iterate over aggregated results
-	for cursor.Next(ctx) {
-		err = cursor.Decode(&cnt)
-		if err != nil {
-			return errs.Wrap(err)
-		}
-	}
-	var newUserIDList []string
-	// If the threshold is exceeded, pop out the previous MaximumSubscription - len(userIDList) and insert it.
-	if cnt.Count+len(userIDList) > MaximumSubscription {
-		newUserIDList, err = u.GetAllSubscribeList(ctx, userID)
-		if err != nil {
-			return err
-		}
-		newUserIDList = newUserIDList[MaximumSubscription-len(userIDList):]
-		_, err = u.userCollection.UpdateOne(
-			ctx,
-			bson.M{"user_id": SubscriptionPrefix + userID},
-			bson.M{"$set": bson.M{"user_id_list": newUserIDList}},
-		)
-		if err != nil {
-			return err
-		}
-		// Another way to subscribe to N before pop,Delete after testing
-		/*for i := 1; i <= MaximumSubscription-len(userIDList); i++ {
-			_, err := u.userCollection.UpdateOne(
-				ctx,
-				bson.M{"user_id": SubscriptionPrefix + userID},
-				bson.M{SubscriptionPrefix + userID: bson.M{"$pop": -1}},
-			)
-			if err != nil {
-				return err
-			}
-		}*/
-	}
-	upsert := true
-	opts := &options.UpdateOptions{
-		Upsert: &upsert,
-	}
-	_, err = u.userCollection.UpdateOne(
-		ctx,
-		bson.M{"user_id": SubscriptionPrefix + userID},
-		bson.M{"$addToSet": bson.M{"user_id_list": bson.M{"$each": userIDList}}},
-		opts,
-	)
-	if err != nil {
-		return errs.Wrap(err)
-	}
-	for _, user := range userIDList {
-		_, err = u.userCollection.UpdateOne(
-			ctx,
-			bson.M{"user_id": SubscribedPrefix + user},
-			bson.M{"$addToSet": bson.M{"user_id_list": userID}},
-			opts,
-		)
-		if err != nil {
-			return errs.WrapMsg(err, "transaction failed")
-		}
-	}
-	return nil
-}
-
-// UnsubscriptionList Handling of unsubscribe.
-func (u *UserMongoDriver) UnsubscriptionList(ctx context.Context, userID string, userIDList []string) error {
-	_, err := u.userCollection.UpdateOne(
-		ctx,
-		bson.M{"user_id": SubscriptionPrefix + userID},
-		bson.M{"$pull": bson.M{"user_id_list": bson.M{"$in": userIDList}}},
-	)
-	if err != nil {
-		return errs.Wrap(err)
-	}
-	err = u.RemoveSubscribedListFromUser(ctx, userID, userIDList)
-	if err != nil {
-		return errs.Wrap(err)
-	}
-	return nil
-}
-
-// RemoveSubscribedListFromUser Among the unsubscribed users, delete the user from the subscribed list.
-func (u *UserMongoDriver) RemoveSubscribedListFromUser(ctx context.Context, userID string, userIDList []string) error {
-	var err error
-	for _, userIDTemp := range userIDList {
-		_, err = u.userCollection.UpdateOne(
-			ctx,
-			bson.M{"user_id": SubscribedPrefix + userIDTemp},
-			bson.M{"$pull": bson.M{"user_id_list": userID}},
-		)
-	}
-	return errs.Wrap(err)
-}
-
-// GetAllSubscribeList Get all users subscribed by this user.
-func (u *UserMongoDriver) GetAllSubscribeList(ctx context.Context, userID string) (userIDList []string, err error) {
-	var user model.SubscribeUser
-	cursor := u.userCollection.FindOne(
-		ctx,
-		bson.M{"user_id": SubscriptionPrefix + userID})
-	err = cursor.Decode(&user)
-	if err != nil {
-		if err == mongo.ErrNoDocuments {
-			return []string{}, nil
-		} else {
-			return nil, errs.Wrap(err)
-		}
-	}
-	return user.UserIDList, nil
-}
-
-// GetSubscribedList Get the user subscribed by those users.
-func (u *UserMongoDriver) GetSubscribedList(ctx context.Context, userID string) (userIDList []string, err error) {
-	var user model.SubscribeUser
-	cursor := u.userCollection.FindOne(
-		ctx,
-		bson.M{"user_id": SubscribedPrefix + userID})
-	err = cursor.Decode(&user)
-	if err != nil {
-		if err == mongo.ErrNoDocuments {
-			return []string{}, nil
-		} else {
-			return nil, errs.Wrap(err)
-		}
-	}
-	return user.UserIDList, nil
-}

pkg/common/storage/database/mgo/version_log.go

@@ -18,8 +18,8 @@ import (
 
 func NewVersionLog(coll *mongo.Collection) (database.VersionLog, error) {
 	lm := &VersionLogMgo{coll: coll}
-	if lm.initIndex(context.Background()) != nil {
-		return nil, errs.ErrInternalServer.WrapMsg("init index failed", "coll", coll.Name())
+	if err := lm.initIndex(context.Background()); err != nil {
+		return nil, errs.WrapMsg(err, "init version log index failed", "coll", coll.Name())
 	}
 	return lm, nil
 }
@@ -33,6 +33,7 @@ func (l *VersionLogMgo) initIndex(ctx context.Context) error {
 		Keys: bson.M{
 			"d_id": 1,
 		},
+		Options: options.Index().SetUnique(true),
 	})
 	return err
 }
@@ -152,8 +153,24 @@ func (l *VersionLogMgo) writeLogBatch2(ctx context.Context, dId string, eIds []s
 			"$unset": "delete_e_ids",
 		},
 	}
-	opt := options.FindOneAndUpdate().SetUpsert(false).SetReturnDocument(options.After).SetProjection(bson.M{"logs": 0})
-	return mongoutil.FindOneAndUpdate[*model.VersionLog](ctx, l.coll, filter, pipeline, opt)
+	projection := bson.M{
+		"logs": 0,
+	}
+	opt := options.FindOneAndUpdate().SetUpsert(false).SetReturnDocument(options.After).SetProjection(projection)
+	res, err := mongoutil.FindOneAndUpdate[*model.VersionLog](ctx, l.coll, filter, pipeline, opt)
+	if err != nil {
+		return nil, err
+	}
+	res.Logs = make([]model.VersionLogElem, 0, len(eIds))
+	for _, id := range eIds {
+		res.Logs = append(res.Logs, model.VersionLogElem{
+			EID:        id,
+			State:      state,
+			Version:    res.Version,
+			LastUpdate: res.LastUpdate,
+		})
+	}
+	return res, nil
 }
 
 func (l *VersionLogMgo) findDoc(ctx context.Context, dId string) (*model.VersionLog, error) {

pkg/common/storage/database/mgo/version_test.go

@@ -9,12 +9,12 @@ import (
 	"time"
 )
 
-func Result[V any](val V, err error) V {
-	if err != nil {
-		panic(err)
-	}
-	return val
-}
+//func Result[V any](val V, err error) V {
+//	if err != nil {
+//		panic(err)
+//	}
+//	return val
+//}
 
 func Check(err error) {
 	if err != nil {
@@ -30,7 +30,7 @@ func TestName(t *testing.T) {
 		panic(err)
 	}
 	vl := tmp.(*VersionLogMgo)
-	res, err := vl.writeLogBatch2(context.Background(), "100", []string{"1000", "1001", "1003"}, model.VersionStateInsert, time.Now())
+	res, err := vl.incrVersionResult(context.Background(), "100", []string{"1000", "1001", "1003"}, model.VersionStateInsert)
 	if err != nil {
 		t.Log(err)
 		return

pkg/common/storage/database/subscribe.go

@@ -1,31 +0,0 @@
-// Copyright © 2023 OpenIM. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package database
-
-import "context"
-
-// SubscribeUser Operation interface of user mongodb.
-type SubscribeUser interface {
-	// AddSubscriptionList Subscriber's handling of thresholds.
-	AddSubscriptionList(ctx context.Context, userID string, userIDList []string) error
-	// UnsubscriptionList Handling of unsubscribe.
-	UnsubscriptionList(ctx context.Context, userID string, userIDList []string) error
-	// RemoveSubscribedListFromUser Among the unsubscribed users, delete the user from the subscribed list.
-	RemoveSubscribedListFromUser(ctx context.Context, userID string, userIDList []string) error
-	// GetAllSubscribeList Get all users subscribed by this user
-	GetAllSubscribeList(ctx context.Context, id string) (userIDList []string, err error)
-	// GetSubscribedList Get the user subscribed by those users
-	GetSubscribedList(ctx context.Context, id string) (userIDList []string, err error)
-}

pkg/common/storage/model/version_log.go

@@ -14,6 +14,11 @@ const (
 	VersionStateUpdate
 )
 
+const (
+	VersionGroupChangeID = ""
+	VersionSortChangeID  = "____S_O_R_T_I_D____"
+)
+
 type VersionLogElem struct {
 	EID        string    `bson:"e_id"`
 	State      int32     `bson:"state"`