feat: new features merged (#2409)

* fix: GroupApplicationAcceptedNotification

* fix: GroupApplicationAcceptedNotification

* fix: NotificationUserInfoUpdate

* cicd: robot automated Change

* fix: component

* fix: getConversationInfo

* feat: cron task

* feat: cron task

* feat: cron task

* feat: cron task

* feat: cron task

* fix: minio config url recognition error

* new mongo

* new mongo

* new mongo

* new mongo

* new mongo

* new mongo

* new mongo

* new mongo

* friend incr sync

* friend incr sync

* friend incr sync

* friend incr sync

* friend incr sync

* mage

* optimization version log

* optimization version log

* sync

* sync

* sync

* group sync

* sync option

* sync option

* refactor: replace `friend` package with `realtion`.

* refactor: update lastest commit to relation.

* sync option

* sync option

* sync option

* sync

* sync

* go.mod

* seq

* update: go mod

* refactor: change incremental to full

* feat: get full friend user ids

* feat: api and config

* seq

* group version

* merge

* seq

* seq

* seq

* fix: sort by id avoid unstable sort friends.

* group

* group

* group

* fix: sort by id avoid unstable sort friends.

* fix: sort by id avoid unstable sort friends.

* fix: sort by id avoid unstable sort friends.

* user version

* seq

* seq

* seq user

* user online

* implement minio expire delete.

* user online

* config

* fix

* fix

* implement minio expire delete logic.

* online cache

* online cache

* online cache

* online cache

* online cache

* online cache

* online cache

* online cache

* online cache

* online cache

* online cache

* online cache

* feat: implement scheduled delete outdated object in minio.

* update gomake version

* update gomake version

* implement FindExpires pagination.

* remove unnesseary incr.

* fix uncorrect args call.

* online push

* online push

* online push

* resolving conflicts

* resolving conflicts

* test

* api prommetrics

* api prommetrics

* api prommetrics

* api prommetrics

* api prommetrics

* rpc prommetrics

* rpc prommetrics

* online status

* online status

* online status

* online status

* sub

* conversation version incremental

* merge seq

* merge online

* merge online

* merge online

* merge seq

* GetOwnerConversation

* fix: change incremental syncer router name.

* rockscache batch get

* rockscache seq batch get

* fix: GetMsgDocModelByIndex bug

* update go.mod

* update go.mod

* merge

* feat: prometheus

* feat: prometheus

---------

Co-authored-by: withchao <withchao@users.noreply.github.com>
Co-authored-by: Monet Lee <monet_lee@163.com>
Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com>
Co-authored-by: icey-yu <1186114839@qq.com>

pkg/common/storage/database/group_member.go

@@ -28,6 +28,8 @@ type GroupMember interface {
 	UpdateUserRoleLevels(ctx context.Context, groupID string, firstUserID string, firstUserRoleLevel int32, secondUserID string, secondUserRoleLevel int32) error
 	FindMemberUserID(ctx context.Context, groupID string) (userIDs []string, err error)
 	Take(ctx context.Context, groupID string, userID string) (groupMember *model.GroupMember, err error)
+	Find(ctx context.Context, groupID string, userIDs []string) ([]*model.GroupMember, error)
+	FindInGroup(ctx context.Context, userID string, groupIDs []string) ([]*model.GroupMember, error)
 	TakeOwner(ctx context.Context, groupID string) (groupMember *model.GroupMember, err error)
 	SearchMember(ctx context.Context, keyword string, groupID string, pagination pagination.Pagination) (total int64, groupList []*model.GroupMember, err error)
 	FindRoleLevelUserIDs(ctx context.Context, groupID string, roleLevel int32) ([]string, error)

pkg/common/storage/database/mgo/group_member.go

@@ -153,6 +153,22 @@ func (g *GroupMemberMgo) FindMemberUserID(ctx context.Context, groupID string) (
 	return mongoutil.Find[string](ctx, g.coll, bson.M{"group_id": groupID}, options.Find().SetProjection(bson.M{"_id": 0, "user_id": 1}).SetSort(g.memberSort()))
 }
 
+func (g *GroupMemberMgo) Find(ctx context.Context, groupID string, userIDs []string) ([]*model.GroupMember, error) {
+	filter := bson.M{"group_id": groupID}
+	if len(userIDs) > 0 {
+		filter["user_id"] = bson.M{"$in": userIDs}
+	}
+	return mongoutil.Find[*model.GroupMember](ctx, g.coll, filter)
+}
+
+func (g *GroupMemberMgo) FindInGroup(ctx context.Context, userID string, groupIDs []string) ([]*model.GroupMember, error) {
+	filter := bson.M{"user_id": userID}
+	if len(groupIDs) > 0 {
+		filter["group_id"] = bson.M{"$in": groupIDs}
+	}
+	return mongoutil.Find[*model.GroupMember](ctx, g.coll, filter)
+}
+
 func (g *GroupMemberMgo) Take(ctx context.Context, groupID string, userID string) (groupMember *model.GroupMember, err error) {
 	return mongoutil.FindOne[*model.GroupMember](ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID})
 }

pkg/common/storage/database/mgo/object.go

@@ -16,10 +16,13 @@ package mgo
 
 import (
 	"context"
+	"time"
+
 	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/database"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
 
 	"github.com/openimsdk/tools/db/mongoutil"
+	"github.com/openimsdk/tools/db/pagination"
 	"github.com/openimsdk/tools/errs"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo"
@@ -68,3 +71,14 @@ func (o *S3Mongo) Take(ctx context.Context, engine string, name string) (*model.
 func (o *S3Mongo) Delete(ctx context.Context, engine string, name string) error {
 	return mongoutil.DeleteOne(ctx, o.coll, bson.M{"name": name, "engine": engine})
 }
+func (o *S3Mongo) FindByExpires(ctx context.Context, duration time.Time, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) {
+	return mongoutil.FindPage[*model.Object](ctx, o.coll, bson.M{
+		"create_time": bson.M{"$lt": duration},
+	}, pagination)
+}
+func (o *S3Mongo) FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error) {
+	return mongoutil.Count(ctx, o.coll, bson.M{
+		"key":         key,
+		"create_time": bson.M{"$gt": duration},
+	})
+}

pkg/common/storage/database/mgo/seq_conversation.go

@@ -0,0 +1,103 @@
+package mgo
+
+import (
+	"context"
+	"errors"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/database"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
+	"github.com/openimsdk/tools/db/mongoutil"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+func NewSeqConversationMongo(db *mongo.Database) (database.SeqConversation, error) {
+	coll := db.Collection(database.SeqConversationName)
+	_, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{
+		Keys: bson.D{
+			{Key: "conversation_id", Value: 1},
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+	return &seqConversationMongo{coll: coll}, nil
+}
+
+type seqConversationMongo struct {
+	coll *mongo.Collection
+}
+
+func (s *seqConversationMongo) setSeq(ctx context.Context, conversationID string, seq int64, field string) error {
+	filter := map[string]any{
+		"conversation_id": conversationID,
+	}
+	insert := bson.M{
+		"conversation_id": conversationID,
+		"min_seq":         0,
+		"max_seq":         0,
+	}
+	delete(insert, field)
+	update := map[string]any{
+		"$set": bson.M{
+			field: seq,
+		},
+		"$setOnInsert": insert,
+	}
+	opt := options.Update().SetUpsert(true)
+	return mongoutil.UpdateOne(ctx, s.coll, filter, update, false, opt)
+}
+
+func (s *seqConversationMongo) Malloc(ctx context.Context, conversationID string, size int64) (int64, error) {
+	if size < 0 {
+		return 0, errors.New("size must be greater than 0")
+	}
+	if size == 0 {
+		return s.GetMaxSeq(ctx, conversationID)
+	}
+	filter := map[string]any{"conversation_id": conversationID}
+	update := map[string]any{
+		"$inc": map[string]any{"max_seq": size},
+		"$set": map[string]any{"min_seq": int64(0)},
+	}
+	opt := options.FindOneAndUpdate().SetUpsert(true).SetReturnDocument(options.After).SetProjection(map[string]any{"_id": 0, "max_seq": 1})
+	lastSeq, err := mongoutil.FindOneAndUpdate[int64](ctx, s.coll, filter, update, opt)
+	if err != nil {
+		return 0, err
+	}
+	return lastSeq - size, nil
+}
+
+func (s *seqConversationMongo) SetMaxSeq(ctx context.Context, conversationID string, seq int64) error {
+	return s.setSeq(ctx, conversationID, seq, "max_seq")
+}
+
+func (s *seqConversationMongo) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) {
+	seq, err := mongoutil.FindOne[int64](ctx, s.coll, bson.M{"conversation_id": conversationID}, options.FindOne().SetProjection(map[string]any{"_id": 0, "max_seq": 1}))
+	if err == nil {
+		return seq, nil
+	} else if IsNotFound(err) {
+		return 0, nil
+	} else {
+		return 0, err
+	}
+}
+
+func (s *seqConversationMongo) GetMinSeq(ctx context.Context, conversationID string) (int64, error) {
+	seq, err := mongoutil.FindOne[int64](ctx, s.coll, bson.M{"conversation_id": conversationID}, options.FindOne().SetProjection(map[string]any{"_id": 0, "min_seq": 1}))
+	if err == nil {
+		return seq, nil
+	} else if IsNotFound(err) {
+		return 0, nil
+	} else {
+		return 0, err
+	}
+}
+
+func (s *seqConversationMongo) SetMinSeq(ctx context.Context, conversationID string, seq int64) error {
+	return s.setSeq(ctx, conversationID, seq, "min_seq")
+}
+
+func (s *seqConversationMongo) GetConversation(ctx context.Context, conversationID string) (*model.SeqConversation, error) {
+	return mongoutil.FindOne[*model.SeqConversation](ctx, s.coll, bson.M{"conversation_id": conversationID})
+}

pkg/common/storage/database/mgo/seq_conversation_test.go

@@ -0,0 +1,37 @@
+package mgo
+
+import (
+	"context"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+	"testing"
+	"time"
+)
+
+func Result[V any](val V, err error) V {
+	if err != nil {
+		panic(err)
+	}
+	return val
+}
+
+func Mongodb() *mongo.Database {
+	return Result(
+		mongo.Connect(context.Background(),
+			options.Client().
+				ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").
+				SetConnectTimeout(5*time.Second)),
+	).Database("openim_v3")
+}
+
+func TestUserSeq(t *testing.T) {
+	uSeq := Result(NewSeqUserMongo(Mongodb())).(*seqUserMongo)
+	t.Log(uSeq.SetMinSeq(context.Background(), "1000", "2000", 4))
+}
+
+func TestConversationSeq(t *testing.T) {
+	cSeq := Result(NewSeqConversationMongo(Mongodb())).(*seqConversationMongo)
+	t.Log(cSeq.SetMaxSeq(context.Background(), "2000", 10))
+	t.Log(cSeq.Malloc(context.Background(), "2000", 10))
+	t.Log(cSeq.GetMaxSeq(context.Background(), "2000"))
+}

pkg/common/storage/database/mgo/seq_user.go

@@ -0,0 +1,110 @@
+package mgo
+
+import (
+	"context"
+	"errors"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/database"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
+	"github.com/openimsdk/tools/db/mongoutil"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+func NewSeqUserMongo(db *mongo.Database) (database.SeqUser, error) {
+	coll := db.Collection(database.SeqUserName)
+	_, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{
+		Keys: bson.D{
+			{Key: "user_id", Value: 1},
+			{Key: "conversation_id", Value: 1},
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+	return &seqUserMongo{coll: coll}, nil
+}
+
+type seqUserMongo struct {
+	coll *mongo.Collection
+}
+
+func (s *seqUserMongo) setSeq(ctx context.Context, conversationID string, userID string, seq int64, field string) error {
+	filter := map[string]any{
+		"user_id":         userID,
+		"conversation_id": conversationID,
+	}
+	insert := bson.M{
+		"user_id":         userID,
+		"conversation_id": conversationID,
+		"min_seq":         0,
+		"max_seq":         0,
+		"read_seq":        0,
+	}
+	delete(insert, field)
+	update := map[string]any{
+		"$set": bson.M{
+			field: seq,
+		},
+		"$setOnInsert": insert,
+	}
+	opt := options.Update().SetUpsert(true)
+	return mongoutil.UpdateOne(ctx, s.coll, filter, update, false, opt)
+}
+
+func (s *seqUserMongo) getSeq(ctx context.Context, conversationID string, userID string, failed string) (int64, error) {
+	filter := map[string]any{
+		"user_id":         userID,
+		"conversation_id": conversationID,
+	}
+	opt := options.FindOne().SetProjection(bson.M{"_id": 0, failed: 1})
+	seq, err := mongoutil.FindOne[int64](ctx, s.coll, filter, opt)
+	if err == nil {
+		return seq, nil
+	} else if errors.Is(err, mongo.ErrNoDocuments) {
+		return 0, nil
+	} else {
+		return 0, err
+	}
+}
+
+func (s *seqUserMongo) GetMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) {
+	return s.getSeq(ctx, conversationID, userID, "max_seq")
+}
+
+func (s *seqUserMongo) SetMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error {
+	return s.setSeq(ctx, conversationID, userID, seq, "max_seq")
+}
+
+func (s *seqUserMongo) GetMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) {
+	return s.getSeq(ctx, conversationID, userID, "min_seq")
+}
+
+func (s *seqUserMongo) SetMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error {
+	return s.setSeq(ctx, conversationID, userID, seq, "min_seq")
+}
+
+func (s *seqUserMongo) GetReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) {
+	return s.getSeq(ctx, conversationID, userID, "read_seq")
+}
+
+func (s *seqUserMongo) GetReadSeqs(ctx context.Context, userID string, conversationID []string) (map[string]int64, error) {
+	if len(conversationID) == 0 {
+		return map[string]int64{}, nil
+	}
+	filter := bson.M{"user_id": userID, "conversation_id": bson.M{"$in": conversationID}}
+	opt := options.Find().SetProjection(bson.M{"_id": 0, "conversation_id": 1, "read_seq": 1})
+	seqs, err := mongoutil.Find[*model.SeqUser](ctx, s.coll, filter, opt)
+	if err != nil {
+		return nil, err
+	}
+	res := make(map[string]int64)
+	for _, seq := range seqs {
+		res[seq.ConversationID] = seq.ReadSeq
+	}
+	return res, nil
+}
+
+func (s *seqUserMongo) SetReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error {
+	return s.setSeq(ctx, conversationID, userID, seq, "read_seq")
+}

pkg/common/storage/database/name.go

@@ -15,4 +15,6 @@ const (
 	LogName                 = "log"
 	ObjectName              = "s3"
 	UserName                = "user"
+	SeqConversationName     = "seq"
+	SeqUserName             = "seq_user"
 )

pkg/common/storage/database/object.go

@@ -16,11 +16,16 @@ package database
 
 import (
 	"context"
+	"time"
+
 	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
+	"github.com/openimsdk/tools/db/pagination"
 )
 
 type ObjectInfo interface {
 	SetObject(ctx context.Context, obj *model.Object) error
 	Take(ctx context.Context, engine string, name string) (*model.Object, error)
 	Delete(ctx context.Context, engine string, name string) error
+	FindByExpires(ctx context.Context, duration time.Time, pagination pagination.Pagination) (total int64, objects []*model.Object, err error)
+	FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error)
 }

pkg/common/storage/database/seq.go

@@ -0,0 +1,11 @@
+package database
+
+import "context"
+
+type SeqConversation interface {
+	Malloc(ctx context.Context, conversationID string, size int64) (int64, error)
+	GetMaxSeq(ctx context.Context, conversationID string) (int64, error)
+	SetMaxSeq(ctx context.Context, conversationID string, seq int64) error
+	GetMinSeq(ctx context.Context, conversationID string) (int64, error)
+	SetMinSeq(ctx context.Context, conversationID string, seq int64) error
+}

pkg/common/storage/database/seq_user.go

@@ -0,0 +1,13 @@
+package database
+
+import "context"
+
+type SeqUser interface {
+	GetMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error)
+	SetMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error
+	GetMinSeq(ctx context.Context, conversationID string, userID string) (int64, error)
+	SetMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error
+	GetReadSeq(ctx context.Context, conversationID string, userID string) (int64, error)
+	SetReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error
+	GetReadSeqs(ctx context.Context, userID string, conversationID []string) (map[string]int64, error)
+}