feat: new features merged (#2409)

* fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * friend incr sync * friend incr sync * friend incr sync * friend incr sync * friend incr sync * mage * optimization version log * optimization version log * sync * sync * sync * group sync * sync option * sync option * refactor: replace `friend` package with `realtion`. * refactor: update lastest commit to relation. * sync option * sync option * sync option * sync * sync * go.mod * seq * update: go mod * refactor: change incremental to full * feat: get full friend user ids * feat: api and config * seq * group version * merge * seq * seq * seq * fix: sort by id avoid unstable sort friends. * group * group * group * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * user version * seq * seq * seq user * user online * implement minio expire delete. * user online * config * fix * fix * implement minio expire delete logic. * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * feat: implement scheduled delete outdated object in minio. * update gomake version * update gomake version * implement FindExpires pagination. * remove unnesseary incr. * fix uncorrect args call. * online push * online push * online push * resolving conflicts * resolving conflicts * test * api prommetrics * api prommetrics * api prommetrics * api prommetrics * api prommetrics * rpc prommetrics * rpc prommetrics * online status * online status * online status * online status * sub * conversation version incremental * merge seq * merge online * merge online * merge online * merge seq * GetOwnerConversation * fix: change incremental syncer router name. * rockscache batch get * rockscache seq batch get * fix: GetMsgDocModelByIndex bug * update go.mod * update go.mod * merge * feat: prometheus * feat: prometheus --------- Co-authored-by: withchao <withchao@users.noreply.github.com> Co-authored-by: Monet Lee <monet_lee@163.com> Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <1186114839@qq.com>
2026-05-10 20:15:59 +08:00 · 2024-07-16 10:46:21 +08:00
parent 5f52fa19bd
commit 4aaf496086
97 changed files with 3743 additions and 1587 deletions
@@ -153,6 +153,22 @@ func (g *GroupMemberMgo) FindMemberUserID(ctx context.Context, groupID string) (
 	return mongoutil.Find[string](ctx, g.coll, bson.M{"group_id": groupID}, options.Find().SetProjection(bson.M{"_id": 0, "user_id": 1}).SetSort(g.memberSort()))
 }

+func (g *GroupMemberMgo) Find(ctx context.Context, groupID string, userIDs []string) ([]*model.GroupMember, error) {
+	filter := bson.M{"group_id": groupID}
+	if len(userIDs) > 0 {
+		filter["user_id"] = bson.M{"$in": userIDs}
+	}
+	return mongoutil.Find[*model.GroupMember](ctx, g.coll, filter)
+}
+
+func (g *GroupMemberMgo) FindInGroup(ctx context.Context, userID string, groupIDs []string) ([]*model.GroupMember, error) {
+	filter := bson.M{"user_id": userID}
+	if len(groupIDs) > 0 {
+		filter["group_id"] = bson.M{"$in": groupIDs}
+	}
+	return mongoutil.Find[*model.GroupMember](ctx, g.coll, filter)
+}
+
 func (g *GroupMemberMgo) Take(ctx context.Context, groupID string, userID string) (groupMember *model.GroupMember, err error) {
 	return mongoutil.FindOne[*model.GroupMember](ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID})
 }
@@ -16,10 +16,13 @@ package mgo

 import (
 	"context"
+	"time"
+
 	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/database"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"

 	"github.com/openimsdk/tools/db/mongoutil"
+	"github.com/openimsdk/tools/db/pagination"
 	"github.com/openimsdk/tools/errs"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo"
@@ -68,3 +71,14 @@ func (o *S3Mongo) Take(ctx context.Context, engine string, name string) (*model.
 func (o *S3Mongo) Delete(ctx context.Context, engine string, name string) error {
 	return mongoutil.DeleteOne(ctx, o.coll, bson.M{"name": name, "engine": engine})
 }
+func (o *S3Mongo) FindByExpires(ctx context.Context, duration time.Time, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) {
+	return mongoutil.FindPage[*model.Object](ctx, o.coll, bson.M{
+		"create_time": bson.M{"$lt": duration},
+	}, pagination)
+}
+func (o *S3Mongo) FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error) {
+	return mongoutil.Count(ctx, o.coll, bson.M{
+		"key":         key,
+		"create_time": bson.M{"$gt": duration},
+	})
+}
@@ -0,0 +1,103 @@
+package mgo
+
+import (
+	"context"
+	"errors"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/database"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
+	"github.com/openimsdk/tools/db/mongoutil"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+func NewSeqConversationMongo(db *mongo.Database) (database.SeqConversation, error) {
+	coll := db.Collection(database.SeqConversationName)
+	_, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{
+		Keys: bson.D{
+			{Key: "conversation_id", Value: 1},
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+	return &seqConversationMongo{coll: coll}, nil
+}
+
+type seqConversationMongo struct {
+	coll *mongo.Collection
+}
+
+func (s *seqConversationMongo) setSeq(ctx context.Context, conversationID string, seq int64, field string) error {
+	filter := map[string]any{
+		"conversation_id": conversationID,
+	}
+	insert := bson.M{
+		"conversation_id": conversationID,
+		"min_seq":         0,
+		"max_seq":         0,
+	}
+	delete(insert, field)
+	update := map[string]any{
+		"$set": bson.M{
+			field: seq,
+		},
+		"$setOnInsert": insert,
+	}
+	opt := options.Update().SetUpsert(true)
+	return mongoutil.UpdateOne(ctx, s.coll, filter, update, false, opt)
+}
+
+func (s *seqConversationMongo) Malloc(ctx context.Context, conversationID string, size int64) (int64, error) {
+	if size < 0 {
+		return 0, errors.New("size must be greater than 0")
+	}
+	if size == 0 {
+		return s.GetMaxSeq(ctx, conversationID)
+	}
+	filter := map[string]any{"conversation_id": conversationID}
+	update := map[string]any{
+		"$inc": map[string]any{"max_seq": size},
+		"$set": map[string]any{"min_seq": int64(0)},
+	}
+	opt := options.FindOneAndUpdate().SetUpsert(true).SetReturnDocument(options.After).SetProjection(map[string]any{"_id": 0, "max_seq": 1})
+	lastSeq, err := mongoutil.FindOneAndUpdate[int64](ctx, s.coll, filter, update, opt)
+	if err != nil {
+		return 0, err
+	}
+	return lastSeq - size, nil
+}
+
+func (s *seqConversationMongo) SetMaxSeq(ctx context.Context, conversationID string, seq int64) error {
+	return s.setSeq(ctx, conversationID, seq, "max_seq")
+}
+
+func (s *seqConversationMongo) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) {
+	seq, err := mongoutil.FindOne[int64](ctx, s.coll, bson.M{"conversation_id": conversationID}, options.FindOne().SetProjection(map[string]any{"_id": 0, "max_seq": 1}))
+	if err == nil {
+		return seq, nil
+	} else if IsNotFound(err) {
+		return 0, nil
+	} else {
+		return 0, err
+	}
+}
+
+func (s *seqConversationMongo) GetMinSeq(ctx context.Context, conversationID string) (int64, error) {
+	seq, err := mongoutil.FindOne[int64](ctx, s.coll, bson.M{"conversation_id": conversationID}, options.FindOne().SetProjection(map[string]any{"_id": 0, "min_seq": 1}))
+	if err == nil {
+		return seq, nil
+	} else if IsNotFound(err) {
+		return 0, nil
+	} else {
+		return 0, err
+	}
+}
+
+func (s *seqConversationMongo) SetMinSeq(ctx context.Context, conversationID string, seq int64) error {
+	return s.setSeq(ctx, conversationID, seq, "min_seq")
+}
+
+func (s *seqConversationMongo) GetConversation(ctx context.Context, conversationID string) (*model.SeqConversation, error) {
+	return mongoutil.FindOne[*model.SeqConversation](ctx, s.coll, bson.M{"conversation_id": conversationID})
+}
@@ -0,0 +1,37 @@
+package mgo
+
+import (
+	"context"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+	"testing"
+	"time"
+)
+
+func Result[V any](val V, err error) V {
+	if err != nil {
+		panic(err)
+	}
+	return val
+}
+
+func Mongodb() *mongo.Database {
+	return Result(
+		mongo.Connect(context.Background(),
+			options.Client().
+				ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").
+				SetConnectTimeout(5*time.Second)),
+	).Database("openim_v3")
+}
+
+func TestUserSeq(t *testing.T) {
+	uSeq := Result(NewSeqUserMongo(Mongodb())).(*seqUserMongo)
+	t.Log(uSeq.SetMinSeq(context.Background(), "1000", "2000", 4))
+}
+
+func TestConversationSeq(t *testing.T) {
+	cSeq := Result(NewSeqConversationMongo(Mongodb())).(*seqConversationMongo)
+	t.Log(cSeq.SetMaxSeq(context.Background(), "2000", 10))
+	t.Log(cSeq.Malloc(context.Background(), "2000", 10))
+	t.Log(cSeq.GetMaxSeq(context.Background(), "2000"))
+}
@@ -0,0 +1,110 @@
+package mgo
+
+import (
+	"context"
+	"errors"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/database"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
+	"github.com/openimsdk/tools/db/mongoutil"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+func NewSeqUserMongo(db *mongo.Database) (database.SeqUser, error) {
+	coll := db.Collection(database.SeqUserName)
+	_, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{
+		Keys: bson.D{
+			{Key: "user_id", Value: 1},
+			{Key: "conversation_id", Value: 1},
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+	return &seqUserMongo{coll: coll}, nil
+}
+
+type seqUserMongo struct {
+	coll *mongo.Collection
+}
+
+func (s *seqUserMongo) setSeq(ctx context.Context, conversationID string, userID string, seq int64, field string) error {
+	filter := map[string]any{
+		"user_id":         userID,
+		"conversation_id": conversationID,
+	}
+	insert := bson.M{
+		"user_id":         userID,
+		"conversation_id": conversationID,
+		"min_seq":         0,
+		"max_seq":         0,
+		"read_seq":        0,
+	}
+	delete(insert, field)
+	update := map[string]any{
+		"$set": bson.M{
+			field: seq,
+		},
+		"$setOnInsert": insert,
+	}
+	opt := options.Update().SetUpsert(true)
+	return mongoutil.UpdateOne(ctx, s.coll, filter, update, false, opt)
+}
+
+func (s *seqUserMongo) getSeq(ctx context.Context, conversationID string, userID string, failed string) (int64, error) {
+	filter := map[string]any{
+		"user_id":         userID,
+		"conversation_id": conversationID,
+	}
+	opt := options.FindOne().SetProjection(bson.M{"_id": 0, failed: 1})
+	seq, err := mongoutil.FindOne[int64](ctx, s.coll, filter, opt)
+	if err == nil {
+		return seq, nil
+	} else if errors.Is(err, mongo.ErrNoDocuments) {
+		return 0, nil
+	} else {
+		return 0, err
+	}
+}
+
+func (s *seqUserMongo) GetMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) {
+	return s.getSeq(ctx, conversationID, userID, "max_seq")
+}
+
+func (s *seqUserMongo) SetMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error {
+	return s.setSeq(ctx, conversationID, userID, seq, "max_seq")
+}
+
+func (s *seqUserMongo) GetMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) {
+	return s.getSeq(ctx, conversationID, userID, "min_seq")
+}
+
+func (s *seqUserMongo) SetMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error {
+	return s.setSeq(ctx, conversationID, userID, seq, "min_seq")
+}
+
+func (s *seqUserMongo) GetReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) {
+	return s.getSeq(ctx, conversationID, userID, "read_seq")
+}
+
+func (s *seqUserMongo) GetReadSeqs(ctx context.Context, userID string, conversationID []string) (map[string]int64, error) {
+	if len(conversationID) == 0 {
+		return map[string]int64{}, nil
+	}
+	filter := bson.M{"user_id": userID, "conversation_id": bson.M{"$in": conversationID}}
+	opt := options.Find().SetProjection(bson.M{"_id": 0, "conversation_id": 1, "read_seq": 1})
+	seqs, err := mongoutil.Find[*model.SeqUser](ctx, s.coll, filter, opt)
+	if err != nil {
+		return nil, err
+	}
+	res := make(map[string]int64)
+	for _, seq := range seqs {
+		res[seq.ConversationID] = seq.ReadSeq
+	}
+	return res, nil
+}
+
+func (s *seqUserMongo) SetReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error {
+	return s.setSeq(ctx, conversationID, userID, seq, "read_seq")
+}