feat: optimize code and support running in single process mode (#3142)

* pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * monolithic * fix: DeleteDoc crash * fix: DeleteDoc crash * fix: monolithic * fix: monolithic * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group * fix: mq * fix: mq * fix: user msg timestamp * fix: mq * 1 * 1 * 1 * 1 * 1 * 1 * 1 * seq read config * seq read config * 1 * 1 * fix: the source message of the reference is withdrawn, and the referenced message is deleted * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1
2026-05-05 17:45:59 +08:00 · 2025-02-14 16:18:27 +08:00
parent e37ea50b94
commit 9ed6200e45
105 changed files with 3385 additions and 1925 deletions
@@ -16,6 +16,7 @@ package database

 import (
 	"context"
+
 	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
 	"github.com/openimsdk/tools/db/pagination"
 )
@@ -29,3 +30,85 @@ type Black interface {
 	FindOwnerBlackInfos(ctx context.Context, ownerUserID string, userIDs []string) (blacks []*model.Black, err error)
 	FindBlackUserIDs(ctx context.Context, ownerUserID string) (blackUserIDs []string, err error)
 }
+
+var (
+	_ Black = (*mgoImpl)(nil)
+	_ Black = (*redisImpl)(nil)
+)
+
+type mgoImpl struct {
+}
+
+func (m *mgoImpl) Create(ctx context.Context, blacks []*model.Black) (err error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (m *mgoImpl) Delete(ctx context.Context, blacks []*model.Black) (err error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (m *mgoImpl) Find(ctx context.Context, blacks []*model.Black) (blackList []*model.Black, err error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (m *mgoImpl) Take(ctx context.Context, ownerUserID, blockUserID string) (black *model.Black, err error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (m *mgoImpl) FindOwnerBlacks(ctx context.Context, ownerUserID string, pagination pagination.Pagination) (total int64, blacks []*model.Black, err error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (m *mgoImpl) FindOwnerBlackInfos(ctx context.Context, ownerUserID string, userIDs []string) (blacks []*model.Black, err error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (m *mgoImpl) FindBlackUserIDs(ctx context.Context, ownerUserID string) (blackUserIDs []string, err error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+type redisImpl struct {
+}
+
+func (r *redisImpl) Create(ctx context.Context, blacks []*model.Black) (err error) {
+
+	//TODO implement me
+	panic("implement me")
+}
+
+func (r *redisImpl) Delete(ctx context.Context, blacks []*model.Black) (err error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (r *redisImpl) Find(ctx context.Context, blacks []*model.Black) (blackList []*model.Black, err error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (r *redisImpl) Take(ctx context.Context, ownerUserID, blockUserID string) (black *model.Black, err error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (r *redisImpl) FindOwnerBlacks(ctx context.Context, ownerUserID string, pagination pagination.Pagination) (total int64, blacks []*model.Black, err error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (r *redisImpl) FindOwnerBlackInfos(ctx context.Context, ownerUserID string, userIDs []string) (blacks []*model.Black, err error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (r *redisImpl) FindBlackUserIDs(ctx context.Context, ownerUserID string) (blackUserIDs []string, err error) {
+	//TODO implement me
+	panic("implement me")
+}
@@ -0,0 +1,16 @@
+package database
+
+import (
+	"context"
+	"time"
+)
+
+type Cache interface {
+	Get(ctx context.Context, key []string) (map[string]string, error)
+	Prefix(ctx context.Context, prefix string) (map[string]string, error)
+	Set(ctx context.Context, key string, value string, expireAt time.Duration) error
+	Incr(ctx context.Context, key string, value int) (int, error)
+	Del(ctx context.Context, key []string) error
+	Lock(ctx context.Context, key string, duration time.Duration) (string, error)
+	Unlock(ctx context.Context, key string, value string) error
+}
@@ -0,0 +1,183 @@
+package mgo
+
+import (
+	"context"
+	"strconv"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/database"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
+	"github.com/openimsdk/tools/db/mongoutil"
+	"github.com/openimsdk/tools/errs"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+func NewCacheMgo(db *mongo.Database) (*CacheMgo, error) {
+	coll := db.Collection(database.CacheName)
+	_, err := coll.Indexes().CreateMany(context.Background(), []mongo.IndexModel{
+		{
+			Keys: bson.D{
+				{Key: "key", Value: 1},
+			},
+			Options: options.Index().SetUnique(true),
+		},
+		{
+			Keys: bson.D{
+				{Key: "expire_at", Value: 1},
+			},
+			Options: options.Index().SetExpireAfterSeconds(0),
+		},
+	})
+	if err != nil {
+		return nil, errs.Wrap(err)
+	}
+	return &CacheMgo{coll: coll}, nil
+}
+
+type CacheMgo struct {
+	coll *mongo.Collection
+}
+
+func (x *CacheMgo) findToMap(res []model.Cache, now time.Time) map[string]string {
+	kv := make(map[string]string)
+	for _, re := range res {
+		if re.ExpireAt != nil && re.ExpireAt.Before(now) {
+			continue
+		}
+		kv[re.Key] = re.Value
+	}
+	return kv
+
+}
+
+func (x *CacheMgo) Get(ctx context.Context, key []string) (map[string]string, error) {
+	if len(key) == 0 {
+		return nil, nil
+	}
+	now := time.Now()
+	res, err := mongoutil.Find[model.Cache](ctx, x.coll, bson.M{
+		"key": bson.M{"$in": key},
+		"$or": []bson.M{
+			{"expire_at": bson.M{"$gt": now}},
+			{"expire_at": nil},
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+	return x.findToMap(res, now), nil
+}
+
+func (x *CacheMgo) Prefix(ctx context.Context, prefix string) (map[string]string, error) {
+	now := time.Now()
+	res, err := mongoutil.Find[model.Cache](ctx, x.coll, bson.M{
+		"key": bson.M{"$regex": "^" + prefix},
+		"$or": []bson.M{
+			{"expire_at": bson.M{"$gt": now}},
+			{"expire_at": nil},
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+	return x.findToMap(res, now), nil
+}
+
+func (x *CacheMgo) Set(ctx context.Context, key string, value string, expireAt time.Duration) error {
+	cv := &model.Cache{
+		Key:   key,
+		Value: value,
+	}
+	if expireAt > 0 {
+		now := time.Now().Add(expireAt)
+		cv.ExpireAt = &now
+	}
+	opt := options.Update().SetUpsert(true)
+	return mongoutil.UpdateOne(ctx, x.coll, bson.M{"key": key}, bson.M{"$set": cv}, false, opt)
+}
+
+func (x *CacheMgo) Incr(ctx context.Context, key string, value int) (int, error) {
+	pipeline := mongo.Pipeline{
+		{
+			{"$set", bson.M{
+				"value": bson.M{
+					"$toString": bson.M{
+						"$add": bson.A{
+							bson.M{"$toInt": "$value"},
+							value,
+						},
+					},
+				},
+			}},
+		},
+	}
+	opt := options.FindOneAndUpdate().SetReturnDocument(options.After)
+	res, err := mongoutil.FindOneAndUpdate[model.Cache](ctx, x.coll, bson.M{"key": key}, pipeline, opt)
+	if err != nil {
+		return 0, err
+	}
+	return strconv.Atoi(res.Value)
+}
+
+func (x *CacheMgo) Del(ctx context.Context, key []string) error {
+	if len(key) == 0 {
+		return nil
+	}
+	_, err := x.coll.DeleteMany(ctx, bson.M{"key": bson.M{"$in": key}})
+	return err
+}
+
+func (x *CacheMgo) lockKey(key string) string {
+	return "LOCK_" + key
+}
+
+func (x *CacheMgo) Lock(ctx context.Context, key string, duration time.Duration) (string, error) {
+	tmp, err := uuid.NewUUID()
+	if err != nil {
+		return "", err
+	}
+	if duration <= 0 || duration > time.Minute*10 {
+		duration = time.Minute * 10
+	}
+	cv := &model.Cache{
+		Key:      x.lockKey(key),
+		Value:    tmp.String(),
+		ExpireAt: nil,
+	}
+	ctx, cancel := context.WithTimeout(ctx, time.Second*30)
+	defer cancel()
+	wait := func() error {
+		timeout := time.NewTimer(time.Millisecond * 100)
+		defer timeout.Stop()
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-timeout.C:
+			return nil
+		}
+	}
+	for {
+		if err := mongoutil.DeleteOne(ctx, x.coll, bson.M{"key": key, "expire_at": bson.M{"$lt": time.Now()}}); err != nil {
+			return "", err
+		}
+		expireAt := time.Now().Add(duration)
+		cv.ExpireAt = &expireAt
+		if err := mongoutil.InsertMany[*model.Cache](ctx, x.coll, []*model.Cache{cv}); err != nil {
+			if mongo.IsDuplicateKeyError(err) {
+				if err := wait(); err != nil {
+					return "", err
+				}
+				continue
+			}
+			return "", err
+		}
+		return cv.Value, nil
+	}
+}
+
+func (x *CacheMgo) Unlock(ctx context.Context, key string, value string) error {
+	return mongoutil.DeleteOne(ctx, x.coll, bson.M{"key": x.lockKey(key), "value": value})
+}
@@ -0,0 +1,133 @@
+package mgo
+
+import (
+	"context"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
+	"github.com/openimsdk/tools/db/mongoutil"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+func TestName1111(t *testing.T) {
+	coll := Mongodb().Collection("temp")
+
+	//updatePipeline := mongo.Pipeline{
+	//	{
+	//		{"$set", bson.M{
+	//			"age": bson.M{
+	//				"$toString": bson.M{
+	//					"$add": bson.A{
+	//						bson.M{"$toInt": "$age"},
+	//						1,
+	//					},
+	//				},
+	//			},
+	//		}},
+	//	},
+	//}
+
+	pipeline := mongo.Pipeline{
+		{
+			{"$set", bson.M{
+				"value": bson.M{
+					"$toString": bson.M{
+						"$add": bson.A{
+							bson.M{"$toInt": "$value"},
+							1,
+						},
+					},
+				},
+			}},
+		},
+	}
+
+	opt := options.FindOneAndUpdate().SetUpsert(false).SetReturnDocument(options.After)
+	res, err := mongoutil.FindOneAndUpdate[model.Cache](context.Background(), coll, bson.M{"key": "123456"}, pipeline, opt)
+	if err != nil {
+		panic(err)
+	}
+	t.Log(res)
+}
+
+func TestName33333(t *testing.T) {
+	c, err := NewCacheMgo(Mongodb())
+	if err != nil {
+		panic(err)
+	}
+	if err := c.Set(context.Background(), "123456", "123456", time.Hour); err != nil {
+		panic(err)
+	}
+
+	if err := c.Set(context.Background(), "123666", "123666", time.Hour); err != nil {
+		panic(err)
+	}
+
+	res1, err := c.Get(context.Background(), []string{"123456"})
+	if err != nil {
+		panic(err)
+	}
+	t.Log(res1)
+
+	res2, err := c.Prefix(context.Background(), "123")
+	if err != nil {
+		panic(err)
+	}
+	t.Log(res2)
+}
+
+func TestName1111aa(t *testing.T) {
+
+	c, err := NewCacheMgo(Mongodb())
+	if err != nil {
+		panic(err)
+	}
+	var count int
+
+	key := "123456"
+
+	doFunc := func() {
+		value, err := c.Lock(context.Background(), key, time.Second*30)
+		if err != nil {
+			t.Log("Lock error", err)
+			return
+		}
+		tmp := count
+		tmp++
+		count = tmp
+		t.Log("count", tmp)
+		if err := c.Unlock(context.Background(), key, value); err != nil {
+			t.Log("Unlock error", err)
+			return
+		}
+	}
+
+	if _, err := c.Lock(context.Background(), key, time.Second*10); err != nil {
+		t.Log(err)
+		return
+	}
+
+	var wg sync.WaitGroup
+	for i := 0; i < 32; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for i := 0; i < 100; i++ {
+				doFunc()
+			}
+		}()
+	}
+
+	wg.Wait()
+
+}
+
+func TestName111111a(t *testing.T) {
+	arr := strings.SplitN("1:testkakskdask:1111", ":", 2)
+	t.Log(arr)
+}
@@ -2,16 +2,17 @@ package mgo

 import (
 	"context"
-	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
-	"github.com/openimsdk/tools/db/mongoutil"
-	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/mongo"
-	"go.mongodb.org/mongo-driver/mongo/options"
 	"math"
 	"math/rand"
 	"strconv"
 	"testing"
 	"time"
+
+	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
+	"github.com/openimsdk/tools/db/mongoutil"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
 )

 func TestName1(t *testing.T) {
@@ -93,7 +94,7 @@ func TestName3(t *testing.T) {
 func TestName4(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*300)
 	defer cancel()
-	cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.66:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second)))
+	cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.135:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second)))

 	msg, err := NewMsgMongo(cli.Database("openim_v3"))
 	if err != nil {
@@ -109,6 +110,41 @@ func TestName4(t *testing.T) {
 }

 func TestName5(t *testing.T) {
-	var v time.Time
-	t.Log(v.UnixMilli())
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*300)
+	defer cancel()
+	cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.135:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second)))
+
+	tmp, err := NewMsgMongo(cli.Database("openim_v3"))
+	if err != nil {
+		panic(err)
+	}
+	msg := tmp.(*MsgMgo)
+	ts := time.Now().Add(-time.Hour * 24 * 5).UnixMilli()
+	t.Log(ts)
+	var seqs []int64
+	for i := 1; i < 256; i++ {
+		seqs = append(seqs, int64(i))
+	}
+	res, err := msg.FindSeqs(ctx, "si_4924054191_9511766539", seqs)
+	if err != nil {
+		panic(err)
+	}
+	t.Log(res)
 }
+
+//func TestName6(t *testing.T) {
+//	ctx, cancel := context.WithTimeout(context.Background(), time.Second*300)
+//	defer cancel()
+//	cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.135:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second)))
+//
+//	tmp, err := NewMsgMongo(cli.Database("openim_v3"))
+//	if err != nil {
+//		panic(err)
+//	}
+//	msg := tmp.(*MsgMgo)
+//	seq, sendTime, err := msg.findBeforeSendTime(ctx, "si_4924054191_9511766539", 1144)
+//	if err != nil {
+//		panic(err)
+//	}
+//	t.Log(seq, sendTime)
+//}
@@ -2,10 +2,11 @@ package mgo

 import (
 	"context"
-	"go.mongodb.org/mongo-driver/mongo"
-	"go.mongodb.org/mongo-driver/mongo/options"
 	"testing"
 	"time"
+
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
 )

 func Result[V any](val V, err error) V {
@@ -19,7 +20,7 @@ func Mongodb() *mongo.Database {
 	return Result(
 		mongo.Connect(context.Background(),
 			options.Client().
-				ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").
+				ApplyURI("mongodb://openIM:openIM123@172.16.8.135:37017/openim_v3?maxPoolSize=100").
 				SetConnectTimeout(5*time.Second)),
 	).Database("openim_v3")
 }
@@ -18,4 +18,5 @@ const (
 	SeqConversationName     = "seq"
 	SeqUserName             = "seq_user"
 	StreamMsgName           = "stream_msg"
+	CacheName               = "cache"
 )