IM/open-im-server
1
0
Fork 0
mirror of https://github.com/openimsdk/open-im-server.git synced 2026-05-05 01:25:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: optimize code and support running in single process mode (#3142)

Browse Source 
* pb

* fix: Modifying other fields while setting IsPrivateChat does not take effect

* fix: quote message error revoke

* refactoring scheduled tasks

* refactoring scheduled tasks

* refactoring scheduled tasks

* refactoring scheduled tasks

* refactoring scheduled tasks

* refactoring scheduled tasks

* upgrading pkg tools

* fix

* fix

* optimize log output

* feat: support GetLastMessage

* feat: support GetLastMessage

* feat: s3 switch

* feat: s3 switch

* fix: GetUsersOnline

* feat: SendBusinessNotification supported configuration parameters

* feat: SendBusinessNotification supported configuration parameters

* feat: SendBusinessNotification supported configuration parameters

* feat: seq conversion failed without exiting

* monolithic

* fix: DeleteDoc crash

* fix: DeleteDoc crash

* fix: monolithic

* fix: monolithic

* fix: fill send time

* fix: fill send time

* fix: crash caused by withdrawing messages from users who have left the group

* fix: mq

* fix: mq

* fix: user msg timestamp

* fix: mq

* 1

* 1

* 1

* 1

* 1

* 1

* 1

* seq read config

* seq read config

* 1

* 1

* fix: the source message of the reference is withdrawn, and the referenced message is deleted

* 1

* 1

* 1

* 1

* 1

* 1

* 1

* 1

* 1

* 1

* 1

* 1

* 1

* 1
This commit is contained in:
chao
2025-02-14 16:18:27 +08:00
committed by GitHub
parent e37ea50b94
commit 9ed6200e45
105 changed files with 3385 additions and 1925 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/common/storage/controller/group.go
+1 -1
View File
@@ -140,7 +140,7 @@ func NewGroupDatabase(
groupMemberDB: groupMemberDB,
groupRequestDB: groupRequestDB,
ctxTx: ctxTx,
cache: redis2.NewGroupCacheRedis(rdb, localCache, groupDB, groupMemberDB, groupRequestDB, groupHash, redis2.GetRocksCacheOptions()),
cache: redis2.NewGroupCacheRedis(rdb, localCache, groupDB, groupMemberDB, groupRequestDB, groupHash),
}
}
pkg/common/storage/controller/msg.go
+14 -18
View File
@@ -18,19 +18,21 @@ import (
"context"
"encoding/json"
"errors"
"github.com/openimsdk/tools/mq"
"github.com/openimsdk/tools/utils/jsonutil"
"google.golang.org/protobuf/proto"
"strconv"
"strings"
"time"
"github.com/openimsdk/tools/utils/jsonutil"
"github.com/openimsdk/open-im-server/v3/pkg/common/storage/database"
"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
"github.com/redis/go-redis/v9"
"go.mongodb.org/mongo-driver/mongo"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
"github.com/openimsdk/open-im-server/v3/pkg/common/convert"
"github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache"
"github.com/openimsdk/protocol/constant"
@@ -38,7 +40,6 @@ import (
"github.com/openimsdk/protocol/sdkws"
"github.com/openimsdk/tools/errs"
"github.com/openimsdk/tools/log"
"github.com/openimsdk/tools/mq/kafka"
"github.com/openimsdk/tools/utils/datautil"
)
@@ -102,22 +103,14 @@ type CommonMsgDatabase interface {
GetLastMessage(ctx context.Context, conversationIDS []string, userID string) (map[string]*sdkws.MsgData, error)
}
func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser cache.SeqUser, seqConversation cache.SeqConversationCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) {
conf, err := kafka.BuildProducerConfig(*kafkaConf.Build())
if err != nil {
return nil, err
}
producerToRedis, err := kafka.NewKafkaProducer(conf, kafkaConf.Address, kafkaConf.ToRedisTopic)
if err != nil {
return nil, err
}
func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser cache.SeqUser, seqConversation cache.SeqConversationCache, producer mq.Producer) CommonMsgDatabase {
return &commonMsgDatabase{
msgDocDatabase: msgDocModel,
msgCache: msg,
seqUser: seqUser,
seqConversation: seqConversation,
producer: producerToRedis,
}, nil
producer: producer,
}
}
type commonMsgDatabase struct {
@@ -126,12 +119,15 @@ type commonMsgDatabase struct {
msgCache cache.MsgCache
seqConversation cache.SeqConversationCache
seqUser cache.SeqUser
producer *kafka.Producer
producer mq.Producer
}
func (db *commonMsgDatabase) MsgToMQ(ctx context.Context, key string, msg2mq *sdkws.MsgData) error {
_, _, err := db.producer.SendMessage(ctx, key, msg2mq)
return err
data, err := proto.Marshal(msg2mq)
if err != nil {
return err
}
return db.producer.SendMessage(ctx, key, data)
}
func (db *commonMsgDatabase) batchInsertBlock(ctx context.Context, conversationID string, fields []any, key int8, firstSeq int64) error {
pkg/common/storage/controller/msg_transfer.go
+33 -26
View File
@@ -2,11 +2,13 @@ package controller
import (
"context"
"github.com/openimsdk/open-im-server/v3/pkg/common/convert"
"github.com/openimsdk/protocol/constant"
"github.com/openimsdk/tools/mq"
"github.com/openimsdk/tools/utils/datautil"
"google.golang.org/protobuf/proto"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
"github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache"
"github.com/openimsdk/open-im-server/v3/pkg/common/storage/database"
"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
@@ -14,7 +16,6 @@ import (
"github.com/openimsdk/protocol/sdkws"
"github.com/openimsdk/tools/errs"
"github.com/openimsdk/tools/log"
"github.com/openimsdk/tools/mq/kafka"
"go.mongodb.org/mongo-driver/mongo"
)
@@ -32,30 +33,30 @@ type MsgTransferDatabase interface {
SetHasReadSeqToDB(ctx context.Context, conversationID string, userSeqMap map[string]int64) error
// to mq
MsgToPushMQ(ctx context.Context, key, conversationID string, msg2mq *sdkws.MsgData) (int32, int64, error)
MsgToPushMQ(ctx context.Context, key, conversationID string, msg2mq *sdkws.MsgData) error
MsgToMongoMQ(ctx context.Context, key, conversationID string, msgs []*sdkws.MsgData, lastSeq int64) error
}
func NewMsgTransferDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser cache.SeqUser, seqConversation cache.SeqConversationCache, kafkaConf *config.Kafka) (MsgTransferDatabase, error) {
conf, err := kafka.BuildProducerConfig(*kafkaConf.Build())
if err != nil {
return nil, err
}
producerToMongo, err := kafka.NewKafkaProducer(conf, kafkaConf.Address, kafkaConf.ToMongoTopic)
if err != nil {
return nil, err
}
producerToPush, err := kafka.NewKafkaProducer(conf, kafkaConf.Address, kafkaConf.ToPushTopic)
if err != nil {
return nil, err
}
func NewMsgTransferDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser cache.SeqUser, seqConversation cache.SeqConversationCache, mongoProducer, pushProducer mq.Producer) (MsgTransferDatabase, error) {
//conf, err := kafka.BuildProducerConfig(*kafkaConf.Build())
//if err != nil {
// return nil, err
//}
//producerToMongo, err := kafka.NewKafkaProducerV2(conf, kafkaConf.Address, kafkaConf.ToMongoTopic)
//if err != nil {
// return nil, err
//}
//producerToPush, err := kafka.NewKafkaProducerV2(conf, kafkaConf.Address, kafkaConf.ToPushTopic)
//if err != nil {
// return nil, err
//}
return &msgTransferDatabase{
msgDocDatabase: msgDocModel,
msgCache: msg,
seqUser: seqUser,
seqConversation: seqConversation,
producerToMongo: producerToMongo,
producerToPush: producerToPush,
producerToMongo: mongoProducer,
producerToPush: pushProducer,
}, nil
}
@@ -65,8 +66,8 @@ type msgTransferDatabase struct {
msgCache cache.MsgCache
seqConversation cache.SeqConversationCache
seqUser cache.SeqUser
producerToMongo *kafka.Producer
producerToPush *kafka.Producer
producerToMongo mq.Producer
producerToPush mq.Producer
}
func (db *msgTransferDatabase) BatchInsertChat2DB(ctx context.Context, conversationID string, msgList []*sdkws.MsgData, currentMaxSeq int64) error {
@@ -281,19 +282,25 @@ func (db *msgTransferDatabase) SetHasReadSeqToDB(ctx context.Context, conversati
return nil
}
func (db *msgTransferDatabase) MsgToPushMQ(ctx context.Context, key, conversationID string, msg2mq *sdkws.MsgData) (int32, int64, error) {
partition, offset, err := db.producerToPush.SendMessage(ctx, key, &pbmsg.PushMsgDataToMQ{MsgData: msg2mq, ConversationID: conversationID})
func (db *msgTransferDatabase) MsgToPushMQ(ctx context.Context, key, conversationID string, msg2mq *sdkws.MsgData) error {
data, err := proto.Marshal(&pbmsg.PushMsgDataToMQ{MsgData: msg2mq, ConversationID: conversationID})
if err != nil {
log.ZError(ctx, "MsgToPushMQ", err, "key", key, "msg2mq", msg2mq)
return 0, 0, err
return err
}
return partition, offset, nil
if err := db.producerToPush.SendMessage(ctx, key, data); err != nil {
log.ZError(ctx, "MsgToPushMQ", err, "key", key, "conversationID", conversationID)
return err
}
return nil
}
func (db *msgTransferDatabase) MsgToMongoMQ(ctx context.Context, key, conversationID string, messages []*sdkws.MsgData, lastSeq int64) error {
if len(messages) > 0 {
_, _, err := db.producerToMongo.SendMessage(ctx, key, &pbmsg.MsgDataToMongoByMQ{LastSeq: lastSeq, ConversationID: conversationID, MsgData: messages})
data, err := proto.Marshal(&pbmsg.MsgDataToMongoByMQ{LastSeq: lastSeq, ConversationID: conversationID, MsgData: messages})
if err != nil {
return err
}
if err := db.producerToMongo.SendMessage(ctx, key, data); err != nil {
log.ZError(ctx, "MsgToMongoMQ", err, "key", key, "conversationID", conversationID, "lastSeq", lastSeq)
return err
}
pkg/common/storage/controller/push.go
+12 -15
View File
@@ -17,12 +17,12 @@ package controller
import (
"context"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
"github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache"
"github.com/openimsdk/protocol/push"
"github.com/openimsdk/protocol/sdkws"
"github.com/openimsdk/tools/log"
"github.com/openimsdk/tools/mq/kafka"
"github.com/openimsdk/tools/mq"
"google.golang.org/protobuf/proto"
)
type PushDatabase interface {
@@ -32,21 +32,13 @@ type PushDatabase interface {
type pushDataBase struct {
cache cache.ThirdCache
producerToOfflinePush *kafka.Producer
producerToOfflinePush mq.Producer
}
func NewPushDatabase(cache cache.ThirdCache, kafkaConf *config.Kafka) PushDatabase {
conf, err := kafka.BuildProducerConfig(*kafkaConf.Build())
if err != nil {
return nil
}
producerToOfflinePush, err := kafka.NewKafkaProducer(conf, kafkaConf.Address, kafkaConf.ToOfflinePushTopic)
if err != nil {
return nil
}
func NewPushDatabase(cache cache.ThirdCache, offlinePushProducer mq.Producer) PushDatabase {
return &pushDataBase{
cache: cache,
producerToOfflinePush: producerToOfflinePush,
producerToOfflinePush: offlinePushProducer,
}
}
@@ -55,7 +47,12 @@ func (p *pushDataBase) DelFcmToken(ctx context.Context, userID string, platformI
}
func (p *pushDataBase) MsgToOfflinePushMQ(ctx context.Context, key string, userIDs []string, msg2mq *sdkws.MsgData) error {
_, _, err := p.producerToOfflinePush.SendMessage(ctx, key, &push.PushMsgReq{MsgData: msg2mq, UserIDs: userIDs})
log.ZInfo(ctx, "message is push to offlinePush topic", "key", key, "userIDs", userIDs, "msg", msg2mq.String())
data, err := proto.Marshal(&push.PushMsgReq{MsgData: msg2mq, UserIDs: userIDs})
if err != nil {
return err
}
if err := p.producerToOfflinePush.SendMessage(ctx, key, data); err != nil {
log.ZError(ctx, "message is push to offlinePush topic", err, "key", key, "userIDs", userIDs, "msg", msg2mq.String())
}
return err
}
pkg/common/storage/controller/s3.go
+2 -2
View File
@@ -30,7 +30,7 @@ import (
)
type S3Database interface {
PartLimit() *s3.PartLimit
PartLimit() (*s3.PartLimit, error)
PartSize(ctx context.Context, size int64) (int64, error)
AuthSign(ctx context.Context, uploadID string, partNumbers []int) (*s3.AuthSignResult, error)
InitiateMultipartUpload(ctx context.Context, hash string, size int64, expire time.Duration, maxParts int) (*cont.InitiateUploadResult, error)
@@ -65,7 +65,7 @@ func (s *s3Database) PartSize(ctx context.Context, size int64) (int64, error) {
return s.s3.PartSize(ctx, size)
}
func (s *s3Database) PartLimit() *s3.PartLimit {
func (s *s3Database) PartLimit() (*s3.PartLimit, error) {
return s.s3.PartLimit()
}