mirror of
https://github.com/openimsdk/open-im-server.git
synced 2026-05-16 14:59:01 +08:00
optimization: change the configuration file from being read globally … (#1935)
* optimization: change the configuration file from being read globally to being read independently. * optimization: change the configuration file from being read globally to being read independently. * optimization: change the configuration file from being read globally to being read independently. * optimization: config file changed to dependency injection. * fix: replace global config with dependency injection * fix: replace global config with dependency injection * fix: import the enough param * fix: import the enough param * fix: import the enough param * fix: fix the component check of path * fix: fix the kafka of tls is nil problem * fix: fix the TLS.CACrt is nil error * fix: fix the valiable shadows problem * fix: fix the comflect * optimization: message remove options. * fix: fix the param pass error * fix: find error * fix: find error * fix: find eror * fix: find error * fix: find error * fix: del the undifined func * fix: find error * fix: fix the error * fix: pass config * fix: find error * fix: find error * fix: find error * fix: find error * fix: find error * fix: fix the config * fix: fix the error * fix: fix the config pass error * fix: fix the eror * fix: fix the error * fix: fix the error * fix: fix the error * fix: find error * fix: fix the error * fix: fix the config * fix: add return err * fix: fix the err2 * fix: err * fix: fix the func * fix: del the chinese comment * fix: fix the func * fix: fix the gateway_test logic * fix: s3 * test * test * fix: not found --------- Co-authored-by: luhaoling <2198702716@qq.com> Co-authored-by: withchao <993506633@qq.com>
This commit is contained in:
Vendored
+21
-21
@@ -38,34 +38,34 @@ const (
|
||||
)
|
||||
|
||||
// NewRedis Initialize redis connection.
|
||||
func NewRedis() (redis.UniversalClient, error) {
|
||||
func NewRedis(config *config.GlobalConfig) (redis.UniversalClient, error) {
|
||||
if redisClient != nil {
|
||||
return redisClient, nil
|
||||
}
|
||||
|
||||
// Read configuration from environment variables
|
||||
overrideConfigFromEnv()
|
||||
overrideConfigFromEnv(config)
|
||||
|
||||
if len(config.Config.Redis.Address) == 0 {
|
||||
return nil, errs.Wrap(errors.New("redis address is empty"), "Redis configuration error")
|
||||
if len(config.Redis.Address) == 0 {
|
||||
return nil, errs.Wrap(errors.New("redis address is empty"))
|
||||
}
|
||||
specialerror.AddReplace(redis.Nil, errs.ErrRecordNotFound)
|
||||
var rdb redis.UniversalClient
|
||||
if len(config.Config.Redis.Address) > 1 || config.Config.Redis.ClusterMode {
|
||||
if len(config.Redis.Address) > 1 || config.Redis.ClusterMode {
|
||||
rdb = redis.NewClusterClient(&redis.ClusterOptions{
|
||||
Addrs: config.Config.Redis.Address,
|
||||
Username: config.Config.Redis.Username,
|
||||
Password: config.Config.Redis.Password, // no password set
|
||||
Addrs: config.Redis.Address,
|
||||
Username: config.Redis.Username,
|
||||
Password: config.Redis.Password, // no password set
|
||||
PoolSize: 50,
|
||||
MaxRetries: maxRetry,
|
||||
})
|
||||
} else {
|
||||
rdb = redis.NewClient(&redis.Options{
|
||||
Addr: config.Config.Redis.Address[0],
|
||||
Username: config.Config.Redis.Username,
|
||||
Password: config.Config.Redis.Password, // no password set
|
||||
DB: 0, // use default DB
|
||||
PoolSize: 100, // connection pool size
|
||||
Addr: config.Redis.Address[0],
|
||||
Username: config.Redis.Username,
|
||||
Password: config.Redis.Password,
|
||||
DB: 0, // use default DB
|
||||
PoolSize: 100, // connection pool size
|
||||
MaxRetries: maxRetry,
|
||||
})
|
||||
}
|
||||
@@ -75,33 +75,33 @@ func NewRedis() (redis.UniversalClient, error) {
|
||||
defer cancel()
|
||||
err = rdb.Ping(ctx).Err()
|
||||
if err != nil {
|
||||
uriFormat := "address:%v, username:%s, clusterMode:%t, enablePipeline:%t"
|
||||
errMsg := fmt.Sprintf(uriFormat, config.Config.Redis.Address, config.Config.Redis.Username, config.Config.Redis.ClusterMode, config.Config.Redis.EnablePipeline)
|
||||
return nil, errs.Wrap(err, "Redis connection failed: %s", errMsg)
|
||||
errMsg := fmt.Sprintf("address:%s, username:%s, password:%s, clusterMode:%t, enablePipeline:%t", config.Redis.Address, config.Redis.Username,
|
||||
config.Redis.Password, config.Redis.ClusterMode, config.Redis.EnablePipeline)
|
||||
return nil, errs.Wrap(err, errMsg)
|
||||
}
|
||||
redisClient = rdb
|
||||
return rdb, err
|
||||
}
|
||||
|
||||
// overrideConfigFromEnv overrides configuration fields with environment variables if present.
|
||||
func overrideConfigFromEnv() {
|
||||
func overrideConfigFromEnv(config *config.GlobalConfig) {
|
||||
if envAddr := os.Getenv("REDIS_ADDRESS"); envAddr != "" {
|
||||
if envPort := os.Getenv("REDIS_PORT"); envPort != "" {
|
||||
addresses := strings.Split(envAddr, ",")
|
||||
for i, addr := range addresses {
|
||||
addresses[i] = addr + ":" + envPort
|
||||
}
|
||||
config.Config.Redis.Address = addresses
|
||||
config.Redis.Address = addresses
|
||||
} else {
|
||||
config.Config.Redis.Address = strings.Split(envAddr, ",")
|
||||
config.Redis.Address = strings.Split(envAddr, ",")
|
||||
}
|
||||
}
|
||||
|
||||
if envUser := os.Getenv("REDIS_USERNAME"); envUser != "" {
|
||||
config.Config.Redis.Username = envUser
|
||||
config.Redis.Username = envUser
|
||||
}
|
||||
|
||||
if envPass := os.Getenv("REDIS_PASSWORD"); envPass != "" {
|
||||
config.Config.Redis.Password = envPass
|
||||
config.Redis.Password = envPass
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
+11
-8
@@ -18,13 +18,16 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/OpenIMSDK/tools/mw/specialerror"
|
||||
|
||||
"github.com/dtm-labs/rockscache"
|
||||
|
||||
"github.com/OpenIMSDK/tools/errs"
|
||||
"github.com/OpenIMSDK/tools/log"
|
||||
"github.com/OpenIMSDK/tools/mw/specialerror"
|
||||
"github.com/OpenIMSDK/tools/utils"
|
||||
"github.com/dtm-labs/rockscache"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -128,7 +131,7 @@ func getCache[T any](ctx context.Context, rcClient *rockscache.Client, key strin
|
||||
v, err := rcClient.Fetch2(ctx, key, expire, func() (s string, err error) {
|
||||
t, err = fn(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", errs.Wrap(err)
|
||||
}
|
||||
bs, err := json.Marshal(t)
|
||||
if err != nil {
|
||||
@@ -139,7 +142,7 @@ func getCache[T any](ctx context.Context, rcClient *rockscache.Client, key strin
|
||||
return string(bs), nil
|
||||
})
|
||||
if err != nil {
|
||||
return t, err
|
||||
return t, errs.Wrap(err)
|
||||
}
|
||||
if write {
|
||||
return t, nil
|
||||
@@ -149,8 +152,8 @@ func getCache[T any](ctx context.Context, rcClient *rockscache.Client, key strin
|
||||
}
|
||||
err = json.Unmarshal([]byte(v), &t)
|
||||
if err != nil {
|
||||
log.ZError(ctx, "cache json.Unmarshal failed", err, "key", key, "value", v, "expire", expire)
|
||||
return t, errs.Wrap(err, "unmarshal failed")
|
||||
errInfo := fmt.Sprintf("cache json.Unmarshal failed, key:%s, value:%s, expire:%s", key, v, expire)
|
||||
return t, errs.Wrap(err, errInfo)
|
||||
}
|
||||
|
||||
return t, nil
|
||||
@@ -203,7 +206,7 @@ func batchGetCache2[T any, K comparable](
|
||||
fns func(ctx context.Context, key K) (T, error),
|
||||
) ([]T, error) {
|
||||
if len(keys) == 0 {
|
||||
return nil, nil
|
||||
return nil, errs.ErrArgs.Wrap("groupID is empty")
|
||||
}
|
||||
res := make([]T, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
@@ -214,7 +217,7 @@ func batchGetCache2[T any, K comparable](
|
||||
if errs.ErrRecordNotFound.Is(specialerror.ErrCode(errs.Unwrap(err))) {
|
||||
continue
|
||||
}
|
||||
return nil, err
|
||||
return nil, errs.Wrap(err)
|
||||
}
|
||||
res = append(res, val)
|
||||
}
|
||||
|
||||
Vendored
+12
-11
@@ -121,13 +121,14 @@ type MsgModel interface {
|
||||
UnLockMessageTypeKey(ctx context.Context, clientMsgID string, TypeKey string) error
|
||||
}
|
||||
|
||||
func NewMsgCacheModel(client redis.UniversalClient) MsgModel {
|
||||
return &msgCache{rdb: client}
|
||||
func NewMsgCacheModel(client redis.UniversalClient, config *config.GlobalConfig) MsgModel {
|
||||
return &msgCache{rdb: client, config: config}
|
||||
}
|
||||
|
||||
type msgCache struct {
|
||||
metaCache
|
||||
rdb redis.UniversalClient
|
||||
rdb redis.UniversalClient
|
||||
config *config.GlobalConfig
|
||||
}
|
||||
|
||||
func (c *msgCache) getMaxSeqKey(conversationID string) string {
|
||||
@@ -315,7 +316,7 @@ func (c *msgCache) allMessageCacheKey(conversationID string) string {
|
||||
}
|
||||
|
||||
func (c *msgCache) GetMessagesBySeq(ctx context.Context, conversationID string, seqs []int64) (seqMsgs []*sdkws.MsgData, failedSeqs []int64, err error) {
|
||||
if config.Config.Redis.EnablePipeline {
|
||||
if c.config.Redis.EnablePipeline {
|
||||
return c.PipeGetMessagesBySeq(ctx, conversationID, seqs)
|
||||
}
|
||||
|
||||
@@ -416,7 +417,7 @@ func (c *msgCache) ParallelGetMessagesBySeq(ctx context.Context, conversationID
|
||||
}
|
||||
|
||||
func (c *msgCache) SetMessageToCache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (int, error) {
|
||||
if config.Config.Redis.EnablePipeline {
|
||||
if c.config.Redis.EnablePipeline {
|
||||
return c.PipeSetMessageToCache(ctx, conversationID, msgs)
|
||||
}
|
||||
return c.ParallelSetMessageToCache(ctx, conversationID, msgs)
|
||||
@@ -431,7 +432,7 @@ func (c *msgCache) PipeSetMessageToCache(ctx context.Context, conversationID str
|
||||
}
|
||||
|
||||
key := c.getMessageCacheKey(conversationID, msg.Seq)
|
||||
_ = pipe.Set(ctx, key, s, time.Duration(config.Config.MsgCacheTimeout)*time.Second)
|
||||
_ = pipe.Set(ctx, key, s, time.Duration(c.config.MsgCacheTimeout)*time.Second)
|
||||
}
|
||||
|
||||
results, err := pipe.Exec(ctx)
|
||||
@@ -461,7 +462,7 @@ func (c *msgCache) ParallelSetMessageToCache(ctx context.Context, conversationID
|
||||
}
|
||||
|
||||
key := c.getMessageCacheKey(conversationID, msg.Seq)
|
||||
if err := c.rdb.Set(ctx, key, s, time.Duration(config.Config.MsgCacheTimeout)*time.Second).Err(); err != nil {
|
||||
if err := c.rdb.Set(ctx, key, s, time.Duration(c.config.MsgCacheTimeout)*time.Second).Err(); err != nil {
|
||||
return errs.Wrap(err)
|
||||
}
|
||||
return nil
|
||||
@@ -496,10 +497,10 @@ func (c *msgCache) UserDeleteMsgs(ctx context.Context, conversationID string, se
|
||||
if err != nil {
|
||||
return errs.Wrap(err)
|
||||
}
|
||||
if err := c.rdb.Expire(ctx, delUserListKey, time.Duration(config.Config.MsgCacheTimeout)*time.Second).Err(); err != nil {
|
||||
if err := c.rdb.Expire(ctx, delUserListKey, time.Duration(c.config.MsgCacheTimeout)*time.Second).Err(); err != nil {
|
||||
return errs.Wrap(err)
|
||||
}
|
||||
if err := c.rdb.Expire(ctx, userDelListKey, time.Duration(config.Config.MsgCacheTimeout)*time.Second).Err(); err != nil {
|
||||
if err := c.rdb.Expire(ctx, userDelListKey, time.Duration(c.config.MsgCacheTimeout)*time.Second).Err(); err != nil {
|
||||
return errs.Wrap(err)
|
||||
}
|
||||
}
|
||||
@@ -604,7 +605,7 @@ func (c *msgCache) DelUserDeleteMsgsList(ctx context.Context, conversationID str
|
||||
}
|
||||
|
||||
func (c *msgCache) DeleteMessages(ctx context.Context, conversationID string, seqs []int64) error {
|
||||
if config.Config.Redis.EnablePipeline {
|
||||
if c.config.Redis.EnablePipeline {
|
||||
return c.PipeDeleteMessages(ctx, conversationID, seqs)
|
||||
}
|
||||
|
||||
@@ -686,7 +687,7 @@ func (c *msgCache) DelMsgFromCache(ctx context.Context, userID string, seqs []in
|
||||
if err != nil {
|
||||
return errs.Wrap(err)
|
||||
}
|
||||
if err := c.rdb.Set(ctx, key, s, time.Duration(config.Config.MsgCacheTimeout)*time.Second).Err(); err != nil {
|
||||
if err := c.rdb.Set(ctx, key, s, time.Duration(c.config.MsgCacheTimeout)*time.Second).Err(); err != nil {
|
||||
return errs.Wrap(err)
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
+20
-12
@@ -22,12 +22,16 @@ import (
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
|
||||
|
||||
"github.com/OpenIMSDK/tools/log"
|
||||
|
||||
"github.com/OpenIMSDK/protocol/constant"
|
||||
|
||||
"github.com/OpenIMSDK/protocol/user"
|
||||
"github.com/OpenIMSDK/tools/errs"
|
||||
"github.com/OpenIMSDK/tools/log"
|
||||
|
||||
"github.com/dtm-labs/rockscache"
|
||||
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
@@ -62,7 +66,11 @@ type UserCacheRedis struct {
|
||||
rcClient *rockscache.Client
|
||||
}
|
||||
|
||||
func NewUserCacheRedis(rdb redis.UniversalClient, userDB relationtb.UserModelInterface, options rockscache.Options) UserCache {
|
||||
func NewUserCacheRedis(
|
||||
rdb redis.UniversalClient,
|
||||
userDB relationtb.UserModelInterface,
|
||||
options rockscache.Options,
|
||||
) UserCache {
|
||||
rcClient := rockscache.NewClient(rdb, options)
|
||||
|
||||
return &UserCacheRedis{
|
||||
@@ -193,13 +201,13 @@ func (u *UserCacheRedis) SetUserStatus(ctx context.Context, userID string, statu
|
||||
Status: constant.Online,
|
||||
PlatformIDs: []int32{platformID},
|
||||
}
|
||||
jsonData, err2 := json.Marshal(&onlineStatus)
|
||||
if err2 != nil {
|
||||
return errs.Wrap(err2)
|
||||
jsonData, err := json.Marshal(&onlineStatus)
|
||||
if err != nil {
|
||||
return errs.Wrap(err)
|
||||
}
|
||||
_, err2 = u.rdb.HSet(ctx, key, userID, string(jsonData)).Result()
|
||||
if err2 != nil {
|
||||
return errs.Wrap(err2)
|
||||
_, err = u.rdb.HSet(ctx, key, userID, string(jsonData)).Result()
|
||||
if err != nil {
|
||||
return errs.Wrap(err)
|
||||
}
|
||||
u.rdb.Expire(ctx, key, userOlineStatusExpireTime)
|
||||
|
||||
@@ -273,9 +281,9 @@ func (u *UserCacheRedis) refreshStatusOffline(ctx context.Context, userID string
|
||||
func (u *UserCacheRedis) refreshStatusOnline(ctx context.Context, userID string, platformID int32, isNil bool, err error, result, key string) error {
|
||||
var onlineStatus user.OnlineStatus
|
||||
if !isNil {
|
||||
err2 := json.Unmarshal([]byte(result), &onlineStatus)
|
||||
if err2 != nil {
|
||||
return errs.Wrap(err, "json.Unmarshal failed")
|
||||
err := json.Unmarshal([]byte(result), &onlineStatus)
|
||||
if err != nil {
|
||||
return errs.Wrap(err)
|
||||
}
|
||||
onlineStatus.PlatformIDs = RemoveRepeatedElementsInList(append(onlineStatus.PlatformIDs, platformID))
|
||||
} else {
|
||||
|
||||
@@ -16,6 +16,7 @@ package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
|
||||
|
||||
"github.com/OpenIMSDK/protocol/constant"
|
||||
"github.com/OpenIMSDK/tools/errs"
|
||||
@@ -33,14 +34,14 @@ type AuthDatabase interface {
|
||||
}
|
||||
|
||||
type authDatabase struct {
|
||||
cache cache.MsgModel
|
||||
|
||||
cache cache.MsgModel
|
||||
accessSecret string
|
||||
accessExpire int64
|
||||
config *config.GlobalConfig
|
||||
}
|
||||
|
||||
func NewAuthDatabase(cache cache.MsgModel, accessSecret string, accessExpire int64) AuthDatabase {
|
||||
return &authDatabase{cache: cache, accessSecret: accessSecret, accessExpire: accessExpire}
|
||||
func NewAuthDatabase(cache cache.MsgModel, accessSecret string, accessExpire int64, config *config.GlobalConfig) AuthDatabase {
|
||||
return &authDatabase{cache: cache, accessSecret: accessSecret, accessExpire: accessExpire, config: config}
|
||||
}
|
||||
|
||||
// If the result is empty.
|
||||
@@ -56,7 +57,7 @@ func (a *authDatabase) CreateToken(ctx context.Context, userID string, platformI
|
||||
}
|
||||
var deleteTokenKey []string
|
||||
for k, v := range tokens {
|
||||
_, err = tokenverify.GetClaimFromToken(k, authverify.Secret())
|
||||
_, err = tokenverify.GetClaimFromToken(k, authverify.Secret(a.config.Secret))
|
||||
if err != nil || v != constant.NormalToken {
|
||||
deleteTokenKey = append(deleteTokenKey, k)
|
||||
}
|
||||
|
||||
@@ -120,16 +120,33 @@ type CommonMsgDatabase interface {
|
||||
ConvertMsgsDocLen(ctx context.Context, conversationIDs []string)
|
||||
}
|
||||
|
||||
func NewCommonMsgDatabase(msgDocModel unrelationtb.MsgDocModelInterface, cacheModel cache.MsgModel) (CommonMsgDatabase, error) {
|
||||
producerToRedis, err := kafka.NewKafkaProducer(config.Config.Kafka.Addr, config.Config.Kafka.LatestMsgToRedis.Topic)
|
||||
func NewCommonMsgDatabase(msgDocModel unrelationtb.MsgDocModelInterface, cacheModel cache.MsgModel, config *config.GlobalConfig) (CommonMsgDatabase, error) {
|
||||
producerConfig := &kafka.ProducerConfig{
|
||||
ProducerAck: config.Kafka.ProducerAck,
|
||||
CompressType: config.Kafka.CompressType,
|
||||
Username: config.Kafka.Username,
|
||||
Password: config.Kafka.Password,
|
||||
}
|
||||
|
||||
var tlsConfig *kafka.TLSConfig
|
||||
if config.Kafka.TLS != nil {
|
||||
tlsConfig = &kafka.TLSConfig{
|
||||
CACrt: config.Kafka.TLS.CACrt,
|
||||
ClientCrt: config.Kafka.TLS.ClientCrt,
|
||||
ClientKey: config.Kafka.TLS.ClientKey,
|
||||
ClientKeyPwd: config.Kafka.TLS.ClientKeyPwd,
|
||||
InsecureSkipVerify: false,
|
||||
}
|
||||
}
|
||||
producerToRedis, err := kafka.NewKafkaProducer(config.Kafka.Addr, config.Kafka.LatestMsgToRedis.Topic, producerConfig, tlsConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
producerToMongo, err := kafka.NewKafkaProducer(config.Config.Kafka.Addr, config.Config.Kafka.MsgToMongo.Topic)
|
||||
producerToMongo, err := kafka.NewKafkaProducer(config.Kafka.Addr, config.Kafka.MsgToMongo.Topic, producerConfig, tlsConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
producerToPush, err := kafka.NewKafkaProducer(config.Config.Kafka.Addr, config.Config.Kafka.MsgToPush.Topic)
|
||||
producerToPush, err := kafka.NewKafkaProducer(config.Kafka.Addr, config.Kafka.MsgToPush.Topic, producerConfig, tlsConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -142,10 +159,10 @@ func NewCommonMsgDatabase(msgDocModel unrelationtb.MsgDocModelInterface, cacheMo
|
||||
}, nil
|
||||
}
|
||||
|
||||
func InitCommonMsgDatabase(rdb redis.UniversalClient, database *mongo.Database) (CommonMsgDatabase, error) {
|
||||
cacheModel := cache.NewMsgCacheModel(rdb)
|
||||
func InitCommonMsgDatabase(rdb redis.UniversalClient, database *mongo.Database, config *config.GlobalConfig) (CommonMsgDatabase, error) {
|
||||
cacheModel := cache.NewMsgCacheModel(rdb, config)
|
||||
msgDocModel := unrelation.NewMsgMongoDriver(database)
|
||||
return NewCommonMsgDatabase(msgDocModel, cacheModel)
|
||||
return NewCommonMsgDatabase(msgDocModel, cacheModel, config)
|
||||
}
|
||||
|
||||
type commonMsgDatabase struct {
|
||||
@@ -397,9 +414,9 @@ func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversa
|
||||
log.ZError(ctx, "db.cache.SetMaxSeq error", err, "conversationID", conversationID)
|
||||
prommetrics.SeqSetFailedCounter.Inc()
|
||||
}
|
||||
err2 := db.cache.SetHasReadSeqs(ctx, conversationID, userSeqMap)
|
||||
err = db.cache.SetHasReadSeqs(ctx, conversationID, userSeqMap)
|
||||
if err != nil {
|
||||
log.ZError(ctx, "SetHasReadSeqs error", err2, "userSeqMap", userSeqMap, "conversationID", conversationID)
|
||||
log.ZError(ctx, "SetHasReadSeqs error", err, "userSeqMap", userSeqMap, "conversationID", conversationID)
|
||||
prommetrics.SeqSetFailedCounter.Inc()
|
||||
}
|
||||
return lastMaxSeq, isNew, errs.Wrap(err)
|
||||
|
||||
@@ -33,27 +33,28 @@ import (
|
||||
)
|
||||
|
||||
func Test_BatchInsertChat2DB(t *testing.T) {
|
||||
config.Config.Mongo.Address = []string{"192.168.44.128:37017"}
|
||||
// config.Config.Mongo.Timeout = 60
|
||||
config.Config.Mongo.Database = "openIM"
|
||||
// config.Config.Mongo.Source = "admin"
|
||||
config.Config.Mongo.Username = "root"
|
||||
config.Config.Mongo.Password = "openIM123"
|
||||
config.Config.Mongo.MaxPoolSize = 100
|
||||
config.Config.RetainChatRecords = 3650
|
||||
config.Config.ChatRecordsClearTime = "0 2 * * 3"
|
||||
conf := config.NewGlobalConfig()
|
||||
conf.Mongo.Address = []string{"192.168.44.128:37017"}
|
||||
// conf.Mongo.Timeout = 60
|
||||
conf.Mongo.Database = "openIM"
|
||||
// conf.Mongo.Source = "admin"
|
||||
conf.Mongo.Username = "root"
|
||||
conf.Mongo.Password = "openIM123"
|
||||
conf.Mongo.MaxPoolSize = 100
|
||||
conf.RetainChatRecords = 3650
|
||||
conf.ChatRecordsClearTime = "0 2 * * 3"
|
||||
|
||||
mongo, err := unrelation.NewMongo()
|
||||
mongo, err := unrelation.NewMongo(conf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = mongo.GetDatabase().Client().Ping(context.Background(), nil)
|
||||
err = mongo.GetDatabase(conf.Mongo.Database).Client().Ping(context.Background(), nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
db := &commonMsgDatabase{
|
||||
msgDocDatabase: unrelation.NewMsgMongoDriver(mongo.GetDatabase()),
|
||||
msgDocDatabase: unrelation.NewMsgMongoDriver(mongo.GetDatabase(conf.Mongo.Database)),
|
||||
}
|
||||
|
||||
//ctx := context.Background()
|
||||
@@ -70,7 +71,7 @@ func Test_BatchInsertChat2DB(t *testing.T) {
|
||||
//}
|
||||
|
||||
_ = db.BatchInsertChat2DB
|
||||
c := mongo.GetDatabase().Collection("msg")
|
||||
c := mongo.GetDatabase(conf.Mongo.Database).Collection("msg")
|
||||
|
||||
ch := make(chan int)
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
@@ -144,26 +145,27 @@ func Test_BatchInsertChat2DB(t *testing.T) {
|
||||
}
|
||||
|
||||
func GetDB() *commonMsgDatabase {
|
||||
config.Config.Mongo.Address = []string{"203.56.175.233:37017"}
|
||||
// config.Config.Mongo.Timeout = 60
|
||||
config.Config.Mongo.Database = "openim_v3"
|
||||
// config.Config.Mongo.Source = "admin"
|
||||
config.Config.Mongo.Username = "root"
|
||||
config.Config.Mongo.Password = "openIM123"
|
||||
config.Config.Mongo.MaxPoolSize = 100
|
||||
config.Config.RetainChatRecords = 3650
|
||||
config.Config.ChatRecordsClearTime = "0 2 * * 3"
|
||||
conf := config.NewGlobalConfig()
|
||||
conf.Mongo.Address = []string{"203.56.175.233:37017"}
|
||||
// conf.Mongo.Timeout = 60
|
||||
conf.Mongo.Database = "openim_v3"
|
||||
// conf.Mongo.Source = "admin"
|
||||
conf.Mongo.Username = "root"
|
||||
conf.Mongo.Password = "openIM123"
|
||||
conf.Mongo.MaxPoolSize = 100
|
||||
conf.RetainChatRecords = 3650
|
||||
conf.ChatRecordsClearTime = "0 2 * * 3"
|
||||
|
||||
mongo, err := unrelation.NewMongo()
|
||||
mongo, err := unrelation.NewMongo(conf)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
err = mongo.GetDatabase().Client().Ping(context.Background(), nil)
|
||||
err = mongo.GetDatabase(conf.Mongo.Database).Client().Ping(context.Background(), nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return &commonMsgDatabase{
|
||||
msgDocDatabase: unrelation.NewMsgMongoDriver(mongo.GetDatabase()),
|
||||
msgDocDatabase: unrelation.NewMsgMongoDriver(mongo.GetDatabase(conf.Mongo.Database)),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,275 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// docURL: https://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html
|
||||
|
||||
package aws
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
sdk "github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
|
||||
)
|
||||
|
||||
const (
|
||||
minPartSize int64 = 1024 * 1024 * 1 // 1MB
|
||||
maxPartSize int64 = 1024 * 1024 * 1024 * 5 // 5GB
|
||||
maxNumSize int64 = 10000
|
||||
)
|
||||
|
||||
// const (
|
||||
// imagePng = "png"
|
||||
// imageJpg = "jpg"
|
||||
// imageJpeg = "jpeg"
|
||||
// imageGif = "gif"
|
||||
// imageWebp = "webp"
|
||||
// )
|
||||
|
||||
// const successCode = http.StatusOK
|
||||
|
||||
// const (
|
||||
// videoSnapshotImagePng = "png"
|
||||
// videoSnapshotImageJpg = "jpg"
|
||||
// )
|
||||
|
||||
func NewAWS() (s3.Interface, error) {
|
||||
conf := config.Config.Object.Aws
|
||||
credential := credentials.NewStaticCredentials(
|
||||
conf.AccessKeyID, // accessKey
|
||||
conf.AccessKeySecret, // secretKey
|
||||
"") // stoken
|
||||
|
||||
sess, err := session.NewSession(&aws.Config{
|
||||
Region: aws.String(conf.Region), // The area where the bucket is located
|
||||
Credentials: credential,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Aws{
|
||||
bucket: conf.Bucket,
|
||||
client: sdk.New(sess),
|
||||
credential: credential,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type Aws struct {
|
||||
bucket string
|
||||
client *sdk.S3
|
||||
credential *credentials.Credentials
|
||||
}
|
||||
|
||||
func (a *Aws) Engine() string {
|
||||
return "aws"
|
||||
}
|
||||
|
||||
func (a *Aws) InitiateMultipartUpload(ctx context.Context, name string) (*s3.InitiateMultipartUploadResult, error) {
|
||||
input := &sdk.CreateMultipartUploadInput{
|
||||
Bucket: aws.String(a.bucket), // TODO: To be verified whether it is required
|
||||
Key: aws.String(name),
|
||||
}
|
||||
result, err := a.client.CreateMultipartUploadWithContext(ctx, input)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3.InitiateMultipartUploadResult{
|
||||
Bucket: *result.Bucket,
|
||||
Key: *result.Key,
|
||||
UploadID: *result.UploadId,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (a *Aws) CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []s3.Part) (*s3.CompleteMultipartUploadResult, error) {
|
||||
sdkParts := make([]*sdk.CompletedPart, len(parts))
|
||||
for i, part := range parts {
|
||||
sdkParts[i] = &sdk.CompletedPart{
|
||||
ETag: aws.String(part.ETag),
|
||||
PartNumber: aws.Int64(int64(part.PartNumber)),
|
||||
}
|
||||
}
|
||||
input := &sdk.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String(a.bucket), // TODO: To be verified whether it is required
|
||||
Key: aws.String(name),
|
||||
UploadId: aws.String(uploadID),
|
||||
MultipartUpload: &sdk.CompletedMultipartUpload{
|
||||
Parts: sdkParts,
|
||||
},
|
||||
}
|
||||
result, err := a.client.CompleteMultipartUploadWithContext(ctx, input)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3.CompleteMultipartUploadResult{
|
||||
Location: *result.Location,
|
||||
Bucket: *result.Bucket,
|
||||
Key: *result.Key,
|
||||
ETag: *result.ETag,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (a *Aws) PartSize(ctx context.Context, size int64) (int64, error) {
|
||||
if size <= 0 {
|
||||
return 0, errors.New("size must be greater than 0")
|
||||
}
|
||||
if size > maxPartSize*maxNumSize {
|
||||
return 0, fmt.Errorf("AWS size must be less than the maximum allowed limit")
|
||||
}
|
||||
if size <= minPartSize*maxNumSize {
|
||||
return minPartSize, nil
|
||||
}
|
||||
partSize := size / maxNumSize
|
||||
if size%maxNumSize != 0 {
|
||||
partSize++
|
||||
}
|
||||
return partSize, nil
|
||||
}
|
||||
|
||||
func (a *Aws) DeleteObject(ctx context.Context, name string) error {
|
||||
_, err := a.client.DeleteObjectWithContext(ctx, &sdk.DeleteObjectInput{
|
||||
Bucket: aws.String(a.bucket),
|
||||
Key: aws.String(name),
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (a *Aws) CopyObject(ctx context.Context, src string, dst string) (*s3.CopyObjectInfo, error) {
|
||||
result, err := a.client.CopyObjectWithContext(ctx, &sdk.CopyObjectInput{
|
||||
Bucket: aws.String(a.bucket),
|
||||
Key: aws.String(dst),
|
||||
CopySource: aws.String(src),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3.CopyObjectInfo{
|
||||
ETag: *result.CopyObjectResult.ETag,
|
||||
Key: dst,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (a *Aws) IsNotFound(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if aerr, ok := err.(awserr.Error); ok {
|
||||
switch aerr.Code() {
|
||||
case sdk.ErrCodeNoSuchKey:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (a *Aws) AbortMultipartUpload(ctx context.Context, uploadID string, name string) error {
|
||||
_, err := a.client.AbortMultipartUploadWithContext(ctx, &sdk.AbortMultipartUploadInput{
|
||||
Bucket: aws.String(a.bucket),
|
||||
Key: aws.String(name),
|
||||
UploadId: aws.String(uploadID),
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (a *Aws) ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*s3.ListUploadedPartsResult, error) {
|
||||
result, err := a.client.ListPartsWithContext(ctx, &sdk.ListPartsInput{
|
||||
Bucket: aws.String(a.bucket),
|
||||
Key: aws.String(name),
|
||||
UploadId: aws.String(uploadID),
|
||||
MaxParts: aws.Int64(int64(maxParts)),
|
||||
PartNumberMarker: aws.Int64(int64(partNumberMarker)),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
parts := make([]s3.UploadedPart, len(result.Parts))
|
||||
for i, part := range result.Parts {
|
||||
parts[i] = s3.UploadedPart{
|
||||
PartNumber: int(*part.PartNumber),
|
||||
LastModified: *part.LastModified,
|
||||
Size: *part.Size,
|
||||
ETag: *part.ETag,
|
||||
}
|
||||
}
|
||||
return &s3.ListUploadedPartsResult{
|
||||
Key: *result.Key,
|
||||
UploadID: *result.UploadId,
|
||||
NextPartNumberMarker: int(*result.NextPartNumberMarker),
|
||||
MaxParts: int(*result.MaxParts),
|
||||
UploadedParts: parts,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (a *Aws) PartLimit() *s3.PartLimit {
|
||||
return &s3.PartLimit{
|
||||
MinPartSize: minPartSize,
|
||||
MaxPartSize: maxPartSize,
|
||||
MaxNumSize: maxNumSize,
|
||||
}
|
||||
}
|
||||
|
||||
func (a *Aws) PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error) {
|
||||
req, _ := a.client.PutObjectRequest(&sdk.PutObjectInput{
|
||||
Bucket: aws.String(a.bucket),
|
||||
Key: aws.String(name),
|
||||
})
|
||||
url, err := req.Presign(expire)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return url, nil
|
||||
}
|
||||
|
||||
func (a *Aws) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) {
|
||||
result, err := a.client.GetObjectWithContext(ctx, &sdk.GetObjectInput{
|
||||
Bucket: aws.String(a.bucket),
|
||||
Key: aws.String(name),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := &s3.ObjectInfo{
|
||||
Key: name,
|
||||
ETag: *result.ETag,
|
||||
Size: *result.ContentLength,
|
||||
LastModified: *result.LastModified,
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// AccessURL todo.
|
||||
func (a *Aws) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
|
||||
// todo
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (a *Aws) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) {
|
||||
// todo
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (a *Aws) AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*s3.AuthSignResult, error) {
|
||||
// todo
|
||||
return nil, nil
|
||||
}
|
||||
@@ -23,13 +23,13 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/OpenIMSDK/tools/errs"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
|
||||
"github.com/tencentyun/cos-go-sdk-v5"
|
||||
)
|
||||
@@ -50,13 +50,15 @@ const (
|
||||
|
||||
const successCode = http.StatusOK
|
||||
|
||||
const (
|
||||
// videoSnapshotImagePng = "png"
|
||||
// videoSnapshotImageJpg = "jpg"
|
||||
)
|
||||
type Config struct {
|
||||
BucketURL string
|
||||
SecretID string
|
||||
SecretKey string
|
||||
SessionToken string
|
||||
PublicRead bool
|
||||
}
|
||||
|
||||
func NewCos() (s3.Interface, error) {
|
||||
conf := config.Config.Object.Cos
|
||||
func NewCos(conf Config) (s3.Interface, error) {
|
||||
u, err := url.Parse(conf.BucketURL)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
@@ -69,6 +71,7 @@ func NewCos() (s3.Interface, error) {
|
||||
},
|
||||
})
|
||||
return &Cos{
|
||||
publicRead: conf.PublicRead,
|
||||
copyURL: u.Host + "/",
|
||||
client: client,
|
||||
credential: client.GetCredential(),
|
||||
@@ -76,6 +79,7 @@ func NewCos() (s3.Interface, error) {
|
||||
}
|
||||
|
||||
type Cos struct {
|
||||
publicRead bool
|
||||
copyURL string
|
||||
client *cos.Client
|
||||
credential *cos.Credential
|
||||
@@ -226,7 +230,7 @@ func (c *Cos) CopyObject(ctx context.Context, src string, dst string) (*s3.CopyO
|
||||
}
|
||||
|
||||
func (c *Cos) IsNotFound(err error) bool {
|
||||
switch e := err.(type) {
|
||||
switch e := errs.Unwrap(err).(type) {
|
||||
case *cos.ErrorResponse:
|
||||
return e.Response.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
|
||||
default:
|
||||
@@ -327,7 +331,7 @@ func (c *Cos) AccessURL(ctx context.Context, name string, expire time.Duration,
|
||||
}
|
||||
|
||||
func (c *Cos) getPresignedURL(ctx context.Context, name string, expire time.Duration, opt *cos.PresignedURLOptions) (*url.URL, error) {
|
||||
if !config.Config.Object.Cos.PublicRead {
|
||||
if !c.publicRead {
|
||||
return c.client.Object.GetPresignedURL(ctx, http.MethodGet, name, c.credential.SecretID, c.credential.SecretKey, expire, opt)
|
||||
}
|
||||
return c.client.Object.GetObjectURL(name), nil
|
||||
|
||||
@@ -42,51 +42,79 @@ func ImageWidthHeight(img image.Image) (int, int) {
|
||||
return bounds.X, bounds.Y
|
||||
}
|
||||
|
||||
// resizeImage resizes an image to a specified maximum width and height, maintaining the aspect ratio.
|
||||
// If both maxWidth and maxHeight are set to 0, the original image is returned.
|
||||
// If both are non-zero, the image is scaled to fit within the constraints while maintaining aspect ratio.
|
||||
// If only one of maxWidth or maxHeight is non-zero, the image is scaled accordingly.
|
||||
func resizeImage(img image.Image, maxWidth, maxHeight int) image.Image {
|
||||
bounds := img.Bounds()
|
||||
imgWidth, imgHeight := bounds.Dx(), bounds.Dy()
|
||||
imgWidth := bounds.Max.X
|
||||
imgHeight := bounds.Max.Y
|
||||
|
||||
// Return original image if no resizing is needed.
|
||||
// 计算缩放比例
|
||||
scaleWidth := float64(maxWidth) / float64(imgWidth)
|
||||
scaleHeight := float64(maxHeight) / float64(imgHeight)
|
||||
|
||||
// 如果都为0,则不缩放,返回原始图片
|
||||
if maxWidth == 0 && maxHeight == 0 {
|
||||
return img
|
||||
}
|
||||
|
||||
var scale float64 = 1
|
||||
// 如果宽度和高度都大于0,则选择较小的缩放比例,以保持宽高比
|
||||
if maxWidth > 0 && maxHeight > 0 {
|
||||
scaleWidth := float64(maxWidth) / float64(imgWidth)
|
||||
scaleHeight := float64(maxHeight) / float64(imgHeight)
|
||||
// Choose the smaller scale to fit both constraints.
|
||||
scale = min(scaleWidth, scaleHeight)
|
||||
} else if maxWidth > 0 {
|
||||
scale = float64(maxWidth) / float64(imgWidth)
|
||||
} else if maxHeight > 0 {
|
||||
scale = float64(maxHeight) / float64(imgHeight)
|
||||
}
|
||||
|
||||
newWidth := int(float64(imgWidth) * scale)
|
||||
newHeight := int(float64(imgHeight) * scale)
|
||||
|
||||
// Resize the image by creating a new image and manually copying pixels.
|
||||
thumbnail := image.NewRGBA(image.Rect(0, 0, newWidth, newHeight))
|
||||
for y := 0; y < newHeight; y++ {
|
||||
for x := 0; x < newWidth; x++ {
|
||||
srcX := int(float64(x) / scale)
|
||||
srcY := int(float64(y) / scale)
|
||||
thumbnail.Set(x, y, img.At(srcX, srcY))
|
||||
scale := scaleWidth
|
||||
if scaleHeight < scaleWidth {
|
||||
scale = scaleHeight
|
||||
}
|
||||
|
||||
// 计算缩略图尺寸
|
||||
thumbnailWidth := int(float64(imgWidth) * scale)
|
||||
thumbnailHeight := int(float64(imgHeight) * scale)
|
||||
|
||||
// 使用"image"库的Resample方法生成缩略图
|
||||
thumbnail := image.NewRGBA(image.Rect(0, 0, thumbnailWidth, thumbnailHeight))
|
||||
for y := 0; y < thumbnailHeight; y++ {
|
||||
for x := 0; x < thumbnailWidth; x++ {
|
||||
srcX := int(float64(x) / scale)
|
||||
srcY := int(float64(y) / scale)
|
||||
thumbnail.Set(x, y, img.At(srcX, srcY))
|
||||
}
|
||||
}
|
||||
|
||||
return thumbnail
|
||||
}
|
||||
|
||||
return thumbnail
|
||||
}
|
||||
// 如果只指定了宽度或高度,则根据最大不超过的规则生成缩略图
|
||||
if maxWidth > 0 {
|
||||
thumbnailWidth := maxWidth
|
||||
thumbnailHeight := int(float64(imgHeight) * scaleWidth)
|
||||
|
||||
// min returns the smaller of x or y.
|
||||
func min(x, y float64) float64 {
|
||||
if x < y {
|
||||
return x
|
||||
// 使用"image"库的Resample方法生成缩略图
|
||||
thumbnail := image.NewRGBA(image.Rect(0, 0, thumbnailWidth, thumbnailHeight))
|
||||
for y := 0; y < thumbnailHeight; y++ {
|
||||
for x := 0; x < thumbnailWidth; x++ {
|
||||
srcX := int(float64(x) / scaleWidth)
|
||||
srcY := int(float64(y) / scaleWidth)
|
||||
thumbnail.Set(x, y, img.At(srcX, srcY))
|
||||
}
|
||||
}
|
||||
|
||||
return thumbnail
|
||||
}
|
||||
return y
|
||||
|
||||
if maxHeight > 0 {
|
||||
thumbnailWidth := int(float64(imgWidth) * scaleHeight)
|
||||
thumbnailHeight := maxHeight
|
||||
|
||||
// 使用"image"库的Resample方法生成缩略图
|
||||
thumbnail := image.NewRGBA(image.Rect(0, 0, thumbnailWidth, thumbnailHeight))
|
||||
for y := 0; y < thumbnailHeight; y++ {
|
||||
for x := 0; x < thumbnailWidth; x++ {
|
||||
srcX := int(float64(x) / scaleHeight)
|
||||
srcY := int(float64(y) / scaleHeight)
|
||||
thumbnail.Set(x, y, img.At(srcX, srcY))
|
||||
}
|
||||
}
|
||||
|
||||
return thumbnail
|
||||
}
|
||||
|
||||
// 默认情况下,返回原始图片
|
||||
return img
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/OpenIMSDK/tools/errs"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -33,7 +34,6 @@ import (
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/minio/minio-go/v7/pkg/signer"
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
|
||||
)
|
||||
@@ -43,7 +43,7 @@ const (
|
||||
)
|
||||
|
||||
const (
|
||||
minPartSize int64 = 1024 * 1024 * 5 // 1MB
|
||||
minPartSize int64 = 1024 * 1024 * 5 // 5MB
|
||||
maxPartSize int64 = 1024 * 1024 * 1024 * 5 // 5GB
|
||||
maxNumSize int64 = 10000
|
||||
)
|
||||
@@ -57,13 +57,23 @@ const (
|
||||
|
||||
const successCode = http.StatusOK
|
||||
|
||||
func NewMinio(cache cache.MinioCache) (s3.Interface, error) {
|
||||
u, err := url.Parse(config.Config.Object.Minio.Endpoint)
|
||||
type Config struct {
|
||||
Bucket string
|
||||
Endpoint string
|
||||
AccessKeyID string
|
||||
SecretAccessKey string
|
||||
SessionToken string
|
||||
SignEndpoint string
|
||||
PublicRead bool
|
||||
}
|
||||
|
||||
func NewMinio(cache cache.MinioCache, conf Config) (s3.Interface, error) {
|
||||
u, err := url.Parse(conf.Endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
opts := &minio.Options{
|
||||
Creds: credentials.NewStaticV4(config.Config.Object.Minio.AccessKeyID, config.Config.Object.Minio.SecretAccessKey, config.Config.Object.Minio.SessionToken),
|
||||
Creds: credentials.NewStaticV4(conf.AccessKeyID, conf.SecretAccessKey, conf.SessionToken),
|
||||
Secure: u.Scheme == "https",
|
||||
}
|
||||
client, err := minio.New(u.Host, opts)
|
||||
@@ -71,26 +81,27 @@ func NewMinio(cache cache.MinioCache) (s3.Interface, error) {
|
||||
return nil, err
|
||||
}
|
||||
m := &Minio{
|
||||
bucket: config.Config.Object.Minio.Bucket,
|
||||
conf: conf,
|
||||
bucket: conf.Bucket,
|
||||
core: &minio.Core{Client: client},
|
||||
lock: &sync.Mutex{},
|
||||
init: false,
|
||||
cache: cache,
|
||||
}
|
||||
if config.Config.Object.Minio.SignEndpoint == "" || config.Config.Object.Minio.SignEndpoint == config.Config.Object.Minio.Endpoint {
|
||||
if conf.SignEndpoint == "" || conf.SignEndpoint == conf.Endpoint {
|
||||
m.opts = opts
|
||||
m.sign = m.core.Client
|
||||
m.prefix = u.Path
|
||||
u.Path = ""
|
||||
config.Config.Object.Minio.Endpoint = u.String()
|
||||
m.signEndpoint = config.Config.Object.Minio.Endpoint
|
||||
conf.Endpoint = u.String()
|
||||
m.signEndpoint = conf.Endpoint
|
||||
} else {
|
||||
su, err := url.Parse(config.Config.Object.Minio.SignEndpoint)
|
||||
su, err := url.Parse(conf.SignEndpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.opts = &minio.Options{
|
||||
Creds: credentials.NewStaticV4(config.Config.Object.Minio.AccessKeyID, config.Config.Object.Minio.SecretAccessKey, config.Config.Object.Minio.SessionToken),
|
||||
Creds: credentials.NewStaticV4(conf.AccessKeyID, conf.SecretAccessKey, conf.SessionToken),
|
||||
Secure: su.Scheme == "https",
|
||||
}
|
||||
m.sign, err = minio.New(su.Host, m.opts)
|
||||
@@ -99,8 +110,8 @@ func NewMinio(cache cache.MinioCache) (s3.Interface, error) {
|
||||
}
|
||||
m.prefix = su.Path
|
||||
su.Path = ""
|
||||
config.Config.Object.Minio.SignEndpoint = su.String()
|
||||
m.signEndpoint = config.Config.Object.Minio.SignEndpoint
|
||||
conf.SignEndpoint = su.String()
|
||||
m.signEndpoint = conf.SignEndpoint
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
@@ -111,6 +122,7 @@ func NewMinio(cache cache.MinioCache) (s3.Interface, error) {
|
||||
}
|
||||
|
||||
type Minio struct {
|
||||
conf Config
|
||||
bucket string
|
||||
signEndpoint string
|
||||
location string
|
||||
@@ -132,31 +144,30 @@ func (m *Minio) initMinio(ctx context.Context) error {
|
||||
if m.init {
|
||||
return nil
|
||||
}
|
||||
conf := config.Config.Object.Minio
|
||||
exists, err := m.core.Client.BucketExists(ctx, conf.Bucket)
|
||||
exists, err := m.core.Client.BucketExists(ctx, m.conf.Bucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("check bucket exists error: %w", err)
|
||||
}
|
||||
if !exists {
|
||||
if err = m.core.Client.MakeBucket(ctx, conf.Bucket, minio.MakeBucketOptions{}); err != nil {
|
||||
if err = m.core.Client.MakeBucket(ctx, m.conf.Bucket, minio.MakeBucketOptions{}); err != nil {
|
||||
return fmt.Errorf("make bucket error: %w", err)
|
||||
}
|
||||
}
|
||||
if conf.PublicRead {
|
||||
if m.conf.PublicRead {
|
||||
policy := fmt.Sprintf(
|
||||
`{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject","s3:PutObject"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::%s/*"],"Sid": ""}]}`,
|
||||
conf.Bucket,
|
||||
m.conf.Bucket,
|
||||
)
|
||||
if err = m.core.Client.SetBucketPolicy(ctx, conf.Bucket, policy); err != nil {
|
||||
if err = m.core.Client.SetBucketPolicy(ctx, m.conf.Bucket, policy); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
m.location, err = m.core.Client.GetBucketLocation(ctx, conf.Bucket)
|
||||
m.location, err = m.core.Client.GetBucketLocation(ctx, m.conf.Bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
func() {
|
||||
if conf.SignEndpoint == "" || conf.SignEndpoint == conf.Endpoint {
|
||||
if m.conf.SignEndpoint == "" || m.conf.SignEndpoint == m.conf.Endpoint {
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
@@ -176,7 +187,7 @@ func (m *Minio) initMinio(ctx context.Context) error {
|
||||
blc := reflect.ValueOf(m.sign).Elem().FieldByName("bucketLocCache")
|
||||
vblc := reflect.New(reflect.PtrTo(blc.Type()))
|
||||
*(*unsafe.Pointer)(vblc.UnsafePointer()) = unsafe.Pointer(blc.UnsafeAddr())
|
||||
vblc.Elem().Elem().Interface().(interface{ Set(string, string) }).Set(conf.Bucket, m.location)
|
||||
vblc.Elem().Elem().Interface().(interface{ Set(string, string) }).Set(m.conf.Bucket, m.location)
|
||||
}()
|
||||
m.init = true
|
||||
return nil
|
||||
@@ -341,10 +352,7 @@ func (m *Minio) CopyObject(ctx context.Context, src string, dst string) (*s3.Cop
|
||||
}
|
||||
|
||||
func (m *Minio) IsNotFound(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
switch e := err.(type) {
|
||||
switch e := errs.Unwrap(err).(type) {
|
||||
case minio.ErrorResponse:
|
||||
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
|
||||
case *minio.ErrorResponse:
|
||||
@@ -397,7 +405,7 @@ func (m *Minio) PresignedGetObject(ctx context.Context, name string, expire time
|
||||
rawURL *url.URL
|
||||
err error
|
||||
)
|
||||
if config.Config.Object.Minio.PublicRead {
|
||||
if m.conf.PublicRead {
|
||||
rawURL, err = makeTargetURL(m.sign, m.bucket, name, m.location, false, query)
|
||||
} else {
|
||||
rawURL, err = m.sign.PresignedGetObject(ctx, m.bucket, name, expire, query)
|
||||
|
||||
+15
-11
@@ -32,7 +32,6 @@ import (
|
||||
|
||||
"github.com/OpenIMSDK/tools/errs"
|
||||
"github.com/aliyun/aliyun-oss-go-sdk/oss"
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
|
||||
)
|
||||
|
||||
@@ -52,13 +51,17 @@ const (
|
||||
|
||||
const successCode = http.StatusOK
|
||||
|
||||
/* const (
|
||||
videoSnapshotImagePng = "png"
|
||||
videoSnapshotImageJpg = "jpg"
|
||||
) */
|
||||
type Config struct {
|
||||
Endpoint string
|
||||
Bucket string
|
||||
BucketURL string
|
||||
AccessKeyID string
|
||||
AccessKeySecret string
|
||||
SessionToken string
|
||||
PublicRead bool
|
||||
}
|
||||
|
||||
func NewOSS() (s3.Interface, error) {
|
||||
conf := config.Config.Object.Oss
|
||||
func NewOSS(conf Config) (s3.Interface, error) {
|
||||
if conf.BucketURL == "" {
|
||||
return nil, errs.Wrap(errors.New("bucket url is empty"))
|
||||
}
|
||||
@@ -78,6 +81,7 @@ func NewOSS() (s3.Interface, error) {
|
||||
bucket: bucket,
|
||||
credentials: client.Config.GetCredentials(),
|
||||
um: *(*urlMaker)(reflect.ValueOf(bucket.Client.Conn).Elem().FieldByName("url").UnsafePointer()),
|
||||
publicRead: conf.PublicRead,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -86,6 +90,7 @@ type OSS struct {
|
||||
bucket *oss.Bucket
|
||||
credentials oss.Credentials
|
||||
um urlMaker
|
||||
publicRead bool
|
||||
}
|
||||
|
||||
func (o *OSS) Engine() string {
|
||||
@@ -236,7 +241,7 @@ func (o *OSS) CopyObject(ctx context.Context, src string, dst string) (*s3.CopyO
|
||||
}
|
||||
|
||||
func (o *OSS) IsNotFound(err error) bool {
|
||||
switch e := err.(type) {
|
||||
switch e := errs.Unwrap(err).(type) {
|
||||
case oss.ServiceError:
|
||||
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
|
||||
case *oss.ServiceError:
|
||||
@@ -282,7 +287,6 @@ func (o *OSS) ListUploadedParts(ctx context.Context, uploadID string, name strin
|
||||
}
|
||||
|
||||
func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
|
||||
publicRead := config.Config.Object.Oss.PublicRead
|
||||
var opts []oss.Option
|
||||
if opt != nil {
|
||||
if opt.Image != nil {
|
||||
@@ -310,7 +314,7 @@ func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration,
|
||||
process += ",format," + format
|
||||
opts = append(opts, oss.Process(process))
|
||||
}
|
||||
if !publicRead {
|
||||
if !o.publicRead {
|
||||
if opt.ContentType != "" {
|
||||
opts = append(opts, oss.ResponseContentType(opt.ContentType))
|
||||
}
|
||||
@@ -324,7 +328,7 @@ func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration,
|
||||
} else if expire < time.Second {
|
||||
expire = time.Second
|
||||
}
|
||||
if !publicRead {
|
||||
if !o.publicRead {
|
||||
return o.bucket.SignURL(name, http.MethodGet, int64(expire/time.Second), opts...)
|
||||
}
|
||||
rawParams, err := oss.GetRawParams(opts)
|
||||
|
||||
@@ -36,13 +36,14 @@ const (
|
||||
)
|
||||
|
||||
type Mongo struct {
|
||||
db *mongo.Client
|
||||
db *mongo.Client
|
||||
config *config.GlobalConfig
|
||||
}
|
||||
|
||||
// NewMongo Initialize MongoDB connection.
|
||||
func NewMongo() (*Mongo, error) {
|
||||
func NewMongo(config *config.GlobalConfig) (*Mongo, error) {
|
||||
specialerror.AddReplace(mongo.ErrNoDocuments, errs.ErrRecordNotFound)
|
||||
uri := buildMongoURI()
|
||||
uri := buildMongoURI(config)
|
||||
|
||||
var mongoClient *mongo.Client
|
||||
var err error
|
||||
@@ -56,7 +57,7 @@ func NewMongo() (*Mongo, error) {
|
||||
if err = mongoClient.Ping(ctx, nil); err != nil {
|
||||
return nil, errs.Wrap(err, uri)
|
||||
}
|
||||
return &Mongo{db: mongoClient}, nil
|
||||
return &Mongo{db: mongoClient, config: config}, nil
|
||||
}
|
||||
if shouldRetry(err) {
|
||||
time.Sleep(time.Second) // exponential backoff could be implemented here
|
||||
@@ -66,14 +67,14 @@ func NewMongo() (*Mongo, error) {
|
||||
return nil, errs.Wrap(err, uri)
|
||||
}
|
||||
|
||||
func buildMongoURI() string {
|
||||
func buildMongoURI(config *config.GlobalConfig) string {
|
||||
uri := os.Getenv("MONGO_URI")
|
||||
if uri != "" {
|
||||
return uri
|
||||
}
|
||||
|
||||
if config.Config.Mongo.Uri != "" {
|
||||
return config.Config.Mongo.Uri
|
||||
if config.Mongo.Uri != "" {
|
||||
return config.Mongo.Uri
|
||||
}
|
||||
|
||||
username := os.Getenv("MONGO_OPENIM_USERNAME")
|
||||
@@ -84,21 +85,21 @@ func buildMongoURI() string {
|
||||
maxPoolSize := os.Getenv("MONGO_MAX_POOL_SIZE")
|
||||
|
||||
if username == "" {
|
||||
username = config.Config.Mongo.Username
|
||||
username = config.Mongo.Username
|
||||
}
|
||||
if password == "" {
|
||||
password = config.Config.Mongo.Password
|
||||
password = config.Mongo.Password
|
||||
}
|
||||
if address == "" {
|
||||
address = strings.Join(config.Config.Mongo.Address, ",")
|
||||
address = strings.Join(config.Mongo.Address, ",")
|
||||
} else if port != "" {
|
||||
address = fmt.Sprintf("%s:%s", address, port)
|
||||
}
|
||||
if database == "" {
|
||||
database = config.Config.Mongo.Database
|
||||
database = config.Mongo.Database
|
||||
}
|
||||
if maxPoolSize == "" {
|
||||
maxPoolSize = fmt.Sprint(config.Config.Mongo.MaxPoolSize)
|
||||
maxPoolSize = fmt.Sprint(config.Mongo.MaxPoolSize)
|
||||
}
|
||||
|
||||
uriFormat := "mongodb://%s/%s?maxPoolSize=%s"
|
||||
@@ -122,8 +123,8 @@ func (m *Mongo) GetClient() *mongo.Client {
|
||||
}
|
||||
|
||||
// GetDatabase returns the specific database from MongoDB.
|
||||
func (m *Mongo) GetDatabase() *mongo.Database {
|
||||
return m.db.Database(config.Config.Mongo.Database)
|
||||
func (m *Mongo) GetDatabase(database string) *mongo.Database {
|
||||
return m.db.Database(database)
|
||||
}
|
||||
|
||||
// CreateMsgIndex creates an index for messages in MongoDB.
|
||||
@@ -133,7 +134,7 @@ func (m *Mongo) CreateMsgIndex() error {
|
||||
|
||||
// createMongoIndex creates an index in a MongoDB collection.
|
||||
func (m *Mongo) createMongoIndex(collection string, isUnique bool, keys ...string) error {
|
||||
db := m.GetDatabase().Collection(collection)
|
||||
db := m.GetDatabase(m.config.Mongo.Database).Collection(collection)
|
||||
opts := options.CreateIndexes().SetMaxTime(10 * time.Second)
|
||||
indexView := db.Indexes()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user