mirror of
https://github.com/openimsdk/open-im-server.git
synced 2026-05-09 11:35:59 +08:00
feat: optimize code and support running in single process mode (#3142)
* pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * monolithic * fix: DeleteDoc crash * fix: DeleteDoc crash * fix: monolithic * fix: monolithic * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group * fix: mq * fix: mq * fix: user msg timestamp * fix: mq * 1 * 1 * 1 * 1 * 1 * 1 * 1 * seq read config * seq read config * 1 * 1 * fix: the source message of the reference is withdrawn, and the referenced message is deleted * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1
This commit is contained in:
chao
+82
-82
@@ -2,7 +2,7 @@ package redis
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/dtm-labs/rockscache"
|
||||
|
||||
"github.com/openimsdk/tools/errs"
|
||||
"github.com/openimsdk/tools/log"
|
||||
"github.com/redis/go-redis/v9"
|
||||
@@ -28,83 +28,83 @@ type Config struct {
|
||||
// Option is a function type for configuring Config
|
||||
type Option func(c *Config)
|
||||
|
||||
// NewRedisShardManager creates a new RedisShardManager instance
|
||||
func NewRedisShardManager(redisClient redis.UniversalClient, opts ...Option) *RedisShardManager {
|
||||
config := &Config{
|
||||
batchSize: defaultBatchSize, // Default batch size is 50 keys
|
||||
continueOnError: false,
|
||||
concurrentLimit: defaultConcurrentLimit, // Default concurrent limit is 3
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(config)
|
||||
}
|
||||
rsm := &RedisShardManager{
|
||||
redisClient: redisClient,
|
||||
config: config,
|
||||
}
|
||||
return rsm
|
||||
}
|
||||
|
||||
// WithBatchSize sets the number of keys to process per batch
|
||||
func WithBatchSize(size int) Option {
|
||||
return func(c *Config) {
|
||||
c.batchSize = size
|
||||
}
|
||||
}
|
||||
|
||||
// WithContinueOnError sets whether to continue processing on error
|
||||
func WithContinueOnError(continueOnError bool) Option {
|
||||
return func(c *Config) {
|
||||
c.continueOnError = continueOnError
|
||||
}
|
||||
}
|
||||
|
||||
// WithConcurrentLimit sets the concurrency limit
|
||||
func WithConcurrentLimit(limit int) Option {
|
||||
return func(c *Config) {
|
||||
c.concurrentLimit = limit
|
||||
}
|
||||
}
|
||||
|
||||
// ProcessKeysBySlot groups keys by their Redis cluster hash slots and processes them using the provided function.
|
||||
func (rsm *RedisShardManager) ProcessKeysBySlot(
|
||||
ctx context.Context,
|
||||
keys []string,
|
||||
processFunc func(ctx context.Context, slot int64, keys []string) error,
|
||||
) error {
|
||||
|
||||
// Group keys by slot
|
||||
slots, err := groupKeysBySlot(ctx, rsm.redisClient, keys)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
g, ctx := errgroup.WithContext(ctx)
|
||||
g.SetLimit(rsm.config.concurrentLimit)
|
||||
|
||||
// Process keys in each slot using the provided function
|
||||
for slot, singleSlotKeys := range slots {
|
||||
batches := splitIntoBatches(singleSlotKeys, rsm.config.batchSize)
|
||||
for _, batch := range batches {
|
||||
slot, batch := slot, batch // Avoid closure capture issue
|
||||
g.Go(func() error {
|
||||
err := processFunc(ctx, slot, batch)
|
||||
if err != nil {
|
||||
log.ZWarn(ctx, "Batch processFunc failed", err, "slot", slot, "keys", batch)
|
||||
if !rsm.config.continueOnError {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if err := g.Wait(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
//// NewRedisShardManager creates a new RedisShardManager instance
|
||||
//func NewRedisShardManager(redisClient redis.UniversalClient, opts ...Option) *RedisShardManager {
|
||||
// config := &Config{
|
||||
// batchSize: defaultBatchSize, // Default batch size is 50 keys
|
||||
// continueOnError: false,
|
||||
// concurrentLimit: defaultConcurrentLimit, // Default concurrent limit is 3
|
||||
// }
|
||||
// for _, opt := range opts {
|
||||
// opt(config)
|
||||
// }
|
||||
// rsm := &RedisShardManager{
|
||||
// redisClient: redisClient,
|
||||
// config: config,
|
||||
// }
|
||||
// return rsm
|
||||
//}
|
||||
//
|
||||
//// WithBatchSize sets the number of keys to process per batch
|
||||
//func WithBatchSize(size int) Option {
|
||||
// return func(c *Config) {
|
||||
// c.batchSize = size
|
||||
// }
|
||||
//}
|
||||
//
|
||||
//// WithContinueOnError sets whether to continue processing on error
|
||||
//func WithContinueOnError(continueOnError bool) Option {
|
||||
// return func(c *Config) {
|
||||
// c.continueOnError = continueOnError
|
||||
// }
|
||||
//}
|
||||
//
|
||||
//// WithConcurrentLimit sets the concurrency limit
|
||||
//func WithConcurrentLimit(limit int) Option {
|
||||
// return func(c *Config) {
|
||||
// c.concurrentLimit = limit
|
||||
// }
|
||||
//}
|
||||
//
|
||||
//// ProcessKeysBySlot groups keys by their Redis cluster hash slots and processes them using the provided function.
|
||||
//func (rsm *RedisShardManager) ProcessKeysBySlot(
|
||||
// ctx context.Context,
|
||||
// keys []string,
|
||||
// processFunc func(ctx context.Context, slot int64, keys []string) error,
|
||||
//) error {
|
||||
//
|
||||
// // Group keys by slot
|
||||
// slots, err := groupKeysBySlot(ctx, rsm.redisClient, keys)
|
||||
// if err != nil {
|
||||
// return err
|
||||
// }
|
||||
//
|
||||
// g, ctx := errgroup.WithContext(ctx)
|
||||
// g.SetLimit(rsm.config.concurrentLimit)
|
||||
//
|
||||
// // Process keys in each slot using the provided function
|
||||
// for slot, singleSlotKeys := range slots {
|
||||
// batches := splitIntoBatches(singleSlotKeys, rsm.config.batchSize)
|
||||
// for _, batch := range batches {
|
||||
// slot, batch := slot, batch // Avoid closure capture issue
|
||||
// g.Go(func() error {
|
||||
// err := processFunc(ctx, slot, batch)
|
||||
// if err != nil {
|
||||
// log.ZWarn(ctx, "Batch processFunc failed", err, "slot", slot, "keys", batch)
|
||||
// if !rsm.config.continueOnError {
|
||||
// return err
|
||||
// }
|
||||
// }
|
||||
// return nil
|
||||
// })
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// if err := g.Wait(); err != nil {
|
||||
// return err
|
||||
// }
|
||||
// return nil
|
||||
//}
|
||||
|
||||
// groupKeysBySlot groups keys by their Redis cluster hash slots.
|
||||
func groupKeysBySlot(ctx context.Context, redisClient redis.UniversalClient, keys []string) (map[int64][]string, error) {
|
||||
@@ -197,15 +197,15 @@ func ProcessKeysBySlot(
|
||||
return nil
|
||||
}
|
||||
|
||||
func DeleteCacheBySlot(ctx context.Context, rcClient *rockscache.Client, keys []string) error {
|
||||
func DeleteCacheBySlot(ctx context.Context, rcClient *rocksCacheClient, keys []string) error {
|
||||
switch len(keys) {
|
||||
case 0:
|
||||
return nil
|
||||
case 1:
|
||||
return rcClient.TagAsDeletedBatch2(ctx, keys)
|
||||
return rcClient.GetClient().TagAsDeletedBatch2(ctx, keys)
|
||||
default:
|
||||
return ProcessKeysBySlot(ctx, getRocksCacheRedisClient(rcClient), keys, func(ctx context.Context, slot int64, keys []string) error {
|
||||
return rcClient.TagAsDeletedBatch2(ctx, keys)
|
||||
return ProcessKeysBySlot(ctx, rcClient.GetRedis(), keys, func(ctx context.Context, slot int64, keys []string) error {
|
||||
return rcClient.GetClient().TagAsDeletedBatch2(ctx, keys)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user