mirror of
https://github.com/openimsdk/open-im-server.git
synced 2026-05-08 11:05:59 +08:00
feat: optimize code and support running in single process mode (#3142)
* pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * monolithic * fix: DeleteDoc crash * fix: DeleteDoc crash * fix: monolithic * fix: monolithic * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group * fix: mq * fix: mq * fix: user msg timestamp * fix: mq * 1 * 1 * 1 * 1 * 1 * 1 * 1 * seq read config * seq read config * 1 * 1 * fix: the source message of the reference is withdrawn, and the referenced message is deleted * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1
This commit is contained in:
chao
@@ -0,0 +1,117 @@
|
||||
package cron
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
|
||||
disetcd "github.com/openimsdk/open-im-server/v3/pkg/common/discovery/etcd"
|
||||
pbconversation "github.com/openimsdk/protocol/conversation"
|
||||
"github.com/openimsdk/protocol/msg"
|
||||
"github.com/openimsdk/protocol/third"
|
||||
"github.com/openimsdk/tools/discovery"
|
||||
"github.com/openimsdk/tools/discovery/etcd"
|
||||
|
||||
"github.com/openimsdk/tools/errs"
|
||||
"github.com/openimsdk/tools/log"
|
||||
"github.com/openimsdk/tools/mcontext"
|
||||
"github.com/openimsdk/tools/utils/runtimeenv"
|
||||
"github.com/robfig/cron/v3"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
CronTask config.CronTask
|
||||
Share config.Share
|
||||
Discovery config.Discovery
|
||||
}
|
||||
|
||||
func Start(ctx context.Context, conf *Config, client discovery.Conn, service grpc.ServiceRegistrar) error {
|
||||
log.CInfo(ctx, "CRON-TASK server is initializing", "runTimeEnv", runtimeenv.RuntimeEnvironment(), "chatRecordsClearTime", conf.CronTask.CronExecuteTime, "msgDestructTime", conf.CronTask.RetainChatRecords)
|
||||
if conf.CronTask.RetainChatRecords < 1 {
|
||||
log.ZInfo(ctx, "disable cron")
|
||||
<-ctx.Done()
|
||||
return nil
|
||||
}
|
||||
ctx = mcontext.SetOpUserID(ctx, conf.Share.IMAdminUserID[0])
|
||||
|
||||
msgConn, err := client.GetConn(ctx, conf.Discovery.RpcService.Msg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
thirdConn, err := client.GetConn(ctx, conf.Discovery.RpcService.Third)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
conversationConn, err := client.GetConn(ctx, conf.Discovery.RpcService.Conversation)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if conf.Discovery.Enable == config.ETCD {
|
||||
cm := disetcd.NewConfigManager(client.(*etcd.SvcDiscoveryRegistryImpl).GetClient(), []string{
|
||||
conf.CronTask.GetConfigFileName(),
|
||||
conf.Share.GetConfigFileName(),
|
||||
conf.Discovery.GetConfigFileName(),
|
||||
})
|
||||
cm.Watch(ctx)
|
||||
}
|
||||
|
||||
srv := &cronServer{
|
||||
ctx: ctx,
|
||||
config: conf,
|
||||
cron: cron.New(),
|
||||
msgClient: msg.NewMsgClient(msgConn),
|
||||
conversationClient: pbconversation.NewConversationClient(conversationConn),
|
||||
thirdClient: third.NewThirdClient(thirdConn),
|
||||
}
|
||||
|
||||
if err := srv.registerClearS3(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := srv.registerDeleteMsg(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := srv.registerClearUserMsg(); err != nil {
|
||||
return err
|
||||
}
|
||||
log.ZDebug(ctx, "start cron task", "CronExecuteTime", conf.CronTask.CronExecuteTime)
|
||||
srv.cron.Start()
|
||||
log.ZDebug(ctx, "cron task server is running")
|
||||
<-ctx.Done()
|
||||
log.ZDebug(ctx, "cron task server is shutting down")
|
||||
return nil
|
||||
}
|
||||
|
||||
type cronServer struct {
|
||||
ctx context.Context
|
||||
config *Config
|
||||
cron *cron.Cron
|
||||
msgClient msg.MsgClient
|
||||
conversationClient pbconversation.ConversationClient
|
||||
thirdClient third.ThirdClient
|
||||
}
|
||||
|
||||
func (c *cronServer) registerClearS3() error {
|
||||
if c.config.CronTask.FileExpireTime <= 0 || len(c.config.CronTask.DeleteObjectType) == 0 {
|
||||
log.ZInfo(c.ctx, "disable scheduled cleanup of s3", "fileExpireTime", c.config.CronTask.FileExpireTime, "deleteObjectType", c.config.CronTask.DeleteObjectType)
|
||||
return nil
|
||||
}
|
||||
_, err := c.cron.AddFunc(c.config.CronTask.CronExecuteTime, c.clearS3)
|
||||
return errs.WrapMsg(err, "failed to register clear s3 cron task")
|
||||
}
|
||||
|
||||
func (c *cronServer) registerDeleteMsg() error {
|
||||
if c.config.CronTask.RetainChatRecords <= 0 {
|
||||
log.ZInfo(c.ctx, "disable scheduled cleanup of chat records", "retainChatRecords", c.config.CronTask.RetainChatRecords)
|
||||
return nil
|
||||
}
|
||||
_, err := c.cron.AddFunc(c.config.CronTask.CronExecuteTime, c.deleteMsg)
|
||||
return errs.WrapMsg(err, "failed to register delete msg cron task")
|
||||
}
|
||||
|
||||
func (c *cronServer) registerClearUserMsg() error {
|
||||
_, err := c.cron.AddFunc(c.config.CronTask.CronExecuteTime, c.clearUserMsg)
|
||||
return errs.WrapMsg(err, "failed to register clear user msg cron task")
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package cron
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
|
||||
kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discovery"
|
||||
pbconversation "github.com/openimsdk/protocol/conversation"
|
||||
"github.com/openimsdk/protocol/msg"
|
||||
"github.com/openimsdk/protocol/third"
|
||||
"github.com/openimsdk/tools/mcontext"
|
||||
"github.com/openimsdk/tools/mw"
|
||||
"github.com/robfig/cron/v3"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
func TestName(t *testing.T) {
|
||||
conf := &config.Discovery{
|
||||
Enable: config.ETCD,
|
||||
Etcd: config.Etcd{
|
||||
RootDirectory: "openim",
|
||||
Address: []string{"localhost:12379"},
|
||||
},
|
||||
}
|
||||
client, err := kdisc.NewDiscoveryRegister(conf, nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
ctx := mcontext.SetOpUserID(context.Background(), "imAdmin")
|
||||
msgConn, err := client.GetConn(ctx, "msg-rpc-service")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
thirdConn, err := client.GetConn(ctx, "third-rpc-service")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
conversationConn, err := client.GetConn(ctx, "conversation-rpc-service")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
srv := &cronServer{
|
||||
ctx: ctx,
|
||||
config: &CronTaskConfig{
|
||||
CronTask: config.CronTask{
|
||||
RetainChatRecords: 1,
|
||||
FileExpireTime: 1,
|
||||
DeleteObjectType: []string{"msg-picture", "msg-file", "msg-voice", "msg-video", "msg-video-snapshot", "sdklog", ""},
|
||||
},
|
||||
},
|
||||
cron: cron.New(),
|
||||
msgClient: msg.NewMsgClient(msgConn),
|
||||
conversationClient: pbconversation.NewConversationClient(conversationConn),
|
||||
thirdClient: third.NewThirdClient(thirdConn),
|
||||
}
|
||||
srv.deleteMsg()
|
||||
//srv.clearS3()
|
||||
//srv.clearUserMsg()
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package cron
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/openimsdk/protocol/msg"
|
||||
"github.com/openimsdk/tools/log"
|
||||
"github.com/openimsdk/tools/mcontext"
|
||||
)
|
||||
|
||||
func (c *cronServer) deleteMsg() {
|
||||
now := time.Now()
|
||||
deltime := now.Add(-time.Hour * 24 * time.Duration(c.config.CronTask.RetainChatRecords))
|
||||
operationID := fmt.Sprintf("cron_msg_%d_%d", os.Getpid(), deltime.UnixMilli())
|
||||
ctx := mcontext.SetOperationID(c.ctx, operationID)
|
||||
log.ZDebug(ctx, "Destruct chat records", "deltime", deltime, "timestamp", deltime.UnixMilli())
|
||||
const (
|
||||
deleteCount = 10000
|
||||
deleteLimit = 50
|
||||
)
|
||||
var count int
|
||||
for i := 1; i <= deleteCount; i++ {
|
||||
ctx := mcontext.SetOperationID(c.ctx, fmt.Sprintf("%s_%d", operationID, i))
|
||||
resp, err := c.msgClient.DestructMsgs(ctx, &msg.DestructMsgsReq{Timestamp: deltime.UnixMilli(), Limit: deleteLimit})
|
||||
if err != nil {
|
||||
log.ZError(ctx, "cron destruct chat records failed", err)
|
||||
break
|
||||
}
|
||||
count += int(resp.Count)
|
||||
if resp.Count < deleteLimit {
|
||||
break
|
||||
}
|
||||
}
|
||||
log.ZDebug(ctx, "cron destruct chat records end", "deltime", deltime, "cont", time.Since(now), "count", count)
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package cron
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/openimsdk/protocol/third"
|
||||
"github.com/openimsdk/tools/log"
|
||||
"github.com/openimsdk/tools/mcontext"
|
||||
)
|
||||
|
||||
func (c *cronServer) clearS3() {
|
||||
start := time.Now()
|
||||
deleteTime := start.Add(-time.Hour * 24 * time.Duration(c.config.CronTask.FileExpireTime))
|
||||
operationID := fmt.Sprintf("cron_s3_%d_%d", os.Getpid(), deleteTime.UnixMilli())
|
||||
ctx := mcontext.SetOperationID(c.ctx, operationID)
|
||||
log.ZDebug(ctx, "deleteoutDatedData", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli())
|
||||
const (
|
||||
deleteCount = 10000
|
||||
deleteLimit = 100
|
||||
)
|
||||
|
||||
var count int
|
||||
for i := 1; i <= deleteCount; i++ {
|
||||
resp, err := c.thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli(), ObjectGroup: c.config.CronTask.DeleteObjectType, Limit: deleteLimit})
|
||||
if err != nil {
|
||||
log.ZError(ctx, "cron deleteoutDatedData failed", err)
|
||||
return
|
||||
}
|
||||
count += int(resp.Count)
|
||||
if resp.Count < deleteLimit {
|
||||
break
|
||||
}
|
||||
}
|
||||
log.ZDebug(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(start), "count", count)
|
||||
}
|
||||
|
||||
// var req *third.DeleteOutdatedDataReq
|
||||
// count1, err := ExtractField(ctx, c.thirdClient.DeleteOutdatedData, req, (*third.DeleteOutdatedDataResp).GetCount)
|
||||
//
|
||||
// c.thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{})
|
||||
// msggateway.GetUsersOnlineStatusCaller.Invoke(ctx, &msggateway.GetUsersOnlineStatusReq{})
|
||||
//
|
||||
// var cli ThirdClient
|
||||
//
|
||||
// c111, err := cli.DeleteOutdatedData(ctx, 100)
|
||||
//
|
||||
// cli.ThirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{})
|
||||
//
|
||||
// cli.AuthSign(ctx, &third.AuthSignReq{})
|
||||
//
|
||||
// cli.SetAppBadge()
|
||||
//
|
||||
//}
|
||||
//
|
||||
//func extractField[A, B, C any](ctx context.Context, fn func(ctx context.Context, req *A, opts ...grpc.CallOption) (*B, error), req *A, get func(*B) C) (C, error) {
|
||||
// resp, err := fn(ctx, req)
|
||||
// if err != nil {
|
||||
// var c C
|
||||
// return c, err
|
||||
// }
|
||||
// return get(resp), nil
|
||||
//}
|
||||
//
|
||||
//func ignore(_ any, err error) error {
|
||||
// return err
|
||||
//}
|
||||
//
|
||||
//type ThirdClient struct {
|
||||
// third.ThirdClient
|
||||
//}
|
||||
//
|
||||
//func (c *ThirdClient) DeleteOutdatedData(ctx context.Context, expireTime int64) (int32, error) {
|
||||
// return extractField(ctx, c.ThirdClient.DeleteOutdatedData, &third.DeleteOutdatedDataReq{ExpireTime: expireTime}, (*third.DeleteOutdatedDataResp).GetCount)
|
||||
//}
|
||||
//
|
||||
//func (c *ThirdClient) DeleteOutdatedData1(ctx context.Context, expireTime int64) error {
|
||||
// return ignore(c.ThirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: expireTime}))
|
||||
//}
|
||||
@@ -0,0 +1,35 @@
|
||||
package cron
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
pbconversation "github.com/openimsdk/protocol/conversation"
|
||||
"github.com/openimsdk/tools/log"
|
||||
"github.com/openimsdk/tools/mcontext"
|
||||
)
|
||||
|
||||
func (c *cronServer) clearUserMsg() {
|
||||
now := time.Now()
|
||||
operationID := fmt.Sprintf("cron_user_msg_%d_%d", os.Getpid(), now.UnixMilli())
|
||||
ctx := mcontext.SetOperationID(c.ctx, operationID)
|
||||
log.ZDebug(ctx, "clear user msg cron start")
|
||||
const (
|
||||
deleteCount = 10000
|
||||
deleteLimit = 100
|
||||
)
|
||||
var count int
|
||||
for i := 1; i <= deleteCount; i++ {
|
||||
resp, err := c.conversationClient.ClearUserConversationMsg(ctx, &pbconversation.ClearUserConversationMsgReq{Timestamp: now.UnixMilli(), Limit: deleteLimit})
|
||||
if err != nil {
|
||||
log.ZError(ctx, "ClearUserConversationMsg failed.", err)
|
||||
return
|
||||
}
|
||||
count += int(resp.Count)
|
||||
if resp.Count < deleteLimit {
|
||||
break
|
||||
}
|
||||
}
|
||||
log.ZDebug(ctx, "clear user msg cron task completed", "cont", time.Since(now), "count", count)
|
||||
}
|
||||
Reference in New Issue
Block a user