IM/open-im-server
1
0
Fork 0
mirror of https://github.com/openimsdk/open-im-server.git synced 2026-04-28 14:29:19 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: improve crontask delete outdated Data. (#2901)

Browse Source 
* fix: improve crontask delete outdated Data.

* remove comments.

* update go pkg

* revert go pkg.

* update gRPC Implement.

* feat: support use config filter in deleteOutdatedData.

* update go pkg.

* update pkg.

* comment GOProxy.
This commit is contained in:
Monet Lee
2024-11-29 16:59:24 +08:00
committed by GitHub
parent cf740a1b1a
commit 83ac4e83b9
19 changed files with 262 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/tools/cron_task.go
+37 -20
View File
@@ -24,6 +24,7 @@ import (
kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister"
pbconversation "github.com/openimsdk/protocol/conversation"
"github.com/openimsdk/protocol/msg"
"github.com/openimsdk/protocol/third"
"github.com/openimsdk/tools/mcontext"
"github.com/openimsdk/tools/mw"
@@ -58,10 +59,10 @@ func Start(ctx context.Context, config *CronTaskConfig) error {
return err
}
// thirdConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Third)
// if err != nil {
// return err
// }
thirdConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Third)
if err != nil {
return err
}
conversationConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Conversation)
if err != nil {
@@ -70,7 +71,7 @@ func Start(ctx context.Context, config *CronTaskConfig) error {
msgClient := msg.NewMsgClient(msgConn)
conversationClient := pbconversation.NewConversationClient(conversationConn)
// thirdClient := third.NewThirdClient(thirdConn)
thirdClient := third.NewThirdClient(thirdConn)
crontab := cron.New()
@@ -114,21 +115,37 @@ func Start(ctx context.Context, config *CronTaskConfig) error {
return errs.Wrap(err)
}
// // scheduled delete outdated file Objects and their datas in specific time.
// deleteObjectFunc := func() {
// now := time.Now()
// deleteTime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.FileExpireTime))
// ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deleteTime.UnixMilli()))
// log.ZDebug(ctx, "deleteoutDatedData ", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli())
// if _, err := thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli()}); err != nil {
// log.ZError(ctx, "cron deleteoutDatedData failed", err, "deleteTime", deleteTime, "cont", time.Since(now))
// return
// }
// log.ZDebug(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(now))
// }
// if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, deleteObjectFunc); err != nil {
// return errs.Wrap(err)
// }
// scheduled delete outdated file Objects and their datas in specific time.
deleteObjectFunc := func() {
now := time.Now()
executeNum := 5
// number of pagination. if need modify, need update value in third.DeleteOutdatedData
pageShowNumber := 500
deleteTime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.FileExpireTime))
ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deleteTime.UnixMilli()))
log.ZDebug(ctx, "deleteoutDatedData", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli())
if len(config.CronTask.DeleteObjectType) == 0 {
log.ZDebug(ctx, "cron deleteoutDatedData not type need delete", "deletetime", deleteTime, "DeleteObjectType", config.CronTask.DeleteObjectType, "cont", time.Since(now))
return
}
for i := 0; i < executeNum; i++ {
resp, err := thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli(), ObjectGroup: config.CronTask.DeleteObjectType})
if err != nil {
log.ZError(ctx, "cron deleteoutDatedData failed", err, "deleteTime", deleteTime, "cont", time.Since(now))
return
}
if resp.Count == 0 || resp.Count < int32(pageShowNumber) {
break
}
}
log.ZDebug(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(now))
}
if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, deleteObjectFunc); err != nil {
return errs.Wrap(err)
}
log.ZDebug(ctx, "start cron task", "CronExecuteTime", config.CronTask.CronExecuteTime)
crontab.Start()