feat: Optimize Scheduled Task (#2985)

* pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks
2026-05-06 10:05:58 +08:00 · 2024-12-20 15:45:37 +08:00
parent 248cb5c107
commit b26b0a422c
22 changed files with 647 additions and 400 deletions
@@ -16,10 +16,6 @@ package tools

 import (
 	"context"
-	"fmt"
-	"os"
-	"time"
-
 	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
 	kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister"
 	pbconversation "github.com/openimsdk/protocol/conversation"
@@ -69,87 +65,58 @@ func Start(ctx context.Context, config *CronTaskConfig) error {
 		return err
 	}

-	msgClient := msg.NewMsgClient(msgConn)
-	conversationClient := pbconversation.NewConversationClient(conversationConn)
-	thirdClient := third.NewThirdClient(thirdConn)
-
-	crontab := cron.New()
-
-	// scheduled hard delete outdated Msgs in specific time.
-	destructMsgsFunc := func() {
-		now := time.Now()
-		deltime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.RetainChatRecords))
-		ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deltime.UnixMilli()))
-		log.ZDebug(ctx, "Destruct chat records", "deltime", deltime, "timestamp", deltime.UnixMilli())
-
-		if _, err := msgClient.DestructMsgs(ctx, &msg.DestructMsgsReq{Timestamp: deltime.UnixMilli()}); err != nil {
-			log.ZError(ctx, "cron destruct chat records failed", err, "deltime", deltime, "cont", time.Since(now))
-			return
-		}
-		log.ZDebug(ctx, "cron destruct chat records success", "deltime", deltime, "cont", time.Since(now))
-	}
-	if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, destructMsgsFunc); err != nil {
-		return errs.Wrap(err)
+	srv := &cronServer{
+		ctx:                ctx,
+		config:             config,
+		cron:               cron.New(),
+		msgClient:          msg.NewMsgClient(msgConn),
+		conversationClient: pbconversation.NewConversationClient(conversationConn),
+		thirdClient:        third.NewThirdClient(thirdConn),
 	}

-	// scheduled soft delete outdated Msgs in specific time when user set `is_msg_destruct` feature.
-	clearMsgFunc := func() {
-		now := time.Now()
-		ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), now.UnixMilli()))
-		log.ZDebug(ctx, "clear msg cron start", "now", now)
-
-		conversations, err := conversationClient.GetConversationsNeedClearMsg(ctx, &pbconversation.GetConversationsNeedClearMsgReq{})
-		if err != nil {
-			log.ZError(ctx, "Get conversation need Destruct msgs failed.", err)
-			return
-		}
-
-		_, err = msgClient.ClearMsg(ctx, &msg.ClearMsgReq{Conversations: conversations.Conversations})
-		if err != nil {
-			log.ZError(ctx, "Clear Msg failed.", err)
-			return
-		}
-
-		log.ZDebug(ctx, "clear msg cron task completed", "cont", time.Since(now))
+	if err := srv.registerClearS3(); err != nil {
+		return err
 	}
-	if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, clearMsgFunc); err != nil {
-		return errs.Wrap(err)
+	if err := srv.registerDeleteMsg(); err != nil {
+		return err
 	}
-
-	// scheduled delete outdated file Objects and their datas in specific time.
-	deleteObjectFunc := func() {
-		now := time.Now()
-		executeNum := 5
-		// number of pagination. if need modify, need update value in third.DeleteOutdatedData
-		pageShowNumber := 500
-		deleteTime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.FileExpireTime))
-		ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deleteTime.UnixMilli()))
-		log.ZDebug(ctx, "deleteoutDatedData", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli())
-
-		if len(config.CronTask.DeleteObjectType) == 0 {
-			log.ZDebug(ctx, "cron deleteoutDatedData not type need delete", "deletetime", deleteTime, "DeleteObjectType", config.CronTask.DeleteObjectType, "cont", time.Since(now))
-			return
-		}
-
-		for i := 0; i < executeNum; i++ {
-			resp, err := thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli(), ObjectGroup: config.CronTask.DeleteObjectType})
-			if err != nil {
-				log.ZError(ctx, "cron deleteoutDatedData failed", err, "deleteTime", deleteTime, "cont", time.Since(now))
-				return
-			}
-			if resp.Count == 0 || resp.Count < int32(pageShowNumber) {
-				break
-			}
-		}
-
-		log.ZDebug(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(now))
+	if err := srv.registerClearUserMsg(); err != nil {
+		return err
 	}
-	if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, deleteObjectFunc); err != nil {
-		return errs.Wrap(err)
-	}
-
 	log.ZDebug(ctx, "start cron task", "CronExecuteTime", config.CronTask.CronExecuteTime)
-	crontab.Start()
+	srv.cron.Start()
 	<-ctx.Done()
 	return nil
 }
+
+type cronServer struct {
+	ctx                context.Context
+	config             *CronTaskConfig
+	cron               *cron.Cron
+	msgClient          msg.MsgClient
+	conversationClient pbconversation.ConversationClient
+	thirdClient        third.ThirdClient
+}
+
+func (c *cronServer) registerClearS3() error {
+	if c.config.CronTask.FileExpireTime <= 0 || len(c.config.CronTask.DeleteObjectType) == 0 {
+		log.ZInfo(c.ctx, "disable scheduled cleanup of s3", "fileExpireTime", c.config.CronTask.FileExpireTime, "deleteObjectType", c.config.CronTask.DeleteObjectType)
+		return nil
+	}
+	_, err := c.cron.AddFunc(c.config.CronTask.CronExecuteTime, c.clearS3)
+	return errs.WrapMsg(err, "failed to register clear s3 cron task")
+}
+
+func (c *cronServer) registerDeleteMsg() error {
+	if c.config.CronTask.RetainChatRecords <= 0 {
+		log.ZInfo(c.ctx, "disable scheduled cleanup of chat records", "retainChatRecords", c.config.CronTask.RetainChatRecords)
+		return nil
+	}
+	_, err := c.cron.AddFunc(c.config.CronTask.CronExecuteTime, c.deleteMsg)
+	return errs.WrapMsg(err, "failed to register delete msg cron task")
+}
+
+func (c *cronServer) registerClearUserMsg() error {
+	_, err := c.cron.AddFunc(c.config.CronTask.CronExecuteTime, c.clearUserMsg)
+	return errs.WrapMsg(err, "failed to register clear user msg cron task")
+}
@@ -0,0 +1,63 @@
+package tools
+
+import (
+	"context"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
+	kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister"
+	pbconversation "github.com/openimsdk/protocol/conversation"
+	"github.com/openimsdk/protocol/msg"
+	"github.com/openimsdk/protocol/third"
+	"github.com/openimsdk/tools/mcontext"
+	"github.com/openimsdk/tools/mw"
+	"github.com/robfig/cron/v3"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"testing"
+)
+
+func TestName(t *testing.T) {
+	conf := &config.Discovery{
+		Enable: config.ETCD,
+		Etcd: config.Etcd{
+			RootDirectory: "openim",
+			Address:       []string{"localhost:12379"},
+		},
+	}
+	client, err := kdisc.NewDiscoveryRegister(conf, "source")
+	if err != nil {
+		panic(err)
+	}
+	client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	ctx := mcontext.SetOpUserID(context.Background(), "imAdmin")
+	msgConn, err := client.GetConn(ctx, "msg-rpc-service")
+	if err != nil {
+		panic(err)
+	}
+	thirdConn, err := client.GetConn(ctx, "third-rpc-service")
+	if err != nil {
+		panic(err)
+	}
+
+	conversationConn, err := client.GetConn(ctx, "conversation-rpc-service")
+	if err != nil {
+		panic(err)
+	}
+
+	srv := &cronServer{
+		ctx: ctx,
+		config: &CronTaskConfig{
+			CronTask: config.CronTask{
+				RetainChatRecords: 1,
+				FileExpireTime:    1,
+				DeleteObjectType:  []string{"msg-picture", "msg-file", "msg-voice", "msg-video", "msg-video-snapshot", "sdklog", ""},
+			},
+		},
+		cron:               cron.New(),
+		msgClient:          msg.NewMsgClient(msgConn),
+		conversationClient: pbconversation.NewConversationClient(conversationConn),
+		thirdClient:        third.NewThirdClient(thirdConn),
+	}
+	srv.deleteMsg()
+	//srv.clearS3()
+	//srv.clearUserMsg()
+}
@@ -0,0 +1,36 @@
+package tools
+
+import (
+	"fmt"
+	"github.com/openimsdk/protocol/msg"
+	"github.com/openimsdk/tools/log"
+	"github.com/openimsdk/tools/mcontext"
+	"os"
+	"time"
+)
+
+func (c *cronServer) deleteMsg() {
+	now := time.Now()
+	deltime := now.Add(-time.Hour * 24 * time.Duration(c.config.CronTask.RetainChatRecords))
+	operationID := fmt.Sprintf("cron_msg_%d_%d", os.Getpid(), deltime.UnixMilli())
+	ctx := mcontext.SetOperationID(c.ctx, operationID)
+	log.ZDebug(ctx, "Destruct chat records", "deltime", deltime, "timestamp", deltime.UnixMilli())
+	const (
+		deleteCount = 10000
+		deleteLimit = 50
+	)
+	var count int
+	for i := 1; i <= deleteCount; i++ {
+		ctx := mcontext.SetOperationID(c.ctx, fmt.Sprintf("%s_%d", operationID, i))
+		resp, err := c.msgClient.DestructMsgs(ctx, &msg.DestructMsgsReq{Timestamp: deltime.UnixMilli(), Limit: deleteLimit})
+		if err != nil {
+			log.ZError(ctx, "cron destruct chat records failed", err)
+			break
+		}
+		count += int(resp.Count)
+		if resp.Count < deleteLimit {
+			break
+		}
+	}
+	log.ZDebug(ctx, "cron destruct chat records end", "deltime", deltime, "cont", time.Since(now), "count", count)
+}
@@ -0,0 +1,79 @@
+package tools
+
+import (
+	"fmt"
+	"github.com/openimsdk/protocol/third"
+	"github.com/openimsdk/tools/log"
+	"github.com/openimsdk/tools/mcontext"
+	"os"
+	"time"
+)
+
+func (c *cronServer) clearS3() {
+	start := time.Now()
+	deleteTime := start.Add(-time.Hour * 24 * time.Duration(c.config.CronTask.FileExpireTime))
+	operationID := fmt.Sprintf("cron_s3_%d_%d", os.Getpid(), deleteTime.UnixMilli())
+	ctx := mcontext.SetOperationID(c.ctx, operationID)
+	log.ZDebug(ctx, "deleteoutDatedData", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli())
+	const (
+		deleteCount = 10000
+		deleteLimit = 100
+	)
+
+	var count int
+	for i := 1; i <= deleteCount; i++ {
+		resp, err := c.thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli(), ObjectGroup: c.config.CronTask.DeleteObjectType, Limit: deleteLimit})
+		if err != nil {
+			log.ZError(ctx, "cron deleteoutDatedData failed", err)
+			return
+		}
+		count += int(resp.Count)
+		if resp.Count < deleteLimit {
+			break
+		}
+	}
+	log.ZDebug(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(start), "count", count)
+}
+
+//	var req *third.DeleteOutdatedDataReq
+//	count1, err := ExtractField(ctx, c.thirdClient.DeleteOutdatedData, req, (*third.DeleteOutdatedDataResp).GetCount)
+//
+//	c.thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{})
+//	msggateway.GetUsersOnlineStatusCaller.Invoke(ctx, &msggateway.GetUsersOnlineStatusReq{})
+//
+//	var cli ThirdClient
+//
+//	c111, err := cli.DeleteOutdatedData(ctx, 100)
+//
+//	cli.ThirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{})
+//
+//	cli.AuthSign(ctx, &third.AuthSignReq{})
+//
+//	cli.SetAppBadge()
+//
+//}
+//
+//func extractField[A, B, C any](ctx context.Context, fn func(ctx context.Context, req *A, opts ...grpc.CallOption) (*B, error), req *A, get func(*B) C) (C, error) {
+//	resp, err := fn(ctx, req)
+//	if err != nil {
+//		var c C
+//		return c, err
+//	}
+//	return get(resp), nil
+//}
+//
+//func ignore(_ any, err error) error {
+//	return err
+//}
+//
+//type ThirdClient struct {
+//	third.ThirdClient
+//}
+//
+//func (c *ThirdClient) DeleteOutdatedData(ctx context.Context, expireTime int64) (int32, error) {
+//	return extractField(ctx, c.ThirdClient.DeleteOutdatedData, &third.DeleteOutdatedDataReq{ExpireTime: expireTime}, (*third.DeleteOutdatedDataResp).GetCount)
+//}
+//
+//func (c *ThirdClient) DeleteOutdatedData1(ctx context.Context, expireTime int64) error {
+//	return ignore(c.ThirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: expireTime}))
+//}
@@ -0,0 +1,34 @@
+package tools
+
+import (
+	"fmt"
+	pbconversation "github.com/openimsdk/protocol/conversation"
+	"github.com/openimsdk/tools/log"
+	"github.com/openimsdk/tools/mcontext"
+	"os"
+	"time"
+)
+
+func (c *cronServer) clearUserMsg() {
+	now := time.Now()
+	operationID := fmt.Sprintf("cron_user_msg_%d_%d", os.Getpid(), now.UnixMilli())
+	ctx := mcontext.SetOperationID(c.ctx, operationID)
+	log.ZDebug(ctx, "clear user msg cron start")
+	const (
+		deleteCount = 10000
+		deleteLimit = 100
+	)
+	var count int
+	for i := 1; i <= deleteCount; i++ {
+		resp, err := c.conversationClient.ClearUserConversationMsg(ctx, &pbconversation.ClearUserConversationMsgReq{Timestamp: now.UnixMilli(), Limit: deleteLimit})
+		if err != nil {
+			log.ZError(ctx, "ClearUserConversationMsg failed.", err)
+			return
+		}
+		count += int(resp.Count)
+		if resp.Count < deleteLimit {
+			break
+		}
+	}
+	log.ZDebug(ctx, "clear user msg cron task completed", "cont", time.Since(now), "count", count)
+}