feat: implement scheduled destruct msgs feature in cron task. (#2466)

* update protocol in go mod.

* add debug log in writePongMsg.

* update log level.

* add Warn log in writePongMsg.

* add debug log.

* feat: update webhookBeforeMemberJoinGroup to batch method.

* feat: update version field implement.

* update webhook implement contents.

* update method field and contents.

* update callbackCommand field.

* fix: add correct fields.

* update struct tags.

* refactor: rename friend module to relation.

* feat: implement scheduled destruct msgs feature in cron task.

* update log contents.

* update func name and comments.

* update waitgroup to errgroup.

* update errgroup wait.

* remove unnecessary contents.

* update clearMsg logic.

internal/rpc/msg/clear.go

@@ -2,16 +2,22 @@ package msg
 
 import (
 	"context"
+	"time"
+
 	"github.com/openimsdk/open-im-server/v3/pkg/authverify"
-	"github.com/openimsdk/protocol/conversation"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/convert"
+	pbconversation "github.com/openimsdk/protocol/conversation"
 	"github.com/openimsdk/protocol/msg"
 	"github.com/openimsdk/protocol/wrapperspb"
 	"github.com/openimsdk/tools/errs"
 	"github.com/openimsdk/tools/log"
-	"strings"
-	"time"
+	"github.com/openimsdk/tools/mcontext"
+	"github.com/openimsdk/tools/utils/idutil"
+	"github.com/openimsdk/tools/utils/stringutil"
+	"golang.org/x/sync/errgroup"
 )
 
+// hard delete in Database.
 func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg.ClearMsgResp, err error) {
 	if err := authverify.CheckAdmin(ctx, m.config.Share.IMAdminUserID); err != nil {
 		return nil, err
@@ -25,18 +31,6 @@ func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg.
 		start  = time.Now()
 	)
 	clearMsg := func(ctx context.Context) (bool, error) {
-		conversationSeqs := make(map[string]struct{})
-		defer func() {
-			req := &conversation.UpdateConversationReq{
-				MsgDestructTime: wrapperspb.Int64(time.Now().UnixMilli()),
-			}
-			for conversationID := range conversationSeqs {
-				req.ConversationID = conversationID
-				if err := m.Conversation.UpdateConversations(ctx, req); err != nil {
-					log.ZError(ctx, "update conversation max seq failed", err, "conversationID", conversationID, "msgDestructTime", req.MsgDestructTime)
-				}
-			}
-		}()
 		msgs, err := m.MsgDatabase.GetBeforeMsg(ctx, req.Timestamp, 100)
 		if err != nil {
 			return false, err
@@ -44,6 +38,7 @@ func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg.
 		if len(msgs) == 0 {
 			return false, nil
 		}
+
 		for _, msg := range msgs {
 			index, err := m.MsgDatabase.DeleteDocMsgBefore(ctx, req.Timestamp, msg)
 			if err != nil {
@@ -52,15 +47,14 @@ func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg.
 			if len(index) == 0 {
 				return false, errs.ErrInternalServer.WrapMsg("delete doc msg failed")
 			}
+
 			docNum++
 			msgNum += len(index)
-			conversationID := msg.DocID[:strings.LastIndex(msg.DocID, ":")]
-			if _, ok := conversationSeqs[conversationID]; !ok {
-				conversationSeqs[conversationID] = struct{}{}
-			}
 		}
+
 		return true, nil
 	}
+
 	for {
 		keep, err := clearMsg(ctx)
 		if err != nil {
@@ -71,7 +65,60 @@ func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg.
 			log.ZInfo(ctx, "clear msg success", "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start))
 			break
 		}
+
 		log.ZInfo(ctx, "clearing message", "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start))
 	}
 	return &msg.ClearMsgResp{}, nil
 }
+
+// soft delete for self
+func (m *msgServer) DestructMsgs(ctx context.Context, req *msg.DestructMsgsReq) (_ *msg.DestructMsgsResp, err error) {
+	temp := convert.ConversationsPb2DB(req.Conversations)
+
+	batchNum := 100
+
+	errg, _ := errgroup.WithContext(ctx)
+	errg.SetLimit(100)
+
+	for i := 0; i < len(temp); i += batchNum {
+		batch := temp[i:min(i+batchNum, len(temp))]
+
+		errg.Go(func() error {
+			for _, conversation := range batch {
+				handleCtx := mcontext.NewCtx(stringutil.GetSelfFuncName() + "-" + idutil.OperationIDGenerator() + "-" + conversation.ConversationID + "-" + conversation.OwnerUserID)
+				log.ZDebug(handleCtx, "User MsgsDestruct",
+					"conversationID", conversation.ConversationID,
+					"ownerUserID", conversation.OwnerUserID,
+					"msgDestructTime", conversation.MsgDestructTime,
+					"lastMsgDestructTime", conversation.LatestMsgDestructTime)
+
+				seqs, err := m.MsgDatabase.UserMsgsDestruct(handleCtx, conversation.OwnerUserID, conversation.ConversationID, conversation.MsgDestructTime, conversation.LatestMsgDestructTime)
+				if err != nil {
+					log.ZError(handleCtx, "user msg destruct failed", err, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID)
+					continue
+				}
+
+				if len(seqs) > 0 {
+					if err := m.Conversation.UpdateConversation(handleCtx,
+						&pbconversation.UpdateConversationReq{
+							UserIDs:               []string{conversation.OwnerUserID},
+							ConversationID:        conversation.ConversationID,
+							LatestMsgDestructTime: wrapperspb.Int64(time.Now().UnixMilli())}); err != nil {
+						log.ZError(handleCtx, "updateUsersConversationField failed", err, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID)
+						continue
+					}
+
+					// if you need Notify SDK client userseq is update.
+					// m.msgNotificationSender.UserDeleteMsgsNotification(handleCtx, conversation.OwnerUserID, conversation.ConversationID, seqs)
+				}
+			}
+			return nil
+		})
+	}
+
+	if err := errg.Wait(); err != nil {
+		return nil, err
+	}
+
+	return nil, nil
+}

internal/rpc/msg/server.go

@@ -16,6 +16,7 @@ package msg
 
 import (
 	"context"
+
 	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo"
@@ -50,6 +51,7 @@ type (
 		ConversationLocalCache *rpccache.ConversationLocalCache // Local cache for conversation data.
 		Handlers               MessageInterceptorChain          // Chain of handlers for processing messages.
 		notificationSender     *rpcclient.NotificationSender    // RPC client for sending notifications.
+		msgNotificationSender  *MsgNotificationSender           // RPC client for sending msg notifications.
 		config                 *Config                          // Global configuration settings.
 		webhookClient          *webhook.Client
 	}
@@ -117,7 +119,10 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg
 	}
 
 	s.notificationSender = rpcclient.NewNotificationSender(&config.NotificationConfig, rpcclient.WithLocalSendMsg(s.SendMsg))
+	s.msgNotificationSender = NewMsgNotificationSender(config, rpcclient.WithLocalSendMsg(s.SendMsg))
+
 	msg.RegisterMsgServer(server, s)
+
 	return nil
 }