errcode

internal/crontask/clear_msg.go

@@ -4,172 +4,97 @@ import (
 	"Open_IM/pkg/common/config"
 	"Open_IM/pkg/common/constant"
 	"Open_IM/pkg/common/db"
+	"Open_IM/pkg/common/db/cache"
+	"Open_IM/pkg/common/db/controller"
 	"Open_IM/pkg/common/db/mongo"
 	"Open_IM/pkg/common/log"
+	"Open_IM/pkg/common/tracelog"
 	sdkws "Open_IM/pkg/proto/sdkws"
 	"Open_IM/pkg/utils"
+	"context"
 	"math"
 	"strconv"
 	"strings"
 
-	goRedis "github.com/go-redis/redis/v8"
+	"github.com/go-redis/redis/v8"
 	"github.com/golang/protobuf/proto"
 )
 
-const oldestList = 0
-const newestList = -1
+type SeqCheckInterface interface {
+	ClearAll() error
+}
 
-func ResetUserGroupMinSeq(operationID, groupID string, userIDList []string) error {
-	var delStruct delMsgRecursionStruct
-	minSeq, err := deleteMongoMsg(operationID, groupID, oldestList, &delStruct)
-	if err != nil {
-		log.NewError(operationID, utils.GetSelfFuncName(), groupID, "deleteMongoMsg failed")
+type ClearMsgCronTask struct {
+	msgModel   controller.MsgInterface
+	userModel  controller.UserInterface
+	groupModel controller.GroupInterface
+	cache      cache.Cache
+}
+
+func (c *ClearMsgCronTask) getCronTaskOperationID() string {
+	return cronTaskOperationID + utils.OperationIDGenerator()
+}
+
+func (c *ClearMsgCronTask) ClearAll() {
+	operationID := c.getCronTaskOperationID()
+	ctx := context.Background()
+	tracelog.SetOperationID(ctx, operationID)
+	log.NewInfo(operationID, "========================= start del cron task =========================")
+	var err error
+	userIDList, err := c.userModel.GetAllUserID(ctx)
+	if err == nil {
+		c.StartClearMsg(operationID, userIDList)
+	} else {
+		log.NewError(operationID, utils.GetSelfFuncName(), err.Error())
 	}
-	if minSeq == 0 {
-		return nil
+	// working group msg clear
+	workingGroupIDList, err := im_mysql_model.GetGroupIDListByGroupType(constant.WorkingGroup)
+	if err == nil {
+		c.StartClearWorkingGroupMsg(operationID, workingGroupIDList)
+	} else {
+		log.NewError(operationID, utils.GetSelfFuncName(), err.Error())
 	}
-	log.NewDebug(operationID, utils.GetSelfFuncName(), "delMsgIDList:", delStruct, "minSeq", minSeq)
+
+	log.NewInfo(operationID, "========================= start del cron finished =========================")
+}
+
+func (c *ClearMsgCronTask) StartClearMsg(operationID string, userIDList []string) {
+	log.NewDebug(operationID, utils.GetSelfFuncName(), "userIDList: ", userIDList)
 	for _, userID := range userIDList {
-		userMinSeq, err := db.DB.GetGroupUserMinSeq(groupID, userID)
-		if err != nil && err != goRedis.Nil {
-			log.NewError(operationID, utils.GetSelfFuncName(), "GetGroupUserMinSeq failed", groupID, userID, err.Error())
+		if err := DeleteUserMsgsAndSetMinSeq(operationID, userID); err != nil {
+			log.NewError(operationID, utils.GetSelfFuncName(), err.Error(), userID)
+		}
+		if err := checkMaxSeqWithMongo(operationID, userID, constant.WriteDiffusion); err != nil {
+			log.NewError(operationID, utils.GetSelfFuncName(), userID, err)
+		}
+	}
+}
+
+func (c *ClearMsgCronTask) StartClearWorkingGroupMsg(operationID string, workingGroupIDList []string) {
+	log.NewDebug(operationID, utils.GetSelfFuncName(), "workingGroupIDList: ", workingGroupIDList)
+	for _, groupID := range workingGroupIDList {
+		userIDList, err := rocksCache.GetGroupMemberIDListFromCache(groupID)
+		if err != nil {
+			log.NewError(operationID, utils.GetSelfFuncName(), err.Error(), groupID)
 			continue
 		}
-		if userMinSeq > uint64(minSeq) {
-			err = db.DB.SetGroupUserMinSeq(groupID, userID, userMinSeq)
-		} else {
-			err = db.DB.SetGroupUserMinSeq(groupID, userID, uint64(minSeq))
+		log.NewDebug(operationID, utils.GetSelfFuncName(), "groupID:", groupID, "workingGroupIDList:", userIDList)
+		if err := DeleteUserSuperGroupMsgsAndSetMinSeq(operationID, groupID, userIDList); err != nil {
+			log.NewError(operationID, utils.GetSelfFuncName(), err.Error(), groupID, userIDList)
 		}
-		if err != nil {
-			log.NewError(operationID, utils.GetSelfFuncName(), err.Error(), groupID, userID, userMinSeq, minSeq)
+		if err := checkMaxSeqWithMongo(operationID, groupID, constant.ReadDiffusion); err != nil {
+			log.NewError(operationID, utils.GetSelfFuncName(), groupID, err)
 		}
 	}
-	return nil
 }
 
-func DeleteMongoMsgAndResetRedisSeq(operationID, userID string) error {
-	var delStruct delMsgRecursionStruct
-	minSeq, err := deleteMongoMsg(operationID, userID, oldestList, &delStruct)
-	if err != nil {
-		return utils.Wrap(err, "")
-	}
-	if minSeq == 0 {
-		return nil
-	}
-	log.NewDebug(operationID, utils.GetSelfFuncName(), "delMsgIDStruct: ", delStruct, "minSeq", minSeq)
-	err = db.DB.SetUserMinSeq(userID, minSeq)
-	return utils.Wrap(err, "")
-}
-
-// del list
-func delMongoMsgsPhysical(uidList []string) error {
-	if len(uidList) > 0 {
-		err := db.DB.DelMongoMsgs(uidList)
-		if err != nil {
-			return utils.Wrap(err, "DelMongoMsgs failed")
-		}
-	}
-	return nil
-}
-
-type delMsgRecursionStruct struct {
-	minSeq     uint32
-	delUidList []string
-}
-
-func (d *delMsgRecursionStruct) getSetMinSeq() uint32 {
-	return d.minSeq
-}
-
-// index 0....19(del) 20...69
-// seq 70
-// set minSeq 21
-// recursion 删除list并且返回设置的最小seq
-func deleteMongoMsg(operationID string, ID string, index int64, delStruct *delMsgRecursionStruct) (uint32, error) {
-	// find from oldest list
-	msgs, err := db.DB.GetUserMsgListByIndex(ID, index)
-	if err != nil || msgs.UID == "" {
-		if err != nil {
-			if err == mongoDB.ErrMsgListNotExist {
-				log.NewInfo(operationID, utils.GetSelfFuncName(), "ID:", ID, "index:", index, err.Error())
-			} else {
-				log.NewError(operationID, utils.GetSelfFuncName(), "GetUserMsgListByIndex failed", err.Error(), index, ID)
-			}
-		}
-		// 获取报错，或者获取不到了，物理删除并且返回seq
-		err = delMongoMsgsPhysical(delStruct.delUidList)
-		if err != nil {
-			return 0, err
-		}
-		return delStruct.getSetMinSeq() + 1, nil
-	}
-	log.NewDebug(operationID, "ID:", ID, "index:", index, "uid:", msgs.UID, "len:", len(msgs.Msg))
-	if len(msgs.Msg) > mongoDB.GetSingleGocMsgNum() {
-		log.NewWarn(operationID, utils.GetSelfFuncName(), "msgs too large", len(msgs.Msg), msgs.UID)
-	}
-	if msgs.Msg[len(msgs.Msg)-1].SendTime+(int64(config.Config.Mongo.DBRetainChatRecords)*24*60*60*1000) > utils.GetCurrentTimestampByMill() && msgListIsFull(msgs) {
-		delStruct.delUidList = append(delStruct.delUidList, msgs.UID)
-		lastMsgPb := &sdkws.MsgData{}
-		err = proto.Unmarshal(msgs.Msg[len(msgs.Msg)-1].Msg, lastMsgPb)
-		if err != nil {
-			log.NewError(operationID, utils.GetSelfFuncName(), err.Error(), len(msgs.Msg)-1, msgs.UID)
-			return 0, utils.Wrap(err, "proto.Unmarshal failed")
-		}
-		delStruct.minSeq = lastMsgPb.Seq
-	} else {
-		var hasMarkDelFlag bool
-		for _, msg := range msgs.Msg {
-			msgPb := &sdkws.MsgData{}
-			err = proto.Unmarshal(msg.Msg, msgPb)
-			if err != nil {
-				log.NewError(operationID, utils.GetSelfFuncName(), err.Error(), len(msgs.Msg)-1, msgs.UID)
-				return 0, utils.Wrap(err, "proto.Unmarshal failed")
-			}
-			if utils.GetCurrentTimestampByMill() > msg.SendTime+(int64(config.Config.Mongo.DBRetainChatRecords)*24*60*60*1000) {
-				msgPb.Status = constant.MsgDeleted
-				bytes, _ := proto.Marshal(msgPb)
-				msg.Msg = bytes
-				msg.SendTime = 0
-				hasMarkDelFlag = true
-			} else {
-				if err := delMongoMsgsPhysical(delStruct.delUidList); err != nil {
-					return 0, err
-				}
-				if hasMarkDelFlag {
-					if err := db.DB.UpdateOneMsgList(msgs); err != nil {
-						return delStruct.getSetMinSeq(), utils.Wrap(err, "")
-					}
-				}
-				return msgPb.Seq + 1, nil
-			}
-		}
-	}
-	log.NewDebug(operationID, ID, "continue to", delStruct)
-	//  继续递归 index+1
-	seq, err := deleteMongoMsg(operationID, ID, index+1, delStruct)
-	return seq, utils.Wrap(err, "deleteMongoMsg failed")
-}
-
-func msgListIsFull(chat *mongoDB.UserChat) bool {
-	index, _ := strconv.Atoi(strings.Split(chat.UID, ":")[1])
-	if index == 0 {
-		if len(chat.Msg) >= 4999 {
-			return true
-		}
-	}
-	if len(chat.Msg) >= 5000 {
-		return true
-	}
-	return false
-}
-
-func checkMaxSeqWithMongo(operationID, ID string, diffusionType int) error {
+func checkMaxSeqWithMongo(operationID, sourceID string, diffusionType int) error {
 	var seqRedis uint64
 	var err error
 	if diffusionType == constant.WriteDiffusion {
-		seqRedis, err = db.DB.GetUserMaxSeq(ID)
+		seqRedis, err = db.DB.GetUserMaxSeq(sourceID)
 	} else {
-		seqRedis, err = db.DB.GetGroupMaxSeq(ID)
+		seqRedis, err = db.DB.GetGroupMaxSeq(sourceID)
 	}
 	if err != nil {
 		if err == goRedis.Nil {
@@ -177,7 +102,7 @@ func checkMaxSeqWithMongo(operationID, ID string, diffusionType int) error {
 		}
 		return utils.Wrap(err, "GetUserMaxSeq failed")
 	}
-	msg, err := db.DB.GetNewestMsg(ID)
+	msg, err := db.DB.GetNewestMsg(sourceID)
 	if err != nil {
 		return utils.Wrap(err, "GetNewestMsg failed")
 	}
@@ -185,9 +110,9 @@ func checkMaxSeqWithMongo(operationID, ID string, diffusionType int) error {
 		return nil
 	}
 	if math.Abs(float64(msg.Seq-uint32(seqRedis))) > 10 {
-		log.NewWarn(operationID, utils.GetSelfFuncName(), "seqMongo, seqRedis", msg.Seq, seqRedis, ID, "redis maxSeq is different with msg.Seq > 10", "status: ", msg.Status, msg.SendTime)
+		log.NewWarn(operationID, utils.GetSelfFuncName(), "seqMongo, seqRedis", msg.Seq, seqRedis, sourceID, "redis maxSeq is different with msg.Seq > 10", "status: ", msg.Status, msg.SendTime)
 	} else {
-		log.NewInfo(operationID, utils.GetSelfFuncName(), "seqMongo, seqRedis", msg.Seq, seqRedis, ID, "seq and msg OK", "status:", msg.Status, msg.SendTime)
+		log.NewInfo(operationID, utils.GetSelfFuncName(), "seqMongo, seqRedis", msg.Seq, seqRedis, sourceID, "seq and msg OK", "status:", msg.Status, msg.SendTime)
 	}
 	return nil
 }

internal/crontask/clear_msg_test.go

@@ -2,8 +2,7 @@ package cronTask
 
 import (
 	"Open_IM/pkg/common/constant"
-	mongo2 "Open_IM/pkg/common/db/mongo"
-	sdkws "Open_IM/pkg/proto/sdkws"
+	"Open_IM/pkg/proto/sdkws"
 	"context"
 	"fmt"
 	"strconv"
@@ -59,7 +58,7 @@ func CreateChat(userChat *mongo2.UserChat) error {
 	return err
 }
 
-func TestDeleteMongoMsgAndResetRedisSeq(t *testing.T) {
+func TestDeleteUserMsgsAndSetMinSeq(t *testing.T) {
 	operationID := getCronTaskOperationID()
 	redisClient = redis.NewClient(&redis.Options{
 		Addr:     "127.0.0.1:16379",
@@ -81,7 +80,7 @@ func TestDeleteMongoMsgAndResetRedisSeq(t *testing.T) {
 	userChat := GenUserChat(1, 500, 200, 0, testUID1)
 	err = CreateChat(userChat)
 
-	if err := DeleteMongoMsgAndResetRedisSeq(operationID, testUID1); err != nil {
+	if err := DeleteUserMsgsAndSetMinSeq(operationID, testUID1); err != nil {
 		t.Error("checkMaxSeqWithMongo failed", testUID1)
 	}
 	if err := checkMaxSeqWithMongo(operationID, testUID1, constant.WriteDiffusion); err != nil {
@@ -94,7 +93,7 @@ func TestDeleteMongoMsgAndResetRedisSeq(t *testing.T) {
 	// for _, groupID := range testWorkingGroupIDList {
 	// 	operationID = groupID + "-" + operationID
 	// 	log.NewDebug(operationID, utils.GetSelfFuncName(), "groupID:", groupID, "userIDList:", testUserIDList)
-	// 	if err := ResetUserGroupMinSeq(operationID, groupID, testUserIDList); err != nil {
+	// 	if err := DeleteUserSuperGroupMsgsAndSetMinSeq(operationID, groupID, testUserIDList); err != nil {
 	// 		t.Error("checkMaxSeqWithMongo failed", groupID)
 	// 	}
 	// 	if err := checkMaxSeqWithMongo(operationID, groupID, constant.ReadDiffusion); err != nil {

internal/crontask/cron_task.go

@@ -2,10 +2,6 @@ package cronTask
 
 import (
 	"Open_IM/pkg/common/config"
-	"Open_IM/pkg/common/constant"
-	"Open_IM/pkg/common/db/controller"
-	"Open_IM/pkg/common/db/mysql_model/im_mysql_model"
-	rocksCache "Open_IM/pkg/common/db/rocks_cache"
 	"Open_IM/pkg/common/log"
 	"Open_IM/pkg/utils"
 	"fmt"
@@ -15,9 +11,10 @@ import (
 )
 
 const cronTaskOperationID = "cronTaskOperationID-"
+const moduleName = "cron"
 
 func StartCronTask(userID, workingGroupID string) {
-	log.NewPrivateLog("cron")
+	log.NewPrivateLog(moduleName)
 	log.NewInfo(utils.OperationIDGenerator(), "start cron task", "cron config", config.Config.Mongo.ChatRecordsClearTime)
 	fmt.Println("cron task start, config", config.Config.Mongo.ChatRecordsClearTime)
 	if userID != "" {
@@ -44,59 +41,3 @@ func StartCronTask(userID, workingGroupID string) {
 		time.Sleep(10 * time.Second)
 	}
 }
-
-func getCronTaskOperationID() string {
-	return cronTaskOperationID + utils.OperationIDGenerator()
-}
-
-func ClearAll() {
-	operationID := getCronTaskOperationID()
-	log.NewInfo(operationID, "========================= start del cron task =========================")
-	var err error
-	userIDList, err := im_mysql_model.SelectAllUserID()
-	if err == nil {
-		StartClearMsg(operationID, userIDList)
-	} else {
-		log.NewError(operationID, utils.GetSelfFuncName(), err.Error())
-	}
-
-	// working group msg clear
-	workingGroupIDList, err := im_mysql_model.GetGroupIDListByGroupType(constant.WorkingGroup)
-	if err == nil {
-		StartClearWorkingGroupMsg(operationID, workingGroupIDList)
-	} else {
-		log.NewError(operationID, utils.GetSelfFuncName(), err.Error())
-	}
-
-	log.NewInfo(operationID, "========================= start del cron finished =========================")
-}
-
-func StartClearMsg(operationID string, userIDList []string) {
-	log.NewDebug(operationID, utils.GetSelfFuncName(), "userIDList: ", userIDList)
-	for _, userID := range userIDList {
-		if err := DeleteMongoMsgAndResetRedisSeq(operationID, userID); err != nil {
-			log.NewError(operationID, utils.GetSelfFuncName(), err.Error(), userID)
-		}
-		if err := checkMaxSeqWithMongo(operationID, userID, constant.WriteDiffusion); err != nil {
-			log.NewError(operationID, utils.GetSelfFuncName(), userID, err)
-		}
-	}
-}
-
-func StartClearWorkingGroupMsg(operationID string, workingGroupIDList []string) {
-	log.NewDebug(operationID, utils.GetSelfFuncName(), "workingGroupIDList: ", workingGroupIDList)
-	for _, groupID := range workingGroupIDList {
-		userIDList, err := rocksCache.GetGroupMemberIDListFromCache(groupID)
-		if err != nil {
-			log.NewError(operationID, utils.GetSelfFuncName(), err.Error(), groupID)
-			continue
-		}
-		log.NewDebug(operationID, utils.GetSelfFuncName(), "groupID:", groupID, "workingGroupIDList:", userIDList)
-		if err := ResetUserGroupMinSeq(operationID, groupID, userIDList); err != nil {
-			log.NewError(operationID, utils.GetSelfFuncName(), err.Error(), groupID, userIDList)
-		}
-		if err := checkMaxSeqWithMongo(operationID, groupID, constant.ReadDiffusion); err != nil {
-			log.NewError(operationID, utils.GetSelfFuncName(), groupID, err)
-		}
-	}
-}