IM/open-im-server
1
0
Fork 0
mirror of https://github.com/openimsdk/open-im-server.git synced 2026-05-02 16:15:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: Optimize Scheduled Task (#2985)

Browse Source 
* pb

* fix: Modifying other fields while setting IsPrivateChat does not take effect

* fix: quote message error revoke

* refactoring scheduled tasks

* refactoring scheduled tasks

* refactoring scheduled tasks

* refactoring scheduled tasks

* refactoring scheduled tasks

* refactoring scheduled tasks
This commit is contained in:
chao
2024-12-20 15:45:37 +08:00
committed by OpenIM-Robot
parent 248cb5c107
commit b26b0a422c
22 changed files with 647 additions and 400 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/common/storage/database/conversation.go
+1
View File
@@ -42,4 +42,5 @@ type Conversation interface {
GetConversationIDsNeedDestruct(ctx context.Context) ([]*model.Conversation, error)
GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error)
FindConversationUserVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error)
FindRandConversation(ctx context.Context, ts int64, limit int) ([]*model.Conversation, error)
}
pkg/common/storage/database/mgo/conversation.go
+32
View File
@@ -228,3 +228,35 @@ func (c *ConversationMgo) GetConversationNotReceiveMessageUserIDs(ctx context.Co
func (c *ConversationMgo) FindConversationUserVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) {
return c.version.FindChangeLog(ctx, userID, version, limit)
}
func (c *ConversationMgo) FindRandConversation(ctx context.Context, ts int64, limit int) ([]*model.Conversation, error) {
pipeline := []bson.M{
{
"$match": bson.M{
"is_msg_destruct": true,
"msg_destruct_time": bson.M{"$ne": 0},
},
},
{
"$addFields": bson.M{
"next_msg_destruct_timestamp": bson.M{
"$add": []any{
bson.M{
"$toLong": "$latest_msg_destruct_time",
}, "$msg_destruct_time"},
},
},
},
{
"$match": bson.M{
"next_msg_destruct_timestamp": bson.M{"$lt": ts},
},
},
{
"$sample": bson.M{
"size": limit,
},
},
}
return mongoutil.Aggregate[*model.Conversation](ctx, c.coll, pipeline)
}
pkg/common/storage/database/mgo/msg.go
+64 -55
View File
@@ -1227,8 +1227,7 @@ func (m *MsgMgo) ConvertMsgsDocLen(ctx context.Context, conversationIDs []string
}
}
func (m *MsgMgo) GetDocIDs(ctx context.Context) ([]string, error) {
limit := 5000
func (m *MsgMgo) GetRandDocIDs(ctx context.Context, limit int) ([]string, error) {
var skip int
var docIDs []string
var offset int
@@ -1267,15 +1266,18 @@ func (m *MsgMgo) GetDocIDs(ctx context.Context) ([]string, error) {
return docIDs, errs.Wrap(err)
}
func (m *MsgMgo) GetBeforeMsg(ctx context.Context, ts int64, docIDs []string, limit int) ([]*model.MsgDocModel, error) {
func (m *MsgMgo) GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) {
return mongoutil.Aggregate[*model.MsgDocModel](ctx, m.coll, []bson.M{
{
"$match": bson.M{
"doc_id": bson.M{
"$in": docIDs,
},
"msgs.msg.send_time": bson.M{
"$lt": ts,
"msgs": bson.M{
"$not": bson.M{
"$elemMatch": bson.M{
"msg.send_time": bson.M{
"$gt": ts,
},
},
},
},
},
},
@@ -1288,7 +1290,9 @@ func (m *MsgMgo) GetBeforeMsg(ctx context.Context, ts int64, docIDs []string, li
},
},
{
"$limit": limit,
"$sample": bson.M{
"size": limit,
},
},
})
}
@@ -1305,53 +1309,58 @@ func (m *MsgMgo) DeleteMsgByIndex(ctx context.Context, docID string, index []int
return mongoutil.UpdateOne(ctx, m.coll, bson.M{"doc_id": docID}, bson.M{"$set": set}, true)
}
//func (m *MsgMgo) ClearMsg(ctx context.Context, t time.Time) (int64, error) {
// ts := t.UnixMilli()
// var count int64
// for {
// msgs, err := m.GetBeforeMsg(ctx, ts, 100)
// if err != nil {
// return count, err
// }
// if len(msgs) == 0 {
// return count, nil
// }
// for _, msg := range msgs {
// num, err := m.deleteOneMsg(ctx, ts, msg)
// count += num
// if err != nil {
// return count, err
// }
// }
// }
//}
func (m *MsgMgo) DeleteDoc(ctx context.Context, docID string) error {
return mongoutil.DeleteOne(ctx, m.coll, bson.M{"doc_id": docID})
}
//func (m *MsgMgo) DeleteDocMsg(ctx context.Context, ts int64, doc *relation.MsgDocModel) (int64, error) {
// var notNull int
// index := make([]int, 0, len(doc.Msg))
// for i, message := range doc.Msg {
// if message.Msg != nil {
// notNull++
// if message.Msg.SendTime < ts {
// index = append(index, i)
// }
// }
// }
// if len(index) == 0 {
// return 0, errs.New("no msg to delete").WrapMsg("deleteOneMsg", "docID", doc.DocID)
// }
// if len(index) == notNull {
// if err := m.DeleteDoc(ctx, doc.DocID); err != nil {
// return 0, err
// }
// } else {
// if err := m.setNullMsg(ctx, doc.DocID, index); err != nil {
// return 0, err
// }
// }
// return int64(len(index)), nil
//}
func (m *MsgMgo) GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) {
pipeline := []bson.M{
{
"$match": bson.M{
"doc_id": bson.M{
"$regex": fmt.Sprintf("^%s", conversationID),
},
},
},
{
"$match": bson.M{
"msgs.msg.send_time": bson.M{
"$lte": time,
},
},
},
{
"$sort": bson.M{
"_id": -1,
},
},
{
"$limit": 1,
},
{
"$project": bson.M{
"_id": 0,
"doc_id": 1,
"msgs.msg.send_time": 1,
"msgs.msg.seq": 1,
},
},
}
res, err := mongoutil.Aggregate[*model.MsgDocModel](ctx, m.coll, pipeline)
if err != nil {
return 0, err
}
if len(res) == 0 {
return 0, nil
}
var seq int64
for _, v := range res[0].Msg {
if v.Msg == nil {
continue
}
if v.Msg.SendTime <= time {
seq = v.Msg.Seq
}
}
return seq, nil
}
pkg/common/storage/database/mgo/msg_test.go
+70 -31
View File
@@ -3,12 +3,11 @@ package mgo
import (
"context"
"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
"github.com/openimsdk/protocol/msg"
"github.com/openimsdk/protocol/sdkws"
"github.com/openimsdk/tools/db/mongoutil"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"math"
"math/rand"
"strconv"
"testing"
@@ -16,35 +15,45 @@ import (
)
func TestName1(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*300)
defer cancel()
cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second)))
v := &MsgMgo{
coll: cli.Database("openim_v3").Collection("msg3"),
}
req := &msg.SearchMessageReq{
//RecvID: "3187706596",
//SendID: "7009965934",
ContentType: 101,
//SendTime: "2024-05-06",
//SessionType: 3,
Pagination: &sdkws.RequestPagination{
PageNumber: 1,
ShowNumber: 10,
},
}
total, res, err := v.SearchMessage(ctx, req)
if err != nil {
panic(err)
}
for i, re := range res {
t.Logf("%d => %d | %+v", i+1, re.Msg.Seq, re.Msg.Content)
}
t.Log(total)
//ctx, cancel := context.WithTimeout(context.Background(), time.Second*300)
//defer cancel()
//cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.66:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second)))
//
//v := &MsgMgo{
// coll: cli.Database("openim_v3").Collection("msg3"),
//}
//
//req := &msg.SearchMessageReq{
// //RecvID: "3187706596",
// //SendID: "7009965934",
// ContentType: 101,
// //SendTime: "2024-05-06",
// //SessionType: 3,
// Pagination: &sdkws.RequestPagination{
// PageNumber: 1,
// ShowNumber: 10,
// },
//}
//total, res, err := v.SearchMessage(ctx, req)
//if err != nil {
// panic(err)
//}
//
//for i, re := range res {
// t.Logf("%d => %d | %+v", i+1, re.Msg.Seq, re.Msg.Content)
//}
//
//t.Log(total)
//
//msg, err := NewMsgMongo(cli.Database("openim_v3"))
//if err != nil {
// panic(err)
//}
//res, err := msg.GetBeforeMsg(ctx, time.Now().UnixMilli(), []string{"1:0"}, 1000)
//if err != nil {
// panic(err)
//}
//t.Log(len(res))
}
func TestName10(t *testing.T) {
@@ -73,3 +82,33 @@ func TestName10(t *testing.T) {
}
}
func TestName3(t *testing.T) {
t.Log(uint64(math.MaxUint64))
t.Log(int64(math.MaxInt64))
t.Log(int64(math.MinInt64))
}
func TestName4(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*300)
defer cancel()
cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.66:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second)))
msg, err := NewMsgMongo(cli.Database("openim_v3"))
if err != nil {
panic(err)
}
ts := time.Now().Add(-time.Hour * 24 * 5).UnixMilli()
t.Log(ts)
res, err := msg.GetLastMessageSeqByTime(ctx, "sg_1523453548", ts)
if err != nil {
panic(err)
}
t.Log(res)
}
func TestName5(t *testing.T) {
var v time.Time
t.Log(v.UnixMilli())
}
pkg/common/storage/database/mgo/object.go
+18 -15
View File
@@ -22,7 +22,6 @@ import (
"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
"github.com/openimsdk/tools/db/mongoutil"
"github.com/openimsdk/tools/db/pagination"
"github.com/openimsdk/tools/errs"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
@@ -91,21 +90,25 @@ func (o *S3Mongo) Take(ctx context.Context, engine string, name string) (*model.
return mongoutil.FindOne[*model.Object](ctx, o.coll, bson.M{"name": name, "engine": engine})
}
func (o *S3Mongo) Delete(ctx context.Context, engine string, name string) error {
return mongoutil.DeleteOne(ctx, o.coll, bson.M{"name": name, "engine": engine})
func (o *S3Mongo) Delete(ctx context.Context, engine string, name []string) error {
if len(name) == 0 {
return nil
}
return mongoutil.DeleteOne(ctx, o.coll, bson.M{"engine": engine, "name": bson.M{"$in": name}})
}
// Find Expires object
func (o *S3Mongo) FindNeedDeleteObjectByDB(ctx context.Context, duration time.Time, needDelType []string, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) {
return mongoutil.FindPage[*model.Object](ctx, o.coll, bson.M{
"create_time": bson.M{"$lt": duration},
"group": bson.M{"$in": needDelType},
}, pagination)
}
// Find object by key
func (o *S3Mongo) FindModelsByKey(ctx context.Context, key string) (objects []*model.Object, err error) {
func (o *S3Mongo) FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) {
opt := options.Find()
if count > 0 {
opt.SetLimit(count)
}
return mongoutil.Find[*model.Object](ctx, o.coll, bson.M{
"key": key,
})
"engine": engine,
"create_time": bson.M{"$lt": expiration},
"group": bson.M{"$in": needDelType},
}, opt)
}
func (o *S3Mongo) GetKeyCount(ctx context.Context, engine string, key string) (int64, error) {
return mongoutil.Count(ctx, o.coll, bson.M{"engine": engine, "key": key})
}
pkg/common/storage/database/msg.go
+4 -2
View File
@@ -45,7 +45,9 @@ type Msg interface {
DeleteDoc(ctx context.Context, docID string) error
DeleteMsgByIndex(ctx context.Context, docID string, index []int) error
GetBeforeMsg(ctx context.Context, ts int64, docIDs []string, limit int) ([]*model.MsgDocModel, error)
GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error)
GetDocIDs(ctx context.Context) ([]string, error)
GetRandDocIDs(ctx context.Context, limit int) ([]string, error)
GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error)
}
pkg/common/storage/database/object.go
+3 -4
View File
@@ -19,13 +19,12 @@ import (
"time"
"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
"github.com/openimsdk/tools/db/pagination"
)
type ObjectInfo interface {
SetObject(ctx context.Context, obj *model.Object) error
Take(ctx context.Context, engine string, name string) (*model.Object, error)
Delete(ctx context.Context, engine string, name string) error
FindNeedDeleteObjectByDB(ctx context.Context, duration time.Time, needDelType []string, pagination pagination.Pagination) (total int64, objects []*model.Object, err error)
FindModelsByKey(ctx context.Context, key string) (objects []*model.Object, err error)
Delete(ctx context.Context, engine string, name []string) error
FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error)
GetKeyCount(ctx context.Context, engine string, key string) (int64, error)
}