Compare commits

...

132 Commits

Author SHA1 Message Date
withchao 2e6ea7b193 Merge remote-tracking branch 'origin/v3dev' into v3dev 2023-07-12 14:27:20 +08:00
withchao fe3e17c3ad s3 InitiateMultipartUpload add ExpireTime 2023-07-12 14:27:06 +08:00
wangchuxiao 38912aed76 Merge branch 'v3dev' of github.com:OpenIMSDK/Open-IM-Server into v3dev 2023-07-12 14:22:23 +08:00
wangchuxiao 98215396f1 log 2023-07-12 14:22:16 +08:00
withchao bdf93565a3 s3 AccessURL 2023-07-12 14:01:48 +08:00
withchao 406a77d5b8 s3 url 2023-07-12 13:58:20 +08:00
withchao 84f8074d6a s3 url 2023-07-12 13:55:17 +08:00
withchao 75af5447ae s3 cos copy 2023-07-12 13:45:45 +08:00
withchao e6d502f0ff Merge remote-tracking branch 'origin/v3dev' into v3dev 2023-07-12 13:39:54 +08:00
withchao 1f6225db41 s3 cos copy 2023-07-12 13:39:08 +08:00
wangchuxiao 16c2028687 kickuserlog 2023-07-12 13:29:40 +08:00
wangchuxiao fd87d70b56 Merge branch 'v3dev' of github.com:OpenIMSDK/Open-IM-Server into v3dev 2023-07-12 13:23:22 +08:00
wangchuxiao 5dbeac7dcf kick user log 2023-07-12 13:21:49 +08:00
withchao c32b437821 s3 cos test 2023-07-12 12:34:00 +08:00
withchao 535736bad1 s3 cos test 2023-07-12 12:30:42 +08:00
withchao 3b9ff16b3b Merge remote-tracking branch 'origin/v3dev' into v3dev 2023-07-12 12:22:32 +08:00
withchao 22b1403819 s3 cos test 2023-07-12 12:22:18 +08:00
wangchuxiao 3658f2b198 Merge branch 'v3dev' of github.com:OpenIMSDK/Open-IM-Server into v3dev 2023-07-12 12:20:44 +08:00
wangchuxiao 37bd1526d7 add kick log 2023-07-12 12:20:38 +08:00
withchao 8a2d0e9e3a Merge remote-tracking branch 'origin/v3dev' into v3dev 2023-07-12 12:10:08 +08:00
withchao c411cc4bc8 s3 cos 2023-07-12 12:09:57 +08:00
wangchuxiao 055853593a Merge branch 'v3dev' of github.com:OpenIMSDK/Open-IM-Server into v3dev 2023-07-12 12:03:58 +08:00
wangchuxiao 31778c9cdf fix kick user bug 2023-07-12 12:03:53 +08:00
withchao 5fd464979d Merge remote-tracking branch 'origin/v3dev' into v3dev 2023-07-12 12:02:50 +08:00
withchao a2fc790cba s3 config 2023-07-12 12:02:40 +08:00
wangchuxiao 20bb77f0f7 cron 2023-07-12 11:49:25 +08:00
wangchuxiao 52fcf70739 add log 2023-07-12 11:41:33 +08:00
wangchuxiao e92a09ecf5 Merge branch 'v3dev' of github.com:OpenIMSDK/Open-IM-Server into v3dev 2023-07-12 11:33:15 +08:00
wangchuxiao a23ecb1710 cron add log and fix cron 2023-07-12 11:33:08 +08:00
withchao d7c66c6f45 s3 debug log 2023-07-12 11:14:06 +08:00
withchao d7802e15a8 Merge remote-tracking branch 'origin/v3dev' into v3dev 2023-07-12 11:06:55 +08:00
withchao 95b1d09a3c s3 debug log 2023-07-12 11:06:41 +08:00
Alan 32b60db21b Update .gitignore (#482) 2023-07-12 09:35:27 +08:00
withchao e62f5d8d7c s3 presigned test 2023-07-10 19:50:47 +08:00
withchao 2f1d9f6265 s3 presigned test 2023-07-10 19:19:33 +08:00
withchao 06a5ef9417 s3 presigned test 2023-07-10 19:13:15 +08:00
withchao 4203705bf5 s3 presigned test 2023-07-10 19:10:12 +08:00
withchao bfd1df751e s3 presigned test 2023-07-10 19:07:00 +08:00
withchao 6716a92085 s3 presigned test 2023-07-10 19:03:18 +08:00
withchao d9c7c57343 s3 presigned test 2023-07-10 18:46:45 +08:00
withchao 5a9ecbf295 s3 presigned test 2023-07-10 18:40:30 +08:00
withchao 70b6cd360d s3 presigned put 2023-07-10 18:32:25 +08:00
withchao 04474c6747 s3 pb 2023-07-10 17:51:57 +08:00
withchao f5dcc6c8a3 s3 pb 2023-07-10 17:34:43 +08:00
withchao aee309097b Merge remote-tracking branch 'origin/v3dev' into v3dev 2023-07-10 17:30:19 +08:00
withchao 732afeab3c s3 pb 2023-07-10 17:30:09 +08:00
wangchuxiao c42a47b6b9 Merge branch 'v3dev' of github.com:OpenIMSDK/Open-IM-Server into v3dev 2023-07-10 17:10:00 +08:00
wangchuxiao cb14fca6c7 remove unuse code 2023-07-10 17:09:55 +08:00
withchao a7a2bd3e4f Merge remote-tracking branch 'origin/v3dev' into v3dev 2023-07-10 17:08:08 +08:00
withchao e55bd7adae s3 route 2023-07-10 17:07:55 +08:00
wangchuxiao 9468589cc7 Merge branch 'v3dev' of github.com:OpenIMSDK/Open-IM-Server into v3dev 2023-07-10 17:07:14 +08:00
wangchuxiao bcf2f0f443 remove extendMsg code 2023-07-10 17:07:08 +08:00
withchao 1c67fab186 s3 route 2023-07-10 16:39:37 +08:00
withchao e82a08bbbd Merge remote-tracking branch 'origin/v3dev' into v3dev 2023-07-10 16:25:53 +08:00
withchao 29dade90ec s3 minio, cos, oss support 2023-07-10 16:24:59 +08:00
BanTanger ec6036b5fa feat: add implement of GetUsersOnlineStatus, #472 (#477) 2023-07-10 16:01:39 +08:00
withchao fea3c5358a s3 minio, cos, oss support 2023-07-10 15:50:57 +08:00
wangchuxiao 4b8af3be2e msgDestruct 2023-07-10 14:54:17 +08:00
wangchuxiao 7059d6b5fb msg destruct 2023-07-10 14:47:07 +08:00
wangchuxiao c02f3cc8d5 set IsMsgDestruct 2023-07-10 11:58:04 +08:00
wangchuxiao ece2498d18 cron use rpc mw 2023-07-10 10:34:44 +08:00
wangchuxiao 7bf9d5a305 msgDestruct 2023-07-10 10:09:51 +08:00
wangchuxiao c64657d0b7 Merge branch 'v3dev' of github.com:OpenIMSDK/Open-IM-Server into v3dev 2023-07-07 19:19:29 +08:00
wangchuxiao c2e1d5a21e remove unuse rpc 2023-07-07 19:19:23 +08:00
Gordon 05889706dd Merge remote-tracking branch 'origin/v3dev' into v3dev 2023-07-07 11:36:22 +08:00
Gordon 91cb0607e3 fix: content update 2023-07-07 11:36:00 +08:00
wangchuxiao 5d82447e92 set privateChat 2023-07-07 11:14:15 +08:00
wangchuxiao 0ae58dac79 Merge branch 'v3dev' of github.com:OpenIMSDK/Open-IM-Server into v3dev 2023-07-07 10:53:00 +08:00
wangchuxiao 4bdf6be372 set conversations update 2023-07-07 10:52:54 +08:00
Gordon f44ba40098 fix: update add model 2023-07-07 10:23:54 +08:00
Gordon ffba76cc6b fix: text update 2023-07-06 18:36:27 +08:00
wangchuxiao e547266755 Merge branch 'v3dev' of github.com:OpenIMSDK/Open-IM-Server into v3dev 2023-07-06 15:22:15 +08:00
wangchuxiao f75bfdabe6 fix sync has read 2023-07-06 15:22:09 +08:00
withchao f6756008b0 api nil slice map 2023-07-06 15:09:41 +08:00
withchao ff22ea7108 chain unary interceptor 2023-07-06 11:23:57 +08:00
wangchuxiao 28fad04d5c change log error 2023-07-06 11:14:50 +08:00
wangchuxiao a3e3dc03c9 Merge branch 'v3dev' of github.com:OpenIMSDK/Open-IM-Server into v3dev 2023-07-06 11:14:28 +08:00
wangchuxiao d50a24e9b1 grpc with detail return error 2023-07-06 11:14:22 +08:00
withchao b494c73328 Merge remote-tracking branch 'origin/v3dev' into v3dev 2023-07-05 20:31:17 +08:00
withchao c2b7d3bc2c a2r call option 2023-07-05 20:30:54 +08:00
wangchuxiao 54bba9a867 privateChat 2023-07-05 19:08:06 +08:00
wangchuxiao 7bb069d435 withname 2023-07-05 13:23:20 +08:00
wangchuxiao 325d898695 Merge branch 'v3dev' of github.com:OpenIMSDK/Open-IM-Server into v3dev 2023-07-05 13:22:14 +08:00
wangchuxiao 7a9e0af8ed sendNotificationWithName 2023-07-05 13:22:06 +08:00
withchao 6bf5b86227 user active 2023-07-04 20:11:51 +08:00
withchao ffe7cc3b10 Merge remote-tracking branch 'origin/v3dev' into v3dev 2023-07-04 18:46:34 +08:00
withchao c2d02cbc4c group active 2023-07-04 18:46:21 +08:00
Gordon 3dba5495a6 feat: group notification show to conversation 2023-07-04 18:25:38 +08:00
Gordon 387ac6e930 Merge remote-tracking branch 'origin/v3dev' into v3dev 2023-07-04 17:38:29 +08:00
Gordon ea4335bb24 feat: group notification show to conversation 2023-07-04 17:38:03 +08:00
withchao 1e93297968 group create 2023-07-04 17:16:38 +08:00
withchao f80372ded4 group create 2023-07-04 17:16:25 +08:00
withchao bb667238f2 active user 2023-07-04 16:55:34 +08:00
withchao be9c8fcab7 active user 2023-07-04 16:36:02 +08:00
withchao 000c68dddd Merge pull request #467 from BanTanger/v3dev
add tx_oss cos
2023-07-04 14:43:38 +08:00
BanTanger 2e99a45dae add tx_oss cos 2023-07-04 12:23:58 +08:00
Gordon 5d115bf651 feat: conn update token 2023-07-04 11:28:43 +08:00
Gordon 49737a2dda feat: conn update token 2023-07-04 11:26:33 +08:00
Gordon 4df0f78af5 Merge remote-tracking branch 'origin/v3dev' into v3dev 2023-07-04 10:33:25 +08:00
Gordon 94b4a18f2d feat: conn update token 2023-07-04 10:32:54 +08:00
withchao 9f32e3a9bc active user 2023-07-04 09:56:02 +08:00
Gordon e416ace264 Merge remote-tracking branch 'origin/v3dev' into v3dev 2023-07-04 09:40:11 +08:00
Gordon dbc4fd5e4a feat: conn update token 2023-07-04 09:39:50 +08:00
withchao 0cb1fc3e52 active user 2023-07-04 09:33:59 +08:00
withchao 898adbb833 Merge remote-tracking branch 'origin/v3dev' into v3dev 2023-07-04 09:32:56 +08:00
withchao aaa1fb29cd active user 2023-07-04 09:32:40 +08:00
Gordon 92f64ddf7c feat: conn update token 2023-07-03 21:30:22 +08:00
Gordon 1f94357498 Merge remote-tracking branch 'origin/v3dev' into v3dev 2023-07-03 21:17:22 +08:00
Gordon e6c2568f6b refactor: errcode update 2023-07-03 21:15:52 +08:00
withchao 80f25c54c9 active user 2023-07-03 21:01:50 +08:00
withchao 807c6eae15 active user 2023-07-03 20:59:05 +08:00
withchao 28b4506395 Merge remote-tracking branch 'origin/v3dev' into v3dev 2023-07-03 20:54:21 +08:00
withchao dd6b47fbbb active user 2023-07-03 20:54:03 +08:00
Gordon 4ca3e757bd Merge remote-tracking branch 'origin/v3dev' into v3dev 2023-07-03 18:29:14 +08:00
Gordon 3a3784329e refactor: errcode update 2023-07-03 18:28:40 +08:00
withchao bedd554269 revoke userID 2023-07-03 18:28:38 +08:00
withchao e2be0fc093 msgs statistics 2023-07-03 18:24:52 +08:00
withchao d787668911 Merge remote-tracking branch 'origin/v3dev' into v3dev 2023-07-03 18:24:34 +08:00
withchao cae8b377b0 msgs statistics 2023-07-03 18:20:01 +08:00
Gordon ba20fa90f7 refactor:pb file 2023-07-03 17:34:46 +08:00
Gordon d325eed4a3 refactor: user pb update 2023-07-03 17:16:39 +08:00
Gordon 48a6265465 Merge remote-tracking branch 'origin/v3dev' into v3dev
# Conflicts:
#	pkg/proto/user/user.pb.go
2023-07-03 17:14:33 +08:00
wangchuxiao 1ac6279569 remove online push close grpc conn 2023-07-03 17:02:42 +08:00
Gordon 2095e0839d refactor: user pb update 2023-07-03 17:02:31 +08:00
wangchuxiao 8bb177a693 Merge branch 'v3dev' of github.com:OpenIMSDK/Open-IM-Server into v3dev 2023-07-03 16:36:57 +08:00
wangchuxiao 01d3fefd5d push use local conn 2023-07-03 16:36:52 +08:00
withchao 3de21d13bd UserRegisterCount 2023-07-03 15:59:42 +08:00
withchao 840e79ca16 Merge remote-tracking branch 'origin/main' into v3dev 2023-07-03 15:49:04 +08:00
withchao 12dd42c60d minio init 2023-07-03 15:09:39 +08:00
Gordon 0642f97e12 Merge remote-tracking branch 'origin/v3dev' into v3dev 2023-07-03 14:30:38 +08:00
Gordon a9569a2a97 refactor: router change 2023-07-03 14:26:45 +08:00
withchao 09ffe40011 statistics user register 2023-07-03 12:11:09 +08:00
126 changed files with 7084 additions and 8072 deletions
+2
View File
@@ -162,6 +162,8 @@ go.work
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
# User-specific stuff
.idea/
.idea
.idea/**/workspace.xml
.idea/**/tasks.xml
.idea/**/usage.statistics.xml
+9
View File
@@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="WEB_MODULE" version="4">
<component name="Go" enabled="true" />
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
+14
View File
@@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="DataSourceManagerImpl" format="xml" multifile-model="true">
<data-source source="LOCAL" name="grafana" uuid="95aae14a-3593-4ff7-ab49-5e4316cbecd1">
<driver-ref>sqlite.xerial</driver-ref>
<synchronize>true</synchronize>
<jdbc-driver>org.sqlite.JDBC</jdbc-driver>
<jdbc-url>jdbc:sqlite:C:\Users\Administrator\Desktop\Open-IM-Server\docker-compose_cfg\grafana.db</jdbc-url>
<driver-properties>
<property name="enable_load_extension" value="true" />
</driver-properties>
</data-source>
</component>
</project>
+6
View File
@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="JavaScriptSettings">
<option name="languageLevel" value="ES6" />
</component>
</project>
+8
View File
@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/Open-IM-Server.iml" filepath="$PROJECT_DIR$/.idea/Open-IM-Server.iml" />
</modules>
</component>
</project>
Generated
+6
View File
@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="" vcs="Git" />
</component>
</project>
+1 -1
View File
@@ -57,7 +57,7 @@ func run(port int) error {
if err != nil {
return err
}
if client.CreateRpcRootNodes(config.GetServiceNames()); err != nil {
if err := client.CreateRpcRootNodes(config.GetServiceNames()); err != nil {
return err
}
fmt.Println("api init discov client success")
+21 -38
View File
@@ -42,14 +42,11 @@ kafka:
topic: "offlineMsgToMongoMysql" #不建议修改
msgToPush:
topic: "msgToPush" #不建议修改
msgToModify:
topic: "msgToModify" #不建议修改
consumerGroupID: #消费者组,不建议修改
msgToRedis: redis #
msgToMongo: mongo #
msgToMySql: mysql #
msgToPush: push #
msgToModify: modify #
rpc:
@@ -62,41 +59,26 @@ api:
listenIP: #默认为0.0.0.0
object:
enable: minio #使用minio
apiURL: http://127.0.0.1:10002/third/object
enable: "minio" #使用minio
apiURL: "http://127.0.0.1:10002/object/"
minio:
tempBucket: "openim" #不建议修改
dataBucket: "openim" #不建议修改
location: us-east-1 #不建议修改
endpoint: http://127.0.0.1:10005 #minio对外服务的ip和端口,app要能访问此ip和端口
accessKeyID: root #ID
secretAccessKey: openIM123 #秘钥
isDistributedMod: false #是否分布式多硬盘部署,如果是多硬盘部署,需要修改为true
tencent: #tencent cos
appID:
region:
bucket:
secretID:
secretKey:
ali: #ali oss
regionID:
accessKeyID:
accessKeySecret:
stsEndpoint:
ossEndpoint:
bucket:
finalHost:
stsDurationSeconds:
OssRoleArn:
aws:
accessKeyID:
accessKeySecret:
region:
bucket:
finalHost:
roleArn:
externalId:
roleSessionName:
bucket: "openim" #不建议修改
endpoint: "http://127.0.0.1:10005" #minio对外服务的ip和端口,app要能访问此ip和端口
accessKeyID: "root" #ID
secretAccessKey: "openIM123" #秘钥
sessionToken: "" #token
cos: #tencent cos
bucketURL: "https://temp-1252357374.cos.ap-chengdu.myqcloud.com"
secretID: ""
secretKey: ""
sessionToken: ""
oss: #ali oss
endpoint: "https://oss-cn-chengdu.aliyuncs.com"
bucket: "demo-9999999"
bucketURL: "https://demo-9999999.oss-cn-chengdu.aliyuncs.com"
accessKeyID: ""
accessKeySecret: ""
sessionToken: ""
rpcPort: #rpc服务端口,不建议修改,端口由脚本读取后传入程序,如启动多个程序,只需要填入多个端口,用逗号隔开,如 [10110, 10111]
openImUserPort: [ 10110 ]
@@ -168,7 +150,8 @@ groupMessageHasReadReceiptEnable: true #群聊已读是否开
singleMessageHasReadReceiptEnable: true #单聊已读是否开启
retainChatRecords: 365 #mongo保存离线消息时间(天)
chatRecordsClearTime: "0 2 * * 3" #每周三凌晨2点清理mongo中的过期(超过retainChatRecords时间)消息
chatRecordsClearTime: "0 2 * * 3" #每周三凌晨2点清理mongo中的过期(超过retainChatRecords时间)消息,这个删除是为了清理满足上个配置retainChatRecords的过期消息,不会发送通知,仅仅作为清理磁盘使用
msgDestructTime: "0 2 * * *" #消息自动删除时间,每天凌晨2点删除过期消息,这个删除是为了删除保留时间超过超过会话字段msg_destruct_time(秒)的消息。
secret: tuoyun #秘钥,获取token时校验
+14 -11
View File
@@ -17,13 +17,13 @@ require (
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
github.com/jinzhu/copier v0.3.5
github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible
github.com/minio/minio-go/v7 v7.0.22
github.com/mitchellh/mapstructure v1.4.2
github.com/minio/minio-go/v7 v7.0.59
github.com/mitchellh/mapstructure v1.5.0
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.13.0
github.com/robfig/cron/v3 v3.0.1
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/sirupsen/logrus v1.9.2 // indirect
github.com/stretchr/testify v1.8.3
github.com/tencentyun/qcloud-cos-sts-sdk v0.0.0-20210325043845-84a0811633ca
go.mongodb.org/mongo-driver v1.8.3
@@ -39,10 +39,12 @@ require (
require github.com/google/uuid v1.3.0
require (
github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible
github.com/go-redis/redis v6.15.9+incompatible
github.com/go-sql-driver/mysql v1.6.0
github.com/go-zookeeper/zk v1.0.3
github.com/redis/go-redis/v9 v9.0.5
github.com/tencentyun/cos-go-sdk-v5 v0.7.41
)
require (
@@ -57,9 +59,10 @@ require (
github.com/bytedance/sonic v1.9.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
github.com/clbanning/mxj v1.8.4 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/eapache/go-resiliency v1.2.0 // indirect
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
github.com/eapache/queue v1.1.0 // indirect
@@ -70,6 +73,7 @@ require (
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/snappy v0.0.3 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
github.com/googleapis/gax-go/v2 v2.7.1 // indirect
github.com/hashicorp/go-uuid v1.0.2 // indirect
@@ -82,17 +86,16 @@ require (
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.13.6 // indirect
github.com/klauspost/cpuid v1.3.1 // indirect
github.com/klauspost/compress v1.16.5 // indirect
github.com/klauspost/cpuid/v2 v2.2.4 // indirect
github.com/leodido/go-urn v1.2.4 // indirect
github.com/lithammer/shortuuid v3.0.0+incompatible // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/minio/md5-simd v1.1.0 // indirect
github.com/minio/sha256-simd v0.1.1 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/minio/sha256-simd v1.0.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mozillazg/go-httpheader v0.4.0 // indirect
github.com/onsi/ginkgo v1.16.5 // indirect
github.com/onsi/gomega v1.18.1 // indirect
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
@@ -102,7 +105,7 @@ require (
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/rs/xid v1.2.1 // indirect
github.com/rs/xid v1.5.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
@@ -136,7 +139,7 @@ require (
go.uber.org/zap v1.24.0
golang.org/x/crypto v0.10.0 // indirect
google.golang.org/genproto v0.0.0-20230525234025-438c736192d0 // indirect
gopkg.in/ini.v1 v1.66.2 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
)
replace github.com/Shopify/sarama => github.com/Shopify/sarama v1.29.0
+36 -22
View File
@@ -51,6 +51,7 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/OpenIMSDK/open_utils v1.0.8 h1:IopxWgJwEF5ZAPsRuiZZOfcxNOQOCt/p8VDENcHN9r4=
github.com/OpenIMSDK/open_utils v1.0.8/go.mod h1:FLoaQblWUVKQgqt2LrNzfSZLT6D3DICBn1kcOMDLUOI=
github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM=
github.com/Shopify/sarama v1.29.0 h1:ARid8o8oieau9XrHI55f/L3EoRAhm9px6sonbD7yuUE=
github.com/Shopify/sarama v1.29.0/go.mod h1:2QpgD79wpdAESqNQMxNc0KYMkycd4slxGdV3TWSVqrU=
github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc=
@@ -60,6 +61,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible h1:KpbJFXwhVeuxNtBJ74MCGbIoaBok2uZvkD7QXp2+Wis=
github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
@@ -83,6 +86,8 @@ github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583j
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I=
github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
@@ -94,8 +99,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/dtm-labs/rockscache v0.1.1 h1:6S1vgaHvGqrLd8Ka4hRTKeKPV7v+tT0MSkTIX81LRyA=
github.com/dtm-labs/rockscache v0.1.1/go.mod h1:c76WX0kyIibmQ2ACxUXvDvaLykoPakivMqIxt+UzE7A=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q=
github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
@@ -202,6 +207,9 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
@@ -216,6 +224,7 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf
github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -274,11 +283,10 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
@@ -306,16 +314,15 @@ github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APP
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
github.com/minio/minio-go/v7 v7.0.22 h1:iXhsiRyYh1ozm/+jN2qGgEIahYjEkvcpuu6NcdpSxcA=
github.com/minio/minio-go/v7 v7.0.22/go.mod h1:ei5JjmxwHaMrgsMrn4U/+Nmg+d8MKS1U2DAn1ou4+Do=
github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo=
github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.59 h1:lxIXwsTIcQkYoEG25rUJbzpmSB/oWeVDmxFo/uWUUsw=
github.com/minio/minio-go/v7 v7.0.59/go.mod h1:NUDy4A4oXPq1l2yK6LTSvCEzAMeIcoz9lcj5dbzSrRE=
github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -324,6 +331,9 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60=
github.com/mozillazg/go-httpheader v0.4.0 h1:aBn6aRXtFzyDLZ4VIRLsZbbJloagQfMnCiYgOq6hK4w=
github.com/mozillazg/go-httpheader v0.4.0/go.mod h1:PuT8h0pw6efvp8ZeUec1Rs7dwjK08bt6gKSReGMqtdA=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
@@ -384,14 +394,14 @@ github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDO
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y=
github.com/sirupsen/logrus v1.9.2/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
@@ -412,6 +422,10 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4=
github.com/tencentyun/cos-go-sdk-v5 v0.7.41 h1:iU0Li/Np78H4SBna0ECQoF3mpgi6ImLXU+doGzPFXGc=
github.com/tencentyun/cos-go-sdk-v5 v0.7.41/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7yZxQuYwgSJM87icDaw=
github.com/tencentyun/qcloud-cos-sts-sdk v0.0.0-20210325043845-84a0811633ca h1:G/aIr3WiUesWHL2YGYgEqjM5tCAJ43Ml+0C18wDkWWs=
github.com/tencentyun/qcloud-cos-sts-sdk v0.0.0-20210325043845-84a0811633ca/go.mod h1:b18KQa4IxHbxeseW1GcZox53d7J0z39VNONTxvvlkXw=
github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
@@ -782,8 +796,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI=
gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-12
View File
@@ -26,18 +26,6 @@ func (o *ConversationApi) GetConversations(c *gin.Context) {
a2r.Call(conversation.ConversationClient.GetConversations, o.Client, c)
}
func (o *ConversationApi) BatchSetConversations(c *gin.Context) {
a2r.Call(conversation.ConversationClient.BatchSetConversations, o.Client, c)
}
func (o *ConversationApi) SetRecvMsgOpt(c *gin.Context) {
a2r.Call(conversation.ConversationClient.SetRecvMsgOpt, o.Client, c)
}
func (o *ConversationApi) ModifyConversationField(c *gin.Context) {
a2r.Call(conversation.ConversationClient.ModifyConversationField, o.Client, c)
}
func (o *ConversationApi) SetConversations(c *gin.Context) {
a2r.Call(conversation.ConversationClient.SetConversations, o.Client, c)
}
+4
View File
@@ -114,3 +114,7 @@ func (o *GroupApi) GetJoinedSuperGroupList(c *gin.Context) {
func (o *GroupApi) GetSuperGroupsInfo(c *gin.Context) {
a2r.Call(group.GroupClient.GetSuperGroupsInfo, o.Client, c)
}
func (o *GroupApi) GroupCreateCount(c *gin.Context) {
a2r.Call(group.GroupClient.GroupCreateCount, o.Client, c)
}
+13 -21
View File
@@ -6,6 +6,7 @@ import (
"github.com/OpenIMSDK/Open-IM-Server/pkg/apistruct"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/tokenverify"
"github.com/OpenIMSDK/Open-IM-Server/pkg/discoveryregistry"
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/msg"
@@ -147,33 +148,16 @@ func (m *MessageApi) DeleteMsgPhysical(c *gin.Context) {
a2r.Call(msg.MsgClient.DeleteMsgPhysical, m.Client, c)
}
func (m *MessageApi) SetMessageReactionExtensions(c *gin.Context) {
a2r.Call(msg.MsgClient.SetMessageReactionExtensions, m.Client, c)
}
func (m *MessageApi) GetMessageListReactionExtensions(c *gin.Context) {
a2r.Call(msg.MsgClient.GetMessagesReactionExtensions, m.Client, c)
}
func (m *MessageApi) AddMessageReactionExtensions(c *gin.Context) {
a2r.Call(msg.MsgClient.AddMessageReactionExtensions, m.Client, c)
}
func (m *MessageApi) DeleteMessageReactionExtensions(c *gin.Context) {
a2r.Call(msg.MsgClient.DeleteMessageReactionExtensions, m.Client, c)
}
func (m *MessageApi) SendMessage(c *gin.Context) {
params := apistruct.ManagementSendMsgReq{}
if err := c.BindJSON(&params); err != nil {
apiresp.GinError(c, errs.ErrArgs.WithDetail(err.Error()).Wrap())
return
}
// todo
//if !tokenverify.IsAppManagerUid(c) {
// apiresp.GinError(c, errs.ErrNoPermission.Wrap("only app manager can send message"))
// return
//}
if !tokenverify.IsAppManagerUid(c) {
apiresp.GinError(c, errs.ErrNoPermission.Wrap("only app manager can send message"))
return
}
var data interface{}
switch params.ContentType {
@@ -238,3 +222,11 @@ func (m *MessageApi) CheckMsgIsSendSuccess(c *gin.Context) {
func (m *MessageApi) GetUsersOnlineStatus(c *gin.Context) {
a2r.Call(msg.MsgClient.GetSendMsgStatus, m.Client, c)
}
func (m *MessageApi) GetActiveUser(c *gin.Context) {
a2r.Call(msg.MsgClient.GetActiveUser, m.Client, c)
}
func (m *MessageApi) GetActiveGroup(c *gin.Context) {
a2r.Call(msg.MsgClient.GetActiveGroup, m.Client, c)
}
+15 -12
View File
@@ -26,6 +26,7 @@ func NewGinRouter(discov discoveryregistry.SvcDiscoveryRegistry, rdb redis.Unive
log.ZInfo(context.Background(), "load config", "config", config.Config)
r.Use(gin.Recovery(), mw.CorsHandler(), mw.GinParseOperationID())
u := NewUserApi(discov)
m := NewMessageApi(discov)
if config.Config.Prometheus.Enable {
prome.NewApiRequestCounter()
prome.NewApiRequestFailedCounter()
@@ -44,6 +45,7 @@ func NewGinRouter(discov discoveryregistry.SvcDiscoveryRegistry, rdb redis.Unive
userRouterGroup.POST("/account_check", ParseToken, u.AccountCheck)
userRouterGroup.POST("/get_users", ParseToken, u.GetUsers)
userRouterGroup.POST("/get_users_online_status", ParseToken, u.GetUsersOnlineStatus)
userRouterGroup.POST("/get_users_online_token_detail", ParseToken, u.GetUsersOnlineTokenDetail)
}
//friend routing group
friendRouterGroup := r.Group("/friend", ParseToken)
@@ -96,7 +98,6 @@ func NewGinRouter(discov discoveryregistry.SvcDiscoveryRegistry, rdb redis.Unive
authRouterGroup := r.Group("/auth")
{
a := NewAuthApi(discov)
authRouterGroup.POST("/user_register", u.UserRegister)
authRouterGroup.POST("/user_token", a.UserToken)
authRouterGroup.POST("/parse_token", a.ParseToken)
authRouterGroup.POST("/force_logout", ParseToken, a.ForceLogout)
@@ -108,17 +109,19 @@ func NewGinRouter(discov discoveryregistry.SvcDiscoveryRegistry, rdb redis.Unive
thirdGroup.POST("/fcm_update_token", t.FcmUpdateToken)
thirdGroup.POST("/set_app_badge", t.SetAppBadge)
thirdGroup.POST("/apply_put", t.ApplyPut)
thirdGroup.POST("/get_put", t.GetPut)
thirdGroup.POST("/confirm_put", t.ConfirmPut)
thirdGroup.POST("/get_hash", t.GetHash)
thirdGroup.POST("/object", t.GetURL)
thirdGroup.GET("/object", t.GetURL)
objectGroup := r.Group("/object", ParseToken)
objectGroup.POST("/part_limit", t.PartLimit)
objectGroup.POST("/part_size", t.PartSize)
objectGroup.POST("/initiate_multipart_upload", t.InitiateMultipartUpload)
objectGroup.POST("/auth_sign", t.AuthSign)
objectGroup.POST("/complete_multipart_upload", t.CompleteMultipartUpload)
objectGroup.POST("/access_url", t.AccessURL)
objectGroup.GET("/*name", t.ObjectRedirect)
}
//Message
msgGroup := r.Group("/msg", ParseToken)
{
m := NewMessageApi(discov)
msgGroup.POST("/newest_seq", m.GetSeq)
msgGroup.POST("/send_msg", m.SendMessage)
msgGroup.POST("/pull_msg_by_seq", m.PullMsgBySeqs)
@@ -144,15 +147,15 @@ func NewGinRouter(discov discoveryregistry.SvcDiscoveryRegistry, rdb redis.Unive
conversationGroup.POST("/get_all_conversations", c.GetAllConversations)
conversationGroup.POST("/get_conversation", c.GetConversation)
conversationGroup.POST("/get_conversations", c.GetConversations)
conversationGroup.POST("/batch_set_conversation", c.BatchSetConversations)
conversationGroup.POST("/set_recv_msg_opt", c.SetRecvMsgOpt)
conversationGroup.POST("/modify_conversation_field", c.ModifyConversationField)
conversationGroup.POST("/set_conversations", c.SetConversations)
}
statisticsGroup := r.Group("/statistics", ParseToken)
{
statisticsGroup.POST("/user_register", u.UserRegisterCount)
statisticsGroup.POST("/user/register", u.UserRegisterCount)
statisticsGroup.POST("/user/active", m.GetActiveUser)
statisticsGroup.POST("/group/create", g.GroupCreateCount)
statisticsGroup.POST("/group/active", m.GetActiveGroup)
}
return r
}
+37 -35
View File
@@ -1,18 +1,16 @@
package api
import (
"math/rand"
"net/http"
"strconv"
"github.com/OpenIMSDK/Open-IM-Server/pkg/a2r"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/mcontext"
"github.com/OpenIMSDK/Open-IM-Server/pkg/discoveryregistry"
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/third"
"github.com/OpenIMSDK/Open-IM-Server/pkg/rpcclient"
"github.com/gin-gonic/gin"
"math/rand"
"net/http"
"strconv"
)
type ThirdApi rpcclient.Third
@@ -21,22 +19,6 @@ func NewThirdApi(discov discoveryregistry.SvcDiscoveryRegistry) ThirdApi {
return ThirdApi(*rpcclient.NewThird(discov))
}
func (o *ThirdApi) ApplyPut(c *gin.Context) {
a2r.Call(third.ThirdClient.ApplyPut, o.Client, c)
}
func (o *ThirdApi) GetPut(c *gin.Context) {
a2r.Call(third.ThirdClient.GetPut, o.Client, c)
}
func (o *ThirdApi) ConfirmPut(c *gin.Context) {
a2r.Call(third.ThirdClient.ConfirmPut, o.Client, c)
}
func (o *ThirdApi) GetHash(c *gin.Context) {
a2r.Call(third.ThirdClient.GetHashInfo, o.Client, c)
}
func (o *ThirdApi) FcmUpdateToken(c *gin.Context) {
a2r.Call(third.ThirdClient.FcmUpdateToken, o.Client, c)
}
@@ -45,27 +27,47 @@ func (o *ThirdApi) SetAppBadge(c *gin.Context) {
a2r.Call(third.ThirdClient.SetAppBadge, o.Client, c)
}
func (o *ThirdApi) GetURL(c *gin.Context) {
if c.Request.Method == http.MethodPost {
a2r.Call(third.ThirdClient.GetUrl, o.Client, c)
return
}
name := c.Query("name")
// #################### s3 ####################
func (o *ThirdApi) PartLimit(c *gin.Context) {
a2r.Call(third.ThirdClient.PartLimit, o.Client, c)
}
func (o *ThirdApi) PartSize(c *gin.Context) {
a2r.Call(third.ThirdClient.PartSize, o.Client, c)
}
func (o *ThirdApi) InitiateMultipartUpload(c *gin.Context) {
a2r.Call(third.ThirdClient.InitiateMultipartUpload, o.Client, c)
}
func (o *ThirdApi) AuthSign(c *gin.Context) {
a2r.Call(third.ThirdClient.AuthSign, o.Client, c)
}
func (o *ThirdApi) CompleteMultipartUpload(c *gin.Context) {
a2r.Call(third.ThirdClient.CompleteMultipartUpload, o.Client, c)
}
func (o *ThirdApi) AccessURL(c *gin.Context) {
a2r.Call(third.ThirdClient.AccessURL, o.Client, c)
}
func (o *ThirdApi) ObjectRedirect(c *gin.Context) {
name := c.Param("name")
if name == "" {
c.String(http.StatusBadRequest, "name is empty")
return
}
if name[0] == '/' {
name = name[1:]
}
operationID := c.Query("operationID")
if operationID == "" {
operationID = "auto_" + strconv.Itoa(rand.Int())
operationID = strconv.Itoa(rand.Int())
}
expires, _ := strconv.ParseInt(c.Query("expires"), 10, 64)
if expires <= 0 {
expires = 3600 * 1000
}
attachment, _ := strconv.ParseBool(c.Query("attachment"))
c.Set(constant.OperationID, operationID)
resp, err := o.Client.GetUrl(mcontext.SetOperationID(c, operationID), &third.GetUrlReq{Name: name, Expires: expires, Attachment: attachment})
ctx := mcontext.SetOperationID(c, operationID)
resp, err := o.Client.AccessURL(ctx, &third.AccessURLReq{Name: name})
if err != nil {
if errs.ErrArgs.Is(err) {
c.String(http.StatusBadRequest, err.Error())
+109 -6
View File
@@ -3,10 +3,12 @@ package api
import (
"github.com/OpenIMSDK/Open-IM-Server/pkg/a2r"
"github.com/OpenIMSDK/Open-IM-Server/pkg/apiresp"
"github.com/OpenIMSDK/Open-IM-Server/pkg/apistruct"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/tokenverify"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
"github.com/OpenIMSDK/Open-IM-Server/pkg/discoveryregistry"
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/msggateway"
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/user"
"github.com/OpenIMSDK/Open-IM-Server/pkg/rpcclient"
"github.com/gin-gonic/gin"
@@ -47,17 +49,118 @@ func (u *UserApi) GetUsers(c *gin.Context) {
}
func (u *UserApi) GetUsersOnlineStatus(c *gin.Context) {
params := apistruct.ManagementSendMsgReq{}
if err := c.BindJSON(&params); err != nil {
var req msggateway.GetUsersOnlineStatusReq
if err := c.BindJSON(&req); err != nil {
apiresp.GinError(c, errs.ErrArgs.WithDetail(err.Error()).Wrap())
return
}
if !tokenverify.IsAppManagerUid(c) {
apiresp.GinError(c, errs.ErrNoPermission.Wrap("only app manager can send message"))
conns, err := u.Discov.GetConns(c, config.Config.RpcRegisterName.OpenImMessageGatewayName)
if err != nil {
apiresp.GinError(c, err)
return
}
var wsResult []*msggateway.GetUsersOnlineStatusResp_SuccessResult
var respResult []*msggateway.GetUsersOnlineStatusResp_SuccessResult
flag := false
//Online push message
for _, v := range conns {
msgClient := msggateway.NewMsgGatewayClient(v)
reply, err := msgClient.GetUsersOnlineStatus(c, &req)
if err != nil {
log.ZWarn(c, "GetUsersOnlineStatus rpc err", err)
continue
} else {
wsResult = append(wsResult, reply.SuccessResult...)
}
}
// 遍历 api 请求体中的 userIDs
for _, v1 := range req.UserIDs {
flag = false
res := new(msggateway.GetUsersOnlineStatusResp_SuccessResult)
// 遍历从各个网关中获取的在线结果
for _, v2 := range wsResult {
// 如果匹配上说明在线,反之
if v2.UserID == v1 {
flag = true
res.UserID = v1
res.Status = constant.OnlineStatus
res.DetailPlatformStatus = append(res.DetailPlatformStatus, v2.DetailPlatformStatus...)
break
}
}
if !flag {
res.UserID = v1
res.Status = constant.OnlineStatus
}
respResult = append(respResult, res)
}
apiresp.GinSuccess(c, respResult)
}
func (u *UserApi) UserRegisterCount(c *gin.Context) {
a2r.Call(user.UserClient.UserRegisterCount, u.Client, c)
}
func (u *UserApi) GetUsersOnlineTokenDetail(c *gin.Context) {
var wsResult []*msggateway.GetUsersOnlineStatusResp_SuccessResult
var respResult []*msggateway.SingleDetail
flag := false
var req msggateway.GetUsersOnlineStatusReq
if err := c.BindJSON(&req); err != nil {
apiresp.GinError(c, errs.ErrArgs.WithDetail(err.Error()).Wrap())
return
}
conns, err := u.Discov.GetConns(c, config.Config.RpcRegisterName.OpenImMessageGatewayName)
if err != nil {
apiresp.GinError(c, err)
return
}
//Online push message
for _, v := range conns {
msgClient := msggateway.NewMsgGatewayClient(v)
reply, err := msgClient.GetUsersOnlineStatus(c, &req)
if err != nil {
log.ZWarn(c, "GetUsersOnlineStatus rpc err", err)
continue
} else {
wsResult = append(wsResult, reply.SuccessResult...)
}
}
for _, v1 := range req.UserIDs {
m := make(map[string][]string, 10)
flag = false
temp := new(msggateway.SingleDetail)
for _, v2 := range wsResult {
if v2.UserID == v1 {
flag = true
temp.UserID = v1
temp.Status = constant.OnlineStatus
for _, status := range v2.DetailPlatformStatus {
if v, ok := m[status.Platform]; ok {
m[status.Platform] = append(v, status.Token)
} else {
m[status.Platform] = []string{status.Token}
}
}
}
}
for p, tokens := range m {
t := new(msggateway.SinglePlatformToken)
t.Platform = p
t.Token = tokens
t.Total = int32(len(tokens))
temp.SinglePlatformToken = append(temp.SinglePlatformToken, t)
}
if flag {
respResult = append(respResult, temp)
}
}
apiresp.GinSuccess(c, respResult)
}
+3 -1
View File
@@ -53,6 +53,7 @@ type Client struct {
longConnServer LongConnServer
closed bool
closedErr error
token string
}
func newClient(ctx *UserConnContext, conn LongConn, isCompress bool) *Client {
@@ -65,7 +66,7 @@ func newClient(ctx *UserConnContext, conn LongConn, isCompress bool) *Client {
ctx: ctx,
}
}
func (c *Client) ResetClient(ctx *UserConnContext, conn LongConn, isBackground, isCompress bool, longConnServer LongConnServer) {
func (c *Client) ResetClient(ctx *UserConnContext, conn LongConn, isBackground, isCompress bool, longConnServer LongConnServer, token string) {
c.w = new(sync.Mutex)
c.conn = conn
c.PlatformID = utils.StringToInt(ctx.GetPlatformID())
@@ -77,6 +78,7 @@ func (c *Client) ResetClient(ctx *UserConnContext, conn LongConn, isBackground,
c.IsBackground = false
c.closed = false
c.closedErr = nil
c.token = token
}
func (c *Client) pongHandler(_ string) error {
c.conn.SetReadDeadline(pongWait)
+5
View File
@@ -2,6 +2,7 @@ package msggateway
import (
"context"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/cache"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
@@ -70,6 +71,7 @@ func (s *Server) GetUsersOnlineStatus(ctx context.Context, req *msggateway.GetUs
ps.Platform = constant.PlatformIDToName(client.PlatformID)
ps.Status = constant.OnlineStatus
ps.ConnID = client.ctx.GetConnID()
ps.Token = client.token
ps.IsBackground = client.IsBackground
temp.Status = constant.OnlineStatus
temp.DetailPlatformStatus = append(temp.DetailPlatformStatus, ps)
@@ -138,11 +140,14 @@ func (s *Server) KickUserOffline(ctx context.Context, req *msggateway.KickUserOf
for _, v := range req.KickUserIDList {
if clients, _, ok := s.LongConnServer.GetUserPlatformCons(v, int(req.PlatformID)); ok {
for _, client := range clients {
log.ZDebug(ctx, "kick user offline", "userID", v, "platformID", req.PlatformID, "client", client)
err := client.KickOnlineMessage()
if err != nil {
return nil, err
}
}
} else {
log.ZWarn(ctx, "conn not exist", nil, "userID", v, "platformID", req.PlatformID)
}
}
return &msggateway.KickUserOfflineResp{}, nil
+1 -1
View File
@@ -312,7 +312,7 @@ func (ws *WsServer) wsHandler(w http.ResponseWriter, r *http.Request) {
}
}
client := ws.clientPool.Get().(*Client)
client.ResetClient(connContext, wsLongConn, connContext.GetBackground(), compression, ws)
client.ResetClient(connContext, wsLongConn, connContext.GetBackground(), compression, ws, token)
ws.registerChan <- client
go client.readMessage()
}
+5 -9
View File
@@ -10,7 +10,6 @@ import (
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/controller"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/relation"
relationTb "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/tx"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/unrelation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/mw"
@@ -25,7 +24,7 @@ type MsgTransfer struct {
persistentCH *PersistentConsumerHandler // 聊天记录持久化到mysql的消费者 订阅的topic: ws2ms_chat
historyCH *OnlineHistoryRedisConsumerHandler // 这个消费者聚合消息, 订阅的topic:ws2ms_chat, 修改通知发往msg_to_modify topic, 消息存入redis后Incr Redis, 再发消息到ms2pschat topic推送, 发消息到msg_to_mongo topic持久化
historyMongoCH *OnlineHistoryMongoConsumerHandler // mongoDB批量插入, 成功后删除redis中消息,以及处理删除通知消息删除的 订阅的topic: msg_to_mongo
modifyCH *ModifyMsgConsumerHandler // 负责消费修改消息通知的consumer, 订阅的topic: msg_to_modify
// modifyCH *ModifyMsgConsumerHandler // 负责消费修改消息通知的consumer, 订阅的topic: msg_to_modify
}
func StartTransfer(prometheusPort int) error {
@@ -59,23 +58,20 @@ func StartTransfer(prometheusPort int) error {
client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()))
msgModel := cache.NewMsgCacheModel(rdb)
msgDocModel := unrelation.NewMsgMongoDriver(mongo.GetDatabase())
extendMsgModel := unrelation.NewExtendMsgSetMongoDriver(mongo.GetDatabase())
extendMsgCache := cache.NewExtendMsgSetCacheRedis(rdb, extendMsgModel, cache.GetDefaultOpt())
chatLogDatabase := controller.NewChatLogDatabase(relation.NewChatLogGorm(db))
extendMsgDatabase := controller.NewExtendMsgDatabase(extendMsgModel, extendMsgCache, tx.NewMongo(mongo.GetClient()))
msgDatabase := controller.NewCommonMsgDatabase(msgDocModel, msgModel)
conversationRpcClient := rpcclient.NewConversationRpcClient(client)
groupRpcClient := rpcclient.NewGroupRpcClient(client)
msgTransfer := NewMsgTransfer(chatLogDatabase, extendMsgDatabase, msgDatabase, &conversationRpcClient, &groupRpcClient)
msgTransfer := NewMsgTransfer(chatLogDatabase, msgDatabase, &conversationRpcClient, &groupRpcClient)
msgTransfer.initPrometheus()
return msgTransfer.Start(prometheusPort)
}
func NewMsgTransfer(chatLogDatabase controller.ChatLogDatabase,
extendMsgDatabase controller.ExtendMsgDatabase, msgDatabase controller.CommonMsgDatabase,
msgDatabase controller.CommonMsgDatabase,
conversationRpcClient *rpcclient.ConversationRpcClient, groupRpcClient *rpcclient.GroupRpcClient) *MsgTransfer {
return &MsgTransfer{persistentCH: NewPersistentConsumerHandler(chatLogDatabase), historyCH: NewOnlineHistoryRedisConsumerHandler(msgDatabase, conversationRpcClient, groupRpcClient),
historyMongoCH: NewOnlineHistoryMongoConsumerHandler(msgDatabase), modifyCH: NewModifyMsgConsumerHandler(extendMsgDatabase)}
historyMongoCH: NewOnlineHistoryMongoConsumerHandler(msgDatabase)}
}
func (m *MsgTransfer) initPrometheus() {
@@ -100,7 +96,7 @@ func (m *MsgTransfer) Start(prometheusPort int) error {
}
go m.historyCH.historyConsumerGroup.RegisterHandleAndConsumer(m.historyCH)
go m.historyMongoCH.historyConsumerGroup.RegisterHandleAndConsumer(m.historyMongoCH)
go m.modifyCH.modifyMsgConsumerGroup.RegisterHandleAndConsumer(m.modifyCH)
// go m.modifyCH.modifyMsgConsumerGroup.RegisterHandleAndConsumer(m.modifyCH)
err := prome.StartPrometheusSrv(prometheusPort)
if err != nil {
return err
-113
View File
@@ -1,113 +0,0 @@
package msgtransfer
import (
"context"
"encoding/json"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/controller"
unRelationTb "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/unrelation"
kfk "github.com/OpenIMSDK/Open-IM-Server/pkg/common/kafka"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/mcontext"
pbMsg "github.com/OpenIMSDK/Open-IM-Server/pkg/proto/msg"
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/sdkws"
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
"github.com/Shopify/sarama"
"google.golang.org/protobuf/proto"
)
type ModifyMsgConsumerHandler struct {
modifyMsgConsumerGroup *kfk.MConsumerGroup
extendMsgDatabase controller.ExtendMsgDatabase
extendSetMsgModel unRelationTb.ExtendMsgSetModel
}
func NewModifyMsgConsumerHandler(database controller.ExtendMsgDatabase) *ModifyMsgConsumerHandler {
return &ModifyMsgConsumerHandler{
modifyMsgConsumerGroup: kfk.NewMConsumerGroup(&kfk.MConsumerGroupConfig{KafkaVersion: sarama.V2_0_0_0,
OffsetsInitial: sarama.OffsetNewest, IsReturnErr: false}, []string{config.Config.Kafka.MsgToModify.Topic},
config.Config.Kafka.Addr, config.Config.Kafka.ConsumerGroupID.MsgToModify),
extendMsgDatabase: database,
}
}
func (ModifyMsgConsumerHandler) Setup(_ sarama.ConsumerGroupSession) error { return nil }
func (ModifyMsgConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) error { return nil }
func (mmc *ModifyMsgConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession,
claim sarama.ConsumerGroupClaim) error {
for msg := range claim.Messages() {
ctx := mmc.modifyMsgConsumerGroup.GetContextFromMsg(msg)
log.ZDebug(ctx, "kafka get info to mysql", "ModifyMsgConsumerHandler", msg.Topic, "msgPartition", msg.Partition, "msg", string(msg.Value), "key", string(msg.Key))
if len(msg.Value) != 0 {
mmc.ModifyMsg(ctx, msg, string(msg.Key), sess)
} else {
log.ZError(ctx, "msg get from kafka but is nil", nil, "key", msg.Key)
}
sess.MarkMessage(msg, "")
}
return nil
}
func (mmc *ModifyMsgConsumerHandler) ModifyMsg(ctx context.Context, cMsg *sarama.ConsumerMessage, msgKey string, _ sarama.ConsumerGroupSession) {
msgFromMQ := pbMsg.MsgDataToModifyByMQ{}
operationID := mcontext.GetOperationID(ctx)
err := proto.Unmarshal(cMsg.Value, &msgFromMQ)
if err != nil {
log.ZError(ctx, "msg_transfer Unmarshal msg err", err, "msg", string(cMsg.Value))
return
}
log.ZDebug(ctx, "proto.Unmarshal MsgDataToMQ", "msgs", msgFromMQ.String())
for _, msg := range msgFromMQ.Messages {
isReactionFromCache := utils.GetSwitchFromOptions(msg.Options, constant.IsReactionFromCache)
if !isReactionFromCache {
continue
}
ctx = mcontext.SetOperationID(ctx, operationID)
if msg.ContentType == constant.ReactionMessageModifier {
notification := &sdkws.ReactionMessageModifierNotification{}
if err := json.Unmarshal(msg.Content, notification); err != nil {
continue
}
if notification.IsExternalExtensions {
continue
}
if !notification.IsReact {
// first time to modify
var reactionExtensionList = make(map[string]unRelationTb.KeyValueModel)
extendMsg := unRelationTb.ExtendMsgModel{
ReactionExtensionList: reactionExtensionList,
ClientMsgID: notification.ClientMsgID,
MsgFirstModifyTime: notification.MsgFirstModifyTime,
}
for _, v := range notification.SuccessReactionExtensions {
reactionExtensionList[v.TypeKey] = unRelationTb.KeyValueModel{
TypeKey: v.TypeKey,
Value: v.Value,
LatestUpdateTime: v.LatestUpdateTime,
}
}
if err := mmc.extendMsgDatabase.InsertExtendMsg(ctx, notification.ConversationID, notification.SessionType, &extendMsg); err != nil {
// log.ZError(ctx, "MsgFirstModify InsertExtendMsg failed", notification.ConversationID, notification.SessionType, extendMsg, err.Error())
continue
}
} else {
if err := mmc.extendMsgDatabase.InsertOrUpdateReactionExtendMsgSet(ctx, notification.ConversationID, notification.SessionType, notification.ClientMsgID, notification.MsgFirstModifyTime, mmc.extendSetMsgModel.Pb2Model(notification.SuccessReactionExtensions)); err != nil {
// log.NewError(operationID, "InsertOrUpdateReactionExtendMsgSet failed")
}
}
} else if msg.ContentType == constant.ReactionMessageDeleter {
notification := &sdkws.ReactionMessageDeleteNotification{}
if err := json.Unmarshal(msg.Content, notification); err != nil {
continue
}
if err := mmc.extendMsgDatabase.DeleteReactionExtendMsgSet(ctx, notification.ConversationID, notification.SessionType, notification.ClientMsgID, notification.MsgFirstModifyTime, mmc.extendSetMsgModel.Pb2Model(notification.SuccessReactionExtensions)); err != nil {
// log.NewError(operationID, "InsertOrUpdateReactionExtendMsgSet failed")
}
}
}
}
-1
View File
@@ -244,7 +244,6 @@ func (p *Pusher) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.MsgData,
for _, v := range conns {
msgClient := msggateway.NewMsgGatewayClient(v)
reply, err := msgClient.SuperGroupOnlineBatchPushOneMsg(ctx, &msggateway.OnlineBatchPushOneMsgReq{MsgData: msg, PushToUserIDs: pushToUserIDs})
p.discov.CloseConn(v)
if err != nil {
continue
}
+3 -5
View File
@@ -41,7 +41,7 @@ func Start(client discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) e
func (s *authServer) UserToken(ctx context.Context, req *pbAuth.UserTokenReq) (*pbAuth.UserTokenResp, error) {
resp := pbAuth.UserTokenResp{}
if req.Secret != config.Config.Secret {
return nil, errs.ErrIdentity.Wrap("secret invalid")
return nil, errs.ErrNoPermission.Wrap("secret invalid")
}
if _, err := s.userRpcClient.GetUserInfo(ctx, req.UserID); err != nil {
return nil, err
@@ -93,14 +93,13 @@ func (s *authServer) ParseToken(ctx context.Context, req *pbAuth.ParseTokenReq)
}
func (s *authServer) ForceLogout(ctx context.Context, req *pbAuth.ForceLogoutReq) (*pbAuth.ForceLogoutResp, error) {
resp := pbAuth.ForceLogoutResp{}
if err := tokenverify.CheckAdmin(ctx); err != nil {
return nil, err
}
if err := s.forceKickOff(ctx, req.UserID, req.PlatformID, mcontext.GetOperationID(ctx)); err != nil {
return nil, err
}
return &resp, nil
return &pbAuth.ForceLogoutResp{}, nil
}
func (s *authServer) forceKickOff(ctx context.Context, userID string, platformID int32, operationID string) error {
@@ -112,8 +111,7 @@ func (s *authServer) forceKickOff(ctx context.Context, userID string, platformID
client := msggateway.NewMsgGatewayClient(v)
kickReq := &msggateway.KickUserOfflineReq{KickUserIDList: []string{userID}, PlatformID: platformID}
_, err := client.KickUserOffline(ctx, kickReq)
s.RegisterCenter.CloseConn(v)
return utils.Wrap(err, "")
}
return errs.ErrInternalServer.Wrap()
return nil
}
+17 -70
View File
@@ -82,16 +82,6 @@ func (c *conversationServer) GetConversations(ctx context.Context, req *pbConver
return resp, nil
}
func (c *conversationServer) BatchSetConversations(ctx context.Context, req *pbConversation.BatchSetConversationsReq) (*pbConversation.BatchSetConversationsResp, error) {
conversations := convert.ConversationsPb2DB(req.Conversations)
err := c.conversationDatabase.SetUserConversations(ctx, req.OwnerUserID, conversations)
if err != nil {
return nil, err
}
_ = c.conversationNotificationSender.ConversationChangeNotification(ctx, req.OwnerUserID)
return &pbConversation.BatchSetConversationsResp{}, nil
}
func (c *conversationServer) SetConversation(ctx context.Context, req *pbConversation.SetConversationReq) (*pbConversation.SetConversationResp, error) {
var conversation tableRelation.ConversationModel
if err := utils.CopyStructFields(&conversation, req.Conversation); err != nil {
@@ -106,61 +96,6 @@ func (c *conversationServer) SetConversation(ctx context.Context, req *pbConvers
return resp, nil
}
func (c *conversationServer) SetRecvMsgOpt(ctx context.Context, req *pbConversation.SetRecvMsgOptReq) (*pbConversation.SetRecvMsgOptResp, error) {
if err := c.conversationDatabase.SetUsersConversationFiledTx(ctx, []string{req.OwnerUserID}, &tableRelation.ConversationModel{OwnerUserID: req.OwnerUserID, ConversationID: req.ConversationID, RecvMsgOpt: req.RecvMsgOpt}, map[string]interface{}{"recv_msg_opt": req.RecvMsgOpt}); err != nil {
return nil, err
}
_ = c.conversationNotificationSender.ConversationChangeNotification(ctx, req.OwnerUserID)
return &pbConversation.SetRecvMsgOptResp{}, nil
}
// deprecated
func (c *conversationServer) ModifyConversationField(ctx context.Context, req *pbConversation.ModifyConversationFieldReq) (*pbConversation.ModifyConversationFieldResp, error) {
resp := &pbConversation.ModifyConversationFieldResp{}
var err error
if req.Conversation.ConversationType == constant.GroupChatType {
groupInfo, err := c.groupRpcClient.GetGroupInfo(ctx, req.Conversation.GroupID)
if err != nil {
return nil, err
}
if groupInfo.Status == constant.GroupStatusDismissed && req.FieldType != constant.FieldUnread {
return nil, err
}
}
conversation := convert.ConversationPb2DB(req.Conversation)
if req.FieldType == constant.FieldIsPrivateChat {
err := c.conversationDatabase.SyncPeerUserPrivateConversationTx(ctx, []*tableRelation.ConversationModel{conversation})
if err != nil {
return nil, err
}
c.conversationNotificationSender.ConversationSetPrivateNotification(ctx, req.Conversation.OwnerUserID, req.Conversation.UserID, req.Conversation.IsPrivateChat)
return resp, nil
}
filedMap := make(map[string]interface{})
switch req.FieldType {
case constant.FieldRecvMsgOpt:
filedMap["recv_msg_opt"] = req.Conversation.RecvMsgOpt
case constant.FieldGroupAtType:
filedMap["group_at_type"] = req.Conversation.GroupAtType
case constant.FieldIsPinned:
filedMap["is_pinned"] = req.Conversation.IsPinned
case constant.FieldEx:
filedMap["ex"] = req.Conversation.Ex
case constant.FieldAttachedInfo:
filedMap["attached_info"] = req.Conversation.AttachedInfo
case constant.FieldBurnDuration:
filedMap["burn_duration"] = req.Conversation.BurnDuration
}
err = c.conversationDatabase.SetUsersConversationFiledTx(ctx, req.UserIDList, conversation, filedMap)
if err != nil {
return nil, err
}
for _, v := range req.UserIDList {
c.conversationNotificationSender.ConversationChangeNotification(ctx, v)
}
return resp, nil
}
func (c *conversationServer) SetConversations(ctx context.Context, req *pbConversation.SetConversationsReq) (*pbConversation.SetConversationsResp, error) {
if req.Conversation == nil {
return nil, errs.ErrArgs.Wrap("conversation must not be nil")
@@ -173,6 +108,12 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbConver
if groupInfo.Status == constant.GroupStatusDismissed {
return nil, err
}
// for _, userID := range req.UserIDs {
// if _, err := c.groupRpcClient.GetGroupMemberCache(ctx, req.Conversation.GroupID, userID); err != nil {
// log.ZError(ctx, "user not in group", err, "userID", userID, "groupID", req.Conversation.GroupID)
// return nil, err
// }
// }
}
var conversation tableRelation.ConversationModel
conversation.ConversationID = req.Conversation.ConversationID
@@ -195,19 +136,25 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbConver
if req.Conversation.GroupAtType != nil {
m["group_at_type"] = req.Conversation.GroupAtType.Value
}
if req.Conversation.IsPrivateChat != nil {
if req.Conversation.MsgDestructTime != nil {
m["msg_destruct_time"] = req.Conversation.MsgDestructTime.Value
}
if req.Conversation.IsMsgDestruct != nil {
m["is_msg_destruct"] = req.Conversation.IsMsgDestruct.Value
}
if req.Conversation.IsPrivateChat != nil && req.Conversation.ConversationType != constant.SuperGroupChatType {
var conversations []*tableRelation.ConversationModel
for _, ownerUserID := range req.UserIDs {
conversation2 := conversation
conversation.OwnerUserID = ownerUserID
conversation.IsPrivateChat = req.Conversation.IsPrivateChat.Value
conversation2.OwnerUserID = ownerUserID
conversation2.IsPrivateChat = req.Conversation.IsPrivateChat.Value
conversations = append(conversations, &conversation2)
}
if err := c.conversationDatabase.SyncPeerUserPrivateConversationTx(ctx, conversations); err != nil {
return nil, err
}
for _, ownerUserID := range req.UserIDs {
c.conversationNotificationSender.ConversationSetPrivateNotification(ctx, ownerUserID, req.Conversation.UserID, req.Conversation.IsPrivateChat.Value)
for _, userID := range req.UserIDs {
c.conversationNotificationSender.ConversationSetPrivateNotification(ctx, userID, req.Conversation.UserID, req.Conversation.IsPrivateChat.Value)
}
}
if req.Conversation.BurnDuration != nil {
+6 -2
View File
@@ -1,19 +1,23 @@
package group
import (
"context"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/mcontext"
"time"
pbGroup "github.com/OpenIMSDK/Open-IM-Server/pkg/proto/group"
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/sdkws"
)
func UpdateGroupInfoMap(group *sdkws.GroupInfoForSet) map[string]any {
func UpdateGroupInfoMap(ctx context.Context, group *sdkws.GroupInfoForSet) map[string]any {
m := make(map[string]any)
if group.GroupName != "" {
m["name"] = group.GroupName
}
if group.Notification != "" {
m["Notification"] = group.Notification
m["notification"] = group.Notification
m["notification_update_time"] = time.Now()
m["notification_user_id"] = mcontext.GetOpUserID(ctx)
}
if group.Introduction != "" {
m["introduction"] = group.Introduction
+21 -2
View File
@@ -3,6 +3,8 @@ package group
import (
"context"
"fmt"
pbConversation "github.com/OpenIMSDK/Open-IM-Server/pkg/proto/conversation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/wrapperspb"
"math/big"
"math/rand"
"strconv"
@@ -55,7 +57,7 @@ func Start(client discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) e
pbGroup.RegisterGroupServer(server, &groupServer{
GroupDatabase: database,
User: userRpcClient,
Notification: notification.NewGroupNotificationSender(database, &msgRpcClient, func(ctx context.Context, userIDs []string) ([]notification.CommonUser, error) {
Notification: notification.NewGroupNotificationSender(database, &msgRpcClient, &userRpcClient, func(ctx context.Context, userIDs []string) ([]notification.CommonUser, error) {
users, err := userRpcClient.GetUsersInfo(ctx, userIDs)
if err != nil {
return nil, err
@@ -844,7 +846,7 @@ func (s *groupServer) SetGroupInfo(ctx context.Context, req *pbGroup.SetGroupInf
if err != nil {
return nil, err
}
data := UpdateGroupInfoMap(req.GroupInfoForSet)
data := UpdateGroupInfoMap(ctx, req.GroupInfoForSet)
if len(data) == 0 {
return resp, nil
}
@@ -865,6 +867,23 @@ func (s *groupServer) SetGroupInfo(ctx context.Context, req *pbGroup.SetGroupInf
}
var num int
if req.GroupInfoForSet.Notification != "" {
go func() {
nctx := mcontext.NewCtx("@@@" + mcontext.GetOperationID(ctx))
conversation := &pbConversation.ConversationReq{
ConversationID: utils.GetConversationIDBySessionType(constant.SuperGroupChatType, req.GroupInfoForSet.GroupID),
ConversationType: constant.SuperGroupChatType,
GroupID: req.GroupInfoForSet.GroupID,
}
resp, err := s.GetGroupMemberUserIDs(nctx, &pbGroup.GetGroupMemberUserIDsReq{GroupID: req.GroupInfoForSet.GroupID})
if err != nil {
log.ZWarn(ctx, "GetGroupMemberIDs", err)
return
}
conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.GroupNotification}
if err := s.conversationRpcClient.SetConversations(nctx, resp.UserIDs, conversation); err != nil {
log.ZWarn(ctx, "SetConversations", err, resp.UserIDs, conversation)
}
}()
num++
s.Notification.GroupInfoSetAnnouncementNotification(ctx, &sdkws.GroupInfoSetAnnouncementTips{Group: tips.Group, OpUser: tips.OpUser})
+28
View File
@@ -0,0 +1,28 @@
package group
import (
"context"
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/group"
"time"
)
func (s *groupServer) GroupCreateCount(ctx context.Context, req *group.GroupCreateCountReq) (*group.GroupCreateCountResp, error) {
if req.Start > req.End {
return nil, errs.ErrArgs.Wrap("start > end")
}
total, err := s.GroupDatabase.CountTotal(ctx, nil)
if err != nil {
return nil, err
}
start := time.UnixMilli(req.Start)
before, err := s.GroupDatabase.CountTotal(ctx, &start)
if err != nil {
return nil, err
}
count, err := s.GroupDatabase.CountRangeEverydayTotal(ctx, start, time.UnixMilli(req.End))
if err != nil {
return nil, err
}
return &group.GroupCreateCountResp{Total: total, Before: before, Count: count}, nil
}
-361
View File
@@ -1,361 +0,0 @@
package msg
import (
"context"
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/msg"
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/sdkws"
)
func (m *msgServer) SetMessageReactionExtensions(ctx context.Context, req *msg.SetMessageReactionExtensionsReq) (resp *msg.SetMessageReactionExtensionsResp, err error) {
//resp = &msg.SetMessageReactionExtensionsResp{}
////resp.ClientMsgID = req.ClientMsgID
////resp.MsgFirstModifyTime = req.MsgFirstModifyTime
//
//if err := CallbackSetMessageReactionExtensions(ctx, req); err != nil {
// return nil, err
//}
////if ExternalExtension
//if req.IsExternalExtensions {
// resp.MsgFirstModifyTime = req.MsgFirstModifyTime
// notification.ExtendMessageUpdatedNotification(req.OperationID, req.OpUserID, req.conversationID, req.SessionType, req, &resp, !req.IsReact, false)
// return resp, nil
//}
//isExists, err := m.MsgDatabase.JudgeMessageReactionExist(ctx, req.ClientMsgID, req.SessionType)
//if err != nil {
// return nil, err
//}
//
//if !isExists {
// if !req.IsReact {
// resp.MsgFirstModifyTime = utils.GetCurrentTimestampByMill()
// for k, v := range req.ReactionExtensions {
// err := m.MessageLocker.LockMessageTypeKey(ctx, req.ClientMsgID, k)
// if err != nil {
// return nil, err
// }
// v.LatestUpdateTime = utils.GetCurrentTimestampByMill()
// if err := m.MsgDatabase.SetMessageTypeKeyValue(ctx, req.ClientMsgID, req.SessionType, k, utils.StructToJsonString(v)); err != nil {
// return nil, err
// }
// }
// resp.IsReact = true
// _, err := m.MsgDatabase.SetMessageReactionExpire(ctx, req.ClientMsgID, req.SessionType, time.Duration(24*3)*time.Hour)
// if err != nil {
// return nil, err
// }
// } else {
// err := m.MessageLocker.LockGlobalMessage(ctx, req.ClientMsgID)
// if err != nil {
// return nil, err
// }
// mongoValue, err := m.MsgDatabase.GetExtendMsg(ctx, req.conversationID, req.SessionType, req.ClientMsgID, req.MsgFirstModifyTime)
// if err != nil {
// return nil, err
// }
// setValue := make(map[string]*sdkws.KeyValue)
// for k, v := range req.ReactionExtensions {
//
// temp := new(sdkws.KeyValue)
// if vv, ok := mongoValue.ReactionExtensions[k]; ok {
// utils.CopyStructFields(temp, &vv)
// if v.LatestUpdateTime != vv.LatestUpdateTime {
// setKeyResultInfo(&resp, 300, "message have update", req.ClientMsgID, k, temp)
// continue
// }
// }
// temp.TypeKey = k
// temp.Value = v.Value
// temp.LatestUpdateTime = utils.GetCurrentTimestampByMill()
// setValue[k] = temp
// }
// err = db.DB.InsertOrUpdateReactionExtendMsgSet(req.conversationID, req.SessionType, req.ClientMsgID, req.MsgFirstModifyTime, setValue)
// if err != nil {
// for _, value := range setValue {
// temp := new(msg.KeyValueResp)
// temp.KeyValue = value
// temp.ErrMsg = err.Error()
// temp.ErrCode = 100
// resp.Result = append(resp.Result, temp)
// }
// } else {
// for _, value := range setValue {
// temp := new(msg.KeyValueResp)
// temp.KeyValue = value
// resp.Result = append(resp.Result, temp)
// }
// }
// lockErr := m.dMessageLocker.UnLockGlobalMessage(req.ClientMsgID)
// if lockErr != nil {
// log.Error(req.OperationID, "UnLockGlobalMessage err:", lockErr.Error())
// }
// }
//
//} else {
// log.Debug(req.OperationID, "redis handle secondly", req.String())
//
// for k, v := range req.Pb2Model {
// err := m.dMessageLocker.LockMessageTypeKey(req.ClientMsgID, k)
// if err != nil {
// setKeyResultInfo(&resp, 100, err.Error(), req.ClientMsgID, k, v)
// continue
// }
// redisValue, err := db.DB.GetMessageTypeKeyValue(req.ClientMsgID, req.SessionType, k)
// if err != nil && err != go_redis.Nil {
// setKeyResultInfo(&resp, 200, err.Error(), req.ClientMsgID, k, v)
// continue
// }
// temp := new(sdkws.KeyValue)
// utils.JsonStringToStruct(redisValue, temp)
// if v.LatestUpdateTime != temp.LatestUpdateTime {
// setKeyResultInfo(&resp, 300, "message have update", req.ClientMsgID, k, temp)
// continue
// } else {
// v.LatestUpdateTime = utils.GetCurrentTimestampByMill()
// newerr := db.DB.SetMessageTypeKeyValue(req.ClientMsgID, req.SessionType, k, utils.StructToJsonString(v))
// if newerr != nil {
// setKeyResultInfo(&resp, 201, newerr.Error(), req.ClientMsgID, k, temp)
// continue
// }
// setKeyResultInfo(&resp, 0, "", req.ClientMsgID, k, v)
// }
//
// }
//}
//if !isExists {
// if !req.IsReact {
// notification.ExtendMessageUpdatedNotification(req.OperationID, req.OpUserID, req.conversationID, req.SessionType, req, &resp, true, true)
// } else {
// notification.ExtendMessageUpdatedNotification(req.OperationID, req.OpUserID, req.conversationID, req.SessionType, req, &resp, false, false)
// }
//} else {
// notification.ExtendMessageUpdatedNotification(req.OperationID, req.OpUserID, req.conversationID, req.SessionType, req, &resp, false, true)
//}
//log.Debug(req.OperationID, utils.GetSelfFuncName(), "m return is:", resp.String())
return resp, nil
}
func (m *msgServer) setKeyResultInfo(ctx context.Context, r *msg.SetMessageReactionExtensionsResp, errCode int32, errMsg, clientMsgID, typeKey string, keyValue *sdkws.KeyValue) {
temp := new(msg.KeyValueResp)
temp.KeyValue = keyValue
temp.ErrCode = errCode
temp.ErrMsg = errMsg
r.Result = append(r.Result, temp)
_ = m.MessageLocker.UnLockMessageTypeKey(ctx, clientMsgID, typeKey)
}
func (m *msgServer) setDeleteKeyResultInfo(ctx context.Context, r *msg.DeleteMessagesReactionExtensionsResp, errCode int32, errMsg, clientMsgID, typeKey string, keyValue *sdkws.KeyValue) {
temp := new(msg.KeyValueResp)
temp.KeyValue = keyValue
temp.ErrCode = errCode
temp.ErrMsg = errMsg
r.Result = append(r.Result, temp)
_ = m.MessageLocker.UnLockMessageTypeKey(ctx, clientMsgID, typeKey)
}
func (m *msgServer) GetMessagesReactionExtensions(ctx context.Context, req *msg.GetMessagesReactionExtensionsReq) (resp *msg.GetMessagesReactionExtensionsResp, err error) {
//log.Debug(req.OperationID, utils.GetSelfFuncName(), "m args is:", req.String())
//var rResp msg.GetMessageListReactionExtensionsResp
//for _, messageValue := range req.MessageReactionKeyList {
// var oneMessage msg.SingleMessageExtensionResult
// oneMessage.ClientMsgID = messageValue.ClientMsgID
//
// isExists, err := db.DB.JudgeMessageReactionExist(messageValue.ClientMsgID, req.SessionType)
// if err != nil {
// rResp.ErrCode = 100
// rResp.ErrMsg = err.Error()
// return &rResp, nil
// }
// if isExists {
// redisValue, err := db.DB.GetOneMessageAllReactionList(messageValue.ClientMsgID, req.SessionType)
// if err != nil {
// oneMessage.ErrCode = 100
// oneMessage.ErrMsg = err.Error()
// rResp.SingleMessageResult = append(rResp.SingleMessageResult, &oneMessage)
// continue
// }
// keyMap := make(map[string]*sdkws.KeyValue)
//
// for k, v := range redisValue {
// temp := new(sdkws.KeyValue)
// utils.JsonStringToStruct(v, temp)
// keyMap[k] = temp
// }
// oneMessage.Pb2Model = keyMap
//
// } else {
// mongoValue, err := db.DB.GetExtendMsg(req.conversationID, req.SessionType, messageValue.ClientMsgID, messageValue.MsgFirstModifyTime)
// if err != nil {
// oneMessage.ErrCode = 100
// oneMessage.ErrMsg = err.Error()
// rResp.SingleMessageResult = append(rResp.SingleMessageResult, &oneMessage)
// continue
// }
// keyMap := make(map[string]*sdkws.KeyValue)
//
// for k, v := range mongoValue.Pb2Model {
// temp := new(sdkws.KeyValue)
// temp.TypeKey = v.TypeKey
// temp.Value = v.Value
// temp.LatestUpdateTime = v.LatestUpdateTime
// keyMap[k] = temp
// }
// oneMessage.Pb2Model = keyMap
// }
// rResp.SingleMessageResult = append(rResp.SingleMessageResult, &oneMessage)
//}
//log.Debug(req.OperationID, utils.GetSelfFuncName(), "m return is:", rResp.String())
return resp, nil
}
func (m *msgServer) AddMessageReactionExtensions(ctx context.Context, req *msg.ModifyMessageReactionExtensionsReq) (resp *msg.ModifyMessageReactionExtensionsResp, err error) {
return
}
func (m *msgServer) DeleteMessageReactionExtensions(ctx context.Context, req *msg.DeleteMessagesReactionExtensionsReq) (resp *msg.DeleteMessagesReactionExtensionsResp, err error) {
//log.Debug(req.OperationID, utils.GetSelfFuncName(), "m args is:", req.String())
//var rResp msg.DeleteMessagesReactionExtensionsResp
//callbackResp := notification.callbackDeleteMessageReactionExtensions(req)
//if callbackResp.ActionCode != constant.ActionAllow || callbackResp.ErrCode != 0 {
// rResp.ErrCode = int32(callbackResp.ErrCode)
// rResp.ErrMsg = callbackResp.ErrMsg
// for _, value := range req.Pb2Model {
// temp := new(msg.KeyValueResp)
// temp.KeyValue = value
// temp.ErrMsg = callbackResp.ErrMsg
// temp.ErrCode = 100
// rResp.Result = append(rResp.Result, temp)
// }
// return &rResp, nil
//}
////if ExternalExtension
//if req.IsExternalExtensions {
// rResp.Result = callbackResp.ResultReactionExtensionList
// notification.ExtendMessageDeleteNotification(req.OperationID, req.OpUserID, req.conversationID, req.SessionType, req, &rResp, false, false)
// return &rResp, nil
//
//}
//for _, v := range callbackResp.ResultReactionExtensions {
// if v.ErrCode != 0 {
// func(req *[]*sdkws.KeyValue, typeKey string) {
// for i := 0; i < len(*req); i++ {
// if (*req)[i].TypeKey == typeKey {
// *req = append((*req)[:i], (*req)[i+1:]...)
// }
// }
// }(&req.Pb2Model, v.KeyValue.TypeKey)
// rResp.Result = append(rResp.Result, v)
// }
//}
//isExists, err := db.DB.JudgeMessageReactionExist(req.ClientMsgID, req.SessionType)
//if err != nil {
// rResp.ErrCode = 100
// rResp.ErrMsg = err.Error()
// for _, value := range req.Pb2Model {
// temp := new(msg.KeyValueResp)
// temp.KeyValue = value
// temp.ErrMsg = err.Error()
// temp.ErrCode = 100
// rResp.Result = append(rResp.Result, temp)
// }
// return &rResp, nil
//}
//
//if isExists {
// log.Debug(req.OperationID, "redis handle this delete", req.String())
// for _, v := range req.Pb2Model {
// err := m.dMessageLocker.LockMessageTypeKey(req.ClientMsgID, v.TypeKey)
// if err != nil {
// setDeleteKeyResultInfo(&rResp, 100, err.Error(), req.ClientMsgID, v.TypeKey, v)
// continue
// }
//
// redisValue, err := db.DB.GetMessageTypeKeyValue(req.ClientMsgID, req.SessionType, v.TypeKey)
// if err != nil && err != go_redis.Nil {
// setDeleteKeyResultInfo(&rResp, 200, err.Error(), req.ClientMsgID, v.TypeKey, v)
// continue
// }
// temp := new(sdkws.KeyValue)
// utils.JsonStringToStruct(redisValue, temp)
// if v.LatestUpdateTime != temp.LatestUpdateTime {
// setDeleteKeyResultInfo(&rResp, 300, "message have update", req.ClientMsgID, v.TypeKey, temp)
// continue
// } else {
// newErr := db.DB.DeleteOneMessageKey(req.ClientMsgID, req.SessionType, v.TypeKey)
// if newErr != nil {
// setDeleteKeyResultInfo(&rResp, 201, newErr.Error(), req.ClientMsgID, v.TypeKey, temp)
// continue
// }
// setDeleteKeyResultInfo(&rResp, 0, "", req.ClientMsgID, v.TypeKey, v)
// }
// }
//} else {
// err := m.dMessageLocker.LockGlobalMessage(req.ClientMsgID)
// if err != nil {
// rResp.ErrCode = 100
// rResp.ErrMsg = err.Error()
// for _, value := range req.Pb2Model {
// temp := new(msg.KeyValueResp)
// temp.KeyValue = value
// temp.ErrMsg = err.Error()
// temp.ErrCode = 100
// rResp.Result = append(rResp.Result, temp)
// }
// return &rResp, nil
// }
// mongoValue, err := db.DB.GetExtendMsg(req.conversationID, req.SessionType, req.ClientMsgID, req.MsgFirstModifyTime)
// if err != nil {
// rResp.ErrCode = 200
// rResp.ErrMsg = err.Error()
// for _, value := range req.Pb2Model {
// temp := new(msg.KeyValueResp)
// temp.KeyValue = value
// temp.ErrMsg = err.Error()
// temp.ErrCode = 100
// rResp.Result = append(rResp.Result, temp)
// }
// return &rResp, nil
// }
// setValue := make(map[string]*sdkws.KeyValue)
// for _, v := range req.Pb2Model {
//
// temp := new(sdkws.KeyValue)
// if vv, ok := mongoValue.Pb2Model[v.TypeKey]; ok {
// utils.CopyStructFields(temp, &vv)
// if v.LatestUpdateTime != vv.LatestUpdateTime {
// setDeleteKeyResultInfo(&rResp, 300, "message have update", req.ClientMsgID, v.TypeKey, temp)
// continue
// }
// } else {
// setDeleteKeyResultInfo(&rResp, 400, "key not in", req.ClientMsgID, v.TypeKey, v)
// continue
// }
// temp.TypeKey = v.TypeKey
// setValue[v.TypeKey] = temp
// }
// err = db.DB.DeleteReactionExtendMsgSet(req.conversationID, req.SessionType, req.ClientMsgID, req.MsgFirstModifyTime, setValue)
// if err != nil {
// for _, value := range setValue {
// temp := new(msg.KeyValueResp)
// temp.KeyValue = value
// temp.ErrMsg = err.Error()
// temp.ErrCode = 100
// rResp.Result = append(rResp.Result, temp)
// }
// } else {
// for _, value := range setValue {
// temp := new(msg.KeyValueResp)
// temp.KeyValue = value
// rResp.Result = append(rResp.Result, temp)
// }
// }
// lockErr := m.dMessageLocker.UnLockGlobalMessage(req.ClientMsgID)
// if lockErr != nil {
// log.Error(req.OperationID, "UnLockGlobalMessage err:", lockErr.Error())
// }
//
//}
//notification.ExtendMessageDeleteNotification(req.OperationID, req.OpUserID, req.conversationID, req.SessionType, req, &rResp, false, isExists)
//log.Debug(req.OperationID, utils.GetSelfFuncName(), "m return is:", rResp.String())
return resp, nil
}
-88
View File
@@ -1,88 +0,0 @@
package msg
import (
"context"
cbapi "github.com/OpenIMSDK/Open-IM-Server/pkg/callbackstruct"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/http"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/mcontext"
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/msg"
)
func callbackSetMessageReactionExtensions(ctx context.Context, setReq *msg.SetMessageReactionExtensionsReq) error {
if !config.Config.Callback.CallbackAfterSendGroupMsg.Enable {
return nil
}
req := &cbapi.CallbackBeforeSetMessageReactionExtReq{
OperationID: mcontext.GetOperationID(ctx),
CallbackCommand: constant.CallbackBeforeSetMessageReactionExtensionCommand,
ConversationID: setReq.ConversationID,
OpUserID: mcontext.GetOpUserID(ctx),
SessionType: setReq.SessionType,
ReactionExtensionList: setReq.ReactionExtensions,
ClientMsgID: setReq.ClientMsgID,
IsReact: setReq.IsReact,
IsExternalExtensions: setReq.IsExternalExtensions,
MsgFirstModifyTime: setReq.MsgFirstModifyTime,
}
resp := &cbapi.CallbackBeforeSetMessageReactionExtResp{}
if err := http.CallBackPostReturn(ctx, cbURL(), req, resp, config.Config.Callback.CallbackAfterSendGroupMsg); err != nil {
return err
}
setReq.MsgFirstModifyTime = resp.MsgFirstModifyTime
return nil
}
func callbackDeleteMessageReactionExtensions(ctx context.Context, setReq *msg.DeleteMessagesReactionExtensionsReq) error {
if !config.Config.Callback.CallbackAfterSendGroupMsg.Enable {
return nil
}
req := &cbapi.CallbackDeleteMessageReactionExtReq{
OperationID: setReq.OperationID,
CallbackCommand: constant.CallbackBeforeDeleteMessageReactionExtensionsCommand,
ConversationID: setReq.ConversationID,
OpUserID: setReq.OpUserID,
SessionType: setReq.SessionType,
ReactionExtensionList: setReq.ReactionExtensions,
ClientMsgID: setReq.ClientMsgID,
IsExternalExtensions: setReq.IsExternalExtensions,
MsgFirstModifyTime: setReq.MsgFirstModifyTime,
}
resp := &cbapi.CallbackDeleteMessageReactionExtResp{}
return http.CallBackPostReturn(ctx, cbURL(), req, resp, config.Config.Callback.CallbackAfterSendGroupMsg)
}
func callbackGetMessageListReactionExtensions(ctx context.Context, getReq *msg.GetMessagesReactionExtensionsReq) error {
if !config.Config.Callback.CallbackAfterSendGroupMsg.Enable {
return nil
}
req := &cbapi.CallbackGetMessageListReactionExtReq{
OperationID: mcontext.GetOperationID(ctx),
CallbackCommand: constant.CallbackGetMessageListReactionExtensionsCommand,
ConversationID: getReq.ConversationID,
OpUserID: mcontext.GetOperationID(ctx),
SessionType: getReq.SessionType,
TypeKeyList: getReq.TypeKeys,
}
resp := &cbapi.CallbackGetMessageListReactionExtResp{}
return http.CallBackPostReturn(ctx, cbURL(), req, resp, config.Config.Callback.CallbackAfterSendGroupMsg)
}
func callbackAddMessageReactionExtensions(ctx context.Context, setReq *msg.ModifyMessageReactionExtensionsReq) error {
req := &cbapi.CallbackAddMessageReactionExtReq{
OperationID: mcontext.GetOperationID(ctx),
CallbackCommand: constant.CallbackAddMessageListReactionExtensionsCommand,
ConversationID: setReq.ConversationID,
OpUserID: mcontext.GetOperationID(ctx),
SessionType: setReq.SessionType,
ReactionExtensionList: setReq.ReactionExtensions,
ClientMsgID: setReq.ClientMsgID,
IsReact: setReq.IsReact,
IsExternalExtensions: setReq.IsExternalExtensions,
MsgFirstModifyTime: setReq.MsgFirstModifyTime,
}
resp := &cbapi.CallbackAddMessageReactionExtResp{}
return http.CallBackPostReturn(ctx, cbURL(), req, resp, config.Config.Callback.CallbackAfterSendGroupMsg)
}
-2
View File
@@ -3,7 +3,6 @@ package msg
import (
"context"
"encoding/json"
"github.com/google/uuid"
"time"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant"
@@ -79,7 +78,6 @@ func (m *msgServer) RevokeMsg(ctx context.Context, req *msg.RevokeMsgReq) (*msg.
}
now := time.Now().UnixMilli()
err = m.MsgDatabase.RevokeMsg(ctx, req.ConversationID, req.Seq, &unRelationTb.RevokeModel{
ID: uuid.New().String(),
Role: role,
UserID: req.UserID,
Nickname: user.Nickname,
+20 -15
View File
@@ -2,6 +2,7 @@ package msg
import (
"context"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/mcontext"
@@ -16,20 +17,24 @@ import (
func (m *msgServer) SendMsg(ctx context.Context, req *pbMsg.SendMsgReq) (resp *pbMsg.SendMsgResp, error error) {
resp = &pbMsg.SendMsgResp{}
flag := isMessageHasReadEnabled(req.MsgData)
if !flag {
return nil, errs.ErrMessageHasReadDisable.Wrap()
}
m.encapsulateMsgData(req.MsgData)
switch req.MsgData.SessionType {
case constant.SingleChatType:
return m.sendMsgSingleChat(ctx, req)
case constant.NotificationChatType:
return m.sendMsgNotification(ctx, req)
case constant.SuperGroupChatType:
return m.sendMsgSuperGroupChat(ctx, req)
default:
return nil, errs.ErrArgs.Wrap("unknown sessionType")
if req.MsgData != nil {
flag := isMessageHasReadEnabled(req.MsgData)
if !flag {
return nil, errs.ErrMessageHasReadDisable.Wrap()
}
m.encapsulateMsgData(req.MsgData)
switch req.MsgData.SessionType {
case constant.SingleChatType:
return m.sendMsgSingleChat(ctx, req)
case constant.NotificationChatType:
return m.sendMsgNotification(ctx, req)
case constant.SuperGroupChatType:
return m.sendMsgSuperGroupChat(ctx, req)
default:
return nil, errs.ErrArgs.Wrap("unknown sessionType")
}
} else {
return nil, errs.ErrArgs.Wrap("msgData is nil")
}
}
@@ -133,7 +138,7 @@ func (m *msgServer) sendMsgSingleChat(ctx context.Context, req *pbMsg.SendMsgReq
}
if !isSend {
promePkg.Inc(promePkg.SingleChatMsgProcessFailedCounter)
return nil, errs.ErrUserNotRecvMsg
return nil, nil
} else {
if err = callbackBeforeSendSingleMsg(ctx, req); err != nil {
return nil, err
-6
View File
@@ -7,7 +7,6 @@ import (
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/cache"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/controller"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/localcache"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/tx"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/unrelation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/prome"
"github.com/OpenIMSDK/Open-IM-Server/pkg/discoveryregistry"
@@ -21,7 +20,6 @@ type MessageInterceptorChain []MessageInterceptorFunc
type msgServer struct {
RegisterCenter discoveryregistry.SvcDiscoveryRegistry
MsgDatabase controller.CommonMsgDatabase
ExtendMsgDatabase controller.ExtendMsgDatabase
Group *rpcclient.GroupRpcClient
User *rpcclient.UserRpcClient
Conversation *rpcclient.ConversationRpcClient
@@ -62,9 +60,6 @@ func Start(client discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) e
}
cacheModel := cache.NewMsgCacheModel(rdb)
msgDocModel := unrelation.NewMsgMongoDriver(mongo.GetDatabase())
extendMsgModel := unrelation.NewExtendMsgSetMongoDriver(mongo.GetDatabase())
extendMsgCacheModel := cache.NewExtendMsgSetCacheRedis(rdb, extendMsgModel, cache.GetDefaultOpt())
extendMsgDatabase := controller.NewExtendMsgDatabase(extendMsgModel, extendMsgCacheModel, tx.NewMongo(mongo.GetClient()))
msgDatabase := controller.NewCommonMsgDatabase(msgDocModel, cacheModel)
conversationClient := rpcclient.NewConversationRpcClient(client)
userRpcClient := rpcclient.NewUserRpcClient(client)
@@ -75,7 +70,6 @@ func Start(client discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) e
User: &userRpcClient,
Group: &groupRpcClient,
MsgDatabase: msgDatabase,
ExtendMsgDatabase: extendMsgDatabase,
RegisterCenter: client,
GroupLocalCache: localcache.NewGroupLocalCache(&groupRpcClient),
ConversationLocalCache: localcache.NewConversationLocalCache(&conversationClient),
+84
View File
@@ -0,0 +1,84 @@
package msg
import (
"context"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/unrelation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/msg"
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/sdkws"
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
"time"
)
func (m *msgServer) GetActiveUser(ctx context.Context, req *msg.GetActiveUserReq) (*msg.GetActiveUserResp, error) {
msgCount, userCount, users, dateCount, err := m.MsgDatabase.RangeUserSendCount(ctx, time.UnixMilli(req.Start), time.UnixMilli(req.End), req.Group, req.Ase, req.Pagination.PageNumber, req.Pagination.ShowNumber)
if err != nil {
return nil, err
}
var pbUsers []*msg.ActiveUser
if len(users) > 0 {
userIDs := utils.Slice(users, func(e *unrelation.UserCount) string { return e.UserID })
userMap, err := m.User.GetUsersInfoMap(ctx, userIDs)
if err != nil {
return nil, err
}
pbUsers = make([]*msg.ActiveUser, 0, len(users))
for _, user := range users {
pbUser := userMap[user.UserID]
if pbUser == nil {
pbUser = &sdkws.UserInfo{
UserID: user.UserID,
Nickname: user.UserID,
}
}
pbUsers = append(pbUsers, &msg.ActiveUser{
User: pbUser,
Count: user.Count,
})
}
}
return &msg.GetActiveUserResp{
MsgCount: msgCount,
UserCount: userCount,
DateCount: dateCount,
Users: pbUsers,
}, nil
}
func (m *msgServer) GetActiveGroup(ctx context.Context, req *msg.GetActiveGroupReq) (*msg.GetActiveGroupResp, error) {
msgCount, groupCount, groups, dateCount, err := m.MsgDatabase.RangeGroupSendCount(ctx, time.UnixMilli(req.Start), time.UnixMilli(req.End), req.Ase, req.Pagination.PageNumber, req.Pagination.ShowNumber)
if err != nil {
return nil, err
}
var pbGroups []*msg.ActiveGroup
if len(groups) > 0 {
groupIDs := utils.Slice(groups, func(e *unrelation.GroupCount) string { return e.GroupID })
resp, err := m.Group.GetGroupInfos(ctx, groupIDs, false)
if err != nil {
return nil, err
}
groupMap := make(map[string]*sdkws.GroupInfo, len(groups))
for i, group := range groups {
groupMap[group.GroupID] = resp[i]
}
pbGroups = make([]*msg.ActiveGroup, 0, len(groups))
for _, group := range groups {
pbGroup := groupMap[group.GroupID]
if pbGroup == nil {
pbGroup = &sdkws.GroupInfo{
GroupID: group.GroupID,
GroupName: group.GroupID,
}
}
pbGroups = append(pbGroups, &msg.ActiveGroup{
Group: pbGroup,
Count: group.Count,
})
}
}
return &msg.GetActiveGroupResp{
MsgCount: msgCount,
GroupCount: groupCount,
DateCount: dateCount,
Groups: pbGroups,
}, nil
}
+132 -18
View File
@@ -2,36 +2,150 @@ package third
import (
"context"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/tokenverify"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3/cont"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/mcontext"
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/third"
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
"time"
)
func (t *thirdServer) ApplyPut(ctx context.Context, req *third.ApplyPutReq) (*third.ApplyPutResp, error) {
return t.s3dataBase.ApplyPut(ctx, req)
func (t *thirdServer) PartLimit(ctx context.Context, req *third.PartLimitReq) (*third.PartLimitResp, error) {
limit := t.s3dataBase.PartLimit()
return &third.PartLimitResp{
MinPartSize: limit.MinPartSize,
MaxPartSize: limit.MaxPartSize,
MaxNumSize: int32(limit.MaxNumSize),
}, nil
}
func (t *thirdServer) GetPut(ctx context.Context, req *third.GetPutReq) (*third.GetPutResp, error) {
return t.s3dataBase.GetPut(ctx, req)
func (t *thirdServer) PartSize(ctx context.Context, req *third.PartSizeReq) (*third.PartSizeResp, error) {
size, err := t.s3dataBase.PartSize(ctx, req.Size)
if err != nil {
return nil, err
}
return &third.PartSizeResp{Size: size}, nil
}
func (t *thirdServer) ConfirmPut(ctx context.Context, req *third.ConfirmPutReq) (*third.ConfirmPutResp, error) {
return t.s3dataBase.ConfirmPut(ctx, req)
}
func (t *thirdServer) GetUrl(ctx context.Context, req *third.GetUrlReq) (*third.GetUrlResp, error) {
if req.Expires <= 0 {
if err := tokenverify.CheckAdmin(ctx); err != nil {
return nil, err
func (t *thirdServer) InitiateMultipartUpload(ctx context.Context, req *third.InitiateMultipartUploadReq) (*third.InitiateMultipartUploadResp, error) {
defer log.ZDebug(ctx, "return")
if err := checkUploadName(ctx, req.Name); err != nil {
return nil, err
}
expireTime := time.Now().Add(t.defaultExpire)
result, err := t.s3dataBase.InitiateMultipartUpload(ctx, req.Hash, req.Size, t.defaultExpire, int(req.MaxParts))
if err != nil {
if haErr, ok := errs.Unwrap(err).(*cont.HashAlreadyExistsError); ok {
obj := &relation.ObjectModel{
Name: req.Name,
UserID: mcontext.GetOpUserID(ctx),
Hash: req.Hash,
Key: haErr.Object.Key,
Size: haErr.Object.Size,
ContentType: req.ContentType,
Cause: req.Cause,
CreateTime: time.Now(),
}
if err := t.s3dataBase.SetObject(ctx, obj); err != nil {
return nil, err
}
return &third.InitiateMultipartUploadResp{
Url: t.apiAddress(obj.Name),
}, nil
}
return nil, err
}
var sign *third.AuthSignParts
if result.Sign != nil && len(result.Sign.Parts) > 0 {
sign = &third.AuthSignParts{
Url: result.Sign.URL,
Query: toPbMapArray(result.Sign.Query),
Header: toPbMapArray(result.Sign.Header),
Parts: make([]*third.SignPart, len(result.Sign.Parts)),
}
for i, part := range result.Sign.Parts {
sign.Parts[i] = &third.SignPart{
PartNumber: int32(part.PartNumber),
Url: part.URL,
Query: toPbMapArray(part.Query),
Header: toPbMapArray(part.Header),
}
}
}
return t.s3dataBase.GetUrl(ctx, req)
return &third.InitiateMultipartUploadResp{
Upload: &third.UploadInfo{
UploadID: result.UploadID,
PartSize: result.PartSize,
Sign: sign,
ExpireTime: expireTime.UnixMilli(),
},
}, nil
}
func (t *thirdServer) GetHashInfo(ctx context.Context, req *third.GetHashInfoReq) (*third.GetHashInfoResp, error) {
return t.s3dataBase.GetHashInfo(ctx, req)
func (t *thirdServer) AuthSign(ctx context.Context, req *third.AuthSignReq) (*third.AuthSignResp, error) {
defer log.ZDebug(ctx, "return")
partNumbers := utils.Slice(req.PartNumbers, func(partNumber int32) int { return int(partNumber) })
result, err := t.s3dataBase.AuthSign(ctx, req.UploadID, partNumbers)
if err != nil {
return nil, err
}
resp := &third.AuthSignResp{
Url: result.URL,
Query: toPbMapArray(result.Query),
Header: toPbMapArray(result.Header),
Parts: make([]*third.SignPart, len(result.Parts)),
}
for i, part := range result.Parts {
resp.Parts[i] = &third.SignPart{
PartNumber: int32(part.PartNumber),
Url: part.URL,
Query: toPbMapArray(part.Query),
Header: toPbMapArray(part.Header),
}
}
return resp, nil
}
func (t *thirdServer) CleanObject(ctx context.Context, now time.Time) {
t.s3dataBase.CleanExpirationObject(ctx, now)
func (t *thirdServer) CompleteMultipartUpload(ctx context.Context, req *third.CompleteMultipartUploadReq) (*third.CompleteMultipartUploadResp, error) {
defer log.ZDebug(ctx, "return")
if err := checkUploadName(ctx, req.Name); err != nil {
return nil, err
}
result, err := t.s3dataBase.CompleteMultipartUpload(ctx, req.UploadID, req.Parts)
if err != nil {
return nil, err
}
obj := &relation.ObjectModel{
Name: req.Name,
UserID: mcontext.GetOpUserID(ctx),
Hash: result.Hash,
Key: result.Key,
Size: result.Size,
ContentType: req.ContentType,
Cause: req.Cause,
CreateTime: time.Now(),
}
if err := t.s3dataBase.SetObject(ctx, obj); err != nil {
return nil, err
}
return &third.CompleteMultipartUploadResp{
Url: t.apiAddress(obj.Name),
}, nil
}
func (t *thirdServer) AccessURL(ctx context.Context, req *third.AccessURLReq) (*third.AccessURLResp, error) {
expireTime, rawURL, err := t.s3dataBase.AccessURL(ctx, req.Name, t.defaultExpire)
if err != nil {
return nil, err
}
return &third.AccessURLResp{
Url: rawURL,
ExpireTime: expireTime.UnixMilli(),
}, nil
}
func (t *thirdServer) apiAddress(name string) string {
return t.apiURL + name
}
+36 -9
View File
@@ -2,12 +2,17 @@ package third
import (
"context"
"fmt"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3/cos"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3/minio"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3/oss"
"net/url"
"time"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/cache"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/controller"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/obj"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/relation"
relationTb "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/discoveryregistry"
@@ -17,37 +22,59 @@ import (
)
func Start(client discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) error {
u, err := url.Parse(config.Config.Object.ApiURL)
if err != nil {
apiURL := config.Config.Object.ApiURL
if apiURL == "" {
return fmt.Errorf("api url is empty")
}
if _, err := url.Parse(config.Config.Object.ApiURL); err != nil {
return err
}
if apiURL[len(apiURL)-1] != '/' {
apiURL += "/"
}
rdb, err := cache.NewRedis()
if err != nil {
return err
}
o, err := obj.NewMinioInterface()
if err != nil {
return err
}
db, err := relation.NewGormDB()
if err != nil {
return err
}
if err := db.AutoMigrate(&relationTb.ObjectHashModel{}, &relationTb.ObjectInfoModel{}, &relationTb.ObjectPutModel{}); err != nil {
if err := db.AutoMigrate(&relationTb.ObjectModel{}); err != nil {
return err
}
// 根据配置文件策略选择 oss 方式
enable := config.Config.Object.Enable
var o s3.Interface
switch config.Config.Object.Enable {
case "minio":
o, err = minio.NewMinio()
case "cos":
o, err = cos.NewCos()
case "oss":
o, err = oss.NewOSS()
default:
err = fmt.Errorf("invalid object enable: %s", enable)
}
if err != nil {
return err
}
third.RegisterThirdServer(server, &thirdServer{
apiURL: apiURL,
thirdDatabase: controller.NewThirdDatabase(cache.NewMsgCacheModel(rdb)),
userRpcClient: rpcclient.NewUserRpcClient(client),
s3dataBase: controller.NewS3Database(o, relation.NewObjectHash(db), relation.NewObjectInfo(db), relation.NewObjectPut(db), u),
s3dataBase: controller.NewS3Database(o, relation.NewObjectInfo(db)),
defaultExpire: time.Hour * 24 * 7,
})
return nil
}
type thirdServer struct {
apiURL string
thirdDatabase controller.ThirdDatabase
s3dataBase controller.S3Database
userRpcClient rpcclient.UserRpcClient
defaultExpire time.Duration
}
func (t *thirdServer) FcmUpdateToken(ctx context.Context, req *third.FcmUpdateTokenReq) (resp *third.FcmUpdateTokenResp, err error) {
+63
View File
@@ -0,0 +1,63 @@
package third
import (
"context"
"errors"
"fmt"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/mcontext"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/tokenverify"
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/third"
"strings"
"unicode/utf8"
)
func toPbMapArray(m map[string][]string) []*third.KeyValues {
res := make([]*third.KeyValues, 0, len(m))
for key := range m {
res = append(res, &third.KeyValues{
Key: key,
Values: m[key],
})
}
return res
}
func checkUploadName(ctx context.Context, name string) error {
if name == "" {
return errs.ErrArgs.Wrap("name is empty")
}
if name[0] == '/' {
return errs.ErrArgs.Wrap("name cannot start with `/`")
}
if err := checkValidObjectName(name); err != nil {
return errs.ErrArgs.Wrap(err.Error())
}
opUserID := mcontext.GetOpUserID(ctx)
if opUserID == "" {
return errs.ErrNoPermission.Wrap("opUserID is empty")
}
if !tokenverify.IsManagerUserID(opUserID) {
if !strings.HasPrefix(name, opUserID+"/") {
return errs.ErrNoPermission.Wrap(fmt.Sprintf("name must start with `%s/`", opUserID))
}
}
return nil
}
func checkValidObjectNamePrefix(objectName string) error {
if len(objectName) > 1024 {
return errors.New("object name cannot be longer than 1024 characters")
}
if !utf8.ValidString(objectName) {
return errors.New("object name with non UTF-8 strings are not supported")
}
return nil
}
func checkValidObjectName(objectName string) error {
if strings.TrimSpace(objectName) == "" {
return errors.New("object name cannot be empty")
}
return checkValidObjectNamePrefix(objectName)
}
+8 -3
View File
@@ -11,13 +11,18 @@ func (s *userServer) UserRegisterCount(ctx context.Context, req *pbuser.UserRegi
if req.Start > req.End {
return nil, errs.ErrArgs.Wrap("start > end")
}
total, err := s.CountTotal(ctx)
total, err := s.CountTotal(ctx, nil)
if err != nil {
return nil, err
}
count, err := s.CountRangeEverydayTotal(ctx, time.UnixMilli(req.Start), time.UnixMilli(req.End))
start := time.UnixMilli(req.Start)
before, err := s.CountTotal(ctx, &start)
if err != nil {
return nil, err
}
return &pbuser.UserRegisterCountResp{Total: total, Count: count}, nil
count, err := s.CountRangeEverydayTotal(ctx, start, time.UnixMilli(req.End))
if err != nil {
return nil, err
}
return &pbuser.UserRegisterCountResp{Total: total, Before: before, Count: count}, nil
}
+3 -2
View File
@@ -2,6 +2,7 @@ package user
import (
"context"
"errors"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
"strings"
"time"
@@ -47,7 +48,7 @@ func Start(client registry.SvcDiscoveryRegistry, server *grpc.Server) error {
}
users := make([]*tablerelation.UserModel, 0)
if len(config.Config.Manager.UserID) != len(config.Config.Manager.Nickname) {
return errs.ErrConfig.Wrap("len(config.Config.Manager.AppManagerUid) != len(config.Config.Manager.Nickname)")
return errors.New("len(config.Config.Manager.AppManagerUid) != len(config.Config.Manager.Nickname)")
}
for k, v := range config.Config.Manager.UserID {
users = append(users, &tablerelation.UserModel{UserID: v, Nickname: config.Config.Manager.Nickname[k]})
@@ -168,7 +169,7 @@ func (s *userServer) UserRegister(ctx context.Context, req *pbuser.UserRegisterR
}
if req.Secret != config.Config.Secret {
log.ZDebug(ctx, "UserRegister", config.Config.Secret, req.Secret)
return nil, errs.ErrIdentity.Wrap("secret invalid")
return nil, errs.ErrNoPermission.Wrap("secret invalid")
}
if utils.DuplicateAny(req.Users, func(e *sdkws.UserInfo) string { return e.UserID }) {
return nil, errs.ErrArgs.Wrap("userID repeated")
+38
View File
@@ -0,0 +1,38 @@
package tools
import (
"context"
"time"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/mcontext"
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
)
func (c *MsgTool) ConversationsDestructMsgs() {
log.ZInfo(context.Background(), "start msg destruct cron task")
ctx := mcontext.NewCtx(utils.GetSelfFuncName())
conversations, err := c.conversationDatabase.GetConversationIDsNeedDestruct(ctx)
if err != nil {
log.ZError(ctx, "get conversation id need destruct failed", err)
return
}
log.ZDebug(context.Background(), "nums conversations need destruct", "nums", len(conversations))
for _, conversation := range conversations {
log.ZDebug(ctx, "UserMsgsDestruct", "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID, "msgDestructTime", conversation.MsgDestructTime, "lastMsgDestructTime", conversation.LatestMsgDestructTime)
seqs, err := c.msgDatabase.UserMsgsDestruct(ctx, conversation.OwnerUserID, conversation.ConversationID, conversation.MsgDestructTime, conversation.LatestMsgDestructTime)
if err != nil {
log.ZError(ctx, "user msg destruct failed", err, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID)
continue
}
if err := c.conversationDatabase.UpdateUsersConversationFiled(ctx, []string{conversation.OwnerUserID}, conversation.ConversationID, map[string]interface{}{"latest_msg_destruct_time": time.Now()}); err != nil {
log.ZError(ctx, "updateUsersConversationFiled failed", err, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID)
continue
}
if len(seqs) > 0 {
if err := c.msgNotificationSender.UserDeleteMsgsNotification(ctx, conversation.OwnerUserID, conversation.ConversationID, seqs); err != nil {
log.ZError(ctx, "userDeleteMsgsNotification failed", err, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID)
}
}
}
}
+9 -3
View File
@@ -11,7 +11,6 @@ import (
)
func StartCronTask() error {
log.ZInfo(context.Background(), "start cron task", "cron config", config.Config.ChatRecordsClearTime)
fmt.Println("cron task start, config", config.Config.ChatRecordsClearTime)
msgTool, err := InitMsgTool()
if err != nil {
@@ -20,10 +19,17 @@ func StartCronTask() error {
c := cron.New()
var wg sync.WaitGroup
wg.Add(1)
log.ZInfo(context.Background(), "start chatRecordsClearTime cron task", "cron config", config.Config.ChatRecordsClearTime)
_, err = c.AddFunc(config.Config.ChatRecordsClearTime, msgTool.AllConversationClearMsgAndFixSeq)
if err != nil {
fmt.Println("start cron failed", err.Error(), config.Config.ChatRecordsClearTime)
return err
fmt.Println("start allConversationClearMsgAndFixSeq cron failed", err.Error(), config.Config.ChatRecordsClearTime)
panic(err)
}
log.ZInfo(context.Background(), "start msgDestruct cron task", "cron config", config.Config.MsgDestructTime)
_, err = c.AddFunc(config.Config.MsgDestructTime, msgTool.ConversationsDestructMsgs)
if err != nil {
fmt.Println("start conversationsDestructMsgs cron failed", err.Error(), config.Config.ChatRecordsClearTime)
panic(err)
}
c.Start()
wg.Wait()
+36 -16
View File
@@ -2,9 +2,9 @@ package tools
import (
"context"
"errors"
"fmt"
"math"
"time"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/cache"
@@ -14,24 +14,33 @@ import (
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/unrelation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/mcontext"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/mw"
"github.com/OpenIMSDK/Open-IM-Server/pkg/discoveryregistry/zookeeper"
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
"github.com/OpenIMSDK/Open-IM-Server/pkg/rpcclient"
"github.com/OpenIMSDK/Open-IM-Server/pkg/rpcclient/notification"
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
"github.com/redis/go-redis/v9"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
type MsgTool struct {
msgDatabase controller.CommonMsgDatabase
conversationDatabase controller.ConversationDatabase
userDatabase controller.UserDatabase
groupDatabase controller.GroupDatabase
msgDatabase controller.CommonMsgDatabase
conversationDatabase controller.ConversationDatabase
userDatabase controller.UserDatabase
groupDatabase controller.GroupDatabase
msgNotificationSender *notification.MsgNotificationSender
}
var errSeq = errors.New("cache max seq and mongo max seq is diff > 10")
func NewMsgTool(msgDatabase controller.CommonMsgDatabase, userDatabase controller.UserDatabase, groupDatabase controller.GroupDatabase, conversationDatabase controller.ConversationDatabase) *MsgTool {
func NewMsgTool(msgDatabase controller.CommonMsgDatabase, userDatabase controller.UserDatabase,
groupDatabase controller.GroupDatabase, conversationDatabase controller.ConversationDatabase, msgNotificationSender *notification.MsgNotificationSender) *MsgTool {
return &MsgTool{
msgDatabase: msgDatabase,
userDatabase: userDatabase,
groupDatabase: groupDatabase,
conversationDatabase: conversationDatabase,
msgDatabase: msgDatabase,
userDatabase: userDatabase,
groupDatabase: groupDatabase,
conversationDatabase: conversationDatabase,
msgNotificationSender: msgNotificationSender,
}
}
@@ -48,12 +57,21 @@ func InitMsgTool() (*MsgTool, error) {
if err != nil {
return nil, err
}
discov, err := zookeeper.NewClient(config.Config.Zookeeper.ZkAddr, config.Config.Zookeeper.Schema,
zookeeper.WithFreq(time.Hour), zookeeper.WithRoundRobin(), zookeeper.WithUserNameAndPassword(config.Config.Zookeeper.Username,
config.Config.Zookeeper.Password), zookeeper.WithTimeout(10), zookeeper.WithLogger(log.NewZkLogger()))
if err != nil {
return nil, err
}
discov.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()))
userDB := relation.NewUserGorm(db)
msgDatabase := controller.InitCommonMsgDatabase(rdb, mongo.GetDatabase())
userDatabase := controller.NewUserDatabase(userDB, cache.NewUserCacheRedis(rdb, relation.NewUserGorm(db), cache.GetDefaultOpt()), tx.NewGorm(db))
groupDatabase := controller.InitGroupDatabase(db, rdb, mongo.GetDatabase())
conversationDatabase := controller.NewConversationDatabase(relation.NewConversationGorm(db), cache.NewConversationRedis(rdb, cache.GetDefaultOpt(), relation.NewConversationGorm(db)), tx.NewGorm(db))
msgTool := NewMsgTool(msgDatabase, userDatabase, groupDatabase, conversationDatabase)
msgRpcClient := rpcclient.NewMessageRpcClient(discov)
msgNotificationSender := notification.NewMsgNotificationSender(rpcclient.WithRpcClient(&msgRpcClient))
msgTool := NewMsgTool(msgDatabase, userDatabase, groupDatabase, conversationDatabase, msgNotificationSender)
return msgTool, nil
}
@@ -80,17 +98,16 @@ func (c *MsgTool) ClearConversationsMsg(ctx context.Context, conversationIDs []s
if err := c.checkMaxSeq(ctx, conversationID); err != nil {
log.ZError(ctx, "fixSeq failed", err, "conversationID", conversationID)
}
}
}
func (c *MsgTool) checkMaxSeqWithMongo(ctx context.Context, conversationID string, maxSeqCache int64) error {
maxSeqMongo, _, err := c.msgDatabase.GetMongoMaxAndMinSeq(ctx, conversationID)
minSeqMongo, maxSeqMongo, err := c.msgDatabase.GetMongoMaxAndMinSeq(ctx, conversationID)
if err != nil {
return err
}
if math.Abs(float64(maxSeqMongo-maxSeqCache)) > 10 {
return errSeq
log.ZError(ctx, "cache max seq and mongo max seq is diff > 10", nil, "maxSeqMongo", maxSeqMongo, "minSeqMongo", minSeqMongo, "maxSeqCache", maxSeqCache, "conversationID", conversationID)
}
return nil
}
@@ -98,6 +115,9 @@ func (c *MsgTool) checkMaxSeqWithMongo(ctx context.Context, conversationID strin
func (c *MsgTool) checkMaxSeq(ctx context.Context, conversationID string) error {
maxSeq, err := c.msgDatabase.GetMaxSeq(ctx, conversationID)
if err != nil {
if errs.Unwrap(err) == redis.Nil {
return nil
}
return err
}
if err := c.checkMaxSeqWithMongo(ctx, conversationID, maxSeq); err != nil {
+5
View File
@@ -0,0 +1,5 @@
package apiresp
type ApiFormat interface {
ApiFormat()
}
+3
View File
@@ -34,6 +34,9 @@ func isAllFieldsPrivate(v any) bool {
}
func ApiSuccess(data any) *ApiResponse {
if format, ok := data.(ApiFormat); ok {
format.ApiFormat()
}
if isAllFieldsPrivate(data) {
return &ApiResponse{}
}
+1 -69
View File
@@ -1,10 +1,5 @@
package apistruct
import (
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/msg"
sdkws "github.com/OpenIMSDK/Open-IM-Server/pkg/proto/sdkws"
)
type DelMsgReq struct {
UserID string `json:"userID,omitempty" binding:"required"`
SeqList []uint32 `json:"seqList,omitempty" binding:"required"`
@@ -49,69 +44,6 @@ type SetMsgMinSeqReq struct {
type SetMsgMinSeqResp struct {
}
type ModifyMessageReactionExtensionsReq struct {
OperationID string `json:"operationID" binding:"required"`
conversationID string `json:"conversationID" binding:"required"`
SessionType int32 `json:"sessionType" binding:"required"`
ReactionExtensionList map[string]*sdkws.KeyValue `json:"reactionExtensionList,omitempty" binding:"required"`
ClientMsgID string `json:"clientMsgID" binding:"required"`
Ex *string `json:"ex"`
AttachedInfo *string `json:"attachedInfo"`
IsReact bool `json:"isReact"`
IsExternalExtensions bool `json:"isExternalExtensions"`
MsgFirstModifyTime int64 `json:"msgFirstModifyTime"`
}
type ModifyMessageReactionExtensionsResp struct {
Data struct {
ResultKeyValue []*msg.KeyValueResp `json:"result"`
MsgFirstModifyTime int64 `json:"msgFirstModifyTime"`
IsReact bool `json:"isReact"`
} `json:"data"`
}
//type OperateMessageListReactionExtensionsReq struct {
// OperationID string `json:"operationID" binding:"required"`
// conversationID string `json:"conversationID" binding:"required"`
// SessionType string `json:"sessionType" binding:"required"`
// MessageReactionKeyList []*msg.GetMessageListReactionExtensionsReq_MessageReactionKey `json:"messageReactionKeyList" binding:"required"`
//}
type OperateMessageListReactionExtensionsResp struct {
Data struct {
SuccessList []*msg.ExtendMsgResp `json:"successList"`
FailedList []*msg.ExtendMsgResp `json:"failedList"`
} `json:"data"`
}
type SetMessageReactionExtensionsCallbackReq ModifyMessageReactionExtensionsReq
type SetMessageReactionExtensionsCallbackResp ModifyMessageReactionExtensionsResp
//type GetMessageListReactionExtensionsReq OperateMessageListReactionExtensionsReq
type GetMessageListReactionExtensionsResp struct {
Data []*msg.SingleMessageExtensionResult `json:"data"`
}
type AddMessageReactionExtensionsReq ModifyMessageReactionExtensionsReq
type AddMessageReactionExtensionsResp ModifyMessageReactionExtensionsResp
type DeleteMessageReactionExtensionsReq struct {
OperationID string `json:"operationID" binding:"required"`
conversationID string `json:"conversationID" binding:"required"`
SessionType int32 `json:"sessionType" binding:"required"`
ClientMsgID string `json:"clientMsgID" binding:"required"`
IsExternalExtensions bool `json:"isExternalExtensions"`
MsgFirstModifyTime int64 `json:"msgFirstModifyTime" binding:"required"`
ReactionExtensionList []*sdkws.KeyValue `json:"reactionExtensionList" binding:"required"`
}
type DeleteMessageReactionExtensionsResp struct {
Data []*msg.KeyValueResp
}
type PictureBaseInfo struct {
UUID string `mapstructure:"uuid"`
Type string `mapstructure:"type" `
@@ -171,7 +103,7 @@ type CustomElem struct {
Extension string `mapstructure:"extension"`
}
type TextElem struct {
Text string `mapstructure:"text" validate:"required"`
Content string `mapstructure:"content" validate:"required"`
}
type RevokeElem struct {
-69
View File
@@ -1,7 +1,6 @@
package callbackstruct
import (
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/msg"
sdkws "github.com/OpenIMSDK/Open-IM-Server/pkg/proto/sdkws"
)
@@ -66,71 +65,3 @@ type CallbackMsgModifyCommandResp struct {
AttachedInfo *string `json:"attachedInfo"`
Ex *string `json:"ex"`
}
type CallbackBeforeSetMessageReactionExtReq struct {
OperationID string `json:"operationID"`
CallbackCommand `json:"callbackCommand"`
ConversationID string `json:"conversationID"`
OpUserID string `json:"opUserID"`
SessionType int32 `json:"sessionType"`
ReactionExtensionList map[string]*sdkws.KeyValue `json:"reactionExtensionList"`
ClientMsgID string `json:"clientMsgID"`
IsReact bool `json:"isReact"`
IsExternalExtensions bool `json:"isExternalExtensions"`
MsgFirstModifyTime int64 `json:"msgFirstModifyTime"`
}
type CallbackBeforeSetMessageReactionExtResp struct {
CommonCallbackResp
ResultReactionExtensionList []*msg.KeyValueResp `json:"resultReactionExtensionList"`
MsgFirstModifyTime int64 `json:"msgFirstModifyTime"`
}
type CallbackDeleteMessageReactionExtReq struct {
CallbackCommand `json:"callbackCommand"`
OperationID string `json:"operationID"`
ConversationID string `json:"conversationID"`
OpUserID string `json:"opUserID"`
SessionType int32 `json:"sessionType"`
ReactionExtensionList []*sdkws.KeyValue `json:"reactionExtensionList"`
ClientMsgID string `json:"clientMsgID"`
IsExternalExtensions bool `json:"isExternalExtensions"`
MsgFirstModifyTime int64 `json:"msgFirstModifyTime"`
}
type CallbackDeleteMessageReactionExtResp struct {
CommonCallbackResp
ResultReactionExtensionList []*msg.KeyValueResp `json:"resultReactionExtensionList"`
MsgFirstModifyTime int64 `json:"msgFirstModifyTime"`
}
type CallbackGetMessageListReactionExtReq struct {
OperationID string `json:"operationID"`
CallbackCommand `json:"callbackCommand"`
ConversationID string `json:"conversationID"`
OpUserID string `json:"opUserID"`
SessionType int32 `json:"sessionType"`
TypeKeyList []string `json:"typeKeyList"`
//MessageKeyList []*msg.GetMessageListReactionExtensionsReq_MessageReactionKey `json:"messageKeyList"`
}
type CallbackGetMessageListReactionExtResp struct {
CommonCallbackResp
MessageResultList []*msg.SingleMessageExtensionResult `json:"messageResultList"`
}
type CallbackAddMessageReactionExtReq struct {
OperationID string `json:"operationID"`
CallbackCommand `json:"callbackCommand"`
ConversationID string `json:"conversationID"`
OpUserID string `json:"opUserID"`
SessionType int32 `json:"sessionType"`
ReactionExtensionList map[string]*sdkws.KeyValue `json:"reactionExtensionList"`
ClientMsgID string `json:"clientMsgID"`
IsReact bool `json:"isReact"`
IsExternalExtensions bool `json:"isExternalExtensions"`
MsgFirstModifyTime int64 `json:"msgFirstModifyTime"`
}
type CallbackAddMessageReactionExtResp struct {
CommonCallbackResp
ResultReactionExtensionList []*msg.KeyValueResp `json:"resultReactionExtensionList"`
IsReact bool `json:"isReact"`
MsgFirstModifyTime int64 `json:"msgFirstModifyTime"`
}
+1 -1
View File
@@ -7,7 +7,7 @@ type CronTaskCmd struct {
}
func NewCronTaskCmd() *CronTaskCmd {
return &CronTaskCmd{NewRootCmd("cronTask")}
return &CronTaskCmd{NewRootCmd("cronTask", WithCronTaskLogName())}
}
func (c *CronTaskCmd) addRunE(f func() error) {
+28 -5
View File
@@ -16,17 +16,40 @@ type RootCmd struct {
prometheusPort int
}
func NewRootCmd(name string) (rootCmd *RootCmd) {
type CmdOpts struct {
loggerPrefixName string
}
func WithCronTaskLogName() func(*CmdOpts) {
return func(opts *CmdOpts) {
opts.loggerPrefixName = "OpenIM.CronTask.log.all"
}
}
func WithLogName(logName string) func(*CmdOpts) {
return func(opts *CmdOpts) {
opts.loggerPrefixName = logName
}
}
func NewRootCmd(name string, opts ...func(*CmdOpts)) (rootCmd *RootCmd) {
rootCmd = &RootCmd{Name: name}
c := cobra.Command{
Use: "start",
Short: fmt.Sprintf(`Start %s server`, name),
Long: fmt.Sprintf(`Start %s server`, name),
Use: "start openIM application",
Short: fmt.Sprintf(`Start %s `, name),
Long: fmt.Sprintf(`Start %s `, name),
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := rootCmd.getConfFromCmdAndInit(cmd); err != nil {
panic(err)
}
if err := log.InitFromConfig("OpenIM.log.all", name, config.Config.Log.RemainLogLevel, config.Config.Log.IsStdout, config.Config.Log.IsJson, config.Config.Log.StorageLocation, config.Config.Log.RemainRotationCount); err != nil {
cmdOpts := &CmdOpts{}
for _, opt := range opts {
opt(cmdOpts)
}
if cmdOpts.loggerPrefixName == "" {
cmdOpts.loggerPrefixName = "OpenIM.log.all"
}
if err := log.InitFromConfig(cmdOpts.loggerPrefixName, name, config.Config.Log.RemainLogLevel, config.Config.Log.IsStdout, config.Config.Log.IsJson, config.Config.Log.StorageLocation, config.Config.Log.RemainRotationCount); err != nil {
panic(err)
}
return nil
+1 -1
View File
@@ -26,7 +26,7 @@ func (a *RpcCmd) Exec() error {
return a.Execute()
}
func (a *RpcCmd) StartSvr(name string, rpcFn func(client discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) error) error {
func (a *RpcCmd) StartSvr(name string, rpcFn func(discov discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) error) error {
if a.GetPortFlag() == 0 {
return errors.New("port is required")
}
+22 -41
View File
@@ -77,15 +77,11 @@ type config struct {
MsgToPush struct {
Topic string `yaml:"topic"`
} `yaml:"msgToPush"`
MsgToModify struct {
Topic string `yaml:"topic"`
} `yaml:"msgToModify"`
ConsumerGroupID struct {
MsgToRedis string `yaml:"msgToRedis"`
MsgToMongo string `yaml:"msgToMongo"`
MsgToMySql string `yaml:"msgToMySql"`
MsgToPush string `yaml:"msgToPush"`
MsgToModify string `yaml:"msgToModify"`
MsgToRedis string `yaml:"msgToRedis"`
MsgToMongo string `yaml:"msgToMongo"`
MsgToMySql string `yaml:"msgToMySql"`
MsgToPush string `yaml:"msgToPush"`
} `yaml:"consumerGroupID"`
} `yaml:"kafka"`
@@ -103,42 +99,26 @@ type config struct {
Enable string `yaml:"enable"`
ApiURL string `yaml:"apiURL"`
Minio struct {
TempBucket string `yaml:"tempBucket"`
DataBucket string `yaml:"dataBucket"`
Location string `yaml:"location"`
Endpoint string `yaml:"endpoint"`
AccessKeyID string `yaml:"accessKeyID"`
SecretAccessKey string `yaml:"secretAccessKey"`
IsDistributedMod bool `yaml:"isDistributedMod"`
Bucket string `yaml:"bucket"`
Endpoint string `yaml:"endpoint"`
AccessKeyID string `yaml:"accessKeyID"`
SecretAccessKey string `yaml:"secretAccessKey"`
SessionToken string `yaml:"sessionToken"`
} `yaml:"minio"`
Tencent struct {
AppID string `yaml:"appID"`
Region string `yaml:"region"`
Bucket string `yaml:"bucket"`
SecretID string `yaml:"secretID"`
SecretKey string `yaml:"secretKey"`
} `yaml:"tencent"`
Ali struct {
RegionID string `yaml:"regionID"`
AccessKeyID string `yaml:"accessKeyID"`
AccessKeySecret string `yaml:"accessKeySecret"`
StsEndpoint string `yaml:"stsEndpoint"`
OssEndpoint string `yaml:"ossEndpoint"`
Bucket string `yaml:"bucket"`
FinalHost string `yaml:"finalHost"`
StsDurationSeconds int64 `yaml:"stsDurationSeconds"`
OssRoleArn string `yaml:"OssRoleArn"`
} `yaml:"ali"`
Aws struct {
Cos struct {
BucketURL string `yaml:"bucketURL"`
SecretID string `yaml:"secretID"`
SecretKey string `yaml:"secretKey"`
SessionToken string `yaml:"sessionToken"`
} `yaml:"cos"`
Oss struct {
Endpoint string `yaml:"endpoint"`
Bucket string `yaml:"bucket"`
BucketURL string `yaml:"bucketURL"`
AccessKeyID string `yaml:"accessKeyID"`
AccessKeySecret string `yaml:"accessKeySecret"`
Region string `yaml:"region"`
Bucket string `yaml:"bucket"`
FinalHost string `yaml:"finalHost"`
RoleArn string `yaml:"roleArn"`
ExternalId string `yaml:"externalId"`
RoleSessionName string `yaml:"roleSessionName"`
} `yaml:"aws"`
SessionToken string `yaml:"sessionToken"`
} `yaml:"oss"`
} `yaml:"object"`
RpcPort struct {
@@ -215,6 +195,7 @@ type config struct {
SingleMessageHasReadReceiptEnable bool `yaml:"singleMessageHasReadReceiptEnable"`
RetainChatRecords int `yaml:"retainChatRecords"`
ChatRecordsClearTime string `yaml:"chatRecordsClearTime"`
MsgDestructTime string `yaml:"msgDestructTime"`
Secret string `yaml:"secret"`
TokenPolicy struct {
Expire int64 `yaml:"expire"`
+2
View File
@@ -8,6 +8,7 @@ import (
func ConversationDB2Pb(conversationDB *relation.ConversationModel) *conversation.Conversation {
conversationPB := &conversation.Conversation{}
conversationPB.LatestMsgDestructTime = conversationDB.LatestMsgDestructTime.Unix()
if err := utils.CopyStructFields(conversationPB, conversationDB); err != nil {
return nil
}
@@ -20,6 +21,7 @@ func ConversationsDB2Pb(conversationsDB []*relation.ConversationModel) (conversa
if err := utils.CopyStructFields(conversationPB, conversationDB); err != nil {
continue
}
conversationPB.LatestMsgDestructTime = conversationDB.LatestMsgDestructTime.Unix()
conversationsPB = append(conversationsPB, conversationPB)
}
return conversationsPB
-64
View File
@@ -1,64 +0,0 @@
package cache
import (
"context"
"time"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/unrelation"
"github.com/dtm-labs/rockscache"
"github.com/redis/go-redis/v9"
)
const (
extendMsgSetCache = "EXTEND_MSG_SET_CACHE:"
extendMsgCache = "EXTEND_MSG_CACHE:"
)
type ExtendMsgSetCache interface {
metaCache
NewCache() ExtendMsgSetCache
GetExtendMsg(ctx context.Context, conversationID string, sessionType int32, clientMsgID string, firstModifyTime int64) (extendMsg *unrelation.ExtendMsgModel, err error)
DelExtendMsg(clientMsgID string) ExtendMsgSetCache
}
type ExtendMsgSetCacheRedis struct {
metaCache
expireTime time.Duration
rcClient *rockscache.Client
extendMsgSetDB unrelation.ExtendMsgSetModelInterface
}
func NewExtendMsgSetCacheRedis(rdb redis.UniversalClient, extendMsgSetDB unrelation.ExtendMsgSetModelInterface, options rockscache.Options) ExtendMsgSetCache {
rcClient := rockscache.NewClient(rdb, options)
return &ExtendMsgSetCacheRedis{
metaCache: NewMetaCacheRedis(rcClient),
expireTime: time.Second * 30 * 60,
extendMsgSetDB: extendMsgSetDB,
rcClient: rcClient,
}
}
func (e *ExtendMsgSetCacheRedis) NewCache() ExtendMsgSetCache {
return &ExtendMsgSetCacheRedis{
metaCache: NewMetaCacheRedis(e.rcClient, e.metaCache.GetPreDelKeys()...),
expireTime: e.expireTime,
extendMsgSetDB: e.extendMsgSetDB,
rcClient: e.rcClient,
}
}
func (e *ExtendMsgSetCacheRedis) getKey(clientMsgID string) string {
return extendMsgCache + clientMsgID
}
func (e *ExtendMsgSetCacheRedis) GetExtendMsg(ctx context.Context, conversationID string, sessionType int32, clientMsgID string, firstModifyTime int64) (extendMsg *unrelation.ExtendMsgModel, err error) {
return getCache(ctx, e.rcClient, e.getKey(clientMsgID), e.expireTime, func(ctx context.Context) (*unrelation.ExtendMsgModel, error) {
return e.extendMsgSetDB.TakeExtendMsg(ctx, conversationID, sessionType, clientMsgID, firstModifyTime)
})
}
func (e *ExtendMsgSetCacheRedis) DelExtendMsg(clientMsgID string) ExtendMsgSetCache {
new := e.NewCache()
new.AddKeys(e.getKey(clientMsgID))
return new
}
+17 -7
View File
@@ -2,6 +2,7 @@ package controller
import (
"context"
"time"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/cache"
@@ -34,6 +35,7 @@ type ConversationDatabase interface {
GetAllConversationIDs(ctx context.Context) ([]string, error)
GetUserAllHasReadSeqs(ctx context.Context, ownerUserID string) (map[string]int64, error)
GetConversationsByConversationID(ctx context.Context, conversationIDs []string) ([]*relationTb.ConversationModel, error)
GetConversationIDsNeedDestruct(ctx context.Context) ([]*relationTb.ConversationModel, error)
}
func NewConversationDatabase(conversation relationTb.ConversationModelInterface, cache cache.ConversationCache, tx tx.Tx) ConversationDatabase {
@@ -73,12 +75,14 @@ func (c *conversationDatabase) SetUsersConversationFiledTx(ctx context.Context,
NotUserIDs := utils.DifferenceString(haveUserIDs, userIDs)
log.ZDebug(ctx, "SetUsersConversationFiledTx", "NotUserIDs", NotUserIDs, "haveUserIDs", haveUserIDs, "userIDs", userIDs)
var conversations []*relationTb.ConversationModel
now := time.Now()
for _, v := range NotUserIDs {
temp := new(relationTb.ConversationModel)
if err := utils.CopyStructFields(temp, conversation); err != nil {
return err
}
temp.OwnerUserID = v
temp.CreateTime = now
conversations = append(conversations, temp)
}
@@ -123,26 +127,28 @@ func (c *conversationDatabase) SyncPeerUserPrivateConversationTx(ctx context.Con
conversationTx := c.conversationDB.NewTx(tx)
for _, conversation := range conversations {
for _, v := range [][2]string{{conversation.OwnerUserID, conversation.UserID}, {conversation.UserID, conversation.OwnerUserID}} {
haveUserIDs, err := conversationTx.FindUserID(ctx, []string{v[0]}, []string{conversation.ConversationID})
ownerUserID := v[0]
userID := v[1]
haveUserIDs, err := conversationTx.FindUserID(ctx, []string{ownerUserID}, []string{conversation.ConversationID})
if err != nil {
return err
}
if len(haveUserIDs) > 0 {
_, err := conversationTx.UpdateByMap(ctx, []string{v[0]}, conversation.ConversationID, map[string]interface{}{"is_private_chat": conversation.IsPrivateChat})
_, err := conversationTx.UpdateByMap(ctx, []string{ownerUserID}, conversation.ConversationID, map[string]interface{}{"is_private_chat": conversation.IsPrivateChat})
if err != nil {
return err
}
cache = cache.DelUsersConversation(conversation.ConversationID, v[0])
cache = cache.DelUsersConversation(conversation.ConversationID, ownerUserID)
} else {
newConversation := *conversation
newConversation.OwnerUserID = v[0]
newConversation.UserID = v[1]
newConversation.OwnerUserID = ownerUserID
newConversation.UserID = userID
newConversation.ConversationID = conversation.ConversationID
newConversation.IsPrivateChat = conversation.IsPrivateChat
if err := conversationTx.Create(ctx, []*relationTb.ConversationModel{&newConversation}); err != nil {
return err
}
cache = cache.DelConversationIDs(v[0]).DelUserConversationIDsHash(v[0])
cache = cache.DelConversationIDs(ownerUserID).DelUserConversationIDsHash(ownerUserID)
}
}
}
@@ -150,7 +156,7 @@ func (c *conversationDatabase) SyncPeerUserPrivateConversationTx(ctx context.Con
}); err != nil {
return err
}
return c.cache.ExecDel(ctx)
return cache.ExecDel(ctx)
}
func (c *conversationDatabase) FindConversations(ctx context.Context, ownerUserID string, conversationIDs []string) ([]*relationTb.ConversationModel, error) {
@@ -270,3 +276,7 @@ func (c *conversationDatabase) GetUserAllHasReadSeqs(ctx context.Context, ownerU
func (c *conversationDatabase) GetConversationsByConversationID(ctx context.Context, conversationIDs []string) ([]*relationTb.ConversationModel, error) {
return c.conversationDB.GetConversationsByConversationID(ctx, conversationIDs)
}
func (c *conversationDatabase) GetConversationIDsNeedDestruct(ctx context.Context) ([]*relationTb.ConversationModel, error) {
return c.conversationDB.GetConversationIDsNeedDestruct(ctx)
}
-58
View File
@@ -1,58 +0,0 @@
package controller
import (
"context"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/cache"
unRelationTb "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/unrelation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/tx"
)
// for mongoDB
type ExtendMsgDatabase interface {
CreateExtendMsgSet(ctx context.Context, set *unRelationTb.ExtendMsgSetModel) error
GetAllExtendMsgSet(ctx context.Context, ID string, opts *unRelationTb.GetAllExtendMsgSetOpts) (sets []*unRelationTb.ExtendMsgSetModel, err error)
GetExtendMsgSet(ctx context.Context, conversationID string, sessionType int32, maxMsgUpdateTime int64) (*unRelationTb.ExtendMsgSetModel, error)
InsertExtendMsg(ctx context.Context, conversationID string, sessionType int32, msg *unRelationTb.ExtendMsgModel) error
InsertOrUpdateReactionExtendMsgSet(ctx context.Context, conversationID string, sessionType int32, clientMsgID string, msgFirstModifyTime int64, reactionExtensionList map[string]*unRelationTb.KeyValueModel) error
DeleteReactionExtendMsgSet(ctx context.Context, conversationID string, sessionType int32, clientMsgID string, msgFirstModifyTime int64, reactionExtensionList map[string]*unRelationTb.KeyValueModel) error
GetExtendMsg(ctx context.Context, conversationID string, sessionType int32, clientMsgID string, maxMsgUpdateTime int64) (extendMsg *unRelationTb.ExtendMsgModel, err error)
}
type extendMsgDatabase struct {
database unRelationTb.ExtendMsgSetModelInterface
cache cache.ExtendMsgSetCache
ctxTx tx.CtxTx
}
func NewExtendMsgDatabase(extendMsgModel unRelationTb.ExtendMsgSetModelInterface, cache cache.ExtendMsgSetCache, ctxTx tx.CtxTx) ExtendMsgDatabase {
return &extendMsgDatabase{database: extendMsgModel, cache: cache, ctxTx: ctxTx}
}
func (e *extendMsgDatabase) CreateExtendMsgSet(ctx context.Context, set *unRelationTb.ExtendMsgSetModel) error {
return e.database.CreateExtendMsgSet(ctx, set)
}
func (e *extendMsgDatabase) GetAllExtendMsgSet(ctx context.Context, conversationID string, opts *unRelationTb.GetAllExtendMsgSetOpts) (sets []*unRelationTb.ExtendMsgSetModel, err error) {
return e.database.GetAllExtendMsgSet(ctx, conversationID, opts)
}
func (e *extendMsgDatabase) GetExtendMsgSet(ctx context.Context, conversationID string, sessionType int32, maxMsgUpdateTime int64) (*unRelationTb.ExtendMsgSetModel, error) {
return e.database.GetExtendMsgSet(ctx, conversationID, sessionType, maxMsgUpdateTime)
}
func (e *extendMsgDatabase) InsertExtendMsg(ctx context.Context, conversationID string, sessionType int32, msg *unRelationTb.ExtendMsgModel) error {
return e.database.InsertExtendMsg(ctx, conversationID, sessionType, msg)
}
func (e *extendMsgDatabase) InsertOrUpdateReactionExtendMsgSet(ctx context.Context, conversationID string, sessionType int32, clientMsgID string, msgFirstModifyTime int64, reactionExtensionList map[string]*unRelationTb.KeyValueModel) error {
return e.database.InsertOrUpdateReactionExtendMsgSet(ctx, conversationID, sessionType, clientMsgID, msgFirstModifyTime, reactionExtensionList)
}
func (e *extendMsgDatabase) DeleteReactionExtendMsgSet(ctx context.Context, conversationID string, sessionType int32, clientMsgID string, msgFirstModifyTime int64, reactionExtensionList map[string]*unRelationTb.KeyValueModel) error {
return e.database.DeleteReactionExtendMsgSet(ctx, conversationID, sessionType, clientMsgID, msgFirstModifyTime, reactionExtensionList)
}
func (e *extendMsgDatabase) GetExtendMsg(ctx context.Context, conversationID string, sessionType int32, clientMsgID string, maxMsgUpdateTime int64) (extendMsg *unRelationTb.ExtendMsgModel, err error) {
return e.cache.GetExtendMsg(ctx, conversationID, sessionType, clientMsgID, maxMsgUpdateTime)
}
+14
View File
@@ -3,6 +3,7 @@ package controller
import (
"context"
"fmt"
"time"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/cache"
@@ -57,6 +58,11 @@ type GroupDatabase interface {
DeleteSuperGroup(ctx context.Context, groupID string) error
DeleteSuperGroupMember(ctx context.Context, groupID string, userIDs []string) error
CreateSuperGroupMember(ctx context.Context, groupID string, userIDs []string) error
// 获取群总数
CountTotal(ctx context.Context, before *time.Time) (count int64, err error)
// 获取范围内群增量
CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error)
}
func NewGroupDatabase(
@@ -419,3 +425,11 @@ func (g *groupDatabase) CreateSuperGroupMember(ctx context.Context, groupID stri
}
return g.cache.DelSuperGroupMemberIDs(groupID).DelJoinedSuperGroupIDs(userIDs...).ExecDel(ctx)
}
func (g *groupDatabase) CountTotal(ctx context.Context, before *time.Time) (count int64, err error) {
return g.groupDB.CountTotal(ctx, before)
}
func (g *groupDatabase) CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error) {
return g.groupDB.CountRangeEverydayTotal(ctx, start, end)
}
+68 -123
View File
@@ -1,10 +1,10 @@
package controller
import (
"fmt"
"github.com/redis/go-redis/v9"
"time"
"github.com/redis/go-redis/v9"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/convert"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/cache"
@@ -48,6 +48,9 @@ type CommonMsgDatabase interface {
GetMsgBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) (minSeq int64, maxSeq int64, seqMsg []*sdkws.MsgData, err error)
// 删除会话消息重置最小seq, remainTime为消息保留的时间单位秒,超时消息删除, 传0删除所有消息(此方法不删除redis cache)
DeleteConversationMsgsAndSetMinSeq(ctx context.Context, conversationID string, remainTime int64) error
// 用户标记删除过期消息返回标记删除的seq列表
UserMsgsDestruct(cte context.Context, userID string, conversationID string, destructTime int64, lastMsgDestructTime time.Time) (seqs []int64, err error)
// 用户根据seq删除消息
DeleteUserMsgsBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) error
// 物理删除消息置空
@@ -71,7 +74,7 @@ type CommonMsgDatabase interface {
GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error)
UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error
GetMongoMaxAndMinSeq(ctx context.Context, conversationID string) (maxSeq, minSeq int64, err error)
GetMongoMaxAndMinSeq(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo int64, err error)
GetConversationMinMaxSeqInMongoAndCache(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache int64, err error)
SetSendMsgStatus(ctx context.Context, id string, status int32) error
GetSendMsgStatus(ctx context.Context, id string) (int32, error)
@@ -82,26 +85,17 @@ type CommonMsgDatabase interface {
MsgToPushMQ(ctx context.Context, key, conversarionID string, msg2mq *sdkws.MsgData) (int32, int64, error)
MsgToMongoMQ(ctx context.Context, key, conversarionID string, msgs []*sdkws.MsgData, lastSeq int64) error
// modify
JudgeMessageReactionExist(ctx context.Context, clientMsgID string, sessionType int32) (bool, error)
SetMessageTypeKeyValue(ctx context.Context, clientMsgID string, sessionType int32, typeKey, value string) error
SetMessageReactionExpire(ctx context.Context, clientMsgID string, sessionType int32, expiration time.Duration) (bool, error)
GetExtendMsg(ctx context.Context, conversationID string, sessionType int32, clientMsgID string, maxMsgUpdateTime int64) (*pbMsg.ExtendMsg, error)
InsertOrUpdateReactionExtendMsgSet(ctx context.Context, conversationID string, sessionType int32, clientMsgID string, msgFirstModifyTime int64, reactionExtensionList map[string]*sdkws.KeyValue) error
GetMessageTypeKeyValue(ctx context.Context, clientMsgID string, sessionType int32, typeKey string) (string, error)
GetOneMessageAllReactionList(ctx context.Context, clientMsgID string, sessionType int32) (map[string]string, error)
DeleteOneMessageKey(ctx context.Context, clientMsgID string, sessionType int32, subKey string) error
DeleteReactionExtendMsgSet(ctx context.Context, conversationID string, sessionType int32, clientMsgID string, msgFirstModifyTime int64, reactionExtensionList map[string]*sdkws.KeyValue) error
RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*unRelationTb.UserCount, dateCount map[string]int64, err error)
RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*unRelationTb.GroupCount, dateCount map[string]int64, err error)
}
func NewCommonMsgDatabase(msgDocModel unRelationTb.MsgDocModelInterface, cacheModel cache.MsgModel) CommonMsgDatabase {
return &commonMsgDatabase{
msgDocDatabase: msgDocModel,
cache: cacheModel,
producer: kafka.NewKafkaProducer(config.Config.Kafka.Addr, config.Config.Kafka.LatestMsgToRedis.Topic),
producerToMongo: kafka.NewKafkaProducer(config.Config.Kafka.Addr, config.Config.Kafka.MsgToMongo.Topic),
producerToPush: kafka.NewKafkaProducer(config.Config.Kafka.Addr, config.Config.Kafka.MsgToPush.Topic),
producerToModify: kafka.NewKafkaProducer(config.Config.Kafka.Addr, config.Config.Kafka.MsgToModify.Topic),
msgDocDatabase: msgDocModel,
cache: cacheModel,
producer: kafka.NewKafkaProducer(config.Config.Kafka.Addr, config.Config.Kafka.LatestMsgToRedis.Topic),
producerToMongo: kafka.NewKafkaProducer(config.Config.Kafka.Addr, config.Config.Kafka.MsgToMongo.Topic),
producerToPush: kafka.NewKafkaProducer(config.Config.Kafka.Addr, config.Config.Kafka.MsgToPush.Topic),
}
}
@@ -113,15 +107,13 @@ func InitCommonMsgDatabase(rdb redis.UniversalClient, database *mongo.Database)
}
type commonMsgDatabase struct {
msgDocDatabase unRelationTb.MsgDocModelInterface
extendMsgDatabase unRelationTb.ExtendMsgSetModelInterface
extendMsgSetModel unRelationTb.ExtendMsgSetModel
msg unRelationTb.MsgDocModel
cache cache.MsgModel
producer *kafka.Producer
producerToMongo *kafka.Producer
producerToModify *kafka.Producer
producerToPush *kafka.Producer
msgDocDatabase unRelationTb.MsgDocModelInterface
msg unRelationTb.MsgDocModel
cache cache.MsgModel
producer *kafka.Producer
producerToMongo *kafka.Producer
producerToModify *kafka.Producer
producerToPush *kafka.Producer
}
func (db *commonMsgDatabase) MsgToMQ(ctx context.Context, key string, msg2mq *sdkws.MsgData) error {
@@ -389,44 +381,6 @@ func (db *commonMsgDatabase) getMsgBySeqs(ctx context.Context, userID, conversat
return totalMsgs, nil
}
// func (db *commonMsgDatabase) refetchDelSeqsMsgs(ctx context.Context, conversationID string, delNums, rangeBegin, begin int64) (seqMsgs []*unRelationTb.MsgDataModel, err error) {
// var reFetchSeqs []int64
// if delNums > 0 {
// newBeginSeq := rangeBegin - delNums
// if newBeginSeq >= begin {
// newEndSeq := rangeBegin - 1
// for i := newBeginSeq; i <= newEndSeq; i++ {
// reFetchSeqs = append(reFetchSeqs, i)
// }
// }
// }
// if len(reFetchSeqs) == 0 {
// return
// }
// if len(reFetchSeqs) > 0 {
// m := db.msg.GetDocIDSeqsMap(conversationID, reFetchSeqs)
// for docID, seqs := range m {
// msgs, _, err := db.findMsgInfoBySeq(ctx, docID, seqs)
// if err != nil {
// return nil, err
// }
// for _, msg := range msgs {
// if msg.Status != constant.MsgDeleted {
// seqMsgs = append(seqMsgs, msg)
// }
// }
// }
// }
// if len(seqMsgs) < int(delNums) {
// seqMsgs2, err := db.refetchDelSeqsMsgs(ctx, conversationID, delNums-int64(len(seqMsgs)), rangeBegin-1, begin)
// if err != nil {
// return seqMsgs, err
// }
// seqMsgs = append(seqMsgs, seqMsgs2...)
// }
// return seqMsgs, nil
// }
func (db *commonMsgDatabase) findMsgInfoBySeq(ctx context.Context, userID, docID string, seqs []int64) (totalMsgs []*unRelationTb.MsgInfoModel, err error) {
msgs, err := db.msgDocDatabase.GetMsgBySeqIndexIn1Doc(ctx, userID, docID, seqs)
for _, msg := range msgs {
@@ -634,6 +588,49 @@ func (db *commonMsgDatabase) DeleteConversationMsgsAndSetMinSeq(ctx context.Cont
return db.cache.SetMinSeq(ctx, conversationID, minSeq)
}
func (db *commonMsgDatabase) UserMsgsDestruct(ctx context.Context, userID string, conversationID string, destructTime int64, lastMsgDestructTime time.Time) (seqs []int64, err error) {
var index int64
for {
// from oldest 2 newest
msgDocModel, err := db.msgDocDatabase.GetMsgDocModelByIndex(ctx, conversationID, index, 1)
if err != nil || msgDocModel.DocID == "" {
if err != nil {
if err == unrelation.ErrMsgListNotExist {
log.ZDebug(ctx, "deleteMsgRecursion finished", "conversationID", conversationID, "userID", userID, "index", index)
} else {
log.ZError(ctx, "deleteMsgRecursion GetUserMsgListByIndex failed", err, "conversationID", conversationID, "index", index)
}
}
// 获取报错,或者获取不到了,物理删除并且返回seq delMongoMsgsPhysical(delStruct.delDocIDList), 结束递归
break
}
index++
//&& msgDocModel.Msg[0].Msg.SendTime > lastMsgDestructTime.UnixMilli()
if len(msgDocModel.Msg) > 0 {
for _, msg := range msgDocModel.Msg {
if msg != nil && msg.Msg != nil && msg.Msg.SendTime+destructTime*1000 <= time.Now().UnixMilli() {
if msg.Msg.SendTime > lastMsgDestructTime.UnixMilli() && !utils.Contain(userID, msg.DelList...) {
seqs = append(seqs, msg.Msg.Seq)
}
} else {
log.ZDebug(ctx, "deleteMsgRecursion finished", "conversationID", conversationID, "userID", userID, "index", index)
break
}
}
}
}
log.ZDebug(ctx, "UserMsgsDestruct", "conversationID", conversationID, "userID", userID, "seqs", seqs)
if len(seqs) > 0 {
latestSeq := seqs[len(seqs)-1]
if err := db.cache.SetConversationUserMinSeq(ctx, conversationID, userID, latestSeq); err != nil {
return nil, err
}
}
return seqs, nil
}
// this is struct for recursion
type delMsgRecursionStruct struct {
minSeq int64
@@ -848,7 +845,7 @@ func (db *commonMsgDatabase) GetConversationMinMaxSeqInMongoAndCache(ctx context
return
}
func (db *commonMsgDatabase) GetMongoMaxAndMinSeq(ctx context.Context, conversationID string) (maxSeq, minSeq int64, err error) {
func (db *commonMsgDatabase) GetMongoMaxAndMinSeq(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo int64, err error) {
return db.GetMinMaxSeqMongo(ctx, conversationID)
}
@@ -866,62 +863,10 @@ func (db *commonMsgDatabase) GetMinMaxSeqMongo(ctx context.Context, conversation
return
}
func (db *commonMsgDatabase) JudgeMessageReactionExist(ctx context.Context, clientMsgID string, sessionType int32) (bool, error) {
return db.cache.JudgeMessageReactionExist(ctx, clientMsgID, sessionType)
func (db *commonMsgDatabase) RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*unRelationTb.UserCount, dateCount map[string]int64, err error) {
return db.msgDocDatabase.RangeUserSendCount(ctx, start, end, group, ase, pageNumber, showNumber)
}
func (db *commonMsgDatabase) SetMessageTypeKeyValue(ctx context.Context, clientMsgID string, sessionType int32, typeKey, value string) error {
return db.cache.SetMessageTypeKeyValue(ctx, clientMsgID, sessionType, typeKey, value)
}
func (db *commonMsgDatabase) SetMessageReactionExpire(ctx context.Context, clientMsgID string, sessionType int32, expiration time.Duration) (bool, error) {
return db.cache.SetMessageReactionExpire(ctx, clientMsgID, sessionType, expiration)
}
func (db *commonMsgDatabase) GetMessageTypeKeyValue(ctx context.Context, clientMsgID string, sessionType int32, typeKey string) (string, error) {
return db.cache.GetMessageTypeKeyValue(ctx, clientMsgID, sessionType, typeKey)
}
func (db *commonMsgDatabase) GetOneMessageAllReactionList(ctx context.Context, clientMsgID string, sessionType int32) (map[string]string, error) {
return db.cache.GetOneMessageAllReactionList(ctx, clientMsgID, sessionType)
}
func (db *commonMsgDatabase) DeleteOneMessageKey(ctx context.Context, clientMsgID string, sessionType int32, subKey string) error {
return db.cache.DeleteOneMessageKey(ctx, clientMsgID, sessionType, subKey)
}
func (db *commonMsgDatabase) InsertOrUpdateReactionExtendMsgSet(ctx context.Context, conversationID string, sessionType int32, clientMsgID string, msgFirstModifyTime int64, reactionExtensions map[string]*sdkws.KeyValue) error {
return db.extendMsgDatabase.InsertOrUpdateReactionExtendMsgSet(ctx, conversationID, sessionType, clientMsgID, msgFirstModifyTime, db.extendMsgSetModel.Pb2Model(reactionExtensions))
}
func (db *commonMsgDatabase) GetExtendMsg(ctx context.Context, conversationID string, sessionType int32, clientMsgID string, maxMsgUpdateTime int64) (*pbMsg.ExtendMsg, error) {
extendMsgSet, err := db.extendMsgDatabase.GetExtendMsgSet(ctx, conversationID, sessionType, maxMsgUpdateTime)
if err != nil {
return nil, err
}
extendMsg, ok := extendMsgSet.ExtendMsgs[clientMsgID]
if !ok {
return nil, errs.ErrRecordNotFound.Wrap(fmt.Sprintf("cant find client msg id: %s", clientMsgID))
}
reactionExtensionList := make(map[string]*pbMsg.KeyValueResp)
for key, model := range extendMsg.ReactionExtensionList {
reactionExtensionList[key] = &pbMsg.KeyValueResp{
KeyValue: &sdkws.KeyValue{
TypeKey: model.TypeKey,
Value: model.Value,
LatestUpdateTime: model.LatestUpdateTime,
},
}
}
return &pbMsg.ExtendMsg{
ReactionExtensions: reactionExtensionList,
ClientMsgID: extendMsg.ClientMsgID,
MsgFirstModifyTime: extendMsg.MsgFirstModifyTime,
AttachedInfo: extendMsg.AttachedInfo,
Ex: extendMsg.Ex,
}, nil
}
func (db *commonMsgDatabase) DeleteReactionExtendMsgSet(ctx context.Context, conversationID string, sessionType int32, clientMsgID string, msgFirstModifyTime int64, reactionExtensions map[string]*sdkws.KeyValue) error {
return db.extendMsgDatabase.DeleteReactionExtendMsgSet(ctx, conversationID, sessionType, clientMsgID, msgFirstModifyTime, db.extendMsgSetModel.Pb2Model(reactionExtensions))
func (db *commonMsgDatabase) RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*unRelationTb.GroupCount, dateCount map[string]int64, err error) {
return db.msgDocDatabase.RangeGroupSendCount(ctx, start, end, ase, pageNumber, showNumber)
}
+76
View File
@@ -0,0 +1,76 @@
package controller
import "C"
import (
"context"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3/cont"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
"path/filepath"
"time"
)
type S3Database interface {
PartLimit() *s3.PartLimit
PartSize(ctx context.Context, size int64) (int64, error)
AuthSign(ctx context.Context, uploadID string, partNumbers []int) (*s3.AuthSignResult, error)
InitiateMultipartUpload(ctx context.Context, hash string, size int64, expire time.Duration, maxParts int) (*cont.InitiateUploadResult, error)
CompleteMultipartUpload(ctx context.Context, uploadID string, parts []string) (*cont.UploadResult, error)
AccessURL(ctx context.Context, name string, expire time.Duration) (time.Time, string, error)
SetObject(ctx context.Context, info *relation.ObjectModel) error
}
func NewS3Database(s3 s3.Interface, obj relation.ObjectInfoModelInterface) S3Database {
return &s3Database{
s3: cont.New(s3),
obj: obj,
}
}
type s3Database struct {
s3 *cont.Controller
obj relation.ObjectInfoModelInterface
}
func (s *s3Database) PartSize(ctx context.Context, size int64) (int64, error) {
return s.s3.PartSize(ctx, size)
}
func (s *s3Database) PartLimit() *s3.PartLimit {
return s.s3.PartLimit()
}
func (s *s3Database) AuthSign(ctx context.Context, uploadID string, partNumbers []int) (*s3.AuthSignResult, error) {
return s.s3.AuthSign(ctx, uploadID, partNumbers)
}
func (s *s3Database) InitiateMultipartUpload(ctx context.Context, hash string, size int64, expire time.Duration, maxParts int) (*cont.InitiateUploadResult, error) {
return s.s3.InitiateUpload(ctx, hash, size, expire, maxParts)
}
func (s *s3Database) CompleteMultipartUpload(ctx context.Context, uploadID string, parts []string) (*cont.UploadResult, error) {
return s.s3.CompleteUpload(ctx, uploadID, parts)
}
func (s *s3Database) SetObject(ctx context.Context, info *relation.ObjectModel) error {
return s.obj.SetObject(ctx, info)
}
func (s *s3Database) AccessURL(ctx context.Context, name string, expire time.Duration) (time.Time, string, error) {
obj, err := s.obj.Take(ctx, name)
if err != nil {
return time.Time{}, "", err
}
opt := &s3.AccessURLOption{
ContentType: obj.ContentType,
}
if filename := filepath.Base(obj.Name); filename != "" {
opt.ContentDisposition = `attachment; filename=` + filename
}
expireTime := time.Now().Add(expire)
rawURL, err := s.s3.AccessURL(ctx, obj.Key, expire, opt)
if err != nil {
return time.Time{}, "", err
}
return expireTime, rawURL, nil
}
-538
View File
@@ -1,538 +0,0 @@
package controller
import "C"
import (
"bytes"
"context"
"crypto/md5"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/obj"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/third"
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
"github.com/google/uuid"
"io"
"net/url"
"path"
"strconv"
"time"
)
const (
hashPrefix = "hash"
tempPrefix = "temp"
fragmentPrefix = "fragment_"
urlsName = "urls.json"
)
type S3Database interface {
ApplyPut(ctx context.Context, req *third.ApplyPutReq) (*third.ApplyPutResp, error)
GetPut(ctx context.Context, req *third.GetPutReq) (*third.GetPutResp, error)
ConfirmPut(ctx context.Context, req *third.ConfirmPutReq) (*third.ConfirmPutResp, error)
GetUrl(ctx context.Context, req *third.GetUrlReq) (*third.GetUrlResp, error)
GetHashInfo(ctx context.Context, req *third.GetHashInfoReq) (*third.GetHashInfoResp, error)
CleanExpirationObject(ctx context.Context, t time.Time)
}
func NewS3Database(obj obj.Interface, hash relation.ObjectHashModelInterface, info relation.ObjectInfoModelInterface, put relation.ObjectPutModelInterface, url *url.URL) S3Database {
return &s3Database{
url: url,
obj: obj,
hash: hash,
info: info,
put: put,
}
}
type s3Database struct {
url *url.URL
obj obj.Interface
hash relation.ObjectHashModelInterface
info relation.ObjectInfoModelInterface
put relation.ObjectPutModelInterface
}
// today 今天的日期
func (c *s3Database) today() string {
return time.Now().Format("20060102")
}
// fragmentName 根据序号生成文件名
func (c *s3Database) fragmentName(index int) string {
return fragmentPrefix + strconv.Itoa(index+1)
}
// getFragmentNum 获取分片大小和分片数量
func (c *s3Database) getFragmentNum(fragmentSize int64, objectSize int64) (int64, int) {
if size := c.obj.MinFragmentSize(); fragmentSize < size {
fragmentSize = size
}
if fragmentSize <= 0 || objectSize <= fragmentSize {
return objectSize, 1
} else {
num := int(objectSize / fragmentSize)
if objectSize%fragmentSize > 0 {
num++
}
if n := c.obj.MaxFragmentNum(); num > n {
num = n
}
return fragmentSize, num
}
}
func (c *s3Database) CheckHash(hash string) error {
val, err := hex.DecodeString(hash)
if err != nil {
return err
}
if len(val) != md5.Size {
return errs.ErrArgs.Wrap("invalid hash")
}
return nil
}
func (c *s3Database) urlName(name string) string {
u := url.URL{
Scheme: c.url.Scheme,
Opaque: c.url.Opaque,
User: c.url.User,
Host: c.url.Host,
Path: c.url.Path,
RawPath: c.url.RawPath,
ForceQuery: c.url.ForceQuery,
RawQuery: c.url.RawQuery,
Fragment: c.url.Fragment,
RawFragment: c.url.RawFragment,
}
v := make(url.Values, 1)
v.Set("name", name)
u.RawQuery = v.Encode()
return u.String()
}
func (c *s3Database) UUID() string {
return uuid.New().String()
}
func (c *s3Database) HashName(hash string) string {
return path.Join(hashPrefix, hash+"_"+c.today()+"_"+c.UUID())
}
func (c *s3Database) isNotFound(err error) bool {
return relation.IsNotFound(err)
}
func (c *s3Database) ApplyPut(ctx context.Context, req *third.ApplyPutReq) (*third.ApplyPutResp, error) {
if err := c.CheckHash(req.Hash); err != nil {
return nil, err
}
if err := c.obj.CheckName(req.Name); err != nil {
return nil, err
}
if req.ValidTime != 0 && req.ValidTime <= time.Now().UnixMilli() {
return nil, errors.New("invalid ValidTime")
}
var expirationTime *time.Time
if req.ValidTime != 0 {
expirationTime = utils.ToPtr(time.UnixMilli(req.ValidTime))
}
if hash, err := c.hash.Take(ctx, req.Hash, c.obj.Name()); err == nil {
o := relation.ObjectInfoModel{
Name: req.Name,
Hash: hash.Hash,
ValidTime: expirationTime,
ContentType: req.ContentType,
CreateTime: time.Now(),
}
if err := c.info.SetObject(ctx, &o); err != nil {
return nil, err
}
return &third.ApplyPutResp{Url: c.urlName(o.Name)}, nil // 服务器已存在
} else if !c.isNotFound(err) {
return nil, err
}
// 新上传
var fragmentNum int
const effective = time.Hour * 24 * 2
req.FragmentSize, fragmentNum = c.getFragmentNum(req.FragmentSize, req.Size)
put := relation.ObjectPutModel{
PutID: req.PutID,
Hash: req.Hash,
Name: req.Name,
ObjectSize: req.Size,
ContentType: req.ContentType,
FragmentSize: req.FragmentSize,
ValidTime: expirationTime,
EffectiveTime: time.Now().Add(effective),
}
if put.PutID == "" {
put.PutID = c.UUID()
}
if v, err := c.put.Take(ctx, put.PutID); err == nil {
now := time.Now().UnixMilli()
if v.EffectiveTime.UnixMilli() <= now {
if err := c.put.DelPut(ctx, []string{v.PutID}); err != nil {
return nil, err
}
} else {
return nil, errs.ErrDuplicateKey.Wrap(fmt.Sprintf("duplicate put id %s", put.PutID))
}
} else if !c.isNotFound(err) {
return nil, err
}
put.Path = path.Join(tempPrefix, c.today(), req.Hash, put.PutID)
putURLs := make([]string, 0, fragmentNum)
for i := 0; i < fragmentNum; i++ {
url, err := c.obj.PresignedPutURL(ctx, &obj.ApplyPutArgs{
Bucket: c.obj.TempBucket(),
Name: path.Join(put.Path, c.fragmentName(i)),
Effective: effective,
MaxObjectSize: req.FragmentSize,
})
if err != nil {
return nil, err
}
putURLs = append(putURLs, url)
}
urlsJsonData, err := json.Marshal(putURLs)
if err != nil {
return nil, err
}
t := md5.Sum(urlsJsonData)
put.PutURLsHash = hex.EncodeToString(t[:])
_, err = c.obj.PutObject(ctx, &obj.BucketObject{Bucket: c.obj.TempBucket(), Name: path.Join(put.Path, urlsName)}, bytes.NewReader(urlsJsonData), int64(len(urlsJsonData)))
if err != nil {
return nil, err
}
put.CreateTime = time.Now()
if err := c.put.Create(ctx, []*relation.ObjectPutModel{&put}); err != nil {
return nil, err
}
return &third.ApplyPutResp{
PutID: put.PutID,
FragmentSize: put.FragmentSize,
PutURLs: putURLs,
ValidTime: put.EffectiveTime.UnixMilli(),
}, nil
}
func (c *s3Database) GetPut(ctx context.Context, req *third.GetPutReq) (*third.GetPutResp, error) {
up, err := c.put.Take(ctx, req.PutID)
if err != nil {
return nil, err
}
reader, err := c.obj.GetObject(ctx, &obj.BucketObject{Bucket: c.obj.TempBucket(), Name: path.Join(up.Path, urlsName)})
if err != nil {
return nil, err
}
urlsData, err := io.ReadAll(reader)
if err != nil {
return nil, err
}
t := md5.Sum(urlsData)
if h := hex.EncodeToString(t[:]); h != up.PutURLsHash {
return nil, fmt.Errorf("invalid put urls hash %s %s", h, up.PutURLsHash)
}
var urls []string
if err := json.Unmarshal(urlsData, &urls); err != nil {
return nil, err
}
_, fragmentNum := c.getFragmentNum(up.FragmentSize, up.ObjectSize)
if len(urls) != fragmentNum {
return nil, fmt.Errorf("invalid urls length %d fragment %d", len(urls), fragmentNum)
}
fragments := make([]*third.GetPutFragment, fragmentNum)
for i := 0; i < fragmentNum; i++ {
name := path.Join(up.Path, c.fragmentName(i))
o, err := c.obj.GetObjectInfo(ctx, &obj.BucketObject{
Bucket: c.obj.TempBucket(),
Name: name,
})
if err != nil {
if c.obj.IsNotFound(err) {
fragments[i] = &third.GetPutFragment{Url: urls[i]}
continue
}
return nil, err
}
fragments[i] = &third.GetPutFragment{Size: o.Size, Hash: o.Hash, Url: urls[i]}
}
var validTime int64
if up.ValidTime != nil {
validTime = up.ValidTime.UnixMilli()
}
return &third.GetPutResp{
FragmentSize: up.FragmentSize,
Size: up.ObjectSize,
Name: up.Name,
Hash: up.Hash,
Fragments: fragments,
PutURLsHash: up.PutURLsHash,
ContentType: up.ContentType,
ValidTime: validTime,
}, nil
}
func (c *s3Database) ConfirmPut(ctx context.Context, req *third.ConfirmPutReq) (_ *third.ConfirmPutResp, _err error) {
put, err := c.put.Take(ctx, req.PutID)
if err != nil {
return nil, err
}
_, pack := c.getFragmentNum(put.FragmentSize, put.ObjectSize)
defer func() {
if _err == nil {
// 清理上传的碎片
err := c.obj.DeleteObject(ctx, &obj.BucketObject{Bucket: c.obj.TempBucket(), Name: put.Path})
if err != nil {
log.ZError(ctx, "deleteObject failed", err, "Bucket", c.obj.TempBucket(), "Path", put.Path)
}
}
}()
now := time.Now().UnixMilli()
if put.EffectiveTime.UnixMilli() < now {
return nil, errs.ErrFileUploadedExpired.Wrap("put expired")
}
if put.ValidTime != nil && put.ValidTime.UnixMilli() < now {
return nil, errs.ErrFileUploadedExpired.Wrap("object expired")
}
if hash, err := c.hash.Take(ctx, put.Hash, c.obj.Name()); err == nil {
o := relation.ObjectInfoModel{
Name: put.Name,
Hash: hash.Hash,
ValidTime: put.ValidTime,
ContentType: put.ContentType,
CreateTime: time.Now(),
}
if err := c.info.SetObject(ctx, &o); err != nil {
return nil, err
}
defer func() {
err := c.obj.DeleteObject(ctx, &obj.BucketObject{
Bucket: c.obj.TempBucket(),
Name: put.Path,
})
if err != nil {
log.ZError(ctx, "DeleteObject", err, "Bucket", c.obj.TempBucket(), "Path", put.Path)
}
}()
// 服务端已存在
return &third.ConfirmPutResp{
Url: c.urlName(o.Name),
}, nil
} else if !c.isNotFound(err) {
return nil, err
}
src := make([]obj.BucketObject, pack)
for i := 0; i < pack; i++ {
name := path.Join(put.Path, c.fragmentName(i))
o, err := c.obj.GetObjectInfo(ctx, &obj.BucketObject{
Bucket: c.obj.TempBucket(),
Name: name,
})
if err != nil {
return nil, err
}
if i+1 == pack { // 最后一个
size := put.ObjectSize - put.FragmentSize*int64(i)
if size != o.Size {
return nil, fmt.Errorf("last fragment %d size %d not equal to %d hash %s", i, o.Size, size, o.Hash)
}
} else {
if o.Size != put.FragmentSize {
return nil, fmt.Errorf("fragment %d size %d not equal to %d hash %s", i, o.Size, put.FragmentSize, o.Hash)
}
}
src[i] = obj.BucketObject{
Bucket: c.obj.TempBucket(),
Name: name,
}
}
dst := &obj.BucketObject{
Bucket: c.obj.DataBucket(),
Name: c.HashName(put.Hash),
}
if len(src) == 1 { // 未分片直接触发copy
// 检查数据完整性,避免脏数据
o, err := c.obj.GetObjectInfo(ctx, &src[0])
if err != nil {
return nil, err
}
if put.ObjectSize != o.Size {
return nil, fmt.Errorf("size mismatching should %d reality %d", put.ObjectSize, o.Size)
}
if put.Hash != o.Hash {
return nil, fmt.Errorf("hash mismatching should %s reality %s", put.Hash, o.Hash)
}
if err := c.obj.CopyObject(ctx, &src[0], dst); err != nil {
return nil, err
}
} else {
tempBucket := &obj.BucketObject{
Bucket: c.obj.TempBucket(),
Name: path.Join(put.Path, "merge_"+c.UUID()),
}
defer func() { // 清理合成的文件
if err := c.obj.DeleteObject(ctx, tempBucket); err != nil {
log.ZError(ctx, "DeleteObject", err, "Bucket", tempBucket.Bucket, "Path", tempBucket.Name)
}
}()
err := c.obj.ComposeObject(ctx, src, tempBucket)
if err != nil {
return nil, err
}
info, err := c.obj.GetObjectInfo(ctx, tempBucket)
if err != nil {
return nil, err
}
if put.ObjectSize != info.Size {
return nil, fmt.Errorf("size mismatch should %d reality %d", put.ObjectSize, info.Size)
}
if put.Hash != info.Hash {
return nil, fmt.Errorf("hash mismatch should %s reality %s", put.Hash, info.Hash)
}
if err := c.obj.CopyObject(ctx, tempBucket, dst); err != nil {
return nil, err
}
}
h := &relation.ObjectHashModel{
Hash: put.Hash,
Engine: c.obj.Name(),
Size: put.ObjectSize,
Bucket: c.obj.DataBucket(),
Name: dst.Name,
CreateTime: time.Now(),
}
if err := c.hash.Create(ctx, []*relation.ObjectHashModel{h}); err != nil {
return nil, err
}
o := &relation.ObjectInfoModel{
Name: put.Name,
Hash: put.Hash,
ContentType: put.ContentType,
ValidTime: put.ValidTime,
CreateTime: time.Now(),
}
if err := c.info.SetObject(ctx, o); err != nil {
return nil, err
}
if err := c.put.DelPut(ctx, []string{put.PutID}); err != nil {
log.ZError(ctx, "DelPut", err, "PutID", put.PutID)
}
return &third.ConfirmPutResp{
Url: c.urlName(o.Name),
}, nil
}
func (c *s3Database) GetUrl(ctx context.Context, req *third.GetUrlReq) (*third.GetUrlResp, error) {
info, err := c.info.Take(ctx, req.Name)
if err != nil {
return nil, err
}
if info.ValidTime != nil && info.ValidTime.Before(time.Now()) {
return nil, errs.ErrRecordNotFound.Wrap("object expired")
}
hash, err := c.hash.Take(ctx, info.Hash, c.obj.Name())
if err != nil {
return nil, err
}
opt := obj.HeaderOption{ContentType: info.ContentType}
if req.Attachment {
opt.Filename = info.Name
}
u, err := c.obj.PresignedGetURL(ctx, hash.Bucket, hash.Name, time.Duration(req.Expires)*time.Millisecond, &opt)
if err != nil {
return nil, err
}
return &third.GetUrlResp{
Url: u,
Size: hash.Size,
Hash: hash.Hash,
}, nil
}
func (c *s3Database) CleanExpirationObject(ctx context.Context, t time.Time) {
// 清理上传产生的临时文件
c.cleanPutTemp(ctx, t, 10)
// 清理hash引用全过期的文件
c.cleanExpirationObject(ctx, t)
// 清理没有引用的hash对象
c.clearNoCitation(ctx, c.obj.Name(), 10)
}
func (c *s3Database) cleanPutTemp(ctx context.Context, t time.Time, num int) {
for {
puts, err := c.put.FindExpirationPut(ctx, t, num)
if err != nil {
log.ZError(ctx, "FindExpirationPut", err, "Time", t, "Num", num)
return
}
if len(puts) == 0 {
return
}
for _, put := range puts {
err := c.obj.DeleteObject(ctx, &obj.BucketObject{Bucket: c.obj.TempBucket(), Name: put.Path})
if err != nil {
log.ZError(ctx, "DeleteObject", err, "Bucket", c.obj.TempBucket(), "Path", put.Path)
return
}
}
ids := utils.Slice(puts, func(e *relation.ObjectPutModel) string { return e.PutID })
err = c.put.DelPut(ctx, ids)
if err != nil {
log.ZError(ctx, "DelPut", err, "PutID", ids)
return
}
}
}
func (c *s3Database) cleanExpirationObject(ctx context.Context, t time.Time) {
err := c.info.DeleteExpiration(ctx, t)
if err != nil {
log.ZError(ctx, "DeleteExpiration", err, "Time", t)
}
}
func (c *s3Database) clearNoCitation(ctx context.Context, engine string, limit int) {
for {
list, err := c.hash.DeleteNoCitation(ctx, engine, limit)
if err != nil {
log.ZError(ctx, "DeleteNoCitation", err, "Engine", engine, "Limit", limit)
return
}
if len(list) == 0 {
return
}
var hasErr bool
for _, h := range list {
err := c.obj.DeleteObject(ctx, &obj.BucketObject{Bucket: h.Bucket, Name: h.Name})
if err != nil {
hasErr = true
log.ZError(ctx, "DeleteObject", err, "Bucket", h.Bucket, "Path", h.Name)
continue
}
}
if hasErr {
return
}
}
}
func (c *s3Database) GetHashInfo(ctx context.Context, req *third.GetHashInfoReq) (*third.GetHashInfoResp, error) {
if err := c.CheckHash(req.Hash); err != nil {
return nil, err
}
o, err := c.hash.Take(ctx, req.Hash, c.obj.Name())
if err != nil {
return nil, err
}
return &third.GetHashInfoResp{
Hash: o.Hash,
Size: o.Size,
}, nil
}
+3 -3
View File
@@ -31,7 +31,7 @@ type UserDatabase interface {
//函数内部先查询db中是否存在,存在则什么都不做;不存在则插入
InitOnce(ctx context.Context, users []*relation.UserModel) (err error)
// 获取用户总数
CountTotal(ctx context.Context) (int64, error)
CountTotal(ctx context.Context, before *time.Time) (int64, error)
// 获取范围内用户增量
CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error)
}
@@ -134,8 +134,8 @@ func (u *userDatabase) GetAllUserID(ctx context.Context) (userIDs []string, err
return u.userDB.GetAllUserID(ctx)
}
func (u *userDatabase) CountTotal(ctx context.Context) (count int64, err error) {
return u.userDB.CountTotal(ctx)
func (u *userDatabase) CountTotal(ctx context.Context, before *time.Time) (count int64, err error) {
return u.userDB.CountTotal(ctx, before)
}
func (u *userDatabase) CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error) {
-231
View File
@@ -1,231 +0,0 @@
package obj
import (
"context"
"errors"
"fmt"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio-go/v7/pkg/s3utils"
"io"
"net/http"
"net/url"
"time"
)
func NewMinioInterface() (Interface, error) {
conf := config.Config.Object.Minio
u, err := url.Parse(conf.Endpoint)
if err != nil {
return nil, fmt.Errorf("minio endpoint parse %w", err)
}
if u.Scheme != "http" && u.Scheme != "https" {
return nil, fmt.Errorf("invalid minio endpoint scheme %s", u.Scheme)
}
client, err := minio.New(u.Host, &minio.Options{
Creds: credentials.NewStaticV4(conf.AccessKeyID, conf.SecretAccessKey, ""),
Secure: u.Scheme == "https",
})
if err != nil {
return nil, fmt.Errorf("minio new client %w", err)
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second*20)
defer cancel()
initBucket := func(ctx context.Context) error {
for _, bucket := range utils.Distinct([]string{conf.TempBucket, conf.DataBucket}) {
exists, err := client.BucketExists(ctx, bucket)
if err != nil {
return fmt.Errorf("minio bucket %s exists %w", bucket, err)
}
if exists {
continue
}
opt := minio.MakeBucketOptions{
Region: conf.Location,
ObjectLocking: conf.IsDistributedMod,
}
if err := client.MakeBucket(ctx, bucket, opt); err != nil {
return fmt.Errorf("minio make bucket %s %w", bucket, err)
}
}
return nil
}
if err := initBucket(ctx); err != nil {
fmt.Println("minio init error:", err)
}
return &minioImpl{
client: client,
tempBucket: conf.TempBucket,
dataBucket: conf.DataBucket,
}, nil
}
type minioImpl struct {
tempBucket string // 上传桶
dataBucket string // 永久桶
urlstr string // 访问地址
client *minio.Client
}
func (m *minioImpl) Name() string {
return "minio"
}
func (m *minioImpl) MinFragmentSize() int64 {
return 1024 * 1024 * 5 // 每个分片最小大小 minio.absMinPartSize
}
func (m *minioImpl) MaxFragmentNum() int {
return 1000 // 最大分片数量 minio.maxPartsCount
}
func (m *minioImpl) MinExpirationTime() time.Duration {
return time.Hour * 24
}
func (m *minioImpl) TempBucket() string {
return m.tempBucket
}
func (m *minioImpl) DataBucket() string {
return m.dataBucket
}
func (m *minioImpl) PresignedGetURL(ctx context.Context, bucket string, name string, expires time.Duration, opt *HeaderOption) (string, error) {
var reqParams url.Values
if opt != nil {
reqParams = make(url.Values)
if opt.ContentType != "" {
reqParams.Set("response-content-type", opt.ContentType)
}
if opt.Filename != "" {
reqParams.Set("response-content-disposition", "attachment;filename="+opt.Filename)
}
}
u, err := m.client.PresignedGetObject(ctx, bucket, name, expires, reqParams)
if err != nil {
return "", err
}
return u.String(), nil
}
func (m *minioImpl) PresignedPutURL(ctx context.Context, args *ApplyPutArgs) (string, error) {
if args.Effective <= 0 {
return "", errors.New("EffectiveTime <= 0")
}
_, err := m.GetObjectInfo(ctx, &BucketObject{
Bucket: m.tempBucket,
Name: args.Name,
})
if err == nil {
return "", fmt.Errorf("minio bucket %s name %s already exists", args.Bucket, args.Name)
} else if !m.IsNotFound(err) {
return "", err
}
u, err := m.client.PresignedPutObject(ctx, m.tempBucket, args.Name, args.Effective)
if err != nil {
return "", fmt.Errorf("minio apply error: %w", err)
}
return u.String(), nil
}
func (m *minioImpl) GetObjectInfo(ctx context.Context, args *BucketObject) (*ObjectInfo, error) {
info, err := m.client.StatObject(ctx, args.Bucket, args.Name, minio.StatObjectOptions{})
if err != nil {
return nil, err
}
return &ObjectInfo{
Size: info.Size,
Hash: info.ETag,
}, nil
}
func (m *minioImpl) CopyObject(ctx context.Context, src *BucketObject, dst *BucketObject) error {
_, err := m.client.CopyObject(ctx, minio.CopyDestOptions{
Bucket: dst.Bucket,
Object: dst.Name,
}, minio.CopySrcOptions{
Bucket: src.Bucket,
Object: src.Name,
})
return err
}
func (m *minioImpl) DeleteObject(ctx context.Context, info *BucketObject) error {
return m.client.RemoveObject(ctx, info.Bucket, info.Name, minio.RemoveObjectOptions{})
}
func (m *minioImpl) MoveObjectInfo(ctx context.Context, src *BucketObject, dst *BucketObject) error {
if err := m.CopyObject(ctx, src, dst); err != nil {
return err
}
return m.DeleteObject(ctx, src)
}
func (m *minioImpl) ComposeObject(ctx context.Context, src []BucketObject, dst *BucketObject) error {
destOptions := minio.CopyDestOptions{
Bucket: dst.Bucket,
Object: dst.Name + ".temp",
}
sources := make([]minio.CopySrcOptions, len(src))
for i, s := range src {
sources[i] = minio.CopySrcOptions{
Bucket: s.Bucket,
Object: s.Name,
}
}
_, err := m.client.ComposeObject(ctx, destOptions, sources...)
if err != nil {
return err
}
return m.MoveObjectInfo(ctx, &BucketObject{
Bucket: destOptions.Bucket,
Name: destOptions.Object,
}, &BucketObject{
Bucket: dst.Bucket,
Name: dst.Name,
})
}
func (m *minioImpl) IsNotFound(err error) bool {
if err == nil {
return false
}
switch e := err.(type) {
case minio.ErrorResponse:
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
case *minio.ErrorResponse:
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
default:
return false
}
}
func (m *minioImpl) PutObject(ctx context.Context, info *BucketObject, reader io.Reader, size int64) (*ObjectInfo, error) {
update, err := m.client.PutObject(ctx, info.Bucket, info.Name, reader, size, minio.PutObjectOptions{})
if err != nil {
return nil, err
}
return &ObjectInfo{
Size: update.Size,
Hash: update.ETag,
}, nil
}
func (m *minioImpl) GetObject(ctx context.Context, info *BucketObject) (SizeReader, error) {
object, err := m.client.GetObject(ctx, info.Bucket, info.Name, minio.GetObjectOptions{})
if err != nil {
return nil, err
}
stat, err := object.Stat()
if err != nil {
return nil, err
}
return NewSizeReader(object, stat.Size), nil
}
func (m *minioImpl) CheckName(name string) error {
return s3utils.CheckValidObjectName(name)
}
-90
View File
@@ -1,90 +0,0 @@
package obj
import (
"context"
"io"
"net/http"
"time"
)
type BucketObject struct {
Bucket string `json:"bucket"`
Name string `json:"name"`
}
type ApplyPutArgs struct {
Bucket string
Name string
Effective time.Duration // 申请有效时间
Header http.Header // header
MaxObjectSize int64
}
type HeaderOption struct {
ContentType string
Filename string
}
type ObjectInfo struct {
Size int64
Hash string
}
type SizeReader interface {
io.ReadCloser
Size() int64
}
func NewSizeReader(r io.ReadCloser, size int64) SizeReader {
if r == nil {
return nil
}
return &sizeReader{
size: size,
ReadCloser: r,
}
}
type sizeReader struct {
size int64
io.ReadCloser
}
func (r *sizeReader) Size() int64 {
return r.size
}
type Interface interface {
// Name 存储名字
Name() string
// MinFragmentSize 最小允许的分片大小
MinFragmentSize() int64
// MaxFragmentNum 最大允许的分片数量
MaxFragmentNum() int
// MinExpirationTime 最小过期时间
MinExpirationTime() time.Duration
// TempBucket 临时桶名,用于上传
TempBucket() string
// DataBucket 永久存储的桶名
DataBucket() string
// PresignedGetURL 通过桶名和对象名返回URL
PresignedGetURL(ctx context.Context, bucket string, name string, expires time.Duration, opt *HeaderOption) (string, error)
// PresignedPutURL 申请上传,返回PUT的上传地址
PresignedPutURL(ctx context.Context, args *ApplyPutArgs) (string, error)
// GetObjectInfo 获取对象信息
GetObjectInfo(ctx context.Context, args *BucketObject) (*ObjectInfo, error)
// CopyObject 复制对象
CopyObject(ctx context.Context, src *BucketObject, dst *BucketObject) error
// DeleteObject 删除对象(不存在返回nil)
DeleteObject(ctx context.Context, info *BucketObject) error
// ComposeObject 合并对象
ComposeObject(ctx context.Context, src []BucketObject, dst *BucketObject) error
// IsNotFound 判断是不是不存在导致的错误
IsNotFound(err error) bool
// CheckName 检查名字是否可用
CheckName(name string) error
// PutObject 上传文件
PutObject(ctx context.Context, info *BucketObject, reader io.Reader, size int64) (*ObjectInfo, error)
// GetObject 下载文件
GetObject(ctx context.Context, info *BucketObject) (SizeReader, error)
}
+5 -7
View File
@@ -82,15 +82,13 @@ func (c *ConversationGorm) GetAllConversationIDs(ctx context.Context) (conversat
}
func (c *ConversationGorm) GetUserAllHasReadSeqs(ctx context.Context, ownerUserID string) (hasReadSeqs map[string]int64, err error) {
var conversations []*relation.ConversationModel
err = utils.Wrap(c.db(ctx).Where("owner_user_id = ?", ownerUserID).Select("conversation_id", "has_read_seq").Find(&conversations).Error, "")
hasReadSeqs = make(map[string]int64, len(conversations))
// for _, conversation := range conversations {
// hasReadSeqs[conversation.ConversationID] = conversation.HasReadSeq
// }
return hasReadSeqs, err
return nil, nil
}
func (c *ConversationGorm) GetConversationsByConversationID(ctx context.Context, conversationIDs []string) (conversations []*relation.ConversationModel, err error) {
return conversations, utils.Wrap(c.db(ctx).Where("conversation_id IN (?)", conversationIDs).Find(&conversations).Error, "")
}
func (c *ConversationGorm) GetConversationIDsNeedDestruct(ctx context.Context) (conversations []*relation.ConversationModel, err error) {
return conversations, utils.Wrap(c.db(ctx).Where("is_msg_destruct = 1 && UNIX_TIMESTAMP(NOW()) > (msg_destruct_time + UNIX_TIMESTAMP(latest_msg_destruct_time)) && msg_destruct_time != 0").Find(&conversations).Error, "")
}
+29
View File
@@ -4,8 +4,10 @@ import (
"context"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/ormutil"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
"gorm.io/gorm"
"time"
)
var _ relation.GroupModelInterface = (*GroupGorm)(nil)
@@ -50,3 +52,30 @@ func (g *GroupGorm) Search(ctx context.Context, keyword string, pageNumber, show
func (g *GroupGorm) GetGroupIDsByGroupType(ctx context.Context, groupType int) (groupIDs []string, err error) {
return groupIDs, utils.Wrap(g.DB.Model(&relation.GroupModel{}).Where("group_type = ? ", groupType).Pluck("group_id", &groupIDs).Error, "")
}
func (g *GroupGorm) CountTotal(ctx context.Context, before *time.Time) (count int64, err error) {
db := g.db(ctx).Model(&relation.GroupModel{})
if before != nil {
db = db.Where("create_time < ?", before)
}
if err := db.Count(&count).Error; err != nil {
return 0, err
}
return count, nil
}
func (g *GroupGorm) CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error) {
var res []struct {
Date time.Time `gorm:"column:date"`
Count int64 `gorm:"column:count"`
}
err := g.db(ctx).Model(&relation.GroupModel{}).Select("DATE(create_time) AS date, count(1) AS count").Where("create_time >= ? and create_time < ?", start, end).Group("date").Find(&res).Error
if err != nil {
return nil, errs.Wrap(err)
}
v := make(map[string]int64)
for _, r := range res {
v[r.Date.Format("2006-01-02")] = r.Count
}
return v, nil
}
@@ -1,42 +0,0 @@
package relation
import (
"context"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
"gorm.io/gorm"
)
type ObjectHashGorm struct {
*MetaDB
}
func NewObjectHash(db *gorm.DB) relation.ObjectHashModelInterface {
return &ObjectHashGorm{
NewMetaDB(db, &relation.ObjectHashModel{}),
}
}
func (o *ObjectHashGorm) NewTx(tx any) relation.ObjectHashModelInterface {
return &ObjectHashGorm{
NewMetaDB(tx.(*gorm.DB), &relation.ObjectHashModel{}),
}
}
func (o *ObjectHashGorm) Take(ctx context.Context, hash string, engine string) (oh *relation.ObjectHashModel, err error) {
oh = &relation.ObjectHashModel{}
return oh, utils.Wrap1(o.DB.Where("hash = ? and engine = ?", hash, engine).Take(oh).Error)
}
func (o *ObjectHashGorm) Create(ctx context.Context, h []*relation.ObjectHashModel) (err error) {
return utils.Wrap1(o.DB.Create(h).Error)
}
func (o *ObjectHashGorm) DeleteNoCitation(ctx context.Context, engine string, num int) (list []*relation.ObjectHashModel, err error) {
err = o.DB.Table(relation.ObjectHashModelTableName, "as h").Select("h.*").
Joins("LEFT JOIN "+relation.ObjectInfoModelTableName+" as i ON h.hash = i.hash").
Where("h.engine = ? AND i.hash IS NULL", engine).
Limit(num).
Find(&list).Error
return list, utils.Wrap1(err)
}
@@ -1,48 +0,0 @@
package relation
import (
"context"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
"gorm.io/gorm"
"time"
)
type ObjectInfoGorm struct {
*MetaDB
}
func NewObjectInfo(db *gorm.DB) relation.ObjectInfoModelInterface {
return &ObjectInfoGorm{
NewMetaDB(db, &relation.ObjectInfoModel{}),
}
}
func (o *ObjectInfoGorm) NewTx(tx any) relation.ObjectInfoModelInterface {
return &ObjectInfoGorm{
NewMetaDB(tx.(*gorm.DB), &relation.ObjectInfoModel{}),
}
}
func (o *ObjectInfoGorm) SetObject(ctx context.Context, obj *relation.ObjectInfoModel) (err error) {
if err := o.DB.WithContext(ctx).Where("name = ?", obj.Name).Delete(&relation.ObjectInfoModel{}).Error; err != nil {
return errs.Wrap(err)
}
return errs.Wrap(o.DB.WithContext(ctx).Create(obj).Error)
//return errs.Wrap(o.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
// if err := tx.Where("name = ?", obj.Name).Delete(&relation.ObjectInfoModel{}).Error; err != nil {
// return errs.Wrap(err)
// }
// return errs.Wrap(tx.Create(obj).Error)
//}))
}
func (o *ObjectInfoGorm) Take(ctx context.Context, name string) (info *relation.ObjectInfoModel, err error) {
info = &relation.ObjectInfoModel{}
return info, utils.Wrap1(o.DB.WithContext(ctx).Where("name = ?", name).Take(info).Error)
}
func (o *ObjectInfoGorm) DeleteExpiration(ctx context.Context, expiration time.Time) (err error) {
return utils.Wrap1(o.DB.WithContext(ctx).Where("expiration_time IS NOT NULL AND expiration_time <= ?", expiration).Delete(&relation.ObjectInfoModel{}).Error)
}
+36
View File
@@ -0,0 +1,36 @@
package relation
import (
"context"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
"gorm.io/gorm"
)
type ObjectInfoGorm struct {
*MetaDB
}
func NewObjectInfo(db *gorm.DB) relation.ObjectInfoModelInterface {
return &ObjectInfoGorm{
NewMetaDB(db, &relation.ObjectModel{}),
}
}
func (o *ObjectInfoGorm) NewTx(tx any) relation.ObjectInfoModelInterface {
return &ObjectInfoGorm{
NewMetaDB(tx.(*gorm.DB), &relation.ObjectModel{}),
}
}
func (o *ObjectInfoGorm) SetObject(ctx context.Context, obj *relation.ObjectModel) (err error) {
if err := o.DB.WithContext(ctx).Where("name = ?", obj.Name).FirstOrCreate(obj).Error; err != nil {
return errs.Wrap(err)
}
return nil
}
func (o *ObjectInfoGorm) Take(ctx context.Context, name string) (info *relation.ObjectModel, err error) {
info = &relation.ObjectModel{}
return info, errs.Wrap(o.DB.WithContext(ctx).Where("name = ?", name).Take(info).Error)
}
@@ -1,47 +0,0 @@
package relation
import (
"context"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
"gorm.io/gorm"
"time"
)
type ObjectPutGorm struct {
*MetaDB
}
func NewObjectPut(db *gorm.DB) relation.ObjectPutModelInterface {
return &ObjectPutGorm{
NewMetaDB(db, &relation.ObjectPutModel{}),
}
}
func (o *ObjectPutGorm) NewTx(tx any) relation.ObjectPutModelInterface {
return &ObjectPutGorm{
NewMetaDB(tx.(*gorm.DB), &relation.ObjectPutModel{}),
}
}
func (o *ObjectPutGorm) Create(ctx context.Context, m []*relation.ObjectPutModel) (err error) {
return utils.Wrap1(o.DB.Create(m).Error)
}
func (o *ObjectPutGorm) Take(ctx context.Context, putID string) (put *relation.ObjectPutModel, err error) {
put = &relation.ObjectPutModel{}
return put, utils.Wrap1(o.DB.Where("put_id = ?", putID).Take(put).Error)
}
func (o *ObjectPutGorm) SetCompleted(ctx context.Context, putID string) (err error) {
return utils.Wrap1(o.DB.Model(&relation.ObjectPutModel{}).Where("put_id = ?", putID).Update("complete", true).Error)
}
func (o *ObjectPutGorm) FindExpirationPut(ctx context.Context, expirationTime time.Time, num int) (list []*relation.ObjectPutModel, err error) {
err = o.DB.Where("effective_time <= ?", expirationTime).Limit(num).Find(&list).Error
return list, utils.Wrap1(err)
}
func (o *ObjectPutGorm) DelPut(ctx context.Context, ids []string) (err error) {
return utils.Wrap1(o.DB.Where("put_id IN ?", ids).Delete(&relation.ObjectPutModel{}).Error)
}
+10 -4
View File
@@ -25,7 +25,7 @@ func (u *UserGorm) Create(ctx context.Context, users []*relation.UserModel) (err
// 更新用户信息 零值
func (u *UserGorm) UpdateByMap(ctx context.Context, userID string, args map[string]interface{}) (err error) {
return utils.Wrap(u.db(ctx).Where("user_id = ?", userID).Updates(args).Error, "")
return utils.Wrap(u.db(ctx).Model(&relation.UserModel{}).Where("user_id = ?", userID).Updates(args).Error, "")
}
// 更新多个用户信息 非零值
@@ -67,9 +67,15 @@ func (u *UserGorm) GetUserGlobalRecvMsgOpt(ctx context.Context, userID string) (
return opt, err
}
func (u *UserGorm) CountTotal(ctx context.Context) (count int64, err error) {
err = u.db(ctx).Model(&relation.UserModel{}).Count(&count).Error
return count, errs.Wrap(err)
func (u *UserGorm) CountTotal(ctx context.Context, before *time.Time) (count int64, err error) {
db := u.db(ctx).Model(&relation.UserModel{})
if before != nil {
db = db.Where("create_time < ?", before)
}
if err := db.Count(&count).Error; err != nil {
return 0, err
}
return count, nil
}
func (u *UserGorm) CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error) {
+9
View File
@@ -0,0 +1,9 @@
package cont
const (
hashPath = "openim/data/hash/"
tempPath = "openim/temp/"
UploadTypeMultipart = 1 // 分片上传
UploadTypePresigned = 2 // 预签名上传
partSeparator = ","
)
+244
View File
@@ -0,0 +1,244 @@
package cont
import (
"context"
"crypto/md5"
"encoding/hex"
"errors"
"fmt"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
"github.com/google/uuid"
"path"
"strings"
"time"
)
func New(impl s3.Interface) *Controller {
return &Controller{impl: impl}
}
type Controller struct {
impl s3.Interface
}
func (c *Controller) HashPath(md5 string) string {
return path.Join(hashPath, md5)
}
func (c *Controller) NowPath() string {
now := time.Now()
return path.Join(
fmt.Sprintf("%04d", now.Year()),
fmt.Sprintf("%02d", now.Month()),
fmt.Sprintf("%02d", now.Day()),
fmt.Sprintf("%02d", now.Hour()),
fmt.Sprintf("%02d", now.Minute()),
fmt.Sprintf("%02d", now.Second()),
)
}
func (c *Controller) UUID() string {
id := uuid.New()
return hex.EncodeToString(id[:])
}
func (c *Controller) PartSize(ctx context.Context, size int64) (int64, error) {
return c.impl.PartSize(ctx, size)
}
func (c *Controller) PartLimit() *s3.PartLimit {
return c.impl.PartLimit()
}
func (c *Controller) GetHashObject(ctx context.Context, hash string) (*s3.ObjectInfo, error) {
return c.impl.StatObject(ctx, c.HashPath(hash))
}
func (c *Controller) InitiateUpload(ctx context.Context, hash string, size int64, expire time.Duration, maxParts int) (*InitiateUploadResult, error) {
defer log.ZDebug(ctx, "return")
if size < 0 {
return nil, errors.New("invalid size")
}
if hashBytes, err := hex.DecodeString(hash); err != nil {
return nil, err
} else if len(hashBytes) != md5.Size {
return nil, errors.New("invalid md5")
}
partSize, err := c.impl.PartSize(ctx, size)
if err != nil {
return nil, err
}
partNumber := int(size / partSize)
if size%partSize > 0 {
partNumber++
}
if maxParts > 0 && partNumber > 0 && partNumber < maxParts {
return nil, errors.New(fmt.Sprintf("too many parts: %d", partNumber))
}
if info, err := c.impl.StatObject(ctx, c.HashPath(hash)); err == nil {
return nil, &HashAlreadyExistsError{Object: info}
} else if !c.impl.IsNotFound(err) {
return nil, err
}
if size <= partSize {
// 预签名上传
key := path.Join(tempPath, c.NowPath(), fmt.Sprintf("%s_%d_%s.presigned", hash, size, c.UUID()))
rawURL, err := c.impl.PresignedPutObject(ctx, key, expire)
if err != nil {
return nil, err
}
return &InitiateUploadResult{
UploadID: newMultipartUploadID(multipartUploadID{
Type: UploadTypePresigned,
ID: "",
Key: key,
Size: size,
Hash: hash,
}),
PartSize: partSize,
Sign: &s3.AuthSignResult{
Parts: []s3.SignPart{
{
PartNumber: 1,
URL: rawURL,
},
},
},
}, nil
} else {
// 分片上传
upload, err := c.impl.InitiateMultipartUpload(ctx, c.HashPath(hash))
if err != nil {
return nil, err
}
if maxParts < 0 {
maxParts = partNumber
}
var authSign *s3.AuthSignResult
if maxParts > 0 {
partNumbers := make([]int, partNumber)
for i := 0; i < maxParts; i++ {
partNumbers[i] = i + 1
}
authSign, err = c.impl.AuthSign(ctx, upload.UploadID, upload.Key, time.Hour*24, partNumbers)
if err != nil {
return nil, err
}
}
return &InitiateUploadResult{
UploadID: newMultipartUploadID(multipartUploadID{
Type: UploadTypeMultipart,
ID: upload.UploadID,
Key: upload.Key,
Size: size,
Hash: hash,
}),
PartSize: partSize,
Sign: authSign,
}, nil
}
}
func (c *Controller) CompleteUpload(ctx context.Context, uploadID string, partHashs []string) (*UploadResult, error) {
defer log.ZDebug(ctx, "return")
upload, err := parseMultipartUploadID(uploadID)
if err != nil {
return nil, err
}
if md5Sum := md5.Sum([]byte(strings.Join(partHashs, partSeparator))); hex.EncodeToString(md5Sum[:]) != upload.Hash {
fmt.Println("CompleteUpload sum:", hex.EncodeToString(md5Sum[:]), "upload hash:", upload.Hash)
return nil, errors.New("md5 mismatching")
}
if info, err := c.impl.StatObject(ctx, c.HashPath(upload.Hash)); err == nil {
return &UploadResult{
Key: info.Key,
Size: info.Size,
Hash: info.ETag,
}, nil
} else if !c.impl.IsNotFound(err) {
return nil, err
}
cleanObject := make(map[string]struct{})
defer func() {
for key := range cleanObject {
_ = c.impl.DeleteObject(ctx, key)
}
}()
var targetKey string
switch upload.Type {
case UploadTypeMultipart:
parts := make([]s3.Part, len(partHashs))
for i, part := range partHashs {
parts[i] = s3.Part{
PartNumber: i + 1,
ETag: part,
}
}
// todo: 验证大小
result, err := c.impl.CompleteMultipartUpload(ctx, upload.ID, upload.Key, parts)
if err != nil {
return nil, err
}
targetKey = result.Key
case UploadTypePresigned:
uploadInfo, err := c.impl.StatObject(ctx, upload.Key)
if err != nil {
return nil, err
}
cleanObject[uploadInfo.Key] = struct{}{}
if uploadInfo.Size != upload.Size {
return nil, errors.New("upload size mismatching")
}
md5Sum := md5.Sum([]byte(strings.Join([]string{uploadInfo.ETag}, partSeparator)))
if md5val := hex.EncodeToString(md5Sum[:]); md5val != upload.Hash {
return nil, errs.ErrArgs.Wrap(fmt.Sprintf("md5 mismatching %s != %s", md5val, upload.Hash))
}
// 防止在这个时候,并发操作,导致文件被覆盖
copyInfo, err := c.impl.CopyObject(ctx, uploadInfo.Key, upload.Key+"."+c.UUID())
if err != nil {
return nil, err
}
cleanObject[copyInfo.Key] = struct{}{}
if copyInfo.ETag != uploadInfo.ETag {
return nil, errors.New("[concurrency]copy md5 mismatching")
}
hashCopyInfo, err := c.impl.CopyObject(ctx, copyInfo.Key, c.HashPath(upload.Hash))
if err != nil {
return nil, err
}
log.ZInfo(ctx, "hashCopyInfo", "value", fmt.Sprintf("%+v", hashCopyInfo))
targetKey = hashCopyInfo.Key
default:
return nil, errors.New("invalid upload id type")
}
return &UploadResult{
Key: targetKey,
Size: upload.Size,
Hash: upload.Hash,
}, nil
}
func (c *Controller) AuthSign(ctx context.Context, uploadID string, partNumbers []int) (*s3.AuthSignResult, error) {
upload, err := parseMultipartUploadID(uploadID)
if err != nil {
return nil, err
}
switch upload.Type {
case UploadTypeMultipart:
return c.impl.AuthSign(ctx, upload.ID, upload.Key, time.Hour*24, partNumbers)
case UploadTypePresigned:
return nil, errors.New("presigned id not support auth sign")
default:
return nil, errors.New("invalid upload id type")
}
}
func (c *Controller) IsNotFound(err error) bool {
return c.impl.IsNotFound(err)
}
func (c *Controller) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
return c.impl.AccessURL(ctx, name, expire, opt)
}
+14
View File
@@ -0,0 +1,14 @@
package cont
import (
"fmt"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
)
type HashAlreadyExistsError struct {
Object *s3.ObjectInfo
}
func (e *HashAlreadyExistsError) Error() string {
return fmt.Sprintf("hash already exists: %s", e.Object.Key)
}
+35
View File
@@ -0,0 +1,35 @@
package cont
import (
"encoding/base64"
"encoding/json"
"fmt"
)
type multipartUploadID struct {
Type int `json:"a,omitempty"`
ID string `json:"b,omitempty"`
Key string `json:"c,omitempty"`
Size int64 `json:"d,omitempty"`
Hash string `json:"e,omitempty"`
}
func newMultipartUploadID(id multipartUploadID) string {
data, err := json.Marshal(id)
if err != nil {
panic(err)
}
return base64.StdEncoding.EncodeToString(data)
}
func parseMultipartUploadID(id string) (*multipartUploadID, error) {
data, err := base64.StdEncoding.DecodeString(id)
if err != nil {
return nil, fmt.Errorf("invalid multipart upload id: %w", err)
}
var upload multipartUploadID
if err := json.Unmarshal(data, &upload); err != nil {
return nil, fmt.Errorf("invalid multipart upload id: %w", err)
}
return &upload, nil
}
+15
View File
@@ -0,0 +1,15 @@
package cont
import "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
type InitiateUploadResult struct {
UploadID string `json:"uploadID"` // 上传ID
PartSize int64 `json:"partSize"` // 分片大小
Sign *s3.AuthSignResult `json:"sign"` // 分片信息
}
type UploadResult struct {
Hash string `json:"hash"`
Size int64 `json:"size"`
Key string `json:"key"`
}
+254
View File
@@ -0,0 +1,254 @@
package cos
import (
"context"
"errors"
"fmt"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
"github.com/tencentyun/cos-go-sdk-v5"
"net/http"
"net/url"
"strconv"
"strings"
"time"
)
const (
minPartSize = 1024 * 1024 * 1 // 1MB
maxPartSize = 1024 * 1024 * 1024 * 5 // 5GB
maxNumSize = 1000
)
func NewCos() (s3.Interface, error) {
conf := config.Config.Object.Cos
u, err := url.Parse(conf.BucketURL)
if err != nil {
panic(err)
}
client := cos.NewClient(&cos.BaseURL{BucketURL: u}, &http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: conf.SecretID,
SecretKey: conf.SecretKey,
SessionToken: conf.SessionToken,
},
})
return &Cos{
copyURL: u.Host + "/",
client: client,
credential: client.GetCredential(),
}, nil
}
type Cos struct {
copyURL string
client *cos.Client
credential *cos.Credential
}
func (c *Cos) Engine() string {
return "tencent-cos"
}
func (c *Cos) PartLimit() *s3.PartLimit {
return &s3.PartLimit{
MinPartSize: minPartSize,
MaxPartSize: maxPartSize,
MaxNumSize: maxNumSize,
}
}
func (c *Cos) InitiateMultipartUpload(ctx context.Context, name string) (*s3.InitiateMultipartUploadResult, error) {
result, _, err := c.client.Object.InitiateMultipartUpload(ctx, name, nil)
if err != nil {
return nil, err
}
return &s3.InitiateMultipartUploadResult{
UploadID: result.UploadID,
Bucket: result.Bucket,
Key: result.Key,
}, nil
}
func (c *Cos) CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []s3.Part) (*s3.CompleteMultipartUploadResult, error) {
opts := &cos.CompleteMultipartUploadOptions{
Parts: make([]cos.Object, len(parts)),
}
for i, part := range parts {
opts.Parts[i] = cos.Object{
PartNumber: part.PartNumber,
ETag: strings.ReplaceAll(part.ETag, `"`, ``),
}
}
result, _, err := c.client.Object.CompleteMultipartUpload(ctx, name, uploadID, opts)
if err != nil {
return nil, err
}
return &s3.CompleteMultipartUploadResult{
Location: result.Location,
Bucket: result.Bucket,
Key: result.Key,
ETag: result.ETag,
}, nil
}
func (c *Cos) PartSize(ctx context.Context, size int64) (int64, error) {
if size <= 0 {
return 0, errors.New("size must be greater than 0")
}
if size > maxPartSize*maxNumSize {
return 0, fmt.Errorf("size must be less than %db", maxPartSize*maxNumSize)
}
if size <= minPartSize*maxNumSize {
return minPartSize, nil
}
partSize := size / maxNumSize
if size%maxNumSize != 0 {
partSize++
}
return partSize, nil
}
func (c *Cos) AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*s3.AuthSignResult, error) {
result := s3.AuthSignResult{
URL: c.client.BaseURL.BucketURL.String() + "/" + cos.EncodeURIComponent(name),
Query: url.Values{"uploadId": {uploadID}},
Header: make(http.Header),
Parts: make([]s3.SignPart, len(partNumbers)),
}
req, err := http.NewRequestWithContext(ctx, http.MethodPut, result.URL, nil)
if err != nil {
return nil, err
}
cos.AddAuthorizationHeader(c.credential.SecretID, c.credential.SecretKey, c.credential.SessionToken, req, cos.NewAuthTime(expire))
result.Header = req.Header
for i, partNumber := range partNumbers {
result.Parts[i] = s3.SignPart{
PartNumber: partNumber,
Query: url.Values{"partNumber": {strconv.Itoa(partNumber)}},
}
}
return &result, nil
}
func (c *Cos) PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error) {
rawURL, err := c.client.Object.GetPresignedURL(ctx, http.MethodPut, name, c.credential.SecretID, c.credential.SecretKey, expire, nil)
if err != nil {
return "", err
}
return rawURL.String(), nil
}
func (c *Cos) DeleteObject(ctx context.Context, name string) error {
_, err := c.client.Object.Delete(ctx, name)
return err
}
func (c *Cos) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) {
if name != "" && name[0] == '/' {
name = name[1:]
}
info, err := c.client.Object.Head(ctx, name, nil)
if err != nil {
return nil, err
}
res := &s3.ObjectInfo{Key: name}
if res.ETag = strings.ToLower(strings.ReplaceAll(info.Header.Get("ETag"), `"`, "")); res.ETag == "" {
return nil, errors.New("StatObject etag not found")
}
if contentLengthStr := info.Header.Get("Content-Length"); contentLengthStr == "" {
return nil, errors.New("StatObject content-length not found")
} else {
res.Size, err = strconv.ParseInt(contentLengthStr, 10, 64)
if err != nil {
return nil, fmt.Errorf("StatObject content-length parse error: %w", err)
}
if res.Size < 0 {
return nil, errors.New("StatObject content-length must be greater than 0")
}
}
if lastModified := info.Header.Get("Last-Modified"); lastModified == "" {
return nil, errors.New("StatObject last-modified not found")
} else {
res.LastModified, err = time.Parse(http.TimeFormat, lastModified)
if err != nil {
return nil, fmt.Errorf("StatObject last-modified parse error: %w", err)
}
}
return res, nil
}
func (c *Cos) CopyObject(ctx context.Context, src string, dst string) (*s3.CopyObjectInfo, error) {
sourceURL := c.copyURL + src
result, _, err := c.client.Object.Copy(ctx, dst, sourceURL, nil)
if err != nil {
return nil, err
}
return &s3.CopyObjectInfo{
Key: dst,
ETag: strings.ReplaceAll(result.ETag, `"`, ``),
}, nil
}
func (c *Cos) IsNotFound(err error) bool {
switch e := err.(type) {
case *cos.ErrorResponse:
return e.Response.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
default:
return false
}
}
func (c *Cos) AbortMultipartUpload(ctx context.Context, uploadID string, name string) error {
_, err := c.client.Object.AbortMultipartUpload(ctx, name, uploadID)
return err
}
func (c *Cos) ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*s3.ListUploadedPartsResult, error) {
result, _, err := c.client.Object.ListParts(ctx, name, uploadID, &cos.ObjectListPartsOptions{
MaxParts: strconv.Itoa(maxParts),
PartNumberMarker: strconv.Itoa(partNumberMarker),
})
if err != nil {
return nil, err
}
res := &s3.ListUploadedPartsResult{
Key: result.Key,
UploadID: result.UploadID,
UploadedParts: make([]s3.UploadedPart, len(result.Parts)),
}
res.MaxParts, _ = strconv.Atoi(result.MaxParts)
res.NextPartNumberMarker, _ = strconv.Atoi(result.NextPartNumberMarker)
for i, part := range result.Parts {
lastModified, _ := time.Parse(http.TimeFormat, part.LastModified)
res.UploadedParts[i] = s3.UploadedPart{
PartNumber: part.PartNumber,
LastModified: lastModified,
ETag: part.ETag,
Size: part.Size,
}
}
return res, nil
}
func (c *Cos) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
//reqParams := make(url.Values)
//if opt != nil {
// if opt.ContentType != "" {
// reqParams.Set("Content-Type", opt.ContentType)
// }
// if opt.ContentDisposition != "" {
// reqParams.Set("Content-Disposition", opt.ContentDisposition)
// }
//}
if expire <= 0 {
expire = time.Hour * 24 * 365 * 99 // 99 years
} else if expire < time.Second {
expire = time.Second
}
rawURL, err := c.client.Object.GetPresignedURL(ctx, http.MethodGet, name, c.credential.SecretID, c.credential.SecretKey, expire, nil)
if err != nil {
return "", err
}
return rawURL.String(), nil
}
+250
View File
@@ -0,0 +1,250 @@
package minio
import (
"context"
"errors"
"fmt"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio-go/v7/pkg/signer"
"net/http"
"net/url"
"strconv"
"strings"
"time"
)
const (
unsignedPayload = "UNSIGNED-PAYLOAD"
)
const (
minPartSize = 1024 * 1024 * 5 // 1MB
maxPartSize = 1024 * 1024 * 1024 * 5 // 5GB
maxNumSize = 10000
)
func NewMinio() (s3.Interface, error) {
conf := config.Config.Object.Minio
u, err := url.Parse(conf.Endpoint)
if err != nil {
return nil, err
}
opts := &minio.Options{
Creds: credentials.NewStaticV4(conf.AccessKeyID, conf.SecretAccessKey, conf.SessionToken),
Secure: u.Scheme == "https",
}
client, err := minio.New(u.Host, opts)
if err != nil {
return nil, err
}
return &Minio{
bucket: conf.Bucket,
bucketURL: conf.Endpoint + "/" + conf.Bucket + "/",
opts: opts,
core: &minio.Core{Client: client},
}, nil
}
type Minio struct {
bucket string
bucketURL string
opts *minio.Options
core *minio.Core
}
func (m *Minio) Engine() string {
return "minio"
}
func (m *Minio) PartLimit() *s3.PartLimit {
return &s3.PartLimit{
MinPartSize: minPartSize,
MaxPartSize: maxPartSize,
MaxNumSize: maxNumSize,
}
}
func (m *Minio) InitiateMultipartUpload(ctx context.Context, name string) (*s3.InitiateMultipartUploadResult, error) {
uploadID, err := m.core.NewMultipartUpload(ctx, m.bucket, name, minio.PutObjectOptions{})
if err != nil {
return nil, err
}
return &s3.InitiateMultipartUploadResult{
Bucket: m.bucket,
Key: name,
UploadID: uploadID,
}, nil
}
func (m *Minio) CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []s3.Part) (*s3.CompleteMultipartUploadResult, error) {
minioParts := make([]minio.CompletePart, len(parts))
for i, part := range parts {
minioParts[i] = minio.CompletePart{
PartNumber: part.PartNumber,
ETag: strings.ToLower(part.ETag),
}
}
upload, err := m.core.CompleteMultipartUpload(ctx, m.bucket, name, uploadID, minioParts, minio.PutObjectOptions{})
if err != nil {
return nil, err
}
return &s3.CompleteMultipartUploadResult{
Location: upload.Location,
Bucket: upload.Bucket,
Key: upload.Key,
ETag: strings.ToLower(upload.ETag),
}, nil
}
func (m *Minio) PartSize(ctx context.Context, size int64) (int64, error) {
if size <= 0 {
return 0, errors.New("size must be greater than 0")
}
if size > maxPartSize*maxNumSize {
return 0, fmt.Errorf("size must be less than %db", maxPartSize*maxNumSize)
}
if size <= minPartSize*maxNumSize {
return minPartSize, nil
}
partSize := size / maxNumSize
if size%maxNumSize != 0 {
partSize++
}
return partSize, nil
}
func (m *Minio) AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*s3.AuthSignResult, error) {
creds, err := m.opts.Creds.Get()
if err != nil {
return nil, err
}
result := s3.AuthSignResult{
URL: m.bucketURL + name,
Query: url.Values{"uploadId": {uploadID}},
Parts: make([]s3.SignPart, len(partNumbers)),
}
for i, partNumber := range partNumbers {
rawURL := result.URL + "?partNumber=" + strconv.Itoa(partNumber) + "&uploadId=" + uploadID
request, err := http.NewRequestWithContext(ctx, http.MethodPut, rawURL, nil)
if err != nil {
return nil, err
}
request.Header.Set("X-Amz-Content-Sha256", unsignedPayload)
request = signer.SignV4Trailer(*request, creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken, "us-east-1", nil)
result.Parts[i] = s3.SignPart{
PartNumber: partNumber,
URL: request.URL.String(),
Query: url.Values{"partNumber": {strconv.Itoa(partNumber)}},
Header: request.Header,
}
}
return &result, nil
}
func (m *Minio) PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error) {
rawURL, err := m.core.Client.PresignedPutObject(ctx, m.bucket, name, expire)
if err != nil {
return "", err
}
return rawURL.String(), nil
}
func (m *Minio) DeleteObject(ctx context.Context, name string) error {
return m.core.Client.RemoveObject(ctx, m.bucket, name, minio.RemoveObjectOptions{})
}
func (m *Minio) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) {
info, err := m.core.Client.StatObject(ctx, m.bucket, name, minio.StatObjectOptions{})
if err != nil {
return nil, err
}
return &s3.ObjectInfo{
ETag: strings.ToLower(info.ETag),
Key: info.Key,
Size: info.Size,
LastModified: info.LastModified,
}, nil
}
func (m *Minio) CopyObject(ctx context.Context, src string, dst string) (*s3.CopyObjectInfo, error) {
result, err := m.core.Client.CopyObject(ctx, minio.CopyDestOptions{
Bucket: m.bucket,
Object: dst,
}, minio.CopySrcOptions{
Bucket: m.bucket,
Object: src,
})
if err != nil {
return nil, err
}
return &s3.CopyObjectInfo{
Key: dst,
ETag: strings.ToLower(result.ETag),
}, nil
}
func (m *Minio) IsNotFound(err error) bool {
if err == nil {
return false
}
switch e := err.(type) {
case minio.ErrorResponse:
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
case *minio.ErrorResponse:
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
default:
return false
}
}
func (m *Minio) AbortMultipartUpload(ctx context.Context, uploadID string, name string) error {
return m.core.AbortMultipartUpload(ctx, m.bucket, name, uploadID)
}
func (m *Minio) ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*s3.ListUploadedPartsResult, error) {
result, err := m.core.ListObjectParts(ctx, m.bucket, name, uploadID, partNumberMarker, maxParts)
if err != nil {
return nil, err
}
res := &s3.ListUploadedPartsResult{
Key: result.Key,
UploadID: result.UploadID,
MaxParts: result.MaxParts,
NextPartNumberMarker: result.NextPartNumberMarker,
UploadedParts: make([]s3.UploadedPart, len(result.ObjectParts)),
}
for i, part := range result.ObjectParts {
res.UploadedParts[i] = s3.UploadedPart{
PartNumber: part.PartNumber,
LastModified: part.LastModified,
ETag: part.ETag,
Size: part.Size,
}
}
return res, nil
}
func (m *Minio) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
//reqParams := make(url.Values)
//if opt != nil {
// if opt.ContentType != "" {
// reqParams.Set("Content-Type", opt.ContentType)
// }
// if opt.ContentDisposition != "" {
// reqParams.Set("Content-Disposition", opt.ContentDisposition)
// }
//}
if expire <= 0 {
expire = time.Hour * 24 * 365 * 99 // 99 years
} else if expire < time.Second {
expire = time.Second
}
u, err := m.core.Client.PresignedGetObject(ctx, m.bucket, name, expire, nil)
if err != nil {
return "", err
}
return u.String(), nil
}
+259
View File
@@ -0,0 +1,259 @@
package oss
import (
"context"
"errors"
"fmt"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
"net/http"
"net/url"
"strconv"
"strings"
"time"
)
const (
minPartSize = 1024 * 1024 * 1 // 1MB
maxPartSize = 1024 * 1024 * 1024 * 5 // 5GB
maxNumSize = 10000
)
func NewOSS() (s3.Interface, error) {
conf := config.Config.Object.Oss
if conf.BucketURL == "" {
return nil, errors.New("bucket url is empty")
}
client, err := oss.New(conf.Endpoint, conf.AccessKeyID, conf.AccessKeySecret)
if err != nil {
return nil, err
}
bucket, err := client.Bucket(conf.Bucket)
if err != nil {
return nil, err
}
if conf.BucketURL[len(conf.BucketURL)-1] != '/' {
conf.BucketURL += "/"
}
return &OSS{
bucketURL: conf.BucketURL,
bucket: bucket,
credentials: client.Config.GetCredentials(),
}, nil
}
type OSS struct {
bucketURL string
bucket *oss.Bucket
credentials oss.Credentials
}
func (o *OSS) Engine() string {
return "ali-oss"
}
func (o *OSS) PartLimit() *s3.PartLimit {
return &s3.PartLimit{
MinPartSize: minPartSize,
MaxPartSize: maxPartSize,
MaxNumSize: maxNumSize,
}
}
func (o *OSS) InitiateMultipartUpload(ctx context.Context, name string) (*s3.InitiateMultipartUploadResult, error) {
result, err := o.bucket.InitiateMultipartUpload(name)
if err != nil {
return nil, err
}
return &s3.InitiateMultipartUploadResult{
UploadID: result.UploadID,
Bucket: result.Bucket,
Key: result.Key,
}, nil
}
func (o *OSS) CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []s3.Part) (*s3.CompleteMultipartUploadResult, error) {
ossParts := make([]oss.UploadPart, len(parts))
for i, part := range parts {
ossParts[i] = oss.UploadPart{
PartNumber: part.PartNumber,
ETag: strings.ToUpper(part.ETag),
}
}
result, err := o.bucket.CompleteMultipartUpload(oss.InitiateMultipartUploadResult{
UploadID: uploadID,
Bucket: o.bucket.BucketName,
Key: name,
}, ossParts)
if err != nil {
return nil, err
}
return &s3.CompleteMultipartUploadResult{
Location: result.Location,
Bucket: result.Bucket,
Key: result.Key,
ETag: strings.ToLower(strings.ReplaceAll(result.ETag, `"`, ``)),
}, nil
}
func (o *OSS) PartSize(ctx context.Context, size int64) (int64, error) {
if size <= 0 {
return 0, errors.New("size must be greater than 0")
}
if size > maxPartSize*maxNumSize {
return 0, fmt.Errorf("size must be less than %db", maxPartSize*maxNumSize)
}
if size <= minPartSize*maxNumSize {
return minPartSize, nil
}
partSize := size / maxNumSize
if size%maxNumSize != 0 {
partSize++
}
return partSize, nil
}
func (o *OSS) AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*s3.AuthSignResult, error) {
result := s3.AuthSignResult{
URL: o.bucketURL + name,
Query: url.Values{"uploadId": {uploadID}},
Header: make(http.Header),
Parts: make([]s3.SignPart, len(partNumbers)),
}
for i, partNumber := range partNumbers {
rawURL := fmt.Sprintf(`%s%s?partNumber=%d&uploadId=%s`, o.bucketURL, name, partNumber, uploadID)
request, err := http.NewRequestWithContext(ctx, http.MethodPut, rawURL, nil)
if err != nil {
return nil, err
}
if o.credentials.GetSecurityToken() != "" {
request.Header.Set(oss.HTTPHeaderOssSecurityToken, o.credentials.GetSecurityToken())
}
request.Header.Set(oss.HTTPHeaderHost, request.Host)
request.Header.Set(oss.HTTPHeaderDate, time.Now().UTC().Format(http.TimeFormat))
authorization := fmt.Sprintf(`OSS %s:%s`, o.credentials.GetAccessKeyID(), o.getSignedStr(request, fmt.Sprintf(`/%s/%s?partNumber=%d&uploadId=%s`, o.bucket.BucketName, name, partNumber, uploadID), o.credentials.GetAccessKeySecret()))
request.Header.Set(oss.HTTPHeaderAuthorization, authorization)
result.Parts[i] = s3.SignPart{
PartNumber: partNumber,
Query: url.Values{"partNumber": {strconv.Itoa(partNumber)}},
URL: request.URL.String(),
Header: request.Header,
}
}
return &result, nil
}
func (o *OSS) PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error) {
return o.bucket.SignURL(name, http.MethodPut, int64(expire/time.Second))
}
func (o *OSS) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) {
header, err := o.bucket.GetObjectMeta(name)
if err != nil {
return nil, err
}
res := &s3.ObjectInfo{Key: name}
if res.ETag = strings.ToLower(strings.ReplaceAll(header.Get("ETag"), `"`, ``)); res.ETag == "" {
return nil, errors.New("StatObject etag not found")
}
if contentLengthStr := header.Get("Content-Length"); contentLengthStr == "" {
return nil, errors.New("StatObject content-length not found")
} else {
res.Size, err = strconv.ParseInt(contentLengthStr, 10, 64)
if err != nil {
return nil, fmt.Errorf("StatObject content-length parse error: %w", err)
}
if res.Size < 0 {
return nil, errors.New("StatObject content-length must be greater than 0")
}
}
if lastModified := header.Get("Last-Modified"); lastModified == "" {
return nil, errors.New("StatObject last-modified not found")
} else {
res.LastModified, err = time.Parse(http.TimeFormat, lastModified)
if err != nil {
return nil, fmt.Errorf("StatObject last-modified parse error: %w", err)
}
}
return res, nil
}
func (o *OSS) DeleteObject(ctx context.Context, name string) error {
return o.bucket.DeleteObject(name)
}
func (o *OSS) CopyObject(ctx context.Context, src string, dst string) (*s3.CopyObjectInfo, error) {
result, err := o.bucket.CopyObject(src, dst)
if err != nil {
return nil, err
}
return &s3.CopyObjectInfo{
Key: dst,
ETag: strings.ToLower(strings.ReplaceAll(result.ETag, `"`, ``)),
}, nil
}
func (o *OSS) IsNotFound(err error) bool {
switch e := err.(type) {
case oss.ServiceError:
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
case *oss.ServiceError:
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
default:
return false
}
}
func (o *OSS) AbortMultipartUpload(ctx context.Context, uploadID string, name string) error {
return o.bucket.AbortMultipartUpload(oss.InitiateMultipartUploadResult{
UploadID: uploadID,
Key: name,
Bucket: o.bucket.BucketName,
})
}
func (o *OSS) ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*s3.ListUploadedPartsResult, error) {
result, err := o.bucket.ListUploadedParts(oss.InitiateMultipartUploadResult{
UploadID: uploadID,
Key: name,
Bucket: o.bucket.BucketName,
}, oss.MaxUploads(100), oss.MaxParts(maxParts), oss.PartNumberMarker(partNumberMarker))
if err != nil {
return nil, err
}
res := &s3.ListUploadedPartsResult{
Key: result.Key,
UploadID: result.UploadID,
MaxParts: result.MaxParts,
UploadedParts: make([]s3.UploadedPart, len(result.UploadedParts)),
}
res.NextPartNumberMarker, _ = strconv.Atoi(result.NextPartNumberMarker)
for i, part := range result.UploadedParts {
res.UploadedParts[i] = s3.UploadedPart{
PartNumber: part.PartNumber,
LastModified: part.LastModified,
ETag: part.ETag,
Size: int64(part.Size),
}
}
return res, nil
}
func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
//var opts []oss.Option
//if opt != nil {
// if opt.ContentType != "" {
// opts = append(opts, oss.ContentType(opt.ContentType))
// }
// if opt.ContentDisposition != "" {
// opts = append(opts, oss.ContentDisposition(opt.ContentDisposition))
// }
//}
if expire <= 0 {
expire = time.Hour * 24 * 365 * 99 // 99 years
} else if expire < time.Second {
expire = time.Second
}
return o.bucket.SignURL(name, http.MethodGet, int64(expire/time.Second))
}
+81
View File
@@ -0,0 +1,81 @@
package oss
import (
"crypto/hmac"
"crypto/sha1"
"crypto/sha256"
"encoding/base64"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
"hash"
"io"
"net/http"
"sort"
"strings"
)
func (o *OSS) getAdditionalHeaderKeys(req *http.Request) ([]string, map[string]string) {
var keysList []string
keysMap := make(map[string]string)
srcKeys := make(map[string]string)
for k := range req.Header {
srcKeys[strings.ToLower(k)] = ""
}
for _, v := range o.bucket.Client.Config.AdditionalHeaders {
if _, ok := srcKeys[strings.ToLower(v)]; ok {
keysMap[strings.ToLower(v)] = ""
}
}
for k := range keysMap {
keysList = append(keysList, k)
}
sort.Strings(keysList)
return keysList, keysMap
}
func (o *OSS) getSignedStr(req *http.Request, canonicalizedResource string, keySecret string) string {
// Find out the "x-oss-"'s address in header of the request
ossHeadersMap := make(map[string]string)
additionalList, additionalMap := o.getAdditionalHeaderKeys(req)
for k, v := range req.Header {
if strings.HasPrefix(strings.ToLower(k), "x-oss-") {
ossHeadersMap[strings.ToLower(k)] = v[0]
} else if o.bucket.Client.Config.AuthVersion == oss.AuthV2 {
if _, ok := additionalMap[strings.ToLower(k)]; ok {
ossHeadersMap[strings.ToLower(k)] = v[0]
}
}
}
hs := newHeaderSorter(ossHeadersMap)
// Sort the ossHeadersMap by the ascending order
hs.Sort()
// Get the canonicalizedOSSHeaders
canonicalizedOSSHeaders := ""
for i := range hs.Keys {
canonicalizedOSSHeaders += hs.Keys[i] + ":" + hs.Vals[i] + "\n"
}
// Give other parameters values
// when sign URL, date is expires
date := req.Header.Get(oss.HTTPHeaderDate)
contentType := req.Header.Get(oss.HTTPHeaderContentType)
contentMd5 := req.Header.Get(oss.HTTPHeaderContentMD5)
// default is v1 signature
signStr := req.Method + "\n" + contentMd5 + "\n" + contentType + "\n" + date + "\n" + canonicalizedOSSHeaders + canonicalizedResource
h := hmac.New(func() hash.Hash { return sha1.New() }, []byte(keySecret))
// v2 signature
if o.bucket.Client.Config.AuthVersion == oss.AuthV2 {
signStr = req.Method + "\n" + contentMd5 + "\n" + contentType + "\n" + date + "\n" + canonicalizedOSSHeaders + strings.Join(additionalList, ";") + "\n" + canonicalizedResource
h = hmac.New(func() hash.Hash { return sha256.New() }, []byte(keySecret))
}
_, _ = io.WriteString(h, signStr)
signedStr := base64.StdEncoding.EncodeToString(h.Sum(nil))
return signedStr
}
+47
View File
@@ -0,0 +1,47 @@
package oss
import (
"bytes"
"sort"
)
// headerSorter defines the key-value structure for storing the sorted data in signHeader.
type headerSorter struct {
Keys []string
Vals []string
}
// newHeaderSorter is an additional function for function SignHeader.
func newHeaderSorter(m map[string]string) *headerSorter {
hs := &headerSorter{
Keys: make([]string, 0, len(m)),
Vals: make([]string, 0, len(m)),
}
for k, v := range m {
hs.Keys = append(hs.Keys, k)
hs.Vals = append(hs.Vals, v)
}
return hs
}
// Sort is an additional function for function SignHeader.
func (hs *headerSorter) Sort() {
sort.Sort(hs)
}
// Len is an additional function for function SignHeader.
func (hs *headerSorter) Len() int {
return len(hs.Vals)
}
// Less is an additional function for function SignHeader.
func (hs *headerSorter) Less(i, j int) bool {
return bytes.Compare([]byte(hs.Keys[i]), []byte(hs.Keys[j])) < 0
}
// Swap is an additional function for function SignHeader.
func (hs *headerSorter) Swap(i, j int) {
hs.Vals[i], hs.Vals[j] = hs.Vals[j], hs.Vals[i]
hs.Keys[i], hs.Keys[j] = hs.Keys[j], hs.Keys[i]
}
+134
View File
@@ -0,0 +1,134 @@
package s3
import (
"context"
"net/http"
"net/url"
"time"
)
type PartLimit struct {
MinPartSize int64 `json:"minPartSize"`
MaxPartSize int64 `json:"maxPartSize"`
MaxNumSize int `json:"maxNumSize"`
}
type InitiateMultipartUploadResult struct {
Bucket string `json:"bucket"`
Key string `json:"key"`
UploadID string `json:"uploadID"`
}
type MultipartUploadRequest struct {
UploadID string `json:"uploadId"`
Bucket string `json:"bucket"`
Key string `json:"key"`
Method string `json:"method"`
URL string `json:"url"`
Query url.Values `json:"query"`
Header http.Header `json:"header"`
PartKey string `json:"partKey"`
PartSize int64 `json:"partSize"`
FirstPart int `json:"firstPart"`
}
type Part struct {
PartNumber int `json:"partNumber"`
ETag string `json:"etag"`
}
type CompleteMultipartUploadResult struct {
Location string `json:"location"`
Bucket string `json:"bucket"`
Key string `json:"key"`
ETag string `json:"etag"`
}
type SignResult struct {
Parts []SignPart `json:"parts"`
}
type ObjectInfo struct {
ETag string `json:"etag"`
Key string `json:"name"`
Size int64 `json:"size"`
LastModified time.Time `json:"lastModified"`
}
type CopyObjectInfo struct {
Key string `json:"name"`
ETag string `json:"etag"`
}
type SignPart struct {
PartNumber int `json:"partNumber"`
URL string `json:"url"`
Query url.Values `json:"query"`
Header http.Header `json:"header"`
}
type AuthSignResult struct {
URL string `json:"url"`
Query url.Values `json:"query"`
Header http.Header `json:"header"`
Parts []SignPart `json:"parts"`
}
type InitiateUpload struct {
UploadID string `json:"uploadId"`
Bucket string `json:"bucket"`
Key string `json:"key"`
Method string `json:"method"`
URL string `json:"url"`
Query url.Values `json:"query"`
Header http.Header `json:"header"`
PartKey string `json:"partKey"`
PartSize int64 `json:"partSize"`
FirstPart int `json:"firstPart"`
}
type UploadedPart struct {
PartNumber int `json:"partNumber"`
LastModified time.Time `json:"lastModified"`
ETag string `json:"etag"`
Size int64 `json:"size"`
}
type ListUploadedPartsResult struct {
Key string `xml:"Key"`
UploadID string `xml:"UploadId"`
NextPartNumberMarker int `xml:"NextPartNumberMarker"`
MaxParts int `xml:"MaxParts"`
UploadedParts []UploadedPart `xml:"Part"`
}
type AccessURLOption struct {
ContentType string `json:"contentType"`
ContentDisposition string `json:"contentDisposition"`
}
type Interface interface {
Engine() string
PartLimit() *PartLimit
InitiateMultipartUpload(ctx context.Context, name string) (*InitiateMultipartUploadResult, error)
CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []Part) (*CompleteMultipartUploadResult, error)
PartSize(ctx context.Context, size int64) (int64, error)
AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*AuthSignResult, error)
PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error)
DeleteObject(ctx context.Context, name string) error
CopyObject(ctx context.Context, src string, dst string) (*CopyObjectInfo, error)
StatObject(ctx context.Context, name string) (*ObjectInfo, error)
IsNotFound(err error) bool
AbortMultipartUpload(ctx context.Context, uploadID string, name string) error
ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*ListUploadedPartsResult, error)
AccessURL(ctx context.Context, name string, expire time.Duration, opt *AccessURLOption) (string, error)
}
+23 -15
View File
@@ -1,26 +1,33 @@
package relation
import "context"
import (
"context"
"time"
)
const (
conversationModelTableName = "conversations"
)
type ConversationModel struct {
OwnerUserID string `gorm:"column:owner_user_id;primary_key;type:char(128)" json:"OwnerUserID"`
ConversationID string `gorm:"column:conversation_id;primary_key;type:char(128)" json:"conversationID"`
ConversationType int32 `gorm:"column:conversation_type" json:"conversationType"`
UserID string `gorm:"column:user_id;type:char(64)" json:"userID"`
GroupID string `gorm:"column:group_id;type:char(128)" json:"groupID"`
RecvMsgOpt int32 `gorm:"column:recv_msg_opt" json:"recvMsgOpt"`
IsPinned bool `gorm:"column:is_pinned" json:"isPinned"`
IsPrivateChat bool `gorm:"column:is_private_chat" json:"isPrivateChat"`
BurnDuration int32 `gorm:"column:burn_duration;default:30" json:"burnDuration"`
GroupAtType int32 `gorm:"column:group_at_type" json:"groupAtType"`
AttachedInfo string `gorm:"column:attached_info;type:varchar(1024)" json:"attachedInfo"`
Ex string `gorm:"column:ex;type:varchar(1024)" json:"ex"`
MaxSeq int64 `gorm:"column:max_seq" json:"maxSeq"`
MinSeq int64 `gorm:"column:min_seq" json:"minSeq"`
OwnerUserID string `gorm:"column:owner_user_id;primary_key;type:char(128)" json:"OwnerUserID"`
ConversationID string `gorm:"column:conversation_id;primary_key;type:char(128)" json:"conversationID"`
ConversationType int32 `gorm:"column:conversation_type" json:"conversationType"`
UserID string `gorm:"column:user_id;type:char(64)" json:"userID"`
GroupID string `gorm:"column:group_id;type:char(128)" json:"groupID"`
RecvMsgOpt int32 `gorm:"column:recv_msg_opt" json:"recvMsgOpt"`
IsPinned bool `gorm:"column:is_pinned" json:"isPinned"`
IsPrivateChat bool `gorm:"column:is_private_chat" json:"isPrivateChat"`
BurnDuration int32 `gorm:"column:burn_duration;default:30" json:"burnDuration"`
GroupAtType int32 `gorm:"column:group_at_type" json:"groupAtType"`
AttachedInfo string `gorm:"column:attached_info;type:varchar(1024)" json:"attachedInfo"`
Ex string `gorm:"column:ex;type:varchar(1024)" json:"ex"`
MaxSeq int64 `gorm:"column:max_seq" json:"maxSeq"`
MinSeq int64 `gorm:"column:min_seq" json:"minSeq"`
CreateTime time.Time `gorm:"column:create_time;index:create_time;autoCreateTime"`
IsMsgDestruct bool `gorm:"column:is_msg_destruct;default:false"`
MsgDestructTime int64 `gorm:"column:msg_destruct_time;default:604800"`
LatestMsgDestructTime time.Time `gorm:"column:latest_msg_destruct_time;autoCreateTime"`
}
func (ConversationModel) TableName() string {
@@ -44,5 +51,6 @@ type ConversationModelInterface interface {
GetAllConversationIDs(ctx context.Context) ([]string, error)
GetUserAllHasReadSeqs(ctx context.Context, ownerUserID string) (hashReadSeqs map[string]int64, err error)
GetConversationsByConversationID(ctx context.Context, conversationIDs []string) ([]*ConversationModel, error)
GetConversationIDsNeedDestruct(ctx context.Context) ([]*ConversationModel, error)
NewTx(tx any) ConversationModelInterface
}
+4
View File
@@ -40,4 +40,8 @@ type GroupModelInterface interface {
Take(ctx context.Context, groupID string) (group *GroupModel, err error)
Search(ctx context.Context, keyword string, pageNumber, showNumber int32) (total uint32, groups []*GroupModel, err error)
GetGroupIDsByGroupType(ctx context.Context, groupType int) (groupIDs []string, err error)
// 获取群总数
CountTotal(ctx context.Context, before *time.Time) (count int64, err error)
// 获取范围内群增量
CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error)
}
+31
View File
@@ -0,0 +1,31 @@
package relation
import (
"context"
"time"
)
const (
ObjectInfoModelTableName = "object"
)
type ObjectModel struct {
Name string `gorm:"column:name;primary_key"`
UserID string `gorm:"column:user_id"`
Hash string `gorm:"column:hash"`
Key string `gorm:"column:key"`
Size int64 `gorm:"column:size"`
ContentType string `gorm:"column:content_type"`
Cause string `gorm:"column:cause"`
CreateTime time.Time `gorm:"column:create_time"`
}
func (ObjectModel) TableName() string {
return ObjectInfoModelTableName
}
type ObjectInfoModelInterface interface {
NewTx(tx any) ObjectInfoModelInterface
SetObject(ctx context.Context, obj *ObjectModel) error
Take(ctx context.Context, name string) (*ObjectModel, error)
}
@@ -1,30 +0,0 @@
package relation
import (
"context"
"time"
)
const (
ObjectHashModelTableName = "object_hash"
)
type ObjectHashModel struct {
Hash string `gorm:"column:hash;primary_key;size:32"`
Engine string `gorm:"column:engine;primary_key;size:16"`
Size int64 `gorm:"column:size"`
Bucket string `gorm:"column:bucket"`
Name string `gorm:"column:name"`
CreateTime time.Time `gorm:"column:create_time"`
}
func (ObjectHashModel) TableName() string {
return ObjectHashModelTableName
}
type ObjectHashModelInterface interface {
NewTx(tx any) ObjectHashModelInterface
Take(ctx context.Context, hash string, engine string) (*ObjectHashModel, error)
Create(ctx context.Context, h []*ObjectHashModel) error
DeleteNoCitation(ctx context.Context, engine string, num int) (list []*ObjectHashModel, err error)
}
@@ -1,29 +0,0 @@
package relation
import (
"context"
"time"
)
const (
ObjectInfoModelTableName = "object_info"
)
type ObjectInfoModel struct {
Name string `gorm:"column:name;primary_key"`
Hash string `gorm:"column:hash"`
ContentType string `gorm:"column:content_type"`
ValidTime *time.Time `gorm:"column:valid_time"`
CreateTime time.Time `gorm:"column:create_time"`
}
func (ObjectInfoModel) TableName() string {
return ObjectInfoModelTableName
}
type ObjectInfoModelInterface interface {
NewTx(tx any) ObjectInfoModelInterface
SetObject(ctx context.Context, obj *ObjectInfoModel) error
Take(ctx context.Context, name string) (*ObjectInfoModel, error)
DeleteExpiration(ctx context.Context, expiration time.Time) error
}
@@ -1,37 +0,0 @@
package relation
import (
"context"
"time"
)
const (
ObjectPutModelTableName = "object_put"
)
type ObjectPutModel struct {
PutID string `gorm:"column:put_id;primary_key"`
Hash string `gorm:"column:hash"`
Path string `gorm:"column:path"`
Name string `gorm:"column:name"`
ContentType string `gorm:"column:content_type"`
ObjectSize int64 `gorm:"column:object_size"`
FragmentSize int64 `gorm:"column:fragment_size"`
PutURLsHash string `gorm:"column:put_urls_hash"`
ValidTime *time.Time `gorm:"column:valid_time"`
EffectiveTime time.Time `gorm:"column:effective_time"`
CreateTime time.Time `gorm:"column:create_time"`
}
func (ObjectPutModel) TableName() string {
return ObjectPutModelTableName
}
type ObjectPutModelInterface interface {
NewTx(tx any) ObjectPutModelInterface
Create(ctx context.Context, m []*ObjectPutModel) error
Take(ctx context.Context, putID string) (*ObjectPutModel, error)
SetCompleted(ctx context.Context, putID string) error
FindExpirationPut(ctx context.Context, expirationTime time.Time, num int) ([]*ObjectPutModel, error)
DelPut(ctx context.Context, ids []string) error
}
+2 -2
View File
@@ -14,7 +14,7 @@ type UserModel struct {
Nickname string `gorm:"column:name;size:255"`
FaceURL string `gorm:"column:face_url;size:255"`
Ex string `gorm:"column:ex;size:1024"`
CreateTime time.Time `gorm:"column:create_time;index:create_time; autoCreateTime"`
CreateTime time.Time `gorm:"column:create_time;index:create_time;autoCreateTime"`
AppMangerLevel int32 `gorm:"column:app_manger_level;default:18"`
GlobalRecvMsgOpt int32 `gorm:"column:global_recv_msg_opt"`
}
@@ -52,7 +52,7 @@ type UserModelInterface interface {
GetAllUserID(ctx context.Context) (userIDs []string, err error)
GetUserGlobalRecvMsgOpt(ctx context.Context, userID string) (opt int, err error)
// 获取用户总数
CountTotal(ctx context.Context) (count int64, err error)
CountTotal(ctx context.Context, before *time.Time) (count int64, err error)
// 获取范围内用户增量
CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error)
}
@@ -1,82 +0,0 @@
package unrelation
import (
"context"
"strconv"
"strings"
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/sdkws"
)
const (
CExtendMsgSet = "extend_msgs"
ExtendMsgMaxNum = 100
)
type ExtendMsgSetModel struct {
ConversationID string `bson:"source_id" json:"conversationID"`
SessionType int32 `bson:"session_type" json:"sessionType"`
ExtendMsgs map[string]ExtendMsgModel `bson:"extend_msgs" json:"extendMsgs"`
ExtendMsgNum int32 `bson:"extend_msg_num" json:"extendMsgNum"`
CreateTime int64 `bson:"create_time" json:"createTime"` // this block's create time
MaxMsgUpdateTime int64 `bson:"max_msg_update_time" json:"maxMsgUpdateTime"` // index find msg
}
type KeyValueModel struct {
TypeKey string `bson:"type_key" json:"typeKey"`
Value string `bson:"value" json:"value"`
LatestUpdateTime int64 `bson:"latest_update_time" json:"latestUpdateTime"`
}
type ExtendMsgModel struct {
ReactionExtensionList map[string]KeyValueModel `bson:"reaction_extension_list" json:"reactionExtensionList"`
ClientMsgID string `bson:"client_msg_id" json:"clientMsgID"`
MsgFirstModifyTime int64 `bson:"msg_first_modify_time" json:"msgFirstModifyTime"` // this extendMsg create time
AttachedInfo string `bson:"attached_info" json:"attachedInfo"`
Ex string `bson:"ex" json:"ex"`
}
type ExtendMsgSetModelInterface interface {
CreateExtendMsgSet(ctx context.Context, set *ExtendMsgSetModel) error
GetAllExtendMsgSet(ctx context.Context, conversationID string, opts *GetAllExtendMsgSetOpts) (sets []*ExtendMsgSetModel, err error)
GetExtendMsgSet(ctx context.Context, conversationID string, sessionType int32, maxMsgUpdateTime int64) (*ExtendMsgSetModel, error)
InsertExtendMsg(ctx context.Context, conversationID string, sessionType int32, msg *ExtendMsgModel) error
InsertOrUpdateReactionExtendMsgSet(ctx context.Context, conversationID string, sessionType int32, clientMsgID string, msgFirstModifyTime int64, reactionExtensionList map[string]*KeyValueModel) error
DeleteReactionExtendMsgSet(ctx context.Context, conversationID string, sessionType int32, clientMsgID string, msgFirstModifyTime int64, reactionExtensionList map[string]*KeyValueModel) error
TakeExtendMsg(ctx context.Context, conversationID string, sessionType int32, clientMsgID string, maxMsgUpdateTime int64) (extendMsg *ExtendMsgModel, err error)
}
func (ExtendMsgSetModel) TableName() string {
return CExtendMsgSet
}
func (ExtendMsgSetModel) GetExtendMsgMaxNum() int32 {
return ExtendMsgMaxNum
}
func (ExtendMsgSetModel) GetConversationID(ID string, index int32) string {
return ID + ":" + strconv.Itoa(int(index))
}
func (e *ExtendMsgSetModel) SplitConversationIDAndGetIndex() int32 {
l := strings.Split(e.ConversationID, ":")
index, _ := strconv.Atoi(l[len(l)-1])
return int32(index)
}
type GetAllExtendMsgSetOpts struct {
ExcludeExtendMsgs bool
}
func (ExtendMsgSetModel) Pb2Model(reactionExtensionList map[string]*sdkws.KeyValue) map[string]*KeyValueModel {
r := make(map[string]*KeyValueModel)
for key, value := range reactionExtensionList {
r[key] = &KeyValueModel{
TypeKey: value.TypeKey,
Value: value.Value,
LatestUpdateTime: value.LatestUpdateTime,
}
}
return r
}
+13 -1
View File
@@ -3,6 +3,7 @@ package unrelation
import (
"context"
"strconv"
"time"
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/sdkws"
"go.mongodb.org/mongo-driver/mongo"
@@ -21,7 +22,6 @@ type MsgDocModel struct {
}
type RevokeModel struct {
ID string `bson:"id"`
Role int32 `bson:"role"`
UserID string `bson:"user_id"`
Nickname string `bson:"nickname"`
@@ -68,6 +68,16 @@ type MsgInfoModel struct {
IsRead bool `bson:"is_read"`
}
type UserCount struct {
UserID string `bson:"user_id"`
Count int64 `bson:"count"`
}
type GroupCount struct {
GroupID string `bson:"group_id"`
Count int64 `bson:"count"`
}
type MsgDocModelInterface interface {
PushMsgsToDoc(ctx context.Context, docID string, msgsToMongo []MsgInfoModel) error
Create(ctx context.Context, model *MsgDocModel) error
@@ -83,6 +93,8 @@ type MsgDocModelInterface interface {
GetMsgDocModelByIndex(ctx context.Context, conversationID string, index, sort int64) (*MsgDocModel, error)
DeleteMsgsInOneDocByIndex(ctx context.Context, docID string, indexes []int) error
MarkSingleChatMsgsAsRead(ctx context.Context, userID string, docID string, indexes []int64) error
RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*UserCount, dateCount map[string]int64, err error)
RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*GroupCount, dateCount map[string]int64, err error)
}
func (MsgDocModel) TableName() string {
-153
View File
@@ -1,153 +0,0 @@
package unrelation
import (
"context"
"errors"
"fmt"
unRelationTb "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/unrelation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
type ExtendMsgSetMongoDriver struct {
mgoDB *mongo.Database
ExtendMsgSetCollection *mongo.Collection
}
func NewExtendMsgSetMongoDriver(mgoDB *mongo.Database) unRelationTb.ExtendMsgSetModelInterface {
return &ExtendMsgSetMongoDriver{mgoDB: mgoDB, ExtendMsgSetCollection: mgoDB.Collection(unRelationTb.CExtendMsgSet)}
}
func (e *ExtendMsgSetMongoDriver) CreateExtendMsgSet(ctx context.Context, set *unRelationTb.ExtendMsgSetModel) error {
_, err := e.ExtendMsgSetCollection.InsertOne(ctx, set)
return err
}
func (e *ExtendMsgSetMongoDriver) GetAllExtendMsgSet(ctx context.Context, ID string, opts *unRelationTb.GetAllExtendMsgSetOpts) (sets []*unRelationTb.ExtendMsgSetModel, err error) {
regex := fmt.Sprintf("^%s", ID)
var findOpts *options.FindOptions
if opts != nil {
if opts.ExcludeExtendMsgs {
findOpts = &options.FindOptions{}
findOpts.SetProjection(bson.M{"extend_msgs": 0})
}
}
cursor, err := e.ExtendMsgSetCollection.Find(ctx, bson.M{"doc_id": primitive.Regex{Pattern: regex}}, findOpts)
if err != nil {
return nil, utils.Wrap(err, "")
}
err = cursor.All(ctx, &sets)
if err != nil {
return nil, utils.Wrap(err, fmt.Sprintf("cursor is %s", cursor.Current.String()))
}
return sets, nil
}
func (e *ExtendMsgSetMongoDriver) GetExtendMsgSet(ctx context.Context, conversationID string, sessionType int32, maxMsgUpdateTime int64) (*unRelationTb.ExtendMsgSetModel, error) {
var err error
findOpts := options.Find().SetLimit(1).SetSkip(0).SetSort(bson.M{"source_id": -1}).SetProjection(bson.M{"extend_msgs": 0})
// update newest
find := bson.M{"source_id": primitive.Regex{Pattern: fmt.Sprintf("^%s", conversationID)}, "session_type": sessionType}
if maxMsgUpdateTime > 0 {
find["max_msg_update_time"] = maxMsgUpdateTime
}
result, err := e.ExtendMsgSetCollection.Find(ctx, find, findOpts)
if err != nil {
return nil, utils.Wrap(err, "")
}
var setList []unRelationTb.ExtendMsgSetModel
if err := result.All(ctx, &setList); err != nil {
return nil, utils.Wrap(err, "")
}
if len(setList) == 0 {
return nil, nil
}
return &setList[0], nil
}
// first modify msg
func (e *ExtendMsgSetMongoDriver) InsertExtendMsg(ctx context.Context, conversationID string, sessionType int32, msg *unRelationTb.ExtendMsgModel) error {
set, err := e.GetExtendMsgSet(ctx, conversationID, sessionType, 0)
if err != nil {
return utils.Wrap(err, "")
}
if set == nil || set.ExtendMsgNum >= set.GetExtendMsgMaxNum() {
var index int32
if set != nil {
index = set.SplitConversationIDAndGetIndex()
}
err = e.CreateExtendMsgSet(ctx, &unRelationTb.ExtendMsgSetModel{
ConversationID: set.GetConversationID(conversationID, index),
SessionType: sessionType,
ExtendMsgs: map[string]unRelationTb.ExtendMsgModel{msg.ClientMsgID: *msg},
ExtendMsgNum: 1,
CreateTime: msg.MsgFirstModifyTime,
MaxMsgUpdateTime: msg.MsgFirstModifyTime,
})
} else {
_, err = e.ExtendMsgSetCollection.UpdateOne(ctx, bson.M{"conversation_id": set.ConversationID, "session_type": sessionType}, bson.M{"$set": bson.M{"max_msg_update_time": msg.MsgFirstModifyTime, "$inc": bson.M{"extend_msg_num": 1}, fmt.Sprintf("extend_msgs.%s", msg.ClientMsgID): msg}})
}
return utils.Wrap(err, "")
}
// insert or update
func (e *ExtendMsgSetMongoDriver) InsertOrUpdateReactionExtendMsgSet(ctx context.Context, conversationID string, sessionType int32, clientMsgID string, msgFirstModifyTime int64, reactionExtensionList map[string]*unRelationTb.KeyValueModel) error {
var updateBson = bson.M{}
for _, v := range reactionExtensionList {
updateBson[fmt.Sprintf("extend_msgs.%s.%s", clientMsgID, v.TypeKey)] = v
}
upsert := true
opt := &options.UpdateOptions{
Upsert: &upsert,
}
set, err := e.GetExtendMsgSet(ctx, conversationID, sessionType, msgFirstModifyTime)
if err != nil {
return utils.Wrap(err, "")
}
if set == nil {
return errors.New(fmt.Sprintf("conversationID %s has no set", conversationID))
}
_, err = e.ExtendMsgSetCollection.UpdateOne(ctx, bson.M{"source_id": set.ConversationID, "session_type": sessionType}, bson.M{"$set": updateBson}, opt)
return utils.Wrap(err, "")
}
// delete TypeKey
func (e *ExtendMsgSetMongoDriver) DeleteReactionExtendMsgSet(ctx context.Context, conversationID string, sessionType int32, clientMsgID string, msgFirstModifyTime int64, reactionExtensionList map[string]*unRelationTb.KeyValueModel) error {
var updateBson = bson.M{}
for _, v := range reactionExtensionList {
updateBson[fmt.Sprintf("extend_msgs.%s.%s", clientMsgID, v.TypeKey)] = ""
}
set, err := e.GetExtendMsgSet(ctx, conversationID, sessionType, msgFirstModifyTime)
if err != nil {
return utils.Wrap(err, "")
}
if set == nil {
return errors.New(fmt.Sprintf("conversationID %s has no set", conversationID))
}
_, err = e.ExtendMsgSetCollection.UpdateOne(ctx, bson.M{"source_id": set.ConversationID, "session_type": sessionType}, bson.M{"$unset": updateBson})
return err
}
func (e *ExtendMsgSetMongoDriver) TakeExtendMsg(ctx context.Context, conversationID string, sessionType int32, clientMsgID string, maxMsgUpdateTime int64) (extendMsg *unRelationTb.ExtendMsgModel, err error) {
findOpts := options.Find().SetLimit(1).SetSkip(0).SetSort(bson.M{"source_id": -1}).SetProjection(bson.M{fmt.Sprintf("extend_msgs.%s", clientMsgID): 1})
regex := fmt.Sprintf("^%s", conversationID)
result, err := e.ExtendMsgSetCollection.Find(ctx, bson.M{"source_id": primitive.Regex{Pattern: regex}, "session_type": sessionType, "max_msg_update_time": bson.M{"$lte": maxMsgUpdateTime}}, findOpts)
if err != nil {
return nil, utils.Wrap(err, "")
}
var setList []unRelationTb.ExtendMsgSetModel
if err := result.All(ctx, &setList); err != nil {
return nil, utils.Wrap(err, "")
}
if len(setList) == 0 {
return nil, utils.Wrap(errors.New("GetExtendMsg failed, len(setList) == 0"), "")
}
if v, ok := setList[0].ExtendMsgs[clientMsgID]; ok {
return &v, nil
}
return nil, errors.New(fmt.Sprintf("cant find client msg id: %s", clientMsgID))
}
-4
View File
@@ -78,10 +78,6 @@ func (m *Mongo) CreateSuperGroupIndex() error {
return nil
}
func (m *Mongo) CreateExtendMsgSetIndex() error {
return m.createMongoIndex(unrelation.CExtendMsgSet, true, "-create_time", "work_moment_id")
}
func (m *Mongo) createMongoIndex(collection string, isUnique bool, keys ...string) error {
db := m.db.Database(config.Config.Mongo.Database).Collection(collection)
opts := options.CreateIndexes().SetMaxTime(10 * time.Second)
+682 -1
View File
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"time"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant"
@@ -247,7 +248,7 @@ func (m *MsgMongoDriver) GetMsgBySeqIndexIn1Doc(ctx context.Context, userID stri
}
if msg.Revoke != nil {
revokeContent := sdkws.MessageRevokedContent{
RevokerID: msg.Revoke.ID,
RevokerID: msg.Revoke.UserID,
RevokerRole: msg.Revoke.Role,
ClientMsgID: msg.Msg.ClientMsgID,
RevokerNickname: msg.Revoke.Nickname,
@@ -308,3 +309,683 @@ func (m *MsgMongoDriver) MarkSingleChatMsgsAsRead(ctx context.Context, userID st
_, err := m.MsgCollection.BulkWrite(ctx, updates)
return err
}
// RangeUserSendCount
// db.msg.aggregate([
//
// {
// $match: {
// "msgs.msg.send_time": {
// "$gte": 0,
// "$lt": 1788122092317
// }
// }
// },
// {
// "$addFields": {
// "msgs": {
// "$filter": {
// "input": "$msgs",
// "as": "item",
// "cond": {
// "$and": [
// {
// $gte: ["$$item.msg.send_time", 0]
// },
// {
// $lt: ["$$item.msg.send_time", 1788122092317]
// }
// ]
// }
// }
// }
// }
// },
// {
// "$project": {
// "_id": 0,
//
// },
//
// },
// {
// "$project": {
// "result": {
// "$map": {
// "input": "$msgs",
// "as": "item",
// "in": {
// user_id: "$$item.msg.send_id",
// send_date: {
// $dateToString: {
// format: "%Y-%m-%d",
// date: {
// $toDate: "$$item.msg.send_time"
// }
// }
// }
// }
// }
// }
// },
//
// },
// {
// "$unwind": "$result"
// },
// {
// "$group": {
// _id: "$result.send_date",
// count: {
// $sum: 1
// },
// original: {
// $push: "$$ROOT"
// }
// }
// },
// {
// "$addFields": {
// "dates": "$$ROOT"
// }
// },
// {
// "$project": {
// "_id": 0,
// "count": 0,
// "dates.original": 0,
//
// },
//
// },
// {
// "$group": {
// _id: null,
// count: {
// $sum: 1
// },
// dates: {
// $push: "$dates"
// },
// original: {
// $push: "$original"
// },
//
// }
// },
// {
// "$unwind": "$original"
// },
// {
// "$unwind": "$original"
// },
// {
// "$group": {
// _id: "$original.result.user_id",
// count: {
// $sum: 1
// },
// original: {
// $push: "$dates"
// },
//
// }
// },
// {
// "$addFields": {
// "dates": {
// $arrayElemAt: ["$original", 0]
// }
// }
// },
// {
// "$project": {
// original: 0
// }
// },
// {
// $sort: {
// count: - 1
// }
// },
// {
// "$group": {
// _id: null,
// user_count: {
// $sum: 1
// },
// users: {
// $push: "$$ROOT"
// },
//
// }
// },
// {
// "$addFields": {
// "dates": {
// $arrayElemAt: ["$users", 0]
// }
// }
// },
// {
// "$addFields": {
// "dates": "$dates.dates"
// }
// },
// {
// "$project": {
// _id: 0,
// "users.dates": 0,
//
// }
// },
// {
// "$addFields": {
// "msg_count": {
// $sum: "$users.count"
// }
// }
// },
// {
// "$addFields": {
// users: {
// $slice: ["$users", 0, 10]
// }
// }
// }
//
// ])
func (m *MsgMongoDriver) RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*table.UserCount, dateCount map[string]int64, err error) {
var sort int
if ase {
sort = 1
} else {
sort = -1
}
type Result struct {
MsgCount int64 `bson:"msg_count"`
UserCount int64 `bson:"user_count"`
Users []struct {
UserID string `bson:"_id"`
Count int64 `bson:"count"`
} `bson:"users"`
Dates []struct {
Date string `bson:"_id"`
Count int64 `bson:"count"`
} `bson:"dates"`
}
or := bson.A{
bson.M{
"doc_id": bson.M{
"$regex": "^si_",
"$options": "i",
},
},
}
if group {
or = append(or,
bson.M{
"doc_id": bson.M{
"$regex": "^g_",
"$options": "i",
},
},
bson.M{
"doc_id": bson.M{
"$regex": "^sg_",
"$options": "i",
},
},
)
}
pipeline := bson.A{
bson.M{
"$match": bson.M{
"$and": bson.A{
bson.M{
"msgs.msg.send_time": bson.M{
"$gte": start.UnixMilli(),
"$lt": end.UnixMilli(),
},
},
bson.M{
"$or": or,
},
},
},
},
bson.M{
"$addFields": bson.M{
"msgs": bson.M{
"$filter": bson.M{
"input": "$msgs",
"as": "item",
"cond": bson.M{
"$and": bson.A{
bson.M{
"$gte": bson.A{
"$$item.msg.send_time", start.UnixMilli(),
},
},
bson.M{
"$lt": bson.A{
"$$item.msg.send_time", end.UnixMilli(),
},
},
},
},
},
},
},
},
bson.M{
"$project": bson.M{
"_id": 0,
},
},
bson.M{
"$project": bson.M{
"result": bson.M{
"$map": bson.M{
"input": "$msgs",
"as": "item",
"in": bson.M{
"user_id": "$$item.msg.send_id",
"send_date": bson.M{
"$dateToString": bson.M{
"format": "%Y-%m-%d",
"date": bson.M{
"$toDate": "$$item.msg.send_time", // 毫秒时间戳
},
},
},
},
},
},
},
},
bson.M{
"$unwind": "$result",
},
bson.M{
"$group": bson.M{
"_id": "$result.send_date",
"count": bson.M{
"$sum": 1,
},
"original": bson.M{
"$push": "$$ROOT",
},
},
},
bson.M{
"$addFields": bson.M{
"dates": "$$ROOT",
},
},
bson.M{
"$project": bson.M{
"_id": 0,
"count": 0,
"dates.original": 0,
},
},
bson.M{
"$group": bson.M{
"_id": nil,
"count": bson.M{
"$sum": 1,
},
"dates": bson.M{
"$push": "$dates",
},
"original": bson.M{
"$push": "$original",
},
},
},
bson.M{
"$unwind": "$original",
},
bson.M{
"$unwind": "$original",
},
bson.M{
"$group": bson.M{
"_id": "$original.result.user_id",
"count": bson.M{
"$sum": 1,
},
"original": bson.M{
"$push": "$dates",
},
},
},
bson.M{
"$addFields": bson.M{
"dates": bson.M{
"$arrayElemAt": bson.A{"$original", 0},
},
},
},
bson.M{
"$project": bson.M{
"original": 0,
},
},
bson.M{
"$sort": bson.M{
"count": sort,
},
},
bson.M{
"$group": bson.M{
"_id": nil,
"user_count": bson.M{
"$sum": 1,
},
"users": bson.M{
"$push": "$$ROOT",
},
},
},
bson.M{
"$addFields": bson.M{
"dates": bson.M{
"$arrayElemAt": bson.A{"$users", 0},
},
},
},
bson.M{
"$addFields": bson.M{
"dates": "$dates.dates",
},
},
bson.M{
"$project": bson.M{
"_id": 0,
"users.dates": 0,
},
},
bson.M{
"$addFields": bson.M{
"msg_count": bson.M{
"$sum": "$users.count",
},
},
},
bson.M{
"$addFields": bson.M{
"users": bson.M{
"$slice": bson.A{"$users", pageNumber - 1, showNumber},
},
},
},
}
cur, err := m.MsgCollection.Aggregate(ctx, pipeline, options.Aggregate().SetAllowDiskUse(true))
if err != nil {
return 0, 0, nil, nil, errs.Wrap(err)
}
defer cur.Close(ctx)
var result []Result
if err := cur.All(ctx, &result); err != nil {
return 0, 0, nil, nil, errs.Wrap(err)
}
if len(result) == 0 {
return 0, 0, nil, nil, errs.Wrap(err)
}
users = make([]*table.UserCount, len(result[0].Users))
for i, r := range result[0].Users {
users[i] = &table.UserCount{
UserID: r.UserID,
Count: r.Count,
}
}
dateCount = make(map[string]int64)
for _, r := range result[0].Dates {
dateCount[r.Date] = r.Count
}
return result[0].MsgCount, result[0].UserCount, users, dateCount, nil
}
func (m *MsgMongoDriver) RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*table.GroupCount, dateCount map[string]int64, err error) {
var sort int
if ase {
sort = 1
} else {
sort = -1
}
type Result struct {
MsgCount int64 `bson:"msg_count"`
UserCount int64 `bson:"user_count"`
Groups []struct {
GroupID string `bson:"_id"`
Count int64 `bson:"count"`
} `bson:"groups"`
Dates []struct {
Date string `bson:"_id"`
Count int64 `bson:"count"`
} `bson:"dates"`
}
pipeline := bson.A{
bson.M{
"$match": bson.M{
"$and": bson.A{
bson.M{
"msgs.msg.send_time": bson.M{
"$gte": start.UnixMilli(),
"$lt": end.UnixMilli(),
},
},
bson.M{
"$or": bson.A{
bson.M{
"doc_id": bson.M{
"$regex": "^g_",
"$options": "i",
},
},
bson.M{
"doc_id": bson.M{
"$regex": "^sg_",
"$options": "i",
},
},
},
},
},
},
},
bson.M{
"$addFields": bson.M{
"msgs": bson.M{
"$filter": bson.M{
"input": "$msgs",
"as": "item",
"cond": bson.M{
"$and": bson.A{
bson.M{
"$gte": bson.A{
"$$item.msg.send_time", start.UnixMilli(),
},
},
bson.M{
"$lt": bson.A{
"$$item.msg.send_time", end.UnixMilli(),
},
},
},
},
},
},
},
},
bson.M{
"$project": bson.M{
"_id": 0,
},
},
bson.M{
"$project": bson.M{
"result": bson.M{
"$map": bson.M{
"input": "$msgs",
"as": "item",
"in": bson.M{
"group_id": "$$item.msg.group_id",
"send_date": bson.M{
"$dateToString": bson.M{
"format": "%Y-%m-%d",
"date": bson.M{
"$toDate": "$$item.msg.send_time", // 毫秒时间戳
},
},
},
},
},
},
},
},
bson.M{
"$unwind": "$result",
},
bson.M{
"$group": bson.M{
"_id": "$result.send_date",
"count": bson.M{
"$sum": 1,
},
"original": bson.M{
"$push": "$$ROOT",
},
},
},
bson.M{
"$addFields": bson.M{
"dates": "$$ROOT",
},
},
bson.M{
"$project": bson.M{
"_id": 0,
"count": 0,
"dates.original": 0,
},
},
bson.M{
"$group": bson.M{
"_id": nil,
"count": bson.M{
"$sum": 1,
},
"dates": bson.M{
"$push": "$dates",
},
"original": bson.M{
"$push": "$original",
},
},
},
bson.M{
"$unwind": "$original",
},
bson.M{
"$unwind": "$original",
},
bson.M{
"$group": bson.M{
"_id": "$original.result.group_id",
"count": bson.M{
"$sum": 1,
},
"original": bson.M{
"$push": "$dates",
},
},
},
bson.M{
"$addFields": bson.M{
"dates": bson.M{
"$arrayElemAt": bson.A{"$original", 0},
},
},
},
bson.M{
"$project": bson.M{
"original": 0,
},
},
bson.M{
"$sort": bson.M{
"count": sort,
},
},
bson.M{
"$group": bson.M{
"_id": nil,
"user_count": bson.M{
"$sum": 1,
},
"groups": bson.M{
"$push": "$$ROOT",
},
},
},
bson.M{
"$addFields": bson.M{
"dates": bson.M{
"$arrayElemAt": bson.A{"$groups", 0},
},
},
},
bson.M{
"$addFields": bson.M{
"dates": "$dates.dates",
},
},
bson.M{
"$project": bson.M{
"_id": 0,
"groups.dates": 0,
},
},
bson.M{
"$addFields": bson.M{
"msg_count": bson.M{
"$sum": "$groups.count",
},
},
},
bson.M{
"$addFields": bson.M{
"groups": bson.M{
"$slice": bson.A{"$groups", pageNumber - 1, showNumber},
},
},
},
}
cur, err := m.MsgCollection.Aggregate(ctx, pipeline, options.Aggregate().SetAllowDiskUse(true))
if err != nil {
return 0, 0, nil, nil, errs.Wrap(err)
}
defer cur.Close(ctx)
var result []Result
if err := cur.All(ctx, &result); err != nil {
return 0, 0, nil, nil, errs.Wrap(err)
}
if len(result) == 0 {
return 0, 0, nil, nil, errs.Wrap(err)
}
groups = make([]*table.GroupCount, len(result[0].Groups))
for i, r := range result[0].Groups {
groups[i] = &table.GroupCount{
GroupID: r.GroupID,
Count: r.Count,
}
}
dateCount = make(map[string]int64)
for _, r := range result[0].Dates {
dateCount[r.Date] = r.Count
}
return result[0].MsgCount, result[0].UserCount, groups, dateCount, nil
}
+2 -2
View File
@@ -60,9 +60,9 @@ func (l *SqlLogger) Trace(ctx context.Context, begin time.Time, fc func() (sql s
sql, rows := fc()
slowLog := fmt.Sprintf("SLOW SQL >= %v", l.SlowThreshold)
if rows == -1 {
ZWarn(ctx, "sql exec detail", nil, "gorm", gormUtils.FileWithLineNum(), nil, "slow sql", slowLog, "elapsed time", fmt.Sprintf("%f(ms)", float64(elapsed.Nanoseconds())/1e6), "sql", sql)
ZWarn(ctx, "sql exec detail", nil, "gorm", gormUtils.FileWithLineNum(), "slow sql", slowLog, "elapsed time", fmt.Sprintf("%f(ms)", float64(elapsed.Nanoseconds())/1e6), "sql", sql)
} else {
ZWarn(ctx, "sql exec detail", nil, "gorm", gormUtils.FileWithLineNum(), nil, "slow sql", slowLog, "elapsed time", fmt.Sprintf("%f(ms)", float64(elapsed.Nanoseconds())/1e6), "rows", rows, "sql", sql)
ZWarn(ctx, "sql exec detail", nil, "gorm", gormUtils.FileWithLineNum(), "slow sql", slowLog, "elapsed time", fmt.Sprintf("%f(ms)", float64(elapsed.Nanoseconds())/1e6), "rows", rows, "sql", sql)
}
case l.LogLevel == gormLogger.Info:
sql, rows := fc()
+1 -1
View File
@@ -16,7 +16,7 @@ import (
)
func GrpcClient() grpc.DialOption {
return grpc.WithUnaryInterceptor(RpcClientInterceptor)
return grpc.WithChainUnaryInterceptor(RpcClientInterceptor)
}
func RpcClientInterceptor(ctx context.Context, method string, req, resp interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) (err error) {
+5 -4
View File
@@ -3,11 +3,12 @@ package mw
import (
"context"
"fmt"
"github.com/OpenIMSDK/Open-IM-Server/pkg/checker"
"math"
"runtime"
"strings"
"github.com/OpenIMSDK/Open-IM-Server/pkg/checker"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
@@ -31,7 +32,6 @@ func rpcString(v interface{}) string {
func RpcServerInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
log.ZDebug(ctx, "rpc server req", "req", rpcString(req))
//defer func() {
// if r := recover(); r != nil {
// log.ZError(ctx, "rpc panic", nil, "FullMethod", info.FullMethod, "type:", fmt.Sprintf("%T", r), "panic:", r)
@@ -145,12 +145,13 @@ func RpcServerInterceptor(ctx context.Context, req interface{}, info *grpc.Unary
}
details, err := grpcStatus.WithDetails(errInfo)
if err != nil {
panic(err)
log.ZWarn(ctx, "rpc server resp WithDetails error", err, "funcName", funcName)
return nil, errs.Wrap(err)
}
log.ZWarn(ctx, "rpc server resp", err, "funcName", funcName)
return nil, details.Err()
}
func GrpcServer() grpc.ServerOption {
return grpc.UnaryInterceptor(RpcServerInterceptor)
return grpc.ChainUnaryInterceptor(RpcServerInterceptor)
}
+2 -2
View File
@@ -68,7 +68,7 @@ func CheckAccessV3(ctx context.Context, ownerUserID string) (err error) {
if opUserID == ownerUserID {
return nil
}
return errs.ErrIdentity.Wrap(utils.GetSelfFuncName())
return errs.ErrNoPermission.Wrap(utils.GetSelfFuncName())
}
func IsAppManagerUid(ctx context.Context) bool {
@@ -79,7 +79,7 @@ func CheckAdmin(ctx context.Context) error {
if utils.IsContain(mcontext.GetOpUserID(ctx), config.Config.Manager.UserID) {
return nil
}
return errs.ErrIdentity.Wrap(fmt.Sprintf("user %s is not admin userID", mcontext.GetOpUserID(ctx)))
return errs.ErrNoPermission.Wrap(fmt.Sprintf("user %s is not admin userID", mcontext.GetOpUserID(ctx)))
}
func ParseRedisInterfaceToken(redisToken interface{}) (*Claims, error) {
+1 -2
View File
@@ -4,7 +4,6 @@ import (
"context"
"google.golang.org/grpc"
"google.golang.org/grpc/resolver"
)
type Conn interface {
@@ -13,7 +12,7 @@ type Conn interface {
AddOption(opts ...grpc.DialOption)
CloseConn(conn grpc.ClientConnInterface)
// do not use this method for call rpc
GetClientLocalConns() map[string][]resolver.Address
GetClientLocalConns() map[string][]grpc.ClientConnInterface
}
type SvcDiscoveryRegistry interface {
+15 -17
View File
@@ -7,6 +7,7 @@ import (
"strings"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
"github.com/pkg/errors"
"github.com/go-zookeeper/zk"
@@ -61,7 +62,7 @@ func (s *ZkClient) GetConnsRemote(serviceName string) (conns []resolver.Address,
}
return nil, errors.Wrap(err, "get children error")
}
log.ZDebug(context.Background(), "get conns from remote", "conn", string(data))
log.ZDebug(context.Background(), "get addrs from remote", "conn", string(data))
conns = append(conns, resolver.Address{Addr: string(data), ServerName: serviceName})
}
}
@@ -70,34 +71,31 @@ func (s *ZkClient) GetConnsRemote(serviceName string) (conns []resolver.Address,
func (s *ZkClient) GetConns(ctx context.Context, serviceName string, opts ...grpc.DialOption) ([]grpc.ClientConnInterface, error) {
s.logger.Printf("get conns from client, serviceName: %s", serviceName)
s.lock.Lock()
opts = append(s.options, opts...)
s.lock.Lock()
defer s.lock.Unlock()
conns := s.localConns[serviceName]
if len(conns) == 0 {
var err error
s.logger.Printf("get conns from zk remote, serviceName: %s", serviceName)
conns, err = s.GetConnsRemote(serviceName)
addrs, err := s.GetConnsRemote(serviceName)
if err != nil {
s.lock.Unlock()
return nil, err
}
if len(conns) == 0 {
if len(addrs) == 0 {
return nil, fmt.Errorf("no conn for service %s, grpc server may not exist, local conn is %v, please check zookeeper server %v, path: %s", serviceName, s.localConns, s.zkServers, s.zkRoot)
}
for _, addr := range addrs {
cc, err := grpc.DialContext(ctx, addr.Addr, append(s.options, opts...)...)
if err != nil {
log.ZError(context.Background(), "dialContext failed", err, "addr", addr.Addr, "opts", append(s.options, opts...))
return nil, errs.Wrap(err)
}
conns = append(conns, cc)
}
s.localConns[serviceName] = conns
}
s.lock.Unlock()
var ret []grpc.ClientConnInterface
s.logger.Printf("get conns from zk success, serviceName: %s", serviceName)
for _, conn := range conns {
cc, err := grpc.DialContext(ctx, conn.Addr, append(s.options, opts...)...)
if err != nil {
return nil, errors.Wrap(err, fmt.Sprintf("conns dialContext error, conn: %s", conn.Addr))
}
ret = append(ret, cc)
}
s.logger.Printf("dial ctx success, serviceName: %s", serviceName)
return ret, nil
return conns, nil
}
func (s *ZkClient) GetConn(ctx context.Context, serviceName string, opts ...grpc.DialOption) (grpc.ClientConnInterface, error) {
+1 -2
View File
@@ -5,7 +5,6 @@ import (
"github.com/go-zookeeper/zk"
"google.golang.org/grpc"
"google.golang.org/grpc/resolver"
)
func (s *ZkClient) CreateRpcRootNodes(serviceNames []string) error {
@@ -43,7 +42,7 @@ func (s *ZkClient) UnRegister() error {
}
time.Sleep(time.Second)
s.node = ""
s.localConns = make(map[string][]resolver.Address)
s.localConns = make(map[string][]grpc.ClientConnInterface)
s.resolvers = make(map[string]*Resolver)
return nil
}
+5 -4
View File
@@ -37,8 +37,9 @@ type ZkClient struct {
lock sync.Locker
options []grpc.DialOption
resolvers map[string]*Resolver
localConns map[string][]resolver.Address
resolvers map[string]*Resolver
localConns map[string][]grpc.ClientConnInterface
balancerName string
logger Logger
@@ -89,7 +90,7 @@ func NewClient(zkServers []string, zkRoot string, options ...ZkOption) (*ZkClien
zkRoot: "/",
scheme: zkRoot,
timeout: timeout,
localConns: make(map[string][]resolver.Address),
localConns: make(map[string][]grpc.ClientConnInterface),
resolvers: make(map[string]*Resolver),
lock: &sync.Mutex{},
}
@@ -197,6 +198,6 @@ func (s *ZkClient) AddOption(opts ...grpc.DialOption) {
s.options = append(s.options, opts...)
}
func (s *ZkClient) GetClientLocalConns() map[string][]resolver.Address {
func (s *ZkClient) GetClientLocalConns() map[string][]grpc.ClientConnInterface {
return s.localConns
}
+26 -40
View File
@@ -22,17 +22,12 @@ const (
// 通用错误码
const (
NoError = 0 //无错误
DatabaseError = 90002 //redis/mysql等db错误
NetworkError = 90004 //网络错误
IdentityError = 90008 // 身份错误 非管理员token,且token中userID与请求userID不一致
GRPCConnIsNil = 90006 //grpc连接空
DefaultOtherError = 90006 //其他错误
DataError = 90007 //数据错误
ConfigError = 90009
CallbackError = 80000
RelationshipAlreadyError = 92001 //已经是好友关系(或者黑名单)
NotRelationshipYetError = 92002 //不是好友关系(或者黑名单)
NoError = 0 //无错误
DatabaseError = 90002 //redis/mysql等db错误
NetworkError = 90004 //网络错误
DataError = 90007 //数据错误
CallbackError = 80000
//通用错误码
ServerInternalError = 500 //服务器内部错误
@@ -43,50 +38,41 @@ const (
// 账号错误码
UserIDNotFoundError = 1101 //UserID不存在 或未注册
UserIDExisted = 1102 //UserID已存在
RegisteredAlreadyError = 1103 //用户已经注册过了
RegisteredAlreadyError = 1102 //用户已经注册过了
// 群组错误码
GroupIDNotFoundError = 1201 //GroupID不存在
GroupIDExisted = 1202 //GroupID已存在
OnlyOneOwnerError = 1203 //只能有一个群主
InGroupAlreadyError = 1204 //已在群组中
NotInGroupYetError = 1205 //不在群组中
DismissedAlreadyError = 1206 //群组已经解散
OwnerNotAllowedQuitError = 1207 //群主不能退群
GroupTypeNotSupport = 1208
GroupNoOwner = 1209
GroupRequestHandled = 1210
GroupIDNotFoundError = 1201 //GroupID不存在
GroupIDExisted = 1202 //GroupID已存在
NotInGroupYetError = 1203 //不在群组中
DismissedAlreadyError = 1204 //群组已经解散
GroupTypeNotSupport = 1205
GroupRequestHandled = 1206
// 关系链错误码
CanNotAddYourselfError = 1301 //不能添加自己为好友
BlockedByPeer = 1302 //被对方拉黑
NotPeersFriend = 1303 //不是对方的好友
CanNotAddYourselfError = 1301 //不能添加自己为好友
BlockedByPeer = 1302 //被对方拉黑
NotPeersFriend = 1303 //不是对方的好友
RelationshipAlreadyError = 1304 //已经是好友关系
// 消息错误码
MessageHasReadDisable = 1401
MutedInGroup = 1402 //群成员被禁言
MutedGroup = 1403 //群被禁言
UserNotRecvMsg = 1404 //用户设置了不接收消息
MsgAlreadyRevoke = 1405 //消息已撤回
MsgAlreadyRevoke = 1404 //消息已撤回
// token错误码
TokenExpiredError = 1501
TokenInvalidError = 1502
TokenMalformedError = 1503
TokenNotValidYetError = 1504
TokenUnknownError = 1505
TokenKickedError = 1506
TokenDifferentPlatformIDError = 1507
TokenDifferentUserIDError = 1508
TokenNotExistError = 1509
TokenExpiredError = 1501
TokenInvalidError = 1502
TokenMalformedError = 1503
TokenNotValidYetError = 1504
TokenUnknownError = 1505
TokenKickedError = 1506
TokenNotExistError = 1507
// 长连接网关错误码
ConnOverMaxNumLimit = 1601
ConnArgsErr = 1602
ConnUpdateErr = 1603
// S3错误码
FileUploadedCompleteError = 1701 // 文件已上传
FileUploadedExpiredError = 1702 // 上传过期
FileUploadedExpiredError = 1701 // 上传过期
)
+17 -34
View File
@@ -2,66 +2,49 @@ package errs
var (
ErrArgs = NewCodeError(ArgsError, "ArgsError")
ErrNoPermission = NewCodeError(NoPermissionError, "NoPermissionError")
ErrDatabase = NewCodeError(DatabaseError, "DatabaseError")
ErrInternalServer = NewCodeError(ServerInternalError, "ServerInternalError")
ErrNetwork = NewCodeError(NetworkError, "NetworkError")
ErrNoPermission = NewCodeError(NoPermissionError, "NoPermissionError")
ErrIdentity = NewCodeError(IdentityError, "IdentityError")
ErrCallback = NewCodeError(CallbackError, "CallbackError")
ErrCallbackContinue = NewCodeError(CallbackError, "ErrCallbackContinue")
ErrUserIDNotFound = NewCodeError(UserIDNotFoundError, "UserIDNotFoundError")
ErrGroupIDNotFound = NewCodeError(GroupIDNotFoundError, "GroupIDNotFoundError")
ErrGroupIDExisted = NewCodeError(GroupIDExisted, "GroupIDExisted")
ErrUserIDExisted = NewCodeError(UserIDExisted, "UserIDExisted")
ErrRecordNotFound = NewCodeError(RecordNotFoundError, "RecordNotFoundError")
ErrRelationshipAlready = NewCodeError(RelationshipAlreadyError, "RelationshipAlreadyError")
ErrNotRelationshipYet = NewCodeError(NotRelationshipYetError, "NotRelationshipYetError")
ErrCanNotAddYourself = NewCodeError(CanNotAddYourselfError, "CanNotAddYourselfError")
ErrOnlyOneOwner = NewCodeError(OnlyOneOwnerError, "OnlyOneOwnerError")
ErrInGroupAlready = NewCodeError(InGroupAlreadyError, "InGroupAlreadyError")
ErrNotInGroupYet = NewCodeError(NotInGroupYetError, "NotInGroupYetError")
ErrDismissedAlready = NewCodeError(DismissedAlreadyError, "DismissedAlreadyError")
ErrOwnerNotAllowedQuit = NewCodeError(OwnerNotAllowedQuitError, "OwnerNotAllowedQuitError")
ErrRegisteredAlready = NewCodeError(RegisteredAlreadyError, "RegisteredAlreadyError")
ErrGroupTypeNotSupport = NewCodeError(GroupTypeNotSupport, "")
ErrGroupNoOwner = NewCodeError(GroupNoOwner, "ErrGroupNoOwner")
ErrGroupRequestHandled = NewCodeError(GroupRequestHandled, "GroupRequestHandled")
ErrDefaultOther = NewCodeError(DefaultOtherError, "DefaultOtherError")
ErrData = NewCodeError(DataError, "DataError")
ErrTokenExpired = NewCodeError(TokenExpiredError, "TokenExpiredError")
ErrTokenInvalid = NewCodeError(TokenInvalidError, "TokenInvalidError") //
ErrTokenMalformed = NewCodeError(TokenMalformedError, "TokenMalformedError") //格式错误
ErrTokenNotValidYet = NewCodeError(TokenNotValidYetError, "TokenNotValidYetError") //还未生效
ErrTokenUnknown = NewCodeError(TokenUnknownError, "TokenUnknownError") //未知错误
ErrTokenKicked = NewCodeError(TokenKickedError, "TokenKickedError")
ErrTokenNotExist = NewCodeError(TokenNotExistError, "TokenNotExistError") //在redis中不存在
ErrTokenDifferentPlatformID = NewCodeError(TokenDifferentPlatformIDError, "TokenDifferentPlatformIDError")
ErrTokenDifferentUserID = NewCodeError(TokenDifferentUserIDError, "TokenDifferentUserIDError")
ErrDuplicateKey = NewCodeError(DuplicateKeyError, "DuplicateKeyError")
ErrData = NewCodeError(DataError, "DataError")
ErrTokenExpired = NewCodeError(TokenExpiredError, "TokenExpiredError")
ErrTokenInvalid = NewCodeError(TokenInvalidError, "TokenInvalidError") //
ErrTokenMalformed = NewCodeError(TokenMalformedError, "TokenMalformedError") //格式错误
ErrTokenNotValidYet = NewCodeError(TokenNotValidYetError, "TokenNotValidYetError") //还未生效
ErrTokenUnknown = NewCodeError(TokenUnknownError, "TokenUnknownError") //未知错误
ErrTokenKicked = NewCodeError(TokenKickedError, "TokenKickedError")
ErrTokenNotExist = NewCodeError(TokenNotExistError, "TokenNotExistError") //在redis中不存在
ErrDuplicateKey = NewCodeError(DuplicateKeyError, "DuplicateKeyError")
ErrMessageHasReadDisable = NewCodeError(MessageHasReadDisable, "MessageHasReadDisable")
ErrBlockedByPeer = NewCodeError(BlockedByPeer, "BlockedByPeer")
//不是对方的好友
ErrNotPeersFriend = NewCodeError(NotPeersFriend, "NotPeersFriend")
ErrCanNotAddYourself = NewCodeError(CanNotAddYourselfError, "CanNotAddYourselfError")
ErrBlockedByPeer = NewCodeError(BlockedByPeer, "BlockedByPeer")
ErrNotPeersFriend = NewCodeError(NotPeersFriend, "NotPeersFriend")
ErrRelationshipAlready = NewCodeError(RelationshipAlreadyError, "RelationshipAlreadyError")
ErrMutedInGroup = NewCodeError(MutedInGroup, "MutedInGroup")
ErrMutedGroup = NewCodeError(MutedGroup, "MutedGroup")
ErrUserNotRecvMsg = NewCodeError(UserNotRecvMsg, "UserNotRecvMsg")
ErrMsgAlreadyRevoke = NewCodeError(MsgAlreadyRevoke, "MsgAlreadyRevoke")
ErrConnOverMaxNumLimit = NewCodeError(ConnOverMaxNumLimit, "ConnOverMaxNumLimit")
ErrConnArgsErr = NewCodeError(ConnArgsErr, "args err, need token, sendID, platformID")
ErrConnUpdateErr = NewCodeError(ConnArgsErr, "upgrade http conn err")
ErrConnArgsErr = NewCodeError(ConnArgsErr, "args err, need token, sendID, platformID")
ErrConfig = NewCodeError(ConfigError, "ConfigError")
ErrFileUploadedComplete = NewCodeError(FileUploadedCompleteError, "FileUploadedComplete")
ErrFileUploadedExpired = NewCodeError(FileUploadedExpiredError, "FileUploadedExpiredError")
ErrGroupRequestHandled = NewCodeError(GroupRequestHandled, "GroupRequestHandled")
ErrFileUploadedExpired = NewCodeError(FileUploadedExpiredError, "FileUploadedExpiredError")
)

Some files were not shown because too many files have changed in this diff Show More