feat: add openim server test about openim web

Signed-off-by: Xinwei Xiong(cubxxw) <3293172751nss@gmail.com>

.env

@@ -30,7 +30,7 @@ MINIO_ENDPOINT=http://172.28.0.1:10005
 
 # Base URL for the application programming interface (API).
 # Default: API_URL=http://172.28.0.1:10002
-API_URL=http://172.28.0.1:10002
+API_URL=http://127.0.0.1:10002
 
 # Directory path for storing data files or related information.
 # Default: DATA_DIR=./

.github/workflows/build-docker-image.yml

@@ -15,8 +15,6 @@
 name: Publish Docker image
 
 on:
-  schedule:
-  - cron: '30 2 * * *'
   push:
     branches:
       - main

.github/workflows/build-openim-web-image.yml

@@ -15,8 +15,6 @@
 name: Build OpenIM Web Docker image
 
 on:
-  schedule:
-  - cron: '30 3 * * *'
   push:
     branches:
       - main
@@ -81,12 +79,12 @@ jobs:
         uses: docker/setup-qemu-action@v2
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
-# registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-web:latest
+# registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-server:latest
       - name: Extract metadata (tags, labels) for Docker
         id: meta2
         uses: docker/metadata-action@v4.6.0
         with:
-          images: registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-web
+          images: registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-server
 
       - name: Log in to AliYun Docker Hub
         uses: docker/login-action@v2

CHANGELOG/CHANGELOG-3.1.md

@@ -8,6 +8,24 @@
 ## [Unreleased]
 
 
+<a name="v3.1.3-beta.1"></a>
+## [v3.1.3-beta.1] - 2023-08-14
+
+<a name="v3.1.3"></a>
+## [v3.1.3] - 2023-08-14
+
+<a name="v3.1.2-beta.3"></a>
+## [v3.1.2-beta.3] - 2023-08-09
+
+<a name="v3.1.2-beta.2"></a>
+## [v3.1.2-beta.2] - 2023-08-09
+
+<a name="v3.1.2-beta.1"></a>
+## [v3.1.2-beta.1] - 2023-08-09
+
+<a name="v3.1.2-beta.0"></a>
+## [v3.1.2-beta.0] - 2023-08-08
+
 <a name="v3.1.0"></a>
 ## v3.1.0 - 2023-07-28
 ### Reverts
@@ -17,4 +35,10 @@
 - Merge branch 'tuoyun'
 
 
-[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v3.1.0...HEAD
+[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v3.1.3-beta.1...HEAD
+[v3.1.3-beta.1]: https://github.com/openimsdk/open-im-server/compare/v3.1.3...v3.1.3-beta.1
+[v3.1.3]: https://github.com/openimsdk/open-im-server/compare/v3.1.2-beta.3...v3.1.3
+[v3.1.2-beta.3]: https://github.com/openimsdk/open-im-server/compare/v3.1.2-beta.2...v3.1.2-beta.3
+[v3.1.2-beta.2]: https://github.com/openimsdk/open-im-server/compare/v3.1.2-beta.1...v3.1.2-beta.2
+[v3.1.2-beta.1]: https://github.com/openimsdk/open-im-server/compare/v3.1.2-beta.0...v3.1.2-beta.1
+[v3.1.2-beta.0]: https://github.com/openimsdk/open-im-server/compare/v3.1.0...v3.1.2-beta.0

CHANGELOG/CHANGELOG-3.2.md

@@ -8,15 +8,39 @@
 ## [Unreleased]
 
 
+<a name="v3.2.2"></a>
+## [v3.2.2] - 2023-09-03
+
+<a name="v3.2.3"></a>
+## [v3.2.3] - 2023-09-03
+
+<a name="v3.2.1"></a>
+## [v3.2.1] - 2023-09-03
+
+<a name="v3.2.2-beta.4"></a>
+## [v3.2.2-beta.4] - 2023-08-28
+
 <a name="v3.2.2-alpha.0"></a>
 ## [v3.2.2-alpha.0] - 2023-08-25
 
+<a name="v3.2.2-beta.3"></a>
+## [v3.2.2-beta.3] - 2023-08-22
+
+<a name="v3.2.2-beta.2"></a>
+## [v3.2.2-beta.2] - 2023-08-21
+
+<a name="v3.2.2-beta.1"></a>
+## [v3.2.2-beta.1] - 2023-08-19
+
 <a name="v3.2.0"></a>
-## [v3.2.0] - 2023-08-19
+## [v3.2.0] - 2023-08-18
 
 <a name="v3.2.0-rc.0"></a>
 ## [v3.2.0-rc.0] - 2023-08-17
 
+<a name="v3.2.2-beta.0"></a>
+## [v3.2.2-beta.0] - 2023-08-17
+
 <a name="v3.2.0-alpha.0"></a>
 ## v3.2.0-alpha.0 - 2023-08-16
 ### Reverts
@@ -26,7 +50,15 @@
 - Merge branch 'tuoyun'
 
 
-[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v3.2.2-alpha.0...HEAD
-[v3.2.2-alpha.0]: https://github.com/openimsdk/open-im-server/compare/v3.2.0...v3.2.2-alpha.0
+[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v3.2.2...HEAD
+[v3.2.2]: https://github.com/openimsdk/open-im-server/compare/v3.2.3...v3.2.2
+[v3.2.3]: https://github.com/openimsdk/open-im-server/compare/v3.2.1...v3.2.3
+[v3.2.1]: https://github.com/openimsdk/open-im-server/compare/v3.2.2-beta.4...v3.2.1
+[v3.2.2-beta.4]: https://github.com/openimsdk/open-im-server/compare/v3.2.2-alpha.0...v3.2.2-beta.4
+[v3.2.2-alpha.0]: https://github.com/openimsdk/open-im-server/compare/v3.2.2-beta.3...v3.2.2-alpha.0
+[v3.2.2-beta.3]: https://github.com/openimsdk/open-im-server/compare/v3.2.2-beta.2...v3.2.2-beta.3
+[v3.2.2-beta.2]: https://github.com/openimsdk/open-im-server/compare/v3.2.2-beta.1...v3.2.2-beta.2
+[v3.2.2-beta.1]: https://github.com/openimsdk/open-im-server/compare/v3.2.0...v3.2.2-beta.1
 [v3.2.0]: https://github.com/openimsdk/open-im-server/compare/v3.2.0-rc.0...v3.2.0
-[v3.2.0-rc.0]: https://github.com/openimsdk/open-im-server/compare/v3.2.0-alpha.0...v3.2.0-rc.0
+[v3.2.0-rc.0]: https://github.com/openimsdk/open-im-server/compare/v3.2.2-beta.0...v3.2.0-rc.0
+[v3.2.2-beta.0]: https://github.com/openimsdk/open-im-server/compare/v3.2.0-alpha.0...v3.2.2-beta.0

CHANGELOG/CHANGELOG-3.3.md

@@ -1,40 +0,0 @@
-# Version logging for OpenIM
-
-<!-- BEGIN MUNGE: GENERATED_TOC -->
-
-<!-- END MUNGE: GENERATED_TOC -->
-
-<a name="unreleased"></a>
-## [Unreleased]
-
-
-<a name="v3.3.1"></a>
-## [v3.3.1] - 2023-09-13
-
-<a name="v3.3.1-beta.0"></a>
-## [v3.3.1-beta.0] - 2023-09-11
-
-<a name="v3.3.0-rc.1"></a>
-## [v3.3.0-rc.1] - 2023-09-11
-
-<a name="v3.3.0-rc.12"></a>
-## [v3.3.0-rc.12] - 2023-09-11
-
-<a name="v3.3.0"></a>
-## [v3.3.0] - 2023-09-09
-
-<a name="v3.3.0-rc.0"></a>
-## v3.3.0-rc.0 - 2023-09-07
-### Reverts
-- update etcd to v3.5.2 ([#206](https://github.com/openimsdk/open-im-server/issues/206))
-
-### Pull Requests
-- Merge branch 'tuoyun'
-
-
-[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v3.3.1...HEAD
-[v3.3.1]: https://github.com/openimsdk/open-im-server/compare/v3.3.1-beta.0...v3.3.1
-[v3.3.1-beta.0]: https://github.com/openimsdk/open-im-server/compare/v3.3.0-rc.1...v3.3.1-beta.0
-[v3.3.0-rc.1]: https://github.com/openimsdk/open-im-server/compare/v3.3.0-rc.12...v3.3.0-rc.1
-[v3.3.0-rc.12]: https://github.com/openimsdk/open-im-server/compare/v3.3.0...v3.3.0-rc.12
-[v3.3.0]: https://github.com/openimsdk/open-im-server/compare/v3.3.0-rc.0...v3.3.0

README.md

@@ -24,7 +24,7 @@
 <p align="center">
     <a href="./README.md"><b> English </b></a> •
     <a href="./README-zh_CN.md"><b> 简体中文 </b></a> •
-<a href="https://www.openim.online/en"><b> Docs </b></a>
+    <a href="https://www.openim.online/en"><b> Docs </b></a>
 </p>
 
 

config/config.yaml

@@ -132,14 +132,14 @@ api:
 # minio.signEndpoint is minio public network address
 object:
   enable: "minio"
-  apiURL: "http://172.28.0.1:10002"
+  apiURL: "http://http://127.0.0.1:10002"
   minio:
     bucket: "openim"
     endpoint: "http://172.28.0.1:10005"
     accessKeyID: "root"
     secretAccessKey: "openIM123"
     sessionToken: ''
-    signEndpoint: "http://172.28.0.1:10002"
+    signEndpoint: "http://127.0.0.1:10005"
   cos:
     bucketURL: https://temp-1252357374.cos.ap-chengdu.myqcloud.com
     secretID: ''

deployments/README.md

@@ -73,7 +73,7 @@ $ SEALOS_VERSION=`curl -s https://api.github.com/repos/labring/sealos/releases/l
 ```bash
 $ export CLUSTER_USERNAME=ubuntu
 $ export CLUSTER_PASSWORD=123456
-$ sudo sealos run labring/kubernetes:v1.25.0 labring/helm:v3.8.2 labring/calico:v3.24.1 \
+$ sealos run labring/kubernetes:v1.25.0 labring/helm:v3.8.2 labring/calico:v3.24.1 \
     --masters 10.0.0.9 \
     --nodes 10.0.0.4,10.0.0.10 \
     -u "$CLUSTER_USERNAME" \

deployments/templates/env_template.yaml

@@ -36,13 +36,6 @@ API_URL=${API_URL}
 # Default: DATA_DIR=./
 DATA_DIR=${DATA_DIR}
 
-# Choose the appropriate image address, the default is GITHUB image, 
-# you can choose docker hub, for Chinese users can choose Ali Cloud
-# export IMAGE_REGISTRY="ghcr.io/openimsdk"
-# export IMAGE_REGISTRY="openim"
-# export IMAGE_REGISTRY="registry.cn-hangzhou.aliyuncs.com/openimsdk"
-IMAGE_REGISTRY=${IMAGE_REGISTRY}
-
 # ======================================
 # ========= Network Configuration ======
 # ======================================
@@ -234,6 +227,7 @@ OPENIM_WEB_PORT=${OPENIM_WEB_PORT}
 # Default: OPENIM_WEB_ADDRESS=172.28.0.1
 OPENIM_WEB_ADDRESS=${OPENIM_WEB_NETWORK_ADDRESS}
 
+
 # ======================================
 # ========= OpenIM Server ==============
 # ======================================

deployments/templates/openim.yaml

@@ -226,18 +226,18 @@ push:
   enable: ${PUSH_ENABLE}
   geTui:
     pushUrl: "${GETUI_PUSH_URL}"
-    masterSecret: ${GETUI_MASTER_SECRET}
-    appKey: ${GETUI_APP_KEY}
-    intent: ${GETUI_INTENT}
-    channelID: ${GETUI_CHANNEL_ID}
-    channelName: ${GETUI_CHANNEL_NAME}
+    masterSecret: ""
+    appKey: ""
+    intent: ""
+    channelID: ""
+    channelName: ""
   fcm:
     serviceAccount: "${FCM_SERVICE_ACCOUNT}"
   jpns:
-    appKey: ${JPNS_APP_KEY}
-    masterSecret: ${JPNS_MASTER_SECRET}
-    pushUrl: ${JPNS_PUSH_URL}
-    pushIntent: ${JPNS_PUSH_INTENT}
+    appKey:
+    masterSecret:
+    pushUrl:
+    pushIntent:
 
 # App manager configuration
 #

docker-compose.yml

@@ -127,10 +127,9 @@ services:
         ipv4_address: ${MINIO_NETWORK_ADDRESS}
 
   openim-web:
-    # image: ghcr.io/openimsdk/openim-web:latest
+    image: ghcr.io/openimsdk/openim-web:latest
     # image: registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-web:latest
     # image: openim/openim-web:latest
-    image: ${IMAGE_REGISTRY}/openim-web:latest
     container_name: openim-web
     environment:
       - OPENIM_WEB_DIST_PATH=${OPENIM_WEB_DIST_PATH}
@@ -143,10 +142,9 @@ services:
         ipv4_address: ${OPENIM_WEB_NETWORK_ADDRESS}
 
 #   openim-server:
-#     # image: ghcr.io/openimsdk/openim-server:main
+#     image: ghcr.io/openimsdk/openim-server:main
 #     # image: registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-server:main
 #     # image: openim/openim-server:main
-#     image: ${IMAGE_REGISTRY}/openim-server:main
 # #    build: .
 #     container_name: openim-server
 #     ports:
@@ -205,4 +203,4 @@ services:
 #       - prometheus
 #     networks:
 #       openim-server:
-#         ipv4_address: ${GRAFANA_NETWORK_ADDRESS}
+#         ipv4_address: ${GRAFANA_NETWORK_ADDRESS}

go.mod

@@ -37,7 +37,7 @@ require github.com/google/uuid v1.3.1
 
 require (
 	github.com/IBM/sarama v1.41.1
-	github.com/OpenIMSDK/protocol v0.0.21
+	github.com/OpenIMSDK/protocol v0.0.18
 	github.com/OpenIMSDK/tools v0.0.14
 	github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible
 	github.com/go-redis/redis v6.15.9+incompatible

go.sum

@@ -19,8 +19,8 @@ firebase.google.com/go v3.13.0+incompatible/go.mod h1:xlah6XbEyW6tbfSklcfe5FHJIw
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/IBM/sarama v1.41.1 h1:B4/TdHce/8Ipza+qrLIeNJ9D1AOxZVp/3uDv6H/dp2M=
 github.com/IBM/sarama v1.41.1/go.mod h1:JFCPURVskaipJdKRFkiE/OZqQHw7jqliaJmRwXCmSSw=
-github.com/OpenIMSDK/protocol v0.0.21 h1:5H6H+hJ9d/VgRqttvxD/zfK9Asd+4M8Eknk5swSbUVY=
-github.com/OpenIMSDK/protocol v0.0.21/go.mod h1:F25dFrwrIx3lkNoiuf6FkCfxuwf8L4Z8UIsdTHP/r0Y=
+github.com/OpenIMSDK/protocol v0.0.18 h1:hXukFiDMLZx7s+hDCQePIK9ABiHyNlobNL4MppvOuMY=
+github.com/OpenIMSDK/protocol v0.0.18/go.mod h1:F25dFrwrIx3lkNoiuf6FkCfxuwf8L4Z8UIsdTHP/r0Y=
 github.com/OpenIMSDK/tools v0.0.14 h1:WLof/+WxyPyRST+QkoTKubYCiV73uCLiL8pgnpH/yKQ=
 github.com/OpenIMSDK/tools v0.0.14/go.mod h1:eg+q4A34Qmu73xkY0mt37FHGMCMfC6CtmOnm0kFEGFI=
 github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM=

go.work

@@ -11,5 +11,4 @@ use (
 	./tools/versionchecker
 	./tools/yamlfmt
 	./tools/component
-	./tools/url2im
 )

internal/api/conversation.go

@@ -44,7 +44,3 @@ func (o *ConversationApi) GetConversations(c *gin.Context) {
 func (o *ConversationApi) SetConversations(c *gin.Context) {
 	a2r.Call(conversation.ConversationClient.SetConversations, o.Client, c)
 }
-
-func (o *ConversationApi) GetConversationOfflinePushUserIDs(c *gin.Context) {
-	a2r.Call(conversation.ConversationClient.GetConversationOfflinePushUserIDs, o.Client, c)
-}

internal/api/route.go

@@ -198,7 +198,6 @@ func NewGinRouter(discov discoveryregistry.SvcDiscoveryRegistry, rdb redis.Unive
 		conversationGroup.POST("/get_conversation", c.GetConversation)
 		conversationGroup.POST("/get_conversations", c.GetConversations)
 		conversationGroup.POST("/set_conversations", c.SetConversations)
-		conversationGroup.POST("/get_conversation_offline_push_user_ids", c.GetConversationOfflinePushUserIDs)
 	}
 
 	statisticsGroup := r.Group("/statistics", ParseToken)

internal/msgtransfer/persistent_msg_handler.go

@@ -60,7 +60,7 @@ func (pc *PersistentConsumerHandler) handleChatWs2Mysql(
 		log.ZError(ctx, "msg_transfer Unmarshal msg err", err)
 		return
 	}
-
+	return
 	log.ZDebug(ctx, "handleChatWs2Mysql", "msg", msgFromMQ.MsgData)
 	// Control whether to store history messages (mysql)
 	isPersist := utils.GetSwitchFromOptions(msgFromMQ.MsgData.Options, constant.IsPersistent)

internal/push/push_to_client.go

@@ -19,8 +19,6 @@ import (
 	"encoding/json"
 	"errors"
 
-	"github.com/OpenIMSDK/protocol/conversation"
-
 	"github.com/openimsdk/open-im-server/v3/pkg/msgprocessor"
 
 	"github.com/OpenIMSDK/protocol/constant"
@@ -119,6 +117,7 @@ func (p *Pusher) Push2User(ctx context.Context, userIDs []string, msg *sdkws.Msg
 				if err != nil {
 					return err
 				}
+				break
 			}
 		}
 	}
@@ -235,23 +234,15 @@ func (p *Pusher) Push2SuperGroup(ctx context.Context, groupID string, msg *sdkws
 			if len(offlinePushUserIDs) > 0 {
 				needOfflinePushUserIDs = offlinePushUserIDs
 			}
-			resp, err := p.conversationRpcClient.Client.GetConversationOfflinePushUserIDs(
-				ctx,
-				&conversation.GetConversationOfflinePushUserIDsReq{ConversationID: utils.GenGroupConversationID(groupID), UserIDs: needOfflinePushUserIDs},
-			)
+			err = p.offlinePushMsg(ctx, groupID, msg, offlinePushUserIDs)
 			if err != nil {
+				log.ZError(ctx, "offlinePushMsg failed", err, "groupID", groupID, "msg", msg)
 				return err
 			}
-			if len(resp.UserIDs) > 0 {
-				err = p.offlinePushMsg(ctx, groupID, msg, resp.UserIDs)
-				if err != nil {
-					log.ZError(ctx, "offlinePushMsg failed", err, "groupID", groupID, "msg", msg)
-					return err
-				}
-				if _, err := p.GetConnsAndOnlinePush(ctx, msg, utils.IntersectString(resp.UserIDs, WebAndPcBackgroundUserIDs)); err != nil {
-					log.ZError(ctx, "offlinePushMsg failed", err, "groupID", groupID, "msg", msg, "userIDs", utils.IntersectString(needOfflinePushUserIDs, WebAndPcBackgroundUserIDs))
-					return err
-				}
+			_, err := p.GetConnsAndOnlinePush(ctx, msg, utils.IntersectString(needOfflinePushUserIDs, WebAndPcBackgroundUserIDs))
+			if err != nil {
+				log.ZError(ctx, "offlinePushMsg failed", err, "groupID", groupID, "msg", msg, "userIDs", utils.IntersectString(needOfflinePushUserIDs, WebAndPcBackgroundUserIDs))
+				return err
 			}
 		}
 	}

internal/rpc/conversation/conversaion.go

@@ -300,30 +300,3 @@ func (c *conversationServer) GetConversationsByConversationID(
 	}
 	return &pbconversation.GetConversationsByConversationIDResp{Conversations: convert.ConversationsDB2Pb(conversations)}, nil
 }
-
-func (c *conversationServer) GetConversationOfflinePushUserIDs(
-	ctx context.Context,
-	req *pbconversation.GetConversationOfflinePushUserIDsReq,
-) (*pbconversation.GetConversationOfflinePushUserIDsResp, error) {
-	if req.ConversationID == "" {
-		return nil, errs.ErrArgs.Wrap("conversationID is empty")
-	}
-	if len(req.UserIDs) == 0 {
-		return &pbconversation.GetConversationOfflinePushUserIDsResp{}, nil
-	}
-	userIDs, err := c.conversationDatabase.GetConversationNotReceiveMessageUserIDs(ctx, req.ConversationID)
-	if err != nil {
-		return nil, err
-	}
-	if len(userIDs) == 0 {
-		return &pbconversation.GetConversationOfflinePushUserIDsResp{UserIDs: req.UserIDs}, nil
-	}
-	userIDSet := make(map[string]struct{})
-	for _, userID := range req.UserIDs {
-		userIDSet[userID] = struct{}{}
-	}
-	for _, userID := range userIDs {
-		delete(userIDSet, userID)
-	}
-	return &pbconversation.GetConversationOfflinePushUserIDsResp{UserIDs: utils.Keys(userIDSet)}, nil
-}

pkg/common/db/cache/conversation.go

@@ -38,7 +38,6 @@ const (
 	recvMsgOptKey                            = "RECV_MSG_OPT:"
 	superGroupRecvMsgNotNotifyUserIDsKey     = "SUPER_GROUP_RECV_MSG_NOT_NOTIFY_USER_IDS:"
 	superGroupRecvMsgNotNotifyUserIDsHashKey = "SUPER_GROUP_RECV_MSG_NOT_NOTIFY_USER_IDS_HASH:"
-	conversationNotReceiveMessageUserIDsKey  = "CONVERSATION_NOT_RECEIVE_MESSAGE_USER_IDS:"
 
 	conversationExpireTime = time.Second * 60 * 60 * 12
 )
@@ -84,8 +83,6 @@ type ConversationCache interface {
 		conversationIDs []string,
 	) ([]*relationtb.ConversationModel, error)
 	DelConversationByConversationID(conversationIDs ...string) ConversationCache
-	GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error)
-	DelConversationNotReceiveMessageUserIDs(conversationIDs ...string) ConversationCache
 }
 
 func NewConversationRedis(
@@ -156,10 +153,6 @@ func (c *ConversationRedisCache) getConversationHasReadSeqKey(ownerUserID, conve
 	return conversationHasReadSeqKey + ownerUserID + ":" + conversationID
 }
 
-func (c *ConversationRedisCache) getConversationNotReceiveMessageUserIDsKey(conversationID string) string {
-	return conversationNotReceiveMessageUserIDsKey + conversationID
-}
-
 func (c *ConversationRedisCache) GetUserConversationIDs(ctx context.Context, ownerUserID string) ([]string, error) {
 	return getCache(
 		ctx,
@@ -439,23 +432,3 @@ func (c *ConversationRedisCache) GetConversationsByConversationID(
 func (c *ConversationRedisCache) DelConversationByConversationID(conversationIDs ...string) ConversationCache {
 	panic("implement me")
 }
-
-func (c *ConversationRedisCache) GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) {
-	return getCache(
-		ctx,
-		c.rcClient,
-		c.getConversationNotReceiveMessageUserIDsKey(conversationID),
-		c.expireTime,
-		func(ctx context.Context) ([]string, error) {
-			return c.conversationDB.GetConversationNotReceiveMessageUserIDs(ctx, conversationID)
-		},
-	)
-}
-
-func (c *ConversationRedisCache) DelConversationNotReceiveMessageUserIDs(conversationIDs ...string) ConversationCache {
-	cache := c.NewCache()
-	for _, conversationID := range conversationIDs {
-		cache.AddKeys(c.getConversationNotReceiveMessageUserIDsKey(conversationID))
-	}
-	return cache
-}

pkg/common/db/controller/conversation.go

@@ -53,7 +53,6 @@ type ConversationDatabase interface {
 	GetUserAllHasReadSeqs(ctx context.Context, ownerUserID string) (map[string]int64, error)
 	GetConversationsByConversationID(ctx context.Context, conversationIDs []string) ([]*relationtb.ConversationModel, error)
 	GetConversationIDsNeedDestruct(ctx context.Context) ([]*relationtb.ConversationModel, error)
-	GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error)
 }
 
 func NewConversationDatabase(conversation relationtb.ConversationModelInterface, cache cache.ConversationCache, tx tx.Tx) ConversationDatabase {
@@ -89,9 +88,6 @@ func (c *conversationDatabase) SetUsersConversationFiledTx(ctx context.Context,
 					cache = cache.DelUserAllHasReadSeqs(userID, conversation.ConversationID)
 				}
 			}
-			if _, ok := filedMap["recv_msg_opt"]; ok {
-				cache = cache.DelConversationNotReceiveMessageUserIDs(conversation.ConversationID)
-			}
 		}
 		NotUserIDs := utils.DifferenceString(haveUserIDs, userIDs)
 		log.ZDebug(ctx, "SetUsersConversationFiledTx", "NotUserIDs", NotUserIDs, "haveUserIDs", haveUserIDs, "userIDs", userIDs)
@@ -125,12 +121,7 @@ func (c *conversationDatabase) UpdateUsersConversationFiled(ctx context.Context,
 	if err != nil {
 		return err
 	}
-	cache := c.cache.NewCache()
-	cache = cache.DelUsersConversation(conversationID, userIDs...)
-	if _, ok := args["recv_msg_opt"]; ok {
-		cache = cache.DelConversationNotReceiveMessageUserIDs(conversationID)
-	}
-	return cache.ExecDel(ctx)
+	return c.cache.DelUsersConversation(conversationID, userIDs...).ExecDel(ctx)
 }
 
 func (c *conversationDatabase) CreateConversation(ctx context.Context, conversations []*relationtb.ConversationModel) error {
@@ -141,7 +132,6 @@ func (c *conversationDatabase) CreateConversation(ctx context.Context, conversat
 	cache := c.cache.NewCache()
 	for _, conversation := range conversations {
 		cache = cache.DelConversations(conversation.OwnerUserID, conversation.ConversationID)
-		cache = cache.DelConversationNotReceiveMessageUserIDs(conversation.ConversationID)
 		userIDs = append(userIDs, conversation.OwnerUserID)
 	}
 	return cache.DelConversationIDs(userIDs...).DelUserConversationIDsHash(userIDs...).ExecDel(ctx)
@@ -234,7 +224,7 @@ func (c *conversationDatabase) SetUserConversations(ctx context.Context, ownerUs
 			if err != nil {
 				return err
 			}
-			cache = cache.DelConversationIDs(ownerUserID).DelUserConversationIDsHash(ownerUserID).DelConversationNotReceiveMessageUserIDs(utils.Slice(notExistConversations, func(e *relationtb.ConversationModel) string { return e.ConversationID })...)
+			cache = cache.DelConversationIDs(ownerUserID).DelUserConversationIDsHash(ownerUserID)
 		}
 		return nil
 	}); err != nil {
@@ -260,7 +250,7 @@ func (c *conversationDatabase) CreateGroupChatConversation(ctx context.Context,
 		for _, v := range notExistUserIDs {
 			conversation := relationtb.ConversationModel{ConversationType: constant.SuperGroupChatType, GroupID: groupID, OwnerUserID: v, ConversationID: conversationID}
 			conversations = append(conversations, &conversation)
-			cache = cache.DelConversations(v, conversationID).DelConversationNotReceiveMessageUserIDs(conversationID)
+			cache = cache.DelConversations(v, conversationID)
 		}
 		cache = cache.DelConversationIDs(notExistUserIDs...).DelUserConversationIDsHash(notExistUserIDs...)
 		if len(conversations) > 0 {
@@ -306,7 +296,3 @@ func (c *conversationDatabase) GetConversationsByConversationID(ctx context.Cont
 func (c *conversationDatabase) GetConversationIDsNeedDestruct(ctx context.Context) ([]*relationtb.ConversationModel, error) {
 	return c.conversationDB.GetConversationIDsNeedDestruct(ctx)
 }
-
-func (c *conversationDatabase) GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) {
-	return c.cache.GetConversationNotReceiveMessageUserIDs(ctx, conversationID)
-}

pkg/common/db/relation/conversation_model.go

@@ -17,8 +17,6 @@ package relation
 import (
 	"context"
 
-	"github.com/OpenIMSDK/tools/errs"
-
 	"gorm.io/gorm"
 
 	"github.com/OpenIMSDK/protocol/constant"
@@ -216,24 +214,3 @@ func (c *ConversationGorm) GetConversationIDsNeedDestruct(
 		"",
 	)
 }
-
-func (c *ConversationGorm) GetConversationRecvMsgOpt(ctx context.Context, userID string, conversationID string) (int32, error) {
-	var recvMsgOpt int32
-	return recvMsgOpt, errs.Wrap(
-		c.db(ctx).
-			Model(&relation.ConversationModel{}).
-			Where("conversation_id = ? and owner_user_id in ?", conversationID, userID).
-			Pluck("recv_msg_opt", &recvMsgOpt).
-			Error,
-	)
-}
-
-func (c *ConversationGorm) GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) {
-	var userIDs []string
-	return userIDs, errs.Wrap(
-		c.db(ctx).
-			Model(&relation.ConversationModel{}).
-			Where("conversation_id = ? and recv_msg_opt <> ?", conversationID, constant.ReceiveMessage).
-			Pluck("owner_user_id", &userIDs).Error,
-	)
-}

pkg/common/db/table/relation/conversation.go

@@ -66,6 +66,5 @@ type ConversationModelInterface interface {
 	GetUserAllHasReadSeqs(ctx context.Context, ownerUserID string) (hashReadSeqs map[string]int64, err error)
 	GetConversationsByConversationID(ctx context.Context, conversationIDs []string) ([]*ConversationModel, error)
 	GetConversationIDsNeedDestruct(ctx context.Context) ([]*ConversationModel, error)
-	GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error)
 	NewTx(tx any) ConversationModelInterface
 }

scripts/check-all.sh

@@ -62,7 +62,6 @@ echo "+++ The port being checked: ${OPENIM_DEPENDENCY_PORT_LISTARIES[@]}"
 
 set +e
 
-# Later, after discarding Docker, the Docker keyword is unreliable, and Kubepods is used
 if grep -qE 'docker|kubepods' /proc/1/cgroup || [ -f /.dockerenv ]; then
     openim::color::echo ${COLOR_BLUE} "Environment in the interior of the container"
 else

scripts/install/environment.sh

@@ -69,15 +69,6 @@ def "ENV_FILE" ""${OPENIM_ROOT}"/scripts/install/environment.sh"
 def "CHAT_BRANCH" "main"
 def "SERVER_BRANCH" "main"
 
-# Choose the appropriate image address, the default is GITHUB image, 
-# you can choose docker hub, for Chinese users can choose Ali Cloud
-# export IMAGE_REGISTRY="ghcr.io/openimsdk"
-# export IMAGE_REGISTRY="openim"
-# export IMAGE_REGISTRY="registry.cn-hangzhou.aliyuncs.com/openimsdk"
-def "IMAGE_REGISTRY" "ghcr.io/openimsdk"
-# def "IMAGE_REGISTRY" "openim"
-# def "IMAGE_REGISTRY" "registry.cn-hangzhou.aliyuncs.com/openimsdk"
-
 ###################### OpenIM Docker Network ######################
 # 设置 Docker 网络的网段
 readonly DOCKER_BRIDGE_SUBNET=${DOCKER_BRIDGE_SUBNET:-'172.28.0.0/16'}
@@ -185,7 +176,7 @@ readonly API_URL=${API_URL:-"http://${IP}:${API_OPENIM_PORT}"}
 
 def "OBJECT_ENABLE" "minio" # 对象是否启用
 # 对象的API地址
-readonly OBJECT_APIURL=${OBJECT_APIURL:-"${API_URL}"}
+readonly OBJECT_APIURL=${OBJECT_APIURL:-"http://${API_URL}"}
 def "MINIO_BUCKET" "openim" # MinIO的存储桶名称
 def "MINIO_PORT" "10005"    # MinIO的端口
 # MinIO的端点URL
@@ -294,16 +285,11 @@ def "WEBSOCKET_TIMEOUT" "10"          # Websocket超时
 def "PUSH_ENABLE" "getui"             # 推送是否启用
 # GeTui推送URL
 readonly GETUI_PUSH_URL=${GETUI_PUSH_URL:-'https://restapi.getui.com/v2/$appId'}
-def "GETUI_MASTER_SECRET" ""          # GeTui主密钥
-def "GETUI_APP_KEY" ""                # GeTui应用密钥
-def "GETUI_INTENT" ""                 # GeTui推送意图
-def "GETUI_CHANNEL_ID" ""             # GeTui渠道ID
-def "GETUI_CHANNEL_NAME" ""           # GeTui渠道名称
 def "FCM_SERVICE_ACCOUNT" "x.json"    # FCM服务账户
-def "JPNS_APP_KEY"  ""                  # JPNS应用密钥
-def "JPNS_MASTER_SECRET" ""             # JPNS主密钥
-def "JPNS_PUSH_URL"  ""                 # JPNS推送URL
-def "JPNS_PUSH_INTENT" ""               # JPNS推送意图
+def "JPNS_APP_KEY"                    # JPNS应用密钥
+def "JPNS_MASTER_SECRET"              # JPNS主密钥
+def "JPNS_PUSH_URL"                   # JPNS推送URL
+def "JPNS_PUSH_INTENT"                # JPNS推送意图
 def "MANAGER_USERID_1" "openIM123456" # 管理员ID 1
 def "MANAGER_USERID_2" "openIM654321" # 管理员ID 2
 def "MANAGER_USERID_3" "openIMAdmin"  # 管理员ID 3

scripts/install/openim-push.sh

@@ -54,14 +54,14 @@ function openim::push::start()
     openim::log::status "Start OpenIM Push, binary root: ${SERVER_NAME}"
     openim::log::info "Start OpenIM Push, path: ${OPENIM_PUSH_BINARY}"
 
+    openim::util::stop_services_with_name ${SERVER_NAME}
+
     openim::log::status "prepare start push process, path: ${OPENIM_PUSH_BINARY}"
     openim::log::status "prepare start push process, port: ${OPENIM_PUSH_PORT}, prometheus port: ${PUSH_PROM_PORT}"
 
     OPENIM_PUSH_PORTS_ARRAY=$(openim::util::list-to-string ${OPENIM_PUSH_PORT} )
     PUSH_PROM_PORTS_ARRAY=$(openim::util::list-to-string ${PUSH_PROM_PORT} )
 
-    openim::util::stop_services_with_name ${SERVER_NAME}
-
     openim::log::status "push port list: ${OPENIM_PUSH_PORTS_ARRAY[@]}"
     openim::log::status "prometheus port list: ${PUSH_PROM_PORTS_ARRAY[@]}"
 
@@ -94,7 +94,7 @@ function openim::push::install()
 
   # 1. Build openim-push
   make build BINS=${SERVER_NAME}
-  openim::common::sudo "cp -r ${OPENIM_OUTPUT_HOSTBIN}/${SERVER_NAME} ${OPENIM_INSTALL_DIR}/bin"
+  openim::common::sudo "cp ${OPENIM_OUTPUT_HOSTBIN}/${SERVER_NAME} ${OPENIM_INSTALL_DIR}/bin"
 
   openim::log::status "${SERVER_NAME} binary: ${OPENIM_INSTALL_DIR}/bin/${SERVER_NAME}"
 
@@ -150,4 +150,4 @@ function openim::push::status()
 
 if [[ "$*" =~ openim::push:: ]];then
   eval $*
-fi
+fi

tools/url2im/go.mod

@@ -1,20 +0,0 @@
-module github.com/openimsdk/open-im-server/v3/tools/url2im
-
-go 1.20
-
-require (
-	github.com/OpenIMSDK/protocol v0.0.21
-	github.com/kelindar/bitmap v1.5.1
-)
-
-require (
-	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/kelindar/simd v1.1.2 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
-	golang.org/x/net v0.9.0 // indirect
-	golang.org/x/sys v0.7.0 // indirect
-	golang.org/x/text v0.9.0 // indirect
-	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
-	google.golang.org/grpc v1.56.2 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
-)

tools/url2im/go.sum

@@ -1,33 +0,0 @@
-github.com/OpenIMSDK/protocol v0.0.21 h1:5H6H+hJ9d/VgRqttvxD/zfK9Asd+4M8Eknk5swSbUVY=
-github.com/OpenIMSDK/protocol v0.0.21/go.mod h1:F25dFrwrIx3lkNoiuf6FkCfxuwf8L4Z8UIsdTHP/r0Y=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
-github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/kelindar/bitmap v1.5.1 h1:+ZmZdwHbJ+CGE+q/aAJ74KJSnp0vOlGD7KY5x51mVzk=
-github.com/kelindar/bitmap v1.5.1/go.mod h1:j3qZjxH9s4OtvsnFTP2bmPkjqil9Y2xQlxPYHexasEA=
-github.com/kelindar/simd v1.1.2 h1:KduKb+M9cMY2HIH8S/cdJyD+5n5EGgq+Aeeleos55To=
-github.com/kelindar/simd v1.1.2/go.mod h1:inq4DFudC7W8L5fhxoeZflLRNpWSs0GNx6MlWFvuvr0=
-github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
-github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
-golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
-golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
-golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
-golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
-golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A=
-google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU=
-google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI=
-google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
-google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

tools/url2im/main.go

@@ -1,84 +0,0 @@
-package main
-
-import (
-	"flag"
-	"log"
-	"os"
-	"path/filepath"
-	"time"
-
-	"github.com/openimsdk/open-im-server/v3/tools/url2im/pkg"
-)
-
-/*take.txt
-{"url":"http://xxx/xxxx","name":"xxxx","contentType":"image/jpeg"}
-{"url":"http://xxx/xxxx","name":"xxxx","contentType":"image/jpeg"}
-{"url":"http://xxx/xxxx","name":"xxxx","contentType":"image/jpeg"}
-*/
-
-func main() {
-	var conf pkg.Config                                                  // 后面带*的为必填项
-	flag.StringVar(&conf.TaskPath, "task", "take.txt", "task path")      // 任务日志文件*
-	flag.StringVar(&conf.ProgressPath, "progress", "", "progress path")  // 进度日志文件
-	flag.IntVar(&conf.Concurrency, "concurrency", 1, "concurrency num")  // 并发数
-	flag.IntVar(&conf.Retry, "retry", 1, "retry num")                    // 重试次数
-	flag.StringVar(&conf.TempDir, "temp", "", "temp dir")                // 临时文件夹
-	flag.Int64Var(&conf.CacheSize, "cache", 1024*1024*100, "cache size") // 缓存大小(超过时,下载到磁盘)
-	flag.Int64Var((*int64)(&conf.Timeout), "timeout", 5000, "timeout")   // 请求超时时间(毫秒)
-	flag.StringVar(&conf.Api, "api", "http://127.0.0.1:10002", "api")    // im地址*
-	flag.StringVar(&conf.UserID, "userID", "openIM123456", "userID")     // im管理员
-	flag.StringVar(&conf.Secret, "secret", "openIM123", "secret")        // im config secret
-	flag.Parse()
-	if !filepath.IsAbs(conf.TaskPath) {
-		var err error
-		conf.TaskPath, err = filepath.Abs(conf.TaskPath)
-		if err != nil {
-			log.Println("get abs path err:", err)
-			return
-		}
-	}
-	if conf.ProgressPath == "" {
-		conf.ProgressPath = conf.TaskPath + ".progress.txt"
-	} else if !filepath.IsAbs(conf.ProgressPath) {
-		var err error
-		conf.ProgressPath, err = filepath.Abs(conf.ProgressPath)
-		if err != nil {
-			log.Println("get abs path err:", err)
-			return
-		}
-	}
-	if conf.TempDir == "" {
-		conf.TempDir = conf.TaskPath + ".temp"
-	}
-	if info, err := os.Stat(conf.TempDir); err == nil {
-		if !info.IsDir() {
-			log.Printf("temp dir %s is not dir\n", err)
-			return
-		}
-	} else if os.IsNotExist(err) {
-		if err := os.MkdirAll(conf.TempDir, os.ModePerm); err != nil {
-			log.Printf("mkdir temp dir %s err %+v\n", conf.TempDir, err)
-			return
-		}
-		defer os.RemoveAll(conf.TempDir)
-	} else {
-		log.Println("get temp dir err:", err)
-		return
-	}
-	if conf.Concurrency <= 0 {
-		conf.Concurrency = 1
-	}
-	if conf.Retry <= 0 {
-		conf.Retry = 1
-	}
-	if conf.CacheSize <= 0 {
-		conf.CacheSize = 1024 * 1024 * 100 // 100M
-	}
-	if conf.Timeout <= 0 {
-		conf.Timeout = 5000
-	}
-	conf.Timeout = conf.Timeout * time.Millisecond
-	if err := pkg.Run(conf); err != nil {
-		log.Println("main err:", err)
-	}
-}

tools/url2im/pkg/api.go

@@ -1,112 +0,0 @@
-package pkg
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-
-	"github.com/OpenIMSDK/protocol/auth"
-	"github.com/OpenIMSDK/protocol/constant"
-	"github.com/OpenIMSDK/protocol/third"
-)
-
-type Api struct {
-	Api    string
-	UserID string
-	Secret string
-	Token  string
-	Client *http.Client
-}
-
-func (a *Api) apiPost(ctx context.Context, path string, req any, resp any) error {
-	operationID, _ := ctx.Value("operationID").(string)
-	if operationID == "" {
-		return errors.New("call api operationID is empty")
-	}
-	reqBody, err := json.Marshal(req)
-	if err != nil {
-		return err
-	}
-	request, err := http.NewRequestWithContext(ctx, http.MethodPost, a.Api+path, bytes.NewReader(reqBody))
-	if err != nil {
-		return err
-	}
-	DefaultRequestHeader(request.Header)
-	request.ContentLength = int64(len(reqBody))
-	request.Header.Set("Content-Type", "application/json")
-	request.Header.Set("operationID", operationID)
-	if a.Token != "" {
-		request.Header.Set("token", a.Token)
-	}
-	response, err := a.Client.Do(request)
-	if err != nil {
-		return err
-	}
-	defer response.Body.Close()
-	body, err := io.ReadAll(response.Body)
-	if err != nil {
-		return err
-	}
-	if response.StatusCode != http.StatusOK {
-		return fmt.Errorf("api %s status %s body %s", path, response.Status, body)
-	}
-	var baseResponse struct {
-		ErrCode int             `json:"errCode"`
-		ErrMsg  string          `json:"errMsg"`
-		ErrDlt  string          `json:"errDlt"`
-		Data    json.RawMessage `json:"data"`
-	}
-	if err := json.Unmarshal(body, &baseResponse); err != nil {
-		return err
-	}
-	if baseResponse.ErrCode != 0 {
-		return fmt.Errorf("api %s errCode %d errMsg %s errDlt %s", path, baseResponse.ErrCode, baseResponse.ErrMsg, baseResponse.ErrDlt)
-	}
-	if resp != nil {
-		if err := json.Unmarshal(baseResponse.Data, resp); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (a *Api) GetToken(ctx context.Context) (string, error) {
-	req := auth.UserTokenReq{
-		UserID:     a.UserID,
-		Secret:     a.Secret,
-		PlatformID: constant.AdminPlatformID,
-	}
-	var resp auth.UserTokenResp
-	if err := a.apiPost(ctx, "/auth/user_token", &req, &resp); err != nil {
-		return "", err
-	}
-	return resp.Token, nil
-}
-
-func (a *Api) GetPartLimit(ctx context.Context) (*third.PartLimitResp, error) {
-	var resp third.PartLimitResp
-	if err := a.apiPost(ctx, "/object/part_limit", &third.PartLimitReq{}, &resp); err != nil {
-		return nil, err
-	}
-	return &resp, nil
-}
-
-func (a *Api) InitiateMultipartUpload(ctx context.Context, req *third.InitiateMultipartUploadReq) (*third.InitiateMultipartUploadResp, error) {
-	var resp third.InitiateMultipartUploadResp
-	if err := a.apiPost(ctx, "/object/initiate_multipart_upload", req, &resp); err != nil {
-		return nil, err
-	}
-	return &resp, nil
-}
-
-func (a *Api) CompleteMultipartUpload(ctx context.Context, req *third.CompleteMultipartUploadReq) (string, error) {
-	var resp third.CompleteMultipartUploadResp
-	if err := a.apiPost(ctx, "/object/complete_multipart_upload", req, &resp); err != nil {
-		return "", err
-	}
-	return resp.Url, nil
-}

tools/url2im/pkg/buffer.go

@@ -1,96 +0,0 @@
-package pkg
-
-import (
-	"bytes"
-	"io"
-	"os"
-)
-
-type ReadSeekSizeCloser interface {
-	io.ReadSeekCloser
-	Size() int64
-}
-
-func NewReader(r io.Reader, max int64, path string) (ReadSeekSizeCloser, error) {
-	buf := make([]byte, max+1)
-	n, err := io.ReadFull(r, buf)
-	if err == nil {
-		f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0666)
-		if err != nil {
-			return nil, err
-		}
-		var ok bool
-		defer func() {
-			if !ok {
-				_ = f.Close()
-				_ = os.Remove(path)
-			}
-		}()
-		if _, err := f.Write(buf[:n]); err != nil {
-			return nil, err
-		}
-		cn, err := io.Copy(f, r)
-		if err != nil {
-			return nil, err
-		}
-		if _, err := f.Seek(0, io.SeekStart); err != nil {
-			return nil, err
-		}
-		ok = true
-		return &fileBuffer{
-			f: f,
-			n: cn + int64(n),
-		}, nil
-	} else if err == io.EOF || err == io.ErrUnexpectedEOF {
-		return &memoryBuffer{
-			r: bytes.NewReader(buf[:n]),
-		}, nil
-	} else {
-		return nil, err
-	}
-}
-
-type fileBuffer struct {
-	n int64
-	f *os.File
-}
-
-func (r *fileBuffer) Read(p []byte) (n int, err error) {
-	return r.f.Read(p)
-}
-
-func (r *fileBuffer) Seek(offset int64, whence int) (int64, error) {
-	return r.f.Seek(offset, whence)
-}
-
-func (r *fileBuffer) Size() int64 {
-	return r.n
-}
-
-func (r *fileBuffer) Close() error {
-	name := r.f.Name()
-	if err := r.f.Close(); err != nil {
-		return err
-	}
-	return os.Remove(name)
-}
-
-type memoryBuffer struct {
-	r *bytes.Reader
-}
-
-func (r *memoryBuffer) Read(p []byte) (n int, err error) {
-	return r.r.Read(p)
-}
-
-func (r *memoryBuffer) Seek(offset int64, whence int) (int64, error) {
-	return r.r.Seek(offset, whence)
-}
-
-func (r *memoryBuffer) Close() error {
-	return nil
-}
-
-func (r *memoryBuffer) Size() int64 {
-	return r.r.Size()
-}

tools/url2im/pkg/config.go

@@ -1,16 +0,0 @@
-package pkg
-
-import "time"
-
-type Config struct {
-	TaskPath     string
-	ProgressPath string
-	Concurrency  int
-	Retry        int
-	Timeout      time.Duration
-	Api          string
-	UserID       string
-	Secret       string
-	TempDir      string
-	CacheSize    int64
-}

tools/url2im/pkg/http.go

@@ -1,7 +0,0 @@
-package pkg
-
-import "net/http"
-
-func DefaultRequestHeader(header http.Header) {
-	header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36")
-}

tools/url2im/pkg/manage.go

@@ -1,385 +0,0 @@
-package pkg
-
-import (
-	"bufio"
-	"context"
-	"crypto/md5"
-	"encoding/hex"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"log"
-	"net/http"
-	"net/url"
-	"os"
-	"path/filepath"
-	"strconv"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/OpenIMSDK/protocol/third"
-)
-
-type Upload struct {
-	URL         string `json:"url"`
-	Name        string `json:"name"`
-	ContentType string `json:"contentType"`
-}
-
-type Task struct {
-	Index  int
-	Upload Upload
-}
-
-type PartInfo struct {
-	ContentType string
-	PartSize    int64
-	PartNum     int
-	FileMd5     string
-	PartMd5     string
-	PartSizes   []int64
-	PartMd5s    []string
-}
-
-func Run(conf Config) error {
-	m := &Manage{
-		prefix: time.Now().Format("20060102150405"),
-		conf:   &conf,
-		ctx:    context.Background(),
-	}
-	return m.Run()
-}
-
-type Manage struct {
-	conf      *Config
-	ctx       context.Context
-	api       *Api
-	partLimit *third.PartLimitResp
-	prefix    string
-	tasks     chan Task
-	id        uint64
-	success   int64
-	failed    int64
-}
-
-func (m *Manage) tempFilePath() string {
-	return filepath.Join(m.conf.TempDir, fmt.Sprintf("%s_%d", m.prefix, atomic.AddUint64(&m.id, 1)))
-}
-
-func (m *Manage) Run() error {
-	defer func(start time.Time) {
-		log.Printf("run time %s\n", time.Since(start))
-	}(time.Now())
-	m.api = &Api{
-		Api:    m.conf.Api,
-		UserID: m.conf.UserID,
-		Secret: m.conf.Secret,
-		Client: &http.Client{Timeout: m.conf.Timeout},
-	}
-	var err error
-	ctx := context.WithValue(m.ctx, "operationID", fmt.Sprintf("%s_init", m.prefix))
-	m.api.Token, err = m.api.GetToken(ctx)
-	if err != nil {
-		return err
-	}
-	m.partLimit, err = m.api.GetPartLimit(ctx)
-	if err != nil {
-		return err
-	}
-	progress, err := ReadProgress(m.conf.ProgressPath)
-	if err != nil {
-		return err
-	}
-	progressFile, err := os.OpenFile(m.conf.ProgressPath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
-	if err != nil {
-		return err
-	}
-	var mutex sync.Mutex
-	writeSuccessIndex := func(index int) {
-		mutex.Lock()
-		defer mutex.Unlock()
-		if _, err := progressFile.Write([]byte(strconv.Itoa(index) + "\n")); err != nil {
-			log.Printf("write progress err: %v\n", err)
-		}
-	}
-	file, err := os.Open(m.conf.TaskPath)
-	if err != nil {
-		return err
-	}
-	m.tasks = make(chan Task, m.conf.Concurrency*2)
-	go func() {
-		defer file.Close()
-		defer close(m.tasks)
-		scanner := bufio.NewScanner(file)
-		var (
-			index int
-			num   int
-		)
-		for scanner.Scan() {
-			line := strings.TrimSpace(scanner.Text())
-			if line == "" {
-				continue
-			}
-			index++
-			if progress.IsUploaded(index) {
-				log.Printf("index: %d already uploaded %s\n", index, line)
-				continue
-			}
-			var upload Upload
-			if err := json.Unmarshal([]byte(line), &upload); err != nil {
-				log.Printf("index: %d json.Unmarshal(%s) err: %v", index, line, err)
-				continue
-			}
-			num++
-			m.tasks <- Task{
-				Index:  index,
-				Upload: upload,
-			}
-		}
-		if num == 0 {
-			log.Println("mark all completed")
-		}
-	}()
-	var wg sync.WaitGroup
-	wg.Add(m.conf.Concurrency)
-	for i := 0; i < m.conf.Concurrency; i++ {
-		go func(tid int) {
-			defer wg.Done()
-			for task := range m.tasks {
-				var success bool
-				for n := 0; n < m.conf.Retry; n++ {
-					ctx := context.WithValue(m.ctx, "operationID", fmt.Sprintf("%s_%d_%d_%d", m.prefix, tid, task.Index, n+1))
-					if urlRaw, err := m.RunTask(ctx, task); err == nil {
-						writeSuccessIndex(task.Index)
-						log.Println("index:", task.Index, "upload success", "urlRaw", urlRaw)
-						success = true
-						break
-					} else {
-						log.Printf("index: %d upload: %+v err: %v", task.Index, task.Upload, err)
-					}
-				}
-				if success {
-					atomic.AddInt64(&m.success, 1)
-				} else {
-					atomic.AddInt64(&m.failed, 1)
-					log.Printf("index: %d upload: %+v failed", task.Index, task.Upload)
-				}
-			}
-		}(i + 1)
-	}
-	wg.Wait()
-	log.Printf("execution completed success %d failed %d\n", m.success, m.failed)
-	return nil
-}
-
-func (m *Manage) RunTask(ctx context.Context, task Task) (string, error) {
-	resp, err := m.HttpGet(ctx, task.Upload.URL)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-	reader, err := NewReader(resp.Body, m.conf.CacheSize, m.tempFilePath())
-	if err != nil {
-		return "", err
-	}
-	defer reader.Close()
-	part, err := m.getPartInfo(ctx, reader, reader.Size())
-	if err != nil {
-		return "", err
-	}
-	var contentType string
-	if task.Upload.ContentType == "" {
-		contentType = part.ContentType
-	} else {
-		contentType = task.Upload.ContentType
-	}
-	initiateMultipartUploadResp, err := m.api.InitiateMultipartUpload(ctx, &third.InitiateMultipartUploadReq{
-		Hash:        part.PartMd5,
-		Size:        reader.Size(),
-		PartSize:    part.PartSize,
-		MaxParts:    -1,
-		Cause:       "batch-import",
-		Name:        task.Upload.Name,
-		ContentType: contentType,
-	})
-	if err != nil {
-		return "", err
-	}
-	if initiateMultipartUploadResp.Upload == nil {
-		return initiateMultipartUploadResp.Url, nil
-	}
-	if _, err := reader.Seek(0, io.SeekStart); err != nil {
-		return "", err
-	}
-	uploadParts := make([]*third.SignPart, part.PartNum)
-	for _, part := range initiateMultipartUploadResp.Upload.Sign.Parts {
-		uploadParts[part.PartNumber-1] = part
-	}
-	for i, currentPartSize := range part.PartSizes {
-		md5Reader := NewMd5Reader(io.LimitReader(reader, currentPartSize))
-		if m.doPut(ctx, m.api.Client, initiateMultipartUploadResp.Upload.Sign, uploadParts[i], md5Reader, currentPartSize); err != nil {
-			return "", err
-		}
-		if md5val := md5Reader.Md5(); md5val != part.PartMd5s[i] {
-			return "", fmt.Errorf("upload part %d failed, md5 not match, expect %s, got %s", i, part.PartMd5s[i], md5val)
-		}
-	}
-	urlRaw, err := m.api.CompleteMultipartUpload(ctx, &third.CompleteMultipartUploadReq{
-		UploadID:    initiateMultipartUploadResp.Upload.UploadID,
-		Parts:       part.PartMd5s,
-		Name:        task.Upload.Name,
-		ContentType: contentType,
-		Cause:       "batch-import",
-	})
-	if err != nil {
-		return "", err
-	}
-	return urlRaw, nil
-}
-
-func (m *Manage) partSize(size int64) (int64, error) {
-	if size <= 0 {
-		return 0, errors.New("size must be greater than 0")
-	}
-	if size > m.partLimit.MaxPartSize*int64(m.partLimit.MaxNumSize) {
-		return 0, fmt.Errorf("size must be less than %db", m.partLimit.MaxPartSize*int64(m.partLimit.MaxNumSize))
-	}
-	if size <= m.partLimit.MinPartSize*int64(m.partLimit.MaxNumSize) {
-		return m.partLimit.MinPartSize, nil
-	}
-	partSize := size / int64(m.partLimit.MaxNumSize)
-	if size%int64(m.partLimit.MaxNumSize) != 0 {
-		partSize++
-	}
-	return partSize, nil
-}
-
-func (m *Manage) partMD5(parts []string) string {
-	s := strings.Join(parts, ",")
-	md5Sum := md5.Sum([]byte(s))
-	return hex.EncodeToString(md5Sum[:])
-}
-
-func (m *Manage) getPartInfo(ctx context.Context, r io.Reader, fileSize int64) (*PartInfo, error) {
-	partSize, err := m.partSize(fileSize)
-	if err != nil {
-		return nil, err
-	}
-	partNum := int(fileSize / partSize)
-	if fileSize%partSize != 0 {
-		partNum++
-	}
-	partSizes := make([]int64, partNum)
-	for i := 0; i < partNum; i++ {
-		partSizes[i] = partSize
-	}
-	partSizes[partNum-1] = fileSize - partSize*(int64(partNum)-1)
-	partMd5s := make([]string, partNum)
-	buf := make([]byte, 1024*8)
-	fileMd5 := md5.New()
-	var contentType string
-	for i := 0; i < partNum; i++ {
-		h := md5.New()
-		r := io.LimitReader(r, partSize)
-		for {
-			if n, err := r.Read(buf); err == nil {
-				if contentType == "" {
-					contentType = http.DetectContentType(buf[:n])
-				}
-				h.Write(buf[:n])
-				fileMd5.Write(buf[:n])
-			} else if err == io.EOF {
-				break
-			} else {
-				return nil, err
-			}
-		}
-		partMd5s[i] = hex.EncodeToString(h.Sum(nil))
-	}
-	partMd5Val := m.partMD5(partMd5s)
-	fileMd5val := hex.EncodeToString(fileMd5.Sum(nil))
-	return &PartInfo{
-		ContentType: contentType,
-		PartSize:    partSize,
-		PartNum:     partNum,
-		FileMd5:     fileMd5val,
-		PartMd5:     partMd5Val,
-		PartSizes:   partSizes,
-		PartMd5s:    partMd5s,
-	}, nil
-}
-
-func (m *Manage) doPut(ctx context.Context, client *http.Client, sign *third.AuthSignParts, part *third.SignPart, reader io.Reader, size int64) error {
-	rawURL := part.Url
-	if rawURL == "" {
-		rawURL = sign.Url
-	}
-	if len(sign.Query)+len(part.Query) > 0 {
-		u, err := url.Parse(rawURL)
-		if err != nil {
-			return err
-		}
-		query := u.Query()
-		for i := range sign.Query {
-			v := sign.Query[i]
-			query[v.Key] = v.Values
-		}
-		for i := range part.Query {
-			v := part.Query[i]
-			query[v.Key] = v.Values
-		}
-		u.RawQuery = query.Encode()
-		rawURL = u.String()
-	}
-	req, err := http.NewRequestWithContext(ctx, http.MethodPut, rawURL, reader)
-	if err != nil {
-		return err
-	}
-	for i := range sign.Header {
-		v := sign.Header[i]
-		req.Header[v.Key] = v.Values
-	}
-	for i := range part.Header {
-		v := part.Header[i]
-		req.Header[v.Key] = v.Values
-	}
-	req.ContentLength = size
-	resp, err := client.Do(req)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		_ = resp.Body.Close()
-	}()
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-	if resp.StatusCode/200 != 1 {
-		return fmt.Errorf("PUT %s part %d failed, status code %d, body %s", rawURL, part.PartNumber, resp.StatusCode, string(body))
-	}
-	return nil
-}
-
-func (m *Manage) HttpGet(ctx context.Context, url string) (*http.Response, error) {
-	reqUrl := url
-	for {
-		request, err := http.NewRequestWithContext(ctx, http.MethodGet, reqUrl, nil)
-		if err != nil {
-			return nil, err
-		}
-		DefaultRequestHeader(request.Header)
-		response, err := m.api.Client.Do(request)
-		if err != nil {
-			return nil, err
-		}
-		if response.StatusCode != http.StatusOK {
-			_ = response.Body.Close()
-			return nil, fmt.Errorf("http get %s status %s", url, response.Status)
-		}
-		return response, nil
-	}
-}

tools/url2im/pkg/md5.go

@@ -1,29 +0,0 @@
-package pkg
-
-import (
-	"crypto/md5"
-	"encoding/hex"
-	"hash"
-	"io"
-)
-
-func NewMd5Reader(r io.Reader) *Md5Reader {
-	return &Md5Reader{h: md5.New(), r: r}
-}
-
-type Md5Reader struct {
-	h hash.Hash
-	r io.Reader
-}
-
-func (r *Md5Reader) Read(p []byte) (n int, err error) {
-	n, err = r.r.Read(p)
-	if err == nil && n > 0 {
-		r.h.Write(p[:n])
-	}
-	return
-}
-
-func (r *Md5Reader) Md5() string {
-	return hex.EncodeToString(r.h.Sum(nil))
-}

tools/url2im/pkg/progress.go

@@ -1,41 +0,0 @@
-package pkg
-
-import (
-	"bufio"
-	"os"
-	"strconv"
-
-	"github.com/kelindar/bitmap"
-)
-
-func ReadProgress(path string) (*Progress, error) {
-	file, err := os.Open(path)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return &Progress{}, nil
-		}
-		return nil, err
-	}
-	defer file.Close()
-	scanner := bufio.NewScanner(file)
-	var upload bitmap.Bitmap
-	for scanner.Scan() {
-		index, err := strconv.Atoi(scanner.Text())
-		if err != nil || index < 0 {
-			continue
-		}
-		upload.Set(uint32(index))
-	}
-	return &Progress{upload: upload}, nil
-}
-
-type Progress struct {
-	upload bitmap.Bitmap
-}
-
-func (p *Progress) IsUploaded(index int) bool {
-	if p == nil {
-		return false
-	}
-	return p.upload.Contains(uint32(index))
-}