fix: conflict resolve main (#537)

* statistics user register

* refactor: router change

* minio init

* UserRegisterCount

* push use local conn

* refactor: user pb update

* remove online push close grpc conn

* refactor: user pb update

* refactor:pb file

* msgs statistics

* msgs statistics

* revoke userID

* refactor: errcode update

* active user

* active user

* active user

* refactor: errcode update

* feat: conn update token

* active user

* active user

* feat: conn update token

* active user

* feat: conn update token

* feat: conn update token

* feat: conn update token

* add tx_oss cos

* active user

* active user

* group create

* group create

* feat: group notification show to conversation

* feat: group notification show to conversation

* group active

* user active

* sendNotificationWithName

* withname

* privateChat

* a2r call option

* grpc with detail return error

* change log error

* chain unary interceptor

* api nil slice map

* fix sync has read

* fix: text update

* fix: update add model

* set conversations update

* set privateChat

* fix: content update

* remove unuse rpc

* msgDestruct

* cron use rpc mw

* set IsMsgDestruct

* msg destruct

* msgDestruct

* s3 minio, cos, oss support

* feat: add implement of GetUsersOnlineStatus, #472 (#477)

* s3 minio, cos, oss support

* s3 route

* remove extendMsg code

* s3 route

* remove unuse code

* s3 pb

* s3 pb

* s3 pb

* s3 presigned put

* s3 presigned test

* s3 presigned test

* s3 presigned test

* s3 presigned test

* s3 presigned test

* s3 presigned test

* s3 presigned test

* s3 presigned test

* Update .gitignore (#482)

* s3 debug log

* s3 debug log

* cron add log and fix cron

* add log

* cron

* s3 config

* fix kick user bug

* s3 cos

* add kick log

* s3 cos test

* s3 cos test

* s3 cos test

* kick user log

* kickuserlog

* s3 cos copy

* s3 cos copy

* s3 url

* s3 url

* s3 AccessURL

* log

* s3 InitiateMultipartUpload add ExpireTime

* feat: regenerate pb file

* feat: regenerate pb file

* Revert "feat: regenerate pb file"

This reverts commit 434f22564a.

* Delete .idea directory

* feat: regenerate pb file

* fix: remove import C

* fix: add msg transfer main file

* fix: get user online status fix

---------

Co-authored-by: withchao <993506633@qq.com>
Co-authored-by: wangchuxiao <wangchuxiao97@outlook.com>
Co-authored-by: BanTanger <88583317+BanTanger@users.noreply.github.com>
Co-authored-by: withchao <48119764+withchao@users.noreply.github.com>
Co-authored-by: Alan <68671759+hanzhixiao@users.noreply.github.com>

pkg/common/db/s3/cont/controller.go

@@ -6,15 +6,13 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
-	"path"
-	"strings"
-	"time"
-
-	"github.com/google/uuid"
-
 	"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
 	"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
 	"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
+	"github.com/google/uuid"
+	"path"
+	"strings"
+	"time"
 )
 
 func New(impl s3.Interface) *Controller {
@@ -58,13 +56,7 @@ func (c *Controller) GetHashObject(ctx context.Context, hash string) (*s3.Object
 	return c.impl.StatObject(ctx, c.HashPath(hash))
 }
 
-func (c *Controller) InitiateUpload(
-	ctx context.Context,
-	hash string,
-	size int64,
-	expire time.Duration,
-	maxParts int,
-) (*InitiateUploadResult, error) {
+func (c *Controller) InitiateUpload(ctx context.Context, hash string, size int64, expire time.Duration, maxParts int) (*InitiateUploadResult, error) {
 	defer log.ZDebug(ctx, "return")
 	if size < 0 {
 		return nil, errors.New("invalid size")
@@ -247,11 +239,6 @@ func (c *Controller) IsNotFound(err error) bool {
 	return c.impl.IsNotFound(err)
 }
 
-func (c *Controller) AccessURL(
-	ctx context.Context,
-	name string,
-	expire time.Duration,
-	opt *s3.AccessURLOption,
-) (string, error) {
+func (c *Controller) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
 	return c.impl.AccessURL(ctx, name, expire, opt)
 }

pkg/common/db/s3/cont/error.go

@@ -2,7 +2,6 @@ package cont
 
 import (
 	"fmt"
-
 	"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
 )
 

pkg/common/db/s3/cos/cos.go

@@ -4,16 +4,14 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
+	"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
+	"github.com/tencentyun/cos-go-sdk-v5"
 	"net/http"
 	"net/url"
 	"strconv"
 	"strings"
 	"time"
-
-	"github.com/tencentyun/cos-go-sdk-v5"
-
-	"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
-	"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
 )
 
 const (
@@ -72,12 +70,7 @@ func (c *Cos) InitiateMultipartUpload(ctx context.Context, name string) (*s3.Ini
 	}, nil
 }
 
-func (c *Cos) CompleteMultipartUpload(
-	ctx context.Context,
-	uploadID string,
-	name string,
-	parts []s3.Part,
-) (*s3.CompleteMultipartUploadResult, error) {
+func (c *Cos) CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []s3.Part) (*s3.CompleteMultipartUploadResult, error) {
 	opts := &cos.CompleteMultipartUploadOptions{
 		Parts: make([]cos.Object, len(parts)),
 	}
@@ -116,13 +109,7 @@ func (c *Cos) PartSize(ctx context.Context, size int64) (int64, error) {
 	return partSize, nil
 }
 
-func (c *Cos) AuthSign(
-	ctx context.Context,
-	uploadID string,
-	name string,
-	expire time.Duration,
-	partNumbers []int,
-) (*s3.AuthSignResult, error) {
+func (c *Cos) AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*s3.AuthSignResult, error) {
 	result := s3.AuthSignResult{
 		URL:    c.client.BaseURL.BucketURL.String() + "/" + cos.EncodeURIComponent(name),
 		Query:  url.Values{"uploadId": {uploadID}},
@@ -133,13 +120,7 @@ func (c *Cos) AuthSign(
 	if err != nil {
 		return nil, err
 	}
-	cos.AddAuthorizationHeader(
-		c.credential.SecretID,
-		c.credential.SecretKey,
-		c.credential.SessionToken,
-		req,
-		cos.NewAuthTime(expire),
-	)
+	cos.AddAuthorizationHeader(c.credential.SecretID, c.credential.SecretKey, c.credential.SessionToken, req, cos.NewAuthTime(expire))
 	result.Header = req.Header
 	for i, partNumber := range partNumbers {
 		result.Parts[i] = s3.SignPart{
@@ -151,15 +132,7 @@ func (c *Cos) AuthSign(
 }
 
 func (c *Cos) PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error) {
-	rawURL, err := c.client.Object.GetPresignedURL(
-		ctx,
-		http.MethodPut,
-		name,
-		c.credential.SecretID,
-		c.credential.SecretKey,
-		expire,
-		nil,
-	)
+	rawURL, err := c.client.Object.GetPresignedURL(ctx, http.MethodPut, name, c.credential.SecretID, c.credential.SecretKey, expire, nil)
 	if err != nil {
 		return "", err
 	}
@@ -231,13 +204,7 @@ func (c *Cos) AbortMultipartUpload(ctx context.Context, uploadID string, name st
 	return err
 }
 
-func (c *Cos) ListUploadedParts(
-	ctx context.Context,
-	uploadID string,
-	name string,
-	partNumberMarker int,
-	maxParts int,
-) (*s3.ListUploadedPartsResult, error) {
+func (c *Cos) ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*s3.ListUploadedPartsResult, error) {
 	result, _, err := c.client.Object.ListParts(ctx, name, uploadID, &cos.ObjectListPartsOptions{
 		MaxParts:         strconv.Itoa(maxParts),
 		PartNumberMarker: strconv.Itoa(partNumberMarker),
@@ -264,12 +231,7 @@ func (c *Cos) ListUploadedParts(
 	return res, nil
 }
 
-func (c *Cos) AccessURL(
-	ctx context.Context,
-	name string,
-	expire time.Duration,
-	opt *s3.AccessURLOption,
-) (string, error) {
+func (c *Cos) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
 	//reqParams := make(url.Values)
 	//if opt != nil {
 	//	if opt.ContentType != "" {
@@ -284,15 +246,7 @@ func (c *Cos) AccessURL(
 	} else if expire < time.Second {
 		expire = time.Second
 	}
-	rawURL, err := c.client.Object.GetPresignedURL(
-		ctx,
-		http.MethodGet,
-		name,
-		c.credential.SecretID,
-		c.credential.SecretKey,
-		expire,
-		nil,
-	)
+	rawURL, err := c.client.Object.GetPresignedURL(ctx, http.MethodGet, name, c.credential.SecretID, c.credential.SecretKey, expire, nil)
 	if err != nil {
 		return "", err
 	}

pkg/common/db/s3/minio/minio.go

@@ -4,18 +4,16 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
+	"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/minio/minio-go/v7/pkg/signer"
 	"net/http"
 	"net/url"
 	"strconv"
 	"strings"
 	"time"
-
-	"github.com/minio/minio-go/v7"
-	"github.com/minio/minio-go/v7/pkg/credentials"
-	"github.com/minio/minio-go/v7/pkg/signer"
-
-	"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
-	"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
 )
 
 const (
@@ -81,12 +79,7 @@ func (m *Minio) InitiateMultipartUpload(ctx context.Context, name string) (*s3.I
 	}, nil
 }
 
-func (m *Minio) CompleteMultipartUpload(
-	ctx context.Context,
-	uploadID string,
-	name string,
-	parts []s3.Part,
-) (*s3.CompleteMultipartUploadResult, error) {
+func (m *Minio) CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []s3.Part) (*s3.CompleteMultipartUploadResult, error) {
 	minioParts := make([]minio.CompletePart, len(parts))
 	for i, part := range parts {
 		minioParts[i] = minio.CompletePart{
@@ -123,13 +116,7 @@ func (m *Minio) PartSize(ctx context.Context, size int64) (int64, error) {
 	return partSize, nil
 }
 
-func (m *Minio) AuthSign(
-	ctx context.Context,
-	uploadID string,
-	name string,
-	expire time.Duration,
-	partNumbers []int,
-) (*s3.AuthSignResult, error) {
+func (m *Minio) AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*s3.AuthSignResult, error) {
 	creds, err := m.opts.Creds.Get()
 	if err != nil {
 		return nil, err
@@ -146,14 +133,7 @@ func (m *Minio) AuthSign(
 			return nil, err
 		}
 		request.Header.Set("X-Amz-Content-Sha256", unsignedPayload)
-		request = signer.SignV4Trailer(
-			*request,
-			creds.AccessKeyID,
-			creds.SecretAccessKey,
-			creds.SessionToken,
-			"us-east-1",
-			nil,
-		)
+		request = signer.SignV4Trailer(*request, creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken, "us-east-1", nil)
 		result.Parts[i] = s3.SignPart{
 			PartNumber: partNumber,
 			URL:        request.URL.String(),
@@ -224,13 +204,7 @@ func (m *Minio) AbortMultipartUpload(ctx context.Context, uploadID string, name
 	return m.core.AbortMultipartUpload(ctx, m.bucket, name, uploadID)
 }
 
-func (m *Minio) ListUploadedParts(
-	ctx context.Context,
-	uploadID string,
-	name string,
-	partNumberMarker int,
-	maxParts int,
-) (*s3.ListUploadedPartsResult, error) {
+func (m *Minio) ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*s3.ListUploadedPartsResult, error) {
 	result, err := m.core.ListObjectParts(ctx, m.bucket, name, uploadID, partNumberMarker, maxParts)
 	if err != nil {
 		return nil, err
@@ -253,12 +227,7 @@ func (m *Minio) ListUploadedParts(
 	return res, nil
 }
 
-func (m *Minio) AccessURL(
-	ctx context.Context,
-	name string,
-	expire time.Duration,
-	opt *s3.AccessURLOption,
-) (string, error) {
+func (m *Minio) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
 	//reqParams := make(url.Values)
 	//if opt != nil {
 	//	if opt.ContentType != "" {

pkg/common/db/s3/oss/oss.go

@@ -4,16 +4,14 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
+	"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
+	"github.com/aliyun/aliyun-oss-go-sdk/oss"
 	"net/http"
 	"net/url"
 	"strconv"
 	"strings"
 	"time"
-
-	"github.com/aliyun/aliyun-oss-go-sdk/oss"
-
-	"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
-	"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
 )
 
 const (
@@ -75,12 +73,7 @@ func (o *OSS) InitiateMultipartUpload(ctx context.Context, name string) (*s3.Ini
 	}, nil
 }
 
-func (o *OSS) CompleteMultipartUpload(
-	ctx context.Context,
-	uploadID string,
-	name string,
-	parts []s3.Part,
-) (*s3.CompleteMultipartUploadResult, error) {
+func (o *OSS) CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []s3.Part) (*s3.CompleteMultipartUploadResult, error) {
 	ossParts := make([]oss.UploadPart, len(parts))
 	for i, part := range parts {
 		ossParts[i] = oss.UploadPart{
@@ -121,13 +114,7 @@ func (o *OSS) PartSize(ctx context.Context, size int64) (int64, error) {
 	return partSize, nil
 }
 
-func (o *OSS) AuthSign(
-	ctx context.Context,
-	uploadID string,
-	name string,
-	expire time.Duration,
-	partNumbers []int,
-) (*s3.AuthSignResult, error) {
+func (o *OSS) AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*s3.AuthSignResult, error) {
 	result := s3.AuthSignResult{
 		URL:    o.bucketURL + name,
 		Query:  url.Values{"uploadId": {uploadID}},
@@ -145,15 +132,7 @@ func (o *OSS) AuthSign(
 		}
 		request.Header.Set(oss.HTTPHeaderHost, request.Host)
 		request.Header.Set(oss.HTTPHeaderDate, time.Now().UTC().Format(http.TimeFormat))
-		authorization := fmt.Sprintf(
-			`OSS %s:%s`,
-			o.credentials.GetAccessKeyID(),
-			o.getSignedStr(
-				request,
-				fmt.Sprintf(`/%s/%s?partNumber=%d&uploadId=%s`, o.bucket.BucketName, name, partNumber, uploadID),
-				o.credentials.GetAccessKeySecret(),
-			),
-		)
+		authorization := fmt.Sprintf(`OSS %s:%s`, o.credentials.GetAccessKeyID(), o.getSignedStr(request, fmt.Sprintf(`/%s/%s?partNumber=%d&uploadId=%s`, o.bucket.BucketName, name, partNumber, uploadID), o.credentials.GetAccessKeySecret()))
 		request.Header.Set(oss.HTTPHeaderAuthorization, authorization)
 		result.Parts[i] = s3.SignPart{
 			PartNumber: partNumber,
@@ -234,13 +213,7 @@ func (o *OSS) AbortMultipartUpload(ctx context.Context, uploadID string, name st
 	})
 }
 
-func (o *OSS) ListUploadedParts(
-	ctx context.Context,
-	uploadID string,
-	name string,
-	partNumberMarker int,
-	maxParts int,
-) (*s3.ListUploadedPartsResult, error) {
+func (o *OSS) ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*s3.ListUploadedPartsResult, error) {
 	result, err := o.bucket.ListUploadedParts(oss.InitiateMultipartUploadResult{
 		UploadID: uploadID,
 		Key:      name,
@@ -267,12 +240,7 @@ func (o *OSS) ListUploadedParts(
 	return res, nil
 }
 
-func (o *OSS) AccessURL(
-	ctx context.Context,
-	name string,
-	expire time.Duration,
-	opt *s3.AccessURLOption,
-) (string, error) {
+func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
 	//var opts []oss.Option
 	//if opt != nil {
 	//	if opt.ContentType != "" {

pkg/common/db/s3/oss/sign.go

@@ -5,13 +5,12 @@ import (
 	"crypto/sha1"
 	"crypto/sha256"
 	"encoding/base64"
+	"github.com/aliyun/aliyun-oss-go-sdk/oss"
 	"hash"
 	"io"
 	"net/http"
 	"sort"
 	"strings"
-
-	"github.com/aliyun/aliyun-oss-go-sdk/oss"
 )
 
 func (o *OSS) getAdditionalHeaderKeys(req *http.Request) ([]string, map[string]string) {
@@ -72,10 +71,7 @@ func (o *OSS) getSignedStr(req *http.Request, canonicalizedResource string, keyS
 
 	// v2 signature
 	if o.bucket.Client.Config.AuthVersion == oss.AuthV2 {
-		signStr = req.Method + "\n" + contentMd5 + "\n" + contentType + "\n" + date + "\n" + canonicalizedOSSHeaders + strings.Join(
-			additionalList,
-			";",
-		) + "\n" + canonicalizedResource
+		signStr = req.Method + "\n" + contentMd5 + "\n" + contentType + "\n" + date + "\n" + canonicalizedOSSHeaders + strings.Join(additionalList, ";") + "\n" + canonicalizedResource
 		h = hmac.New(func() hash.Hash { return sha256.New() }, []byte(keySecret))
 	}
 	_, _ = io.WriteString(h, signStr)

pkg/common/db/s3/s3.go

@@ -112,21 +112,10 @@ type Interface interface {
 	PartLimit() *PartLimit
 
 	InitiateMultipartUpload(ctx context.Context, name string) (*InitiateMultipartUploadResult, error)
-	CompleteMultipartUpload(
-		ctx context.Context,
-		uploadID string,
-		name string,
-		parts []Part,
-	) (*CompleteMultipartUploadResult, error)
+	CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []Part) (*CompleteMultipartUploadResult, error)
 
 	PartSize(ctx context.Context, size int64) (int64, error)
-	AuthSign(
-		ctx context.Context,
-		uploadID string,
-		name string,
-		expire time.Duration,
-		partNumbers []int,
-	) (*AuthSignResult, error)
+	AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*AuthSignResult, error)
 
 	PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error)
 
@@ -139,13 +128,7 @@ type Interface interface {
 	IsNotFound(err error) bool
 
 	AbortMultipartUpload(ctx context.Context, uploadID string, name string) error
-	ListUploadedParts(
-		ctx context.Context,
-		uploadID string,
-		name string,
-		partNumberMarker int,
-		maxParts int,
-	) (*ListUploadedPartsResult, error)
+	ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*ListUploadedPartsResult, error)
 
 	AccessURL(ctx context.Context, name string, expire time.Duration, opt *AccessURLOption) (string, error)
 }