feat: Remove MySQL and implement it all with Mongo (#1508)

* fix: GetUserReqApplicationList error when there is a disbanded group chat

* fix: error when querying some information about disbanded group

* fix: GetUserReqApplicationList dismissed group error

* fix: the original message referenced by the pull message processing is withdrawn

* fix: the original message referenced by the pull message processing is withdrawn

* fix: the original message referenced by the pull message processing is withdrawn

* fix: the original message referenced by the pull message processing is withdrawn

* fix: the original message referenced by the pull message processing is withdrawn

* fix: the original message referenced by the pull message processing is withdrawn

* fix: the original message referenced by the pull message processing is withdrawn

* fix: the original message referenced by the pull message processing is withdrawn

* fix: the original message referenced by the pull message processing is withdrawn

* merge

* cicd: robot automated Change

* sdkws.MsgData

* user

* interface{} -> any

* user

* third

* group

* group

* group

* group

* group

* group

* conversation

* standalone mysql db model

* tx

* s3

* group

* mongo

* group

* group

* group

* group

* group

* group

* refactor: add openim mysql to mongo refactor

Signed-off-by: Xinwei Xiong(cubxxw) <3293172751nss@gmail.com>

* refactor: add openim mysql to mongo refactor

Signed-off-by: Xinwei Xiong(cubxxw) <3293172751nss@gmail.com>

* remove mysql

* remove mysql

* friend

* friend

* friend

* friend

* friend

* friend

* group

* convert

* index

* index

* all

* all

* mysql2mongo

* data conversion

* up35

* up35

* feat: add format set

Signed-off-by: Xinwei Xiong(cubxxw) <3293172751nss@gmail.com>

* fix: fix scripts

Signed-off-by: Xinwei Xiong(cubxxw) <3293172751nss@gmail.com>

* merge main

* merge main

* Update init-config.sh

* fix: user args check

---------

Signed-off-by: Xinwei Xiong(cubxxw) <3293172751nss@gmail.com>
Co-authored-by: withchao <withchao@users.noreply.github.com>
Co-authored-by: Xinwei Xiong(cubxxw) <3293172751nss@gmail.com>
Co-authored-by: Xinwei Xiong <3293172751@qq.com>

pkg/common/db/s3/cont/controller.go

@@ -46,6 +46,10 @@ type Controller struct {
 	impl  s3.Interface
 }
 
+func (c *Controller) Engine() string {
+	return c.impl.Engine()
+}
+
 func (c *Controller) HashPath(md5 string) string {
 	return path.Join(hashPath, md5)
 }

pkg/common/db/s3/cos/internal.go

@@ -10,4 +10,4 @@ import (
 )
 
 //go:linkname newRequest github.com/tencentyun/cos-go-sdk-v5.(*Client).newRequest
-func newRequest(c *cos.Client, ctx context.Context, baseURL *url.URL, uri, method string, body interface{}, optQuery interface{}, optHeader interface{}) (req *http.Request, err error)
+func newRequest(c *cos.Client, ctx context.Context, baseURL *url.URL, uri, method string, body any, optQuery any, optHeader any) (req *http.Request, err error)

pkg/common/db/s3/kodo/internal.go

@@ -1 +0,0 @@
-package kodo

pkg/common/db/s3/kodo/kodo.go

@@ -1,323 +0,0 @@
-package kodo
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"net/http"
-	"net/url"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/aws/aws-sdk-go-v2/aws"
-	awss3config "github.com/aws/aws-sdk-go-v2/config"
-	"github.com/aws/aws-sdk-go-v2/credentials"
-	awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
-	awss3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
-	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
-	"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
-	"github.com/qiniu/go-sdk/v7/auth"
-	"github.com/qiniu/go-sdk/v7/storage"
-)
-
-const (
-	minPartSize = 1024 * 1024 * 1        // 1MB
-	maxPartSize = 1024 * 1024 * 1024 * 5 // 5GB
-	maxNumSize  = 10000
-)
-
-type Kodo struct {
-	AccessKey     string
-	SecretKey     string
-	Region        string
-	Token         string
-	Endpoint      string
-	BucketURL     string
-	Auth          *auth.Credentials
-	Client        *awss3.Client
-	PresignClient *awss3.PresignClient
-}
-
-func NewKodo() (s3.Interface, error) {
-	conf := config.Config.Object.Kodo
-	//init client
-	cfg, err := awss3config.LoadDefaultConfig(context.TODO(),
-		awss3config.WithRegion(conf.Bucket),
-		awss3config.WithEndpointResolverWithOptions(
-			aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
-				return aws.Endpoint{URL: conf.Endpoint}, nil
-			})),
-		awss3config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
-			conf.AccessKeyID,
-			conf.AccessKeySecret,
-			conf.SessionToken),
-		),
-	)
-	if err != nil {
-		panic(err)
-	}
-	client := awss3.NewFromConfig(cfg)
-	presignClient := awss3.NewPresignClient(client)
-
-	return &Kodo{
-		AccessKey:     conf.AccessKeyID,
-		SecretKey:     conf.AccessKeySecret,
-		Region:        conf.Bucket,
-		BucketURL:     conf.BucketURL,
-		Auth:          auth.New(conf.AccessKeyID, conf.AccessKeySecret),
-		Client:        client,
-		PresignClient: presignClient,
-	}, nil
-}
-
-func (k Kodo) Engine() string {
-	return "kodo"
-}
-
-func (k Kodo) PartLimit() *s3.PartLimit {
-	return &s3.PartLimit{
-		MinPartSize: minPartSize,
-		MaxPartSize: maxPartSize,
-		MaxNumSize:  maxNumSize,
-	}
-}
-
-func (k Kodo) InitiateMultipartUpload(ctx context.Context, name string) (*s3.InitiateMultipartUploadResult, error) {
-	result, err := k.Client.CreateMultipartUpload(ctx, &awss3.CreateMultipartUploadInput{
-		Bucket: aws.String(k.Region),
-		Key:    aws.String(name),
-	})
-	if err != nil {
-		return nil, err
-	}
-	return &s3.InitiateMultipartUploadResult{
-		UploadID: aws.ToString(result.UploadId),
-		Bucket:   aws.ToString(result.Bucket),
-		Key:      aws.ToString(result.Key),
-	}, nil
-}
-
-func (k Kodo) CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []s3.Part) (*s3.CompleteMultipartUploadResult, error) {
-	kodoParts := make([]awss3types.CompletedPart, len(parts))
-	for i, part := range parts {
-		kodoParts[i] = awss3types.CompletedPart{
-			PartNumber: aws.Int32(int32(part.PartNumber)),
-			ETag:       aws.String(part.ETag),
-		}
-	}
-	result, err := k.Client.CompleteMultipartUpload(ctx, &awss3.CompleteMultipartUploadInput{
-		Bucket:          aws.String(k.Region),
-		Key:             aws.String(name),
-		UploadId:        aws.String(uploadID),
-		MultipartUpload: &awss3types.CompletedMultipartUpload{Parts: kodoParts},
-	})
-	if err != nil {
-		return nil, err
-	}
-	return &s3.CompleteMultipartUploadResult{
-		Location: aws.ToString(result.Location),
-		Bucket:   aws.ToString(result.Bucket),
-		Key:      aws.ToString(result.Key),
-		ETag:     strings.ToLower(strings.ReplaceAll(aws.ToString(result.ETag), `"`, ``)),
-	}, nil
-}
-
-func (k Kodo) PartSize(ctx context.Context, size int64) (int64, error) {
-	if size <= 0 {
-		return 0, errors.New("size must be greater than 0")
-	}
-	if size > maxPartSize*maxNumSize {
-		return 0, fmt.Errorf("size must be less than %db", maxPartSize*maxNumSize)
-	}
-	if size <= minPartSize*maxNumSize {
-		return minPartSize, nil
-	}
-	partSize := size / maxNumSize
-	if size%maxNumSize != 0 {
-		partSize++
-	}
-	return partSize, nil
-}
-
-func (k Kodo) AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*s3.AuthSignResult, error) {
-	result := s3.AuthSignResult{
-		URL:    k.BucketURL + "/" + name,
-		Query:  url.Values{"uploadId": {uploadID}},
-		Header: make(http.Header),
-		Parts:  make([]s3.SignPart, len(partNumbers)),
-	}
-	for i, partNumber := range partNumbers {
-		part, _ := k.PresignClient.PresignUploadPart(ctx, &awss3.UploadPartInput{
-			Bucket:     aws.String(k.Region),
-			UploadId:   aws.String(uploadID),
-			Key:        aws.String(name),
-			PartNumber: aws.Int32(int32(partNumber)),
-		})
-		result.Parts[i] = s3.SignPart{
-			PartNumber: partNumber,
-			URL:        part.URL,
-			Header:     part.SignedHeader,
-		}
-	}
-	return &result, nil
-
-}
-
-func (k Kodo) PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error) {
-	object, err := k.PresignClient.PresignPutObject(ctx, &awss3.PutObjectInput{
-		Bucket: aws.String(k.Region),
-		Key:    aws.String(name),
-	}, func(po *awss3.PresignOptions) {
-		po.Expires = expire
-	})
-	return object.URL, err
-
-}
-
-func (k Kodo) DeleteObject(ctx context.Context, name string) error {
-	_, err := k.Client.DeleteObject(ctx, &awss3.DeleteObjectInput{
-		Bucket: aws.String(k.Region),
-		Key:    aws.String(name),
-	})
-	return err
-}
-
-func (k Kodo) CopyObject(ctx context.Context, src string, dst string) (*s3.CopyObjectInfo, error) {
-	result, err := k.Client.CopyObject(ctx, &awss3.CopyObjectInput{
-		Bucket:     aws.String(k.Region),
-		CopySource: aws.String(k.Region + "/" + src),
-		Key:        aws.String(dst),
-	})
-	if err != nil {
-		return nil, err
-	}
-	return &s3.CopyObjectInfo{
-		Key:  dst,
-		ETag: strings.ToLower(strings.ReplaceAll(aws.ToString(result.CopyObjectResult.ETag), `"`, ``)),
-	}, nil
-}
-
-func (k Kodo) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) {
-	info, err := k.Client.HeadObject(ctx, &awss3.HeadObjectInput{
-		Bucket: aws.String(k.Region),
-		Key:    aws.String(name),
-	})
-	if err != nil {
-		return nil, err
-	}
-	res := &s3.ObjectInfo{Key: name}
-	res.Size = aws.ToInt64(info.ContentLength)
-	res.ETag = strings.ToLower(strings.ReplaceAll(aws.ToString(info.ETag), `"`, ``))
-	return res, nil
-}
-
-func (k Kodo) IsNotFound(err error) bool {
-	return true
-}
-
-func (k Kodo) AbortMultipartUpload(ctx context.Context, uploadID string, name string) error {
-	_, err := k.Client.AbortMultipartUpload(ctx, &awss3.AbortMultipartUploadInput{
-		UploadId: aws.String(uploadID),
-		Bucket:   aws.String(k.Region),
-		Key:      aws.String(name),
-	})
-	return err
-}
-
-func (k Kodo) ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*s3.ListUploadedPartsResult, error) {
-	result, err := k.Client.ListParts(ctx, &awss3.ListPartsInput{
-		Key:              aws.String(name),
-		UploadId:         aws.String(uploadID),
-		Bucket:           aws.String(k.Region),
-		MaxParts:         aws.Int32(int32(maxParts)),
-		PartNumberMarker: aws.String(strconv.Itoa(partNumberMarker)),
-	})
-	if err != nil {
-		return nil, err
-	}
-	res := &s3.ListUploadedPartsResult{
-		Key:           aws.ToString(result.Key),
-		UploadID:      aws.ToString(result.UploadId),
-		MaxParts:      int(aws.ToInt32(result.MaxParts)),
-		UploadedParts: make([]s3.UploadedPart, len(result.Parts)),
-	}
-	// int to string
-	NextPartNumberMarker, err := strconv.Atoi(aws.ToString(result.NextPartNumberMarker))
-	if err != nil {
-		return nil, err
-	}
-	res.NextPartNumberMarker = NextPartNumberMarker
-	for i, part := range result.Parts {
-		res.UploadedParts[i] = s3.UploadedPart{
-			PartNumber:   int(aws.ToInt32(part.PartNumber)),
-			LastModified: aws.ToTime(part.LastModified),
-			ETag:         aws.ToString(part.ETag),
-			Size:         aws.ToInt64(part.Size),
-		}
-	}
-	return res, nil
-}
-
-func (k Kodo) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
-	//get object head
-	info, err := k.Client.HeadObject(ctx, &awss3.HeadObjectInput{
-		Bucket: aws.String(k.Region),
-		Key:    aws.String(name),
-	})
-	if err != nil {
-		return "", errors.New("AccessURL object not found")
-	}
-	if opt != nil {
-		if opt.ContentType != aws.ToString(info.ContentType) {
-			//修改文件类型
-			err := k.SetObjectContentType(ctx, name, opt.ContentType)
-			if err != nil {
-				return "", errors.New("AccessURL setContentType error")
-			}
-		}
-	}
-	imageMogr := ""
-	//image dispose
-	if opt != nil {
-		if opt.Image != nil {
-			//https://developer.qiniu.com/dora/8255/the-zoom
-			process := ""
-			if opt.Image.Width > 0 {
-				process += strconv.Itoa(opt.Image.Width) + "x"
-			}
-			if opt.Image.Height > 0 {
-				if opt.Image.Width > 0 {
-					process += strconv.Itoa(opt.Image.Height)
-				} else {
-					process += "x" + strconv.Itoa(opt.Image.Height)
-				}
-			}
-			imageMogr = "imageMogr2/thumbnail/" + process
-		}
-	}
-	//expire
-	deadline := time.Now().Add(time.Second * expire).Unix()
-	domain := k.BucketURL
-	query := url.Values{}
-	if opt != nil && opt.Filename != "" {
-		query.Add("attname", opt.Filename)
-	}
-	privateURL := storage.MakePrivateURLv2WithQuery(k.Auth, domain, name, query, deadline)
-	if imageMogr != "" {
-		privateURL += "&" + imageMogr
-	}
-	return privateURL, nil
-}
-
-func (k *Kodo) SetObjectContentType(ctx context.Context, name string, contentType string) error {
-	//set object content-type
-	_, err := k.Client.CopyObject(ctx, &awss3.CopyObjectInput{
-		Bucket:            aws.String(k.Region),
-		CopySource:        aws.String(k.Region + "/" + name),
-		Key:               aws.String(name),
-		ContentType:       aws.String(contentType),
-		MetadataDirective: awss3types.MetadataDirectiveReplace,
-	})
-	return err
-}

pkg/common/db/s3/oss/internal.go

@@ -26,7 +26,7 @@ import (
 func signHeader(c oss.Conn, req *http.Request, canonicalizedResource string)
 
 //go:linkname getURLParams github.com/aliyun/aliyun-oss-go-sdk/oss.Conn.getURLParams
-func getURLParams(c oss.Conn, params map[string]interface{}) string
+func getURLParams(c oss.Conn, params map[string]any) string
 
 //go:linkname getURL github.com/aliyun/aliyun-oss-go-sdk/oss.urlMaker.getURL
 func getURL(um urlMaker, bucket, object, params string) *url.URL