feat: s3 FormData upload (#1614)

* upgrade package and rtc convert

* upgrade package and rtc convert

* upgrade package and rtc convert

* upgrade package and rtc convert

* friend user

* s3 form data

* s3 form data

* s3 form data

* s3 form data

* s3 form data

* s3 form data

* s3 form data

* s3 form data

* s3 form data

pkg/common/db/controller/s3.go

@@ -35,6 +35,8 @@ type S3Database interface {
 	CompleteMultipartUpload(ctx context.Context, uploadID string, parts []string) (*cont.UploadResult, error)
 	AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (time.Time, string, error)
 	SetObject(ctx context.Context, info *relation.ObjectModel) error
+	StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error)
+	FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error)
 }
 
 func NewS3Database(rdb redis.UniversalClient, s3 s3.Interface, obj relation.ObjectInfoModelInterface) S3Database {
@@ -100,3 +102,11 @@ func (s *s3Database) AccessURL(ctx context.Context, name string, expire time.Dur
 	}
 	return expireTime, rawURL, nil
 }
+
+func (s *s3Database) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) {
+	return s.s3.StatObject(ctx, name)
+}
+
+func (s *s3Database) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) {
+	return s.s3.FormData(ctx, name, size, contentType, duration)
+}

pkg/common/db/s3/cont/consts.go

@@ -17,6 +17,7 @@ package cont
 const (
 	hashPath            = "openim/data/hash/"
 	tempPath            = "openim/temp/"
+	DirectPath          = "openim/direct"
 	UploadTypeMultipart = 1 // 分片上传
 	UploadTypePresigned = 2 // 预签名上传
 	partSeparator       = ","

pkg/common/db/s3/cont/controller.go

@@ -279,3 +279,7 @@ func (c *Controller) AccessURL(ctx context.Context, name string, expire time.Dur
 	}
 	return c.impl.AccessURL(ctx, name, expire, opt)
 }
+
+func (c *Controller) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) {
+	return c.impl.FormData(ctx, name, size, contentType, duration)
+}

pkg/common/db/s3/cos/cos.go

@@ -16,6 +16,11 @@ package cos
 
 import (
 	"context"
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
@@ -44,6 +49,8 @@ const (
 	imageWebp = "webp"
 )
 
+const successCode = http.StatusOK
+
 const (
 	videoSnapshotImagePng = "png"
 	videoSnapshotImageJpg = "jpg"
@@ -326,3 +333,65 @@ func (c *Cos) getPresignedURL(ctx context.Context, name string, expire time.Dura
 	}
 	return c.client.Object.GetObjectURL(name), nil
 }
+
+func (c *Cos) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) {
+	// https://cloud.tencent.com/document/product/436/14690
+	now := time.Now()
+	expiration := now.Add(duration)
+	keyTime := fmt.Sprintf("%d;%d", now.Unix(), expiration.Unix())
+	conditions := []any{
+		map[string]string{"q-sign-algorithm": "sha1"},
+		map[string]string{"q-ak": c.credential.SecretID},
+		map[string]string{"q-sign-time": keyTime},
+		map[string]string{"key": name},
+	}
+	if contentType != "" {
+		conditions = append(conditions, map[string]string{"Content-Type": contentType})
+	}
+	policy := map[string]any{
+		"expiration": expiration.Format("2006-01-02T15:04:05.000Z"),
+		"conditions": conditions,
+	}
+	policyJson, err := json.Marshal(policy)
+	if err != nil {
+		return nil, err
+	}
+	signKey := hmacSha1val(c.credential.SecretKey, keyTime)
+	strToSign := sha1val(string(policyJson))
+	signature := hmacSha1val(signKey, strToSign)
+
+	fd := &s3.FormData{
+		URL:     c.client.BaseURL.BucketURL.String(),
+		File:    "file",
+		Expires: expiration,
+		FormData: map[string]string{
+			"policy":                base64.StdEncoding.EncodeToString(policyJson),
+			"q-sign-algorithm":      "sha1",
+			"q-ak":                  c.credential.SecretID,
+			"q-key-time":            keyTime,
+			"q-signature":           signature,
+			"key":                   name,
+			"success_action_status": strconv.Itoa(successCode),
+		},
+		SuccessCodes: []int{successCode},
+	}
+	if contentType != "" {
+		fd.FormData["Content-Type"] = contentType
+	}
+	if c.credential.SessionToken != "" {
+		fd.FormData["x-cos-security-token"] = c.credential.SessionToken
+	}
+	return fd, nil
+}
+
+func hmacSha1val(key, msg string) string {
+	v := hmac.New(sha1.New, []byte(key))
+	v.Write([]byte(msg))
+	return hex.EncodeToString(v.Sum(nil))
+}
+
+func sha1val(msg string) string {
+	sha1Hash := sha1.New()
+	sha1Hash.Write([]byte(msg))
+	return hex.EncodeToString(sha1Hash.Sum(nil))
+}

pkg/common/db/s3/minio/minio.go

@@ -57,6 +57,8 @@ const (
 	imageThumbnailPath = "openim/thumbnail"
 )
 
+const successCode = http.StatusOK
+
 func NewMinio(cache cache.MinioCache) (s3.Interface, error) {
 	u, err := url.Parse(config.Config.Object.Minio.Endpoint)
 	if err != nil {
@@ -441,3 +443,51 @@ func (m *Minio) getObjectData(ctx context.Context, name string, limit int64) ([]
 	}
 	return io.ReadAll(io.LimitReader(object, limit))
 }
+
+func (m *Minio) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) {
+	if err := m.initMinio(ctx); err != nil {
+		return nil, err
+	}
+	policy := minio.NewPostPolicy()
+	if err := policy.SetKey(name); err != nil {
+		return nil, err
+	}
+	expires := time.Now().Add(duration)
+	if err := policy.SetExpires(expires); err != nil {
+		return nil, err
+	}
+	if size > 0 {
+		if err := policy.SetContentLengthRange(0, size); err != nil {
+			return nil, err
+		}
+	}
+	if err := policy.SetSuccessStatusAction(strconv.Itoa(successCode)); err != nil {
+		return nil, err
+	}
+	if contentType != "" {
+		if err := policy.SetContentType(contentType); err != nil {
+			return nil, err
+		}
+	}
+	if err := policy.SetBucket(m.bucket); err != nil {
+		return nil, err
+	}
+	u, fd, err := m.core.PresignedPostPolicy(ctx, policy)
+	if err != nil {
+		return nil, err
+	}
+	sign, err := url.Parse(m.signEndpoint)
+	if err != nil {
+		return nil, err
+	}
+	u.Scheme = sign.Scheme
+	u.Host = sign.Host
+	return &s3.FormData{
+		URL:          u.String(),
+		File:         "file",
+		Header:       nil,
+		FormData:     fd,
+		Expires:      expires,
+		SuccessCodes: []int{successCode},
+	}, nil
+}

pkg/common/db/s3/oss/oss.go

@@ -16,8 +16,13 @@ package oss
 
 import (
 	"context"
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/base64"
+	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"net/url"
 	"reflect"
@@ -45,6 +50,8 @@ const (
 	imageWebp = "webp"
 )
 
+const successCode = http.StatusOK
+
 const (
 	videoSnapshotImagePng = "png"
 	videoSnapshotImageJpg = "jpg"
@@ -327,3 +334,45 @@ func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration,
 	params := getURLParams(*o.bucket.Client.Conn, rawParams)
 	return getURL(o.um, o.bucket.BucketName, name, params).String(), nil
 }
+
+func (o *OSS) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) {
+	// https://help.aliyun.com/zh/oss/developer-reference/postobject?spm=a2c4g.11186623.0.0.1cb83cebkP55nn
+	expires := time.Now().Add(duration)
+	conditions := []any{
+		map[string]string{"bucket": o.bucket.BucketName},
+		map[string]string{"key": name},
+	}
+	if size > 0 {
+		conditions = append(conditions, []any{"content-length-range", 0, size})
+	}
+	policy := map[string]any{
+		"expiration": expires.Format("2006-01-02T15:04:05.000Z"),
+		"conditions": conditions,
+	}
+	policyJson, err := json.Marshal(policy)
+	if err != nil {
+		return nil, err
+	}
+	policyStr := base64.StdEncoding.EncodeToString(policyJson)
+	h := hmac.New(sha1.New, []byte(o.credentials.GetAccessKeySecret()))
+	if _, err := io.WriteString(h, policyStr); err != nil {
+		return nil, err
+	}
+	fd := &s3.FormData{
+		URL:     o.bucketURL,
+		File:    "file",
+		Expires: expires,
+		FormData: map[string]string{
+			"key":                   name,
+			"policy":                policyStr,
+			"OSSAccessKeyId":        o.credentials.GetAccessKeyID(),
+			"success_action_status": strconv.Itoa(successCode),
+			"signature":             base64.StdEncoding.EncodeToString(h.Sum(nil)),
+		},
+		SuccessCodes: []int{successCode},
+	}
+	if contentType != "" {
+		fd.FormData["x-oss-content-type"] = contentType
+	}
+	return fd, nil
+}

pkg/common/db/s3/s3.go

@@ -74,6 +74,15 @@ type CopyObjectInfo struct {
 	ETag string `json:"etag"`
 }
 
+type FormData struct {
+	URL          string            `json:"url"`
+	File         string            `json:"file"`
+	Header       http.Header       `json:"header"`
+	FormData     map[string]string `json:"form"`
+	Expires      time.Time         `json:"expires"`
+	SuccessCodes []int             `json:"successActionStatus"`
+}
+
 type SignPart struct {
 	PartNumber int         `json:"partNumber"`
 	URL        string      `json:"url"`
@@ -152,4 +161,6 @@ type Interface interface {
 	ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*ListUploadedPartsResult, error)
 
 	AccessURL(ctx context.Context, name string, expire time.Duration, opt *AccessURLOption) (string, error)
+
+	FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*FormData, error)
 }