feat: S3 server cache (#1329)

* optimize scheduled deletion

* optimize scheduled deletion

* optimize scheduled deletion

* optimize scheduled deletion

* minio cache

* fix: conflicts

* feat: minio cache

* feat: cache optimize

* feat: cache optimize

* feat: cache optimize

* feat: cache optimize

* feat: cache optimize

pkg/common/db/s3/cont/controller.go

@@ -20,6 +20,7 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
 	"path"
 	"strings"
 	"time"
@@ -32,12 +33,16 @@ import (
 	"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
 )
 
-func New(impl s3.Interface) *Controller {
-	return &Controller{impl: impl}
+func New(cache cache.S3Cache, impl s3.Interface) *Controller {
+	return &Controller{
+		cache: cache,
+		impl:  impl,
+	}
 }
 
 type Controller struct {
-	impl s3.Interface
+	cache cache.S3Cache
+	impl  s3.Interface
 }
 
 func (c *Controller) HashPath(md5 string) string {
@@ -69,8 +74,12 @@ func (c *Controller) PartLimit() *s3.PartLimit {
 	return c.impl.PartLimit()
 }
 
+func (c *Controller) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) {
+	return c.cache.GetKey(ctx, c.impl.Engine(), name)
+}
+
 func (c *Controller) GetHashObject(ctx context.Context, hash string) (*s3.ObjectInfo, error) {
-	return c.impl.StatObject(ctx, c.HashPath(hash))
+	return c.StatObject(ctx, c.HashPath(hash))
 }
 
 func (c *Controller) InitiateUpload(ctx context.Context, hash string, size int64, expire time.Duration, maxParts int) (*InitiateUploadResult, error) {
@@ -94,7 +103,7 @@ func (c *Controller) InitiateUpload(ctx context.Context, hash string, size int64
 	if maxParts > 0 && partNumber > 0 && partNumber < maxParts {
 		return nil, errors.New(fmt.Sprintf("too many parts: %d", partNumber))
 	}
-	if info, err := c.impl.StatObject(ctx, c.HashPath(hash)); err == nil {
+	if info, err := c.StatObject(ctx, c.HashPath(hash)); err == nil {
 		return nil, &HashAlreadyExistsError{Object: info}
 	} else if !c.impl.IsNotFound(err) {
 		return nil, err
@@ -168,13 +177,13 @@ func (c *Controller) CompleteUpload(ctx context.Context, uploadID string, partHa
 		fmt.Println("CompleteUpload sum:", hex.EncodeToString(md5Sum[:]), "upload hash:", upload.Hash)
 		return nil, errors.New("md5 mismatching")
 	}
-	if info, err := c.impl.StatObject(ctx, c.HashPath(upload.Hash)); err == nil {
+	if info, err := c.StatObject(ctx, c.HashPath(upload.Hash)); err == nil {
 		return &UploadResult{
 			Key:  info.Key,
 			Size: info.Size,
 			Hash: info.ETag,
 		}, nil
-	} else if !c.impl.IsNotFound(err) {
+	} else if !c.IsNotFound(err) {
 		return nil, err
 	}
 	cleanObject := make(map[string]struct{})
@@ -200,7 +209,7 @@ func (c *Controller) CompleteUpload(ctx context.Context, uploadID string, partHa
 		}
 		targetKey = result.Key
 	case UploadTypePresigned:
-		uploadInfo, err := c.impl.StatObject(ctx, upload.Key)
+		uploadInfo, err := c.StatObject(ctx, upload.Key)
 		if err != nil {
 			return nil, err
 		}
@@ -230,6 +239,9 @@ func (c *Controller) CompleteUpload(ctx context.Context, uploadID string, partHa
 	default:
 		return nil, errors.New("invalid upload id type")
 	}
+	if err := c.cache.DelS3Key(c.impl.Engine(), targetKey).ExecDel(ctx); err != nil {
+		return nil, err
+	}
 	return &UploadResult{
 		Key:  targetKey,
 		Size: upload.Size,
@@ -253,7 +265,7 @@ func (c *Controller) AuthSign(ctx context.Context, uploadID string, partNumbers
 }
 
 func (c *Controller) IsNotFound(err error) bool {
-	return c.impl.IsNotFound(err)
+	return c.impl.IsNotFound(err) || errs.ErrRecordNotFound.Is(err)
 }
 
 func (c *Controller) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {

pkg/common/db/s3/minio/minio.go

@@ -15,20 +15,14 @@
 package minio
 
 import (
-	"bytes"
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
-	"image"
-	"image/gif"
-	"image/jpeg"
-	"image/png"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
 	"io"
 	"net/http"
 	"net/url"
 	"path"
-	"path/filepath"
 	"reflect"
 	"strconv"
 	"strings"
@@ -56,13 +50,13 @@ const (
 )
 
 const (
-	maxImageWidth  = 1024
-	maxImageHeight = 1024
-	maxImageSize   = 1024 * 1024 * 50
-	pathInfo       = "openim/thumbnail"
+	maxImageWidth      = 1024
+	maxImageHeight     = 1024
+	maxImageSize       = 1024 * 1024 * 50
+	imageThumbnailPath = "openim/thumbnail"
 )
 
-func NewMinio() (s3.Interface, error) {
+func NewMinio(cache cache.MinioCache) (s3.Interface, error) {
 	u, err := url.Parse(config.Config.Object.Minio.Endpoint)
 	if err != nil {
 		return nil, err
@@ -80,6 +74,7 @@ func NewMinio() (s3.Interface, error) {
 		core:   &minio.Core{Client: client},
 		lock:   &sync.Mutex{},
 		init:   false,
+		cache:  cache,
 	}
 	if config.Config.Object.Minio.SignEndpoint == "" || config.Config.Object.Minio.SignEndpoint == config.Config.Object.Minio.Endpoint {
 		m.opts = opts
@@ -124,6 +119,7 @@ type Minio struct {
 	lock         sync.Locker
 	init         bool
 	prefix       string
+	cache        cache.MinioCache
 }
 
 func (m *Minio) initMinio(ctx context.Context) error {
@@ -227,6 +223,7 @@ func (m *Minio) CompleteMultipartUpload(ctx context.Context, uploadID string, na
 	if err != nil {
 		return nil, err
 	}
+	m.delObjectImageInfoKey(ctx, name, upload.Size)
 	return &s3.CompleteMultipartUploadResult{
 		Location: upload.Location,
 		Bucket:   upload.Bucket,
@@ -389,7 +386,7 @@ func (m *Minio) ListUploadedParts(ctx context.Context, uploadID string, name str
 	return res, nil
 }
 
-func (m *Minio) presignedGetObject(ctx context.Context, name string, expire time.Duration, query url.Values) (string, error) {
+func (m *Minio) PresignedGetObject(ctx context.Context, name string, expire time.Duration, query url.Values) (string, error) {
 	if expire <= 0 {
 		expire = time.Hour * 24 * 365 * 99 // 99 years
 	} else if expire < time.Second {
@@ -427,109 +424,9 @@ func (m *Minio) AccessURL(ctx context.Context, name string, expire time.Duration
 		}
 	}
 	if opt.Image == nil || (opt.Image.Width < 0 && opt.Image.Height < 0 && opt.Image.Format == "") || (opt.Image.Width > maxImageWidth || opt.Image.Height > maxImageHeight) {
-		return m.presignedGetObject(ctx, name, expire, reqParams)
+		return m.PresignedGetObject(ctx, name, expire, reqParams)
 	}
-	fileInfo, err := m.StatObject(ctx, name)
-	if err != nil {
-		return "", err
-	}
-	if fileInfo.Size > maxImageSize {
-		return "", errors.New("file size too large")
-	}
-	objectInfoPath := path.Join(pathInfo, fileInfo.ETag, "image.json")
-	var (
-		img  image.Image
-		info minioImageInfo
-	)
-	data, err := m.getObjectData(ctx, objectInfoPath, 1024)
-	if err == nil {
-		if err := json.Unmarshal(data, &info); err != nil {
-			return "", fmt.Errorf("unmarshal minio image info.json error: %w", err)
-		}
-		if info.NotImage {
-			return "", errors.New("not image")
-		}
-	} else if m.IsNotFound(err) {
-		reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
-		if err != nil {
-			return "", err
-		}
-		defer reader.Close()
-		imageInfo, format, err := ImageStat(reader)
-		if err == nil {
-			info.NotImage = false
-			info.Format = format
-			info.Width, info.Height = ImageWidthHeight(imageInfo)
-			img = imageInfo
-		} else {
-			info.NotImage = true
-		}
-		data, err := json.Marshal(&info)
-		if err != nil {
-			return "", err
-		}
-		if _, err := m.core.Client.PutObject(ctx, m.bucket, objectInfoPath, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{}); err != nil {
-			return "", err
-		}
-	} else {
-		return "", err
-	}
-	if opt.Image.Width > info.Width || opt.Image.Width <= 0 {
-		opt.Image.Width = info.Width
-	}
-	if opt.Image.Height > info.Height || opt.Image.Height <= 0 {
-		opt.Image.Height = info.Height
-	}
-	opt.Image.Format = strings.ToLower(opt.Image.Format)
-	if opt.Image.Format == formatJpg {
-		opt.Image.Format = formatJpeg
-	}
-	switch opt.Image.Format {
-	case formatPng:
-	case formatJpeg:
-	case formatGif:
-	default:
-		if info.Format == formatGif {
-			opt.Image.Format = formatGif
-		} else {
-			opt.Image.Format = formatJpeg
-		}
-	}
-	reqParams.Set("response-content-type", "image/"+opt.Image.Format)
-	if opt.Image.Width == info.Width && opt.Image.Height == info.Height && opt.Image.Format == info.Format {
-		return m.presignedGetObject(ctx, name, expire, reqParams)
-	}
-	cacheKey := filepath.Join(pathInfo, fileInfo.ETag, fmt.Sprintf("image_w%d_h%d.%s", opt.Image.Width, opt.Image.Height, opt.Image.Format))
-	if _, err := m.core.Client.StatObject(ctx, m.bucket, cacheKey, minio.StatObjectOptions{}); err == nil {
-		return m.presignedGetObject(ctx, cacheKey, expire, reqParams)
-	} else if !m.IsNotFound(err) {
-		return "", err
-	}
-	if img == nil {
-		reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
-		if err != nil {
-			return "", err
-		}
-		defer reader.Close()
-		img, _, err = ImageStat(reader)
-		if err != nil {
-			return "", err
-		}
-	}
-	thumbnail := resizeImage(img, opt.Image.Width, opt.Image.Height)
-	buf := bytes.NewBuffer(nil)
-	switch opt.Image.Format {
-	case formatPng:
-		err = png.Encode(buf, thumbnail)
-	case formatJpeg:
-		err = jpeg.Encode(buf, thumbnail, nil)
-	case formatGif:
-		err = gif.Encode(buf, thumbnail, nil)
-	}
-	if _, err := m.core.Client.PutObject(ctx, m.bucket, cacheKey, buf, int64(buf.Len()), minio.PutObjectOptions{}); err != nil {
-		return "", err
-	}
-	return m.presignedGetObject(ctx, cacheKey, expire, reqParams)
+	return m.getImageThumbnailURL(ctx, name, expire, opt.Image)
 }
 
 func (m *Minio) getObjectData(ctx context.Context, name string, limit int64) ([]byte, error) {
@@ -541,5 +438,5 @@ func (m *Minio) getObjectData(ctx context.Context, name string, limit int64) ([]
 	if limit < 0 {
 		return io.ReadAll(object)
 	}
-	return io.ReadAll(io.LimitReader(object, 1024))
+	return io.ReadAll(io.LimitReader(object, limit))
 }

pkg/common/db/s3/minio/struct.go

@@ -1,22 +0,0 @@
-// Copyright © 2023 OpenIM. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package minio
-
-type minioImageInfo struct {
-	NotImage bool   `json:"notImage,omitempty"`
-	Width    int    `json:"width,omitempty"`
-	Height   int    `json:"height,omitempty"`
-	Format   string `json:"format,omitempty"`
-}

pkg/common/db/s3/minio/thumbnail.go

@@ -0,0 +1,134 @@
+package minio
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"github.com/OpenIMSDK/tools/errs"
+	"github.com/OpenIMSDK/tools/log"
+	"github.com/minio/minio-go/v7"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
+	"image"
+	"image/gif"
+	"image/jpeg"
+	"image/png"
+	"net/url"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+func (m *Minio) getImageThumbnailURL(ctx context.Context, name string, expire time.Duration, opt *s3.Image) (string, error) {
+	var img image.Image
+	info, err := m.cache.GetImageObjectKeyInfo(ctx, name, func(ctx context.Context) (info *cache.MinioImageInfo, err error) {
+		info, img, err = m.getObjectImageInfo(ctx, name)
+		return
+	})
+	if err != nil {
+		return "", err
+	}
+	if !info.IsImg {
+		return "", errs.ErrData.Wrap("object not image")
+	}
+	if opt.Width > info.Width || opt.Width <= 0 {
+		opt.Width = info.Width
+	}
+	if opt.Height > info.Height || opt.Height <= 0 {
+		opt.Height = info.Height
+	}
+	opt.Format = strings.ToLower(opt.Format)
+	if opt.Format == formatJpg {
+		opt.Format = formatJpeg
+	}
+	switch opt.Format {
+	case formatPng, formatJpeg, formatGif:
+	default:
+		opt.Format = ""
+	}
+	reqParams := make(url.Values)
+	if opt.Width == info.Width && opt.Height == info.Height && (opt.Format == info.Format || opt.Format == "") {
+		reqParams.Set("response-content-type", "image/"+info.Format)
+		return m.PresignedGetObject(ctx, name, expire, reqParams)
+	}
+	if opt.Format == "" {
+		switch opt.Format {
+		case formatGif:
+			opt.Format = formatGif
+		case formatJpeg:
+			opt.Format = formatJpeg
+		case formatPng:
+			opt.Format = formatPng
+		default:
+			opt.Format = formatPng
+		}
+	}
+	key, err := m.cache.GetThumbnailKey(ctx, name, opt.Format, opt.Width, opt.Height, func(ctx context.Context) (string, error) {
+		if img == nil {
+			reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
+			if err != nil {
+				return "", err
+			}
+			defer reader.Close()
+			img, _, err = ImageStat(reader)
+			if err != nil {
+				return "", err
+			}
+		}
+		thumbnail := resizeImage(img, opt.Width, opt.Height)
+		buf := bytes.NewBuffer(nil)
+		switch opt.Format {
+		case formatPng:
+			err = png.Encode(buf, thumbnail)
+		case formatJpeg:
+			err = jpeg.Encode(buf, thumbnail, nil)
+		case formatGif:
+			err = gif.Encode(buf, thumbnail, nil)
+		}
+		cacheKey := filepath.Join(imageThumbnailPath, info.Etag, fmt.Sprintf("image_w%d_h%d.%s", opt.Width, opt.Height, opt.Format))
+		if _, err := m.core.Client.PutObject(ctx, m.bucket, cacheKey, buf, int64(buf.Len()), minio.PutObjectOptions{}); err != nil {
+			return "", err
+		}
+		return cacheKey, nil
+	})
+	if err != nil {
+		return "", err
+	}
+	reqParams.Set("response-content-type", "image/"+opt.Format)
+	return m.PresignedGetObject(ctx, key, expire, reqParams)
+}
+
+func (m *Minio) getObjectImageInfo(ctx context.Context, name string) (*cache.MinioImageInfo, image.Image, error) {
+	fileInfo, err := m.StatObject(ctx, name)
+	if err != nil {
+		return nil, nil, err
+	}
+	if fileInfo.Size > maxImageSize {
+		return nil, nil, errors.New("file size too large")
+	}
+	imageData, err := m.getObjectData(ctx, name, fileInfo.Size)
+	if err != nil {
+		return nil, nil, err
+	}
+	var info cache.MinioImageInfo
+	imageInfo, format, err := ImageStat(bytes.NewReader(imageData))
+	if err == nil {
+		info.IsImg = true
+		info.Format = format
+		info.Width, info.Height = ImageWidthHeight(imageInfo)
+	} else {
+		info.IsImg = false
+	}
+	info.Etag = fileInfo.ETag
+	return &info, imageInfo, nil
+}
+
+func (m *Minio) delObjectImageInfoKey(ctx context.Context, key string, size int64) {
+	if size > 0 && size > maxImageSize {
+		return
+	}
+	if err := m.cache.DelObjectImageInfoKey(key).ExecDel(ctx); err != nil {
+		log.ZError(ctx, "DelObjectImageInfoKey failed", err, "key", key)
+	}
+}