api rpc third

pkg/common/db/controller/s3.go

@@ -0,0 +1,353 @@
+package controller
+
+import "C"
+import (
+	"OpenIM/pkg/common/db/obj"
+	"OpenIM/pkg/common/db/table/relation"
+	"OpenIM/pkg/proto/third"
+	"OpenIM/pkg/utils"
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"github.com/google/uuid"
+	"log"
+	"path"
+	"strconv"
+	"time"
+)
+
+type S3Database interface {
+	ApplyPut(ctx context.Context, req *third.ApplyPutReq) (*third.ApplyPutResp, error)
+	GetPut(ctx context.Context, req *third.GetPutReq) (*third.GetPutResp, error)
+	ConfirmPut(ctx context.Context, req *third.ConfirmPutReq) (*third.ConfirmPutResp, error)
+}
+
+func NewS3Database(obj obj.Interface, hash relation.ObjectHashModelInterface, info relation.ObjectInfoModelInterface, put relation.ObjectPutModelInterface) S3Database {
+	return &s3Database{
+		obj:  obj,
+		hash: hash,
+		info: info,
+		put:  put,
+	}
+}
+
+type s3Database struct {
+	obj  obj.Interface
+	hash relation.ObjectHashModelInterface
+	info relation.ObjectInfoModelInterface
+	put  relation.ObjectPutModelInterface
+}
+
+// today 今天的日期
+func (c *s3Database) today() string {
+	return time.Now().Format("20060102")
+}
+
+// fragmentName 根据序号生成文件名
+func (c *s3Database) fragmentName(index int) string {
+	return "fragment_" + strconv.Itoa(index+1)
+}
+
+// getFragmentNum 获取分片大小和分片数量
+func (c *s3Database) getFragmentNum(fragmentSize int64, objectSize int64) (int64, int) {
+	if size := c.obj.MinFragmentSize(); fragmentSize < size {
+		fragmentSize = size
+	}
+	if fragmentSize <= 0 || objectSize <= fragmentSize {
+		return objectSize, 1
+	} else {
+		num := int(objectSize / fragmentSize)
+		if objectSize%fragmentSize > 0 {
+			num++
+		}
+		if n := c.obj.MaxFragmentNum(); num > n {
+			num = n
+		}
+		return fragmentSize, num
+	}
+}
+
+func (c *s3Database) CheckHash(hash string) error {
+	val, err := hex.DecodeString(hash)
+	if err != nil {
+		return err
+	}
+	if len(val) != md5.Size {
+		return errors.New("hash value error")
+	}
+	return nil
+}
+
+func (c *s3Database) urlName(name string) string {
+	if name[0] != '/' {
+		name = "/" + name
+	}
+	return "http://127.0.0.1:8080" + name
+}
+
+func (c *s3Database) UUID() string {
+	return uuid.New().String()
+}
+
+func (c *s3Database) HashName(hash string) string {
+	return path.Join("hash", hash)
+}
+
+func (c *s3Database) isNotFound(err error) bool {
+	return false
+}
+
+func (c *s3Database) ApplyPut(ctx context.Context, req *third.ApplyPutReq) (*third.ApplyPutResp, error) {
+	if err := c.CheckHash(req.Hash); err != nil {
+		return nil, err
+	}
+	if err := c.obj.CheckName(req.Name); err != nil {
+		return nil, err
+	}
+	if req.CleanTime != 0 && req.CleanTime <= time.Now().UnixMilli() {
+		return nil, errors.New("invalid CleanTime")
+	}
+	var expirationTime *time.Time
+	if req.CleanTime != 0 {
+		expirationTime = utils.ToPtr(time.UnixMilli(req.CleanTime))
+	}
+	if hash, err := c.hash.Take(ctx, req.Hash, c.obj.Name()); err == nil {
+		o := relation.ObjectInfoModel{
+			Name:           req.Name,
+			Hash:           hash.Hash,
+			ExpirationTime: expirationTime,
+			CreateTime:     time.Now(),
+		}
+		if err := c.info.SetObject(ctx, &o); err != nil {
+			return nil, err
+		}
+		return &third.ApplyPutResp{Url: c.urlName(o.Name)}, nil // 服务器已存在
+	} else if !c.isNotFound(err) {
+		return nil, err
+	}
+	// 新上传
+	var pack int
+	const effective = time.Hour * 24 * 2
+	req.FragmentSize, pack = c.getFragmentNum(req.FragmentSize, req.Size)
+	put := relation.ObjectPutModel{
+		PutID:          c.UUID(),
+		Hash:           req.Hash,
+		Name:           req.Name,
+		ObjectSize:     req.Size,
+		FragmentSize:   req.FragmentSize,
+		ExpirationTime: expirationTime,
+		EffectiveTime:  time.Now().Add(effective),
+	}
+	put.Path = path.Join("upload", c.today(), req.Hash, put.PutID)
+	putURLs := make([]string, 0, pack)
+	for i := 0; i < pack; i++ {
+		url, err := c.obj.PresignedPutURL(ctx, &obj.ApplyPutArgs{
+			Bucket:        c.obj.TempBucket(),
+			Name:          path.Join(put.Path, c.fragmentName(i)),
+			Effective:     effective,
+			MaxObjectSize: req.FragmentSize,
+		})
+		if err != nil {
+			return nil, err
+		}
+		putURLs = append(putURLs, url)
+	}
+	put.CreateTime = time.Now()
+	if err := c.put.Create(ctx, []*relation.ObjectPutModel{&put}); err != nil {
+		return nil, err
+	}
+	return &third.ApplyPutResp{
+		PutID:        put.PutID,
+		FragmentSize: put.FragmentSize,
+		PutURLs:      putURLs,
+	}, nil
+}
+
+func (c *s3Database) GetPut(ctx context.Context, req *third.GetPutReq) (*third.GetPutResp, error) {
+	up, err := c.put.Take(ctx, req.PutID)
+	if err != nil {
+		return nil, err
+	}
+	if up.Complete {
+		return nil, errors.New("up completed")
+	}
+	_, pack := c.getFragmentNum(up.FragmentSize, up.ObjectSize)
+	fragments := make([]*third.GetPutFragment, pack)
+	for i := 0; i < pack; i++ {
+		name := path.Join(up.Path, c.fragmentName(i))
+		o, err := c.obj.GetObjectInfo(ctx, &obj.BucketObject{
+			Bucket: c.obj.TempBucket(),
+			Name:   name,
+		})
+		if err != nil {
+			if c.obj.IsNotFound(err) {
+				fragments[i] = &third.GetPutFragment{}
+				continue
+			}
+			return nil, err
+		}
+		fragments[i] = &third.GetPutFragment{Size: o.Size, Hash: o.Hash}
+	}
+	var cleanTime int64
+	if up.ExpirationTime != nil {
+		cleanTime = up.ExpirationTime.UnixMilli()
+	}
+	return &third.GetPutResp{
+		FragmentSize: up.FragmentSize,
+		Size:         up.ObjectSize,
+		Name:         up.Name,
+		Hash:         up.Hash,
+		Fragments:    fragments,
+		CleanTime:    cleanTime,
+	}, nil
+}
+
+func (c *s3Database) ConfirmPut(ctx context.Context, req *third.ConfirmPutReq) (_ *third.ConfirmPutResp, _err error) {
+	up, err := c.put.Take(ctx, req.PutID)
+	if err != nil {
+		return nil, err
+	}
+	_, pack := c.getFragmentNum(up.FragmentSize, up.ObjectSize)
+	defer func() {
+		if _err == nil {
+			// 清理上传的碎片
+			for i := 0; i < pack; i++ {
+				name := path.Join(up.Path, c.fragmentName(i))
+				err := c.obj.DeleteObjet(ctx, &obj.BucketObject{
+					Bucket: c.obj.TempBucket(),
+					Name:   name,
+				})
+				if err != nil {
+					log.Printf("delete fragment %d %s %s failed %s\n", i, c.obj.TempBucket(), name, err)
+				}
+			}
+		}
+	}()
+	if up.Complete {
+		return nil, errors.New("put completed")
+	}
+	now := time.Now().UnixMilli()
+	if up.EffectiveTime.UnixMilli() < now {
+		return nil, errors.New("upload expired")
+	}
+	if up.ExpirationTime != nil && up.ExpirationTime.UnixMilli() < now {
+		return nil, errors.New("object expired")
+	}
+	if hash, err := c.hash.Take(ctx, up.Hash, c.obj.Name()); err == nil {
+		o := relation.ObjectInfoModel{
+			Name:           up.Name,
+			Hash:           hash.Hash,
+			ExpirationTime: up.ExpirationTime,
+			CreateTime:     time.Now(),
+		}
+		if err := c.info.SetObject(ctx, &o); err != nil {
+			return nil, err
+		}
+		// 服务端已存在
+		return &third.ConfirmPutResp{
+			Url: c.urlName(o.Name),
+		}, nil
+	} else if c.isNotFound(err) {
+		return nil, err
+	}
+	src := make([]obj.BucketObject, pack)
+	for i := 0; i < pack; i++ {
+		name := path.Join(up.Path, c.fragmentName(i))
+		o, err := c.obj.GetObjectInfo(ctx, &obj.BucketObject{
+			Bucket: c.obj.TempBucket(),
+			Name:   name,
+		})
+		if err != nil {
+			return nil, err
+		}
+		if i+1 == pack { // 最后一个
+			size := up.ObjectSize - up.FragmentSize*int64(i)
+			if size != o.Size {
+				return nil, fmt.Errorf("last fragment %d size %d not equal to %d hash %s", i, o.Size, size, o.Hash)
+			}
+		} else {
+			if o.Size != up.FragmentSize {
+				return nil, fmt.Errorf("fragment %d size %d not equal to %d hash %s", i, o.Size, up.FragmentSize, o.Hash)
+			}
+		}
+		src[i] = obj.BucketObject{
+			Bucket: c.obj.TempBucket(),
+			Name:   name,
+		}
+	}
+	dst := &obj.BucketObject{
+		Bucket: c.obj.DataBucket(),
+		Name:   c.HashName(up.Hash),
+	}
+	if len(src) == 1 { // 未分片直接触发copy
+		// 检查数据完整性,避免脏数据
+		o, err := c.obj.GetObjectInfo(ctx, &src[0])
+		if err != nil {
+			return nil, err
+		}
+		if up.ObjectSize != o.Size {
+			return nil, fmt.Errorf("size mismatching should %d reality %d", up.ObjectSize, o.Size)
+		}
+		if up.Hash != o.Hash {
+			return nil, fmt.Errorf("hash mismatching should %s reality %s", up.Hash, o.Hash)
+		}
+		if err := c.obj.CopyObjet(ctx, &src[0], dst); err != nil {
+			return nil, err
+		}
+	} else {
+		tempBucket := &obj.BucketObject{
+			Bucket: c.obj.TempBucket(),
+			Name:   path.Join("merge", c.today(), req.PutID, c.UUID()),
+		}
+		defer func() { // 清理合成的文件
+			if err := c.obj.DeleteObjet(ctx, tempBucket); err != nil {
+				log.Printf("delete %s %s %s failed %s\n", c.obj.Name(), tempBucket.Bucket, tempBucket.Name, err)
+			}
+		}()
+		err := c.obj.ComposeObject(ctx, src, tempBucket)
+		if err != nil {
+			return nil, err
+		}
+		info, err := c.obj.GetObjectInfo(ctx, tempBucket)
+		if err != nil {
+			return nil, err
+		}
+		if up.ObjectSize != info.Size {
+			return nil, fmt.Errorf("size mismatch should %d reality %d", up.ObjectSize, info.Size)
+		}
+		if up.Hash != info.Hash {
+			return nil, fmt.Errorf("hash mismatch should %s reality %s", up.Hash, info.Hash)
+		}
+		if err := c.obj.CopyObjet(ctx, tempBucket, dst); err != nil {
+			return nil, err
+		}
+	}
+	o := &relation.ObjectInfoModel{
+		Name:           up.Name,
+		Hash:           up.Hash,
+		ExpirationTime: up.ExpirationTime,
+		CreateTime:     time.Now(),
+	}
+	h := &relation.ObjectHashModel{
+		Hash:       up.Hash,
+		Size:       up.ObjectSize,
+		Engine:     c.obj.Name(),
+		Bucket:     c.obj.DataBucket(),
+		Name:       c.HashName(up.Hash),
+		CreateTime: time.Now(),
+	}
+	if err := c.hash.Create(ctx, []*relation.ObjectHashModel{h}); err != nil {
+		return nil, err
+	}
+	if err := c.info.SetObject(ctx, o); err != nil {
+		return nil, err
+	}
+	if err := c.put.SetCompleted(ctx, up.PutID); err != nil {
+		log.Printf("set uploaded %s failed %s\n", up.PutID, err)
+	}
+	return &third.ConfirmPutResp{
+		Url: c.urlName(o.Name),
+	}, nil
+}

pkg/common/db/obj/minio.go

@@ -0,0 +1,183 @@
+package obj
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/s3utils"
+	"net/http"
+	"time"
+)
+
+func NewMinioClient() {
+
+}
+
+func NewMinioInterface() (Interface, error) {
+	//client, err := minio.New("127.0.0.1:9000", &minio.Options{
+	//	Creds:  credentials.NewStaticV4("minioadmin", "minioadmin", ""),
+	//	Secure: false,
+	//})
+
+	return &minioImpl{}, nil
+}
+
+type minioImpl struct {
+	tempBucket      string // 上传桶
+	permanentBucket string // 永久桶
+	clearBucket     string // 自动清理桶
+	urlstr          string // 访问地址
+	client          *minio.Client
+}
+
+//func (m *minioImpl) Init() error {
+//	client, err := minio.New("127.0.0.1:9000", &minio.Options{
+//		Creds:  credentials.NewStaticV4("minioadmin", "minioadmin", ""),
+//		Secure: false,
+//	})
+//	if err != nil {
+//		return fmt.Errorf("minio client error: %w", err)
+//	}
+//	m.urlstr = "http://127.0.0.1:9000"
+//	m.client = client
+//	m.tempBucket = "temp"
+//	m.permanentBucket = "permanent"
+//	m.clearBucket = "clear"
+//	return nil
+//}
+
+func (m *minioImpl) Name() string {
+	return "minio"
+}
+
+func (m *minioImpl) MinFragmentSize() int64 {
+	return 1024 * 1024 * 5 // 每个分片最小大小 minio.absMinPartSize
+}
+
+func (m *minioImpl) MaxFragmentNum() int {
+	return 1000 // 最大分片数量 minio.maxPartsCount
+}
+
+func (m *minioImpl) MinExpirationTime() time.Duration {
+	return time.Hour * 24
+}
+
+func (m *minioImpl) AppendHeader() http.Header {
+	return map[string][]string{
+		"x-amz-object-append": {"true"},
+	}
+}
+
+func (m *minioImpl) TempBucket() string {
+	return m.tempBucket
+}
+
+func (m *minioImpl) DataBucket() string {
+	return m.permanentBucket
+}
+
+func (m *minioImpl) ClearBucket() string {
+	return m.clearBucket
+}
+
+func (m *minioImpl) GetURL(bucket string, name string) string {
+	return fmt.Sprintf("%s/%s/%s", m.urlstr, bucket, name)
+}
+
+func (m *minioImpl) PresignedPutURL(ctx context.Context, args *ApplyPutArgs) (string, error) {
+	if args.Effective <= 0 {
+		return "", errors.New("EffectiveTime <= 0")
+	}
+	_, err := m.GetObjectInfo(ctx, &BucketObject{
+		Bucket: m.tempBucket,
+		Name:   args.Name,
+	})
+	if err == nil {
+		return "", fmt.Errorf("minio bucket %s name %s already exists", args.Bucket, args.Name)
+	} else if !m.IsNotFound(err) {
+		return "", err
+	}
+	u, err := m.client.PresignedPutObject(ctx, m.tempBucket, args.Name, args.Effective)
+	if err != nil {
+		return "", fmt.Errorf("minio apply error: %w", err)
+	}
+	return u.String(), nil
+}
+
+func (m *minioImpl) GetObjectInfo(ctx context.Context, args *BucketObject) (*ObjectInfo, error) {
+	info, err := m.client.StatObject(ctx, args.Bucket, args.Name, minio.StatObjectOptions{})
+	if err != nil {
+		return nil, err
+	}
+	return &ObjectInfo{
+		URL:  m.GetURL(args.Bucket, args.Name),
+		Size: info.Size,
+		Hash: info.ETag,
+	}, nil
+}
+
+func (m *minioImpl) CopyObjet(ctx context.Context, src *BucketObject, dst *BucketObject) error {
+	_, err := m.client.CopyObject(ctx, minio.CopyDestOptions{
+		Bucket: dst.Bucket,
+		Object: dst.Name,
+	}, minio.CopySrcOptions{
+		Bucket: src.Bucket,
+		Object: src.Name,
+	})
+	return err
+}
+
+func (m *minioImpl) DeleteObjet(ctx context.Context, info *BucketObject) error {
+	return m.client.RemoveObject(ctx, info.Bucket, info.Name, minio.RemoveObjectOptions{})
+}
+
+func (m *minioImpl) MoveObjetInfo(ctx context.Context, src *BucketObject, dst *BucketObject) error {
+	if err := m.CopyObjet(ctx, src, dst); err != nil {
+		return err
+	}
+	return m.DeleteObjet(ctx, src)
+}
+
+func (m *minioImpl) ComposeObject(ctx context.Context, src []BucketObject, dst *BucketObject) error {
+	destOptions := minio.CopyDestOptions{
+		Bucket: dst.Bucket,
+		Object: dst.Name + ".temp",
+	}
+	sources := make([]minio.CopySrcOptions, len(src))
+	for i, s := range src {
+		sources[i] = minio.CopySrcOptions{
+			Bucket: s.Bucket,
+			Object: s.Name,
+		}
+	}
+	_, err := m.client.ComposeObject(ctx, destOptions, sources...)
+	if err != nil {
+		return err
+	}
+	return m.MoveObjetInfo(ctx, &BucketObject{
+		Bucket: destOptions.Bucket,
+		Name:   destOptions.Object,
+	}, &BucketObject{
+		Bucket: dst.Bucket,
+		Name:   dst.Name,
+	})
+}
+
+func (m *minioImpl) IsNotFound(err error) bool {
+	if err == nil {
+		return false
+	}
+	switch e := err.(type) {
+	case minio.ErrorResponse:
+		return e.StatusCode == 404 && e.Code == "NoSuchKey"
+	case *minio.ErrorResponse:
+		return e.StatusCode == 404 && e.Code == "NoSuchKey"
+	default:
+		return false
+	}
+}
+
+func (m *minioImpl) CheckName(name string) error {
+	return s3utils.CheckValidObjectName(name)
+}

pkg/common/db/obj/obj.go

@@ -0,0 +1,58 @@
+package obj
+
+import (
+	"context"
+	"net/http"
+	"time"
+)
+
+type BucketObject struct {
+	Bucket string `json:"bucket"`
+	Name   string `json:"name"`
+}
+
+type ApplyPutArgs struct {
+	Bucket        string
+	Name          string
+	Effective     time.Duration // 申请有效时间
+	Header        http.Header   // header
+	MaxObjectSize int64
+}
+
+type ObjectInfo struct {
+	URL        string
+	Size       int64
+	Hash       string
+	Expiration time.Time
+}
+
+type Interface interface {
+	// Name 存储名字
+	Name() string
+	// MinFragmentSize 最小允许的分片大小
+	MinFragmentSize() int64
+	// MaxFragmentNum 最大允许的分片数量
+	MaxFragmentNum() int
+	// MinExpirationTime 最小过期时间
+	MinExpirationTime() time.Duration
+	// TempBucket 临时桶名，用于上传
+	TempBucket() string
+	// DataBucket 永久存储的桶名
+	DataBucket() string
+	// GetURL 通过桶名和对象名返回URL
+	GetURL(bucket string, name string) string
+	// PresignedPutURL 申请上传,返回PUT的上传地址
+	PresignedPutURL(ctx context.Context, args *ApplyPutArgs) (string, error)
+	// GetObjectInfo 获取对象信息
+	GetObjectInfo(ctx context.Context, args *BucketObject) (*ObjectInfo, error)
+	// CopyObjet 复制对象
+	CopyObjet(ctx context.Context, src *BucketObject, dst *BucketObject) error
+	// DeleteObjet 删除对象
+	DeleteObjet(ctx context.Context, info *BucketObject) error
+	// ComposeObject 合并对象
+	ComposeObject(ctx context.Context, src []BucketObject, dst *BucketObject) error
+	// IsNotFound 判断是不是不存在导致的错误
+	IsNotFound(err error) bool
+	// CheckName 检查名字是否可用
+	CheckName(name string) error
+}

pkg/common/db/relation/object_hash_model.go

@@ -0,0 +1,40 @@
+package relation
+
+import (
+	"OpenIM/pkg/common/db/table/relation"
+	"OpenIM/pkg/common/tracelog"
+	"OpenIM/pkg/utils"
+	"context"
+	"gorm.io/gorm"
+)
+
+func NewObjectHash(db *gorm.DB) relation.ObjectHashModelInterface {
+	return &ObjectHashGorm{
+		DB: db,
+	}
+}
+
+type ObjectHashGorm struct {
+	DB *gorm.DB
+}
+
+func (o *ObjectHashGorm) NewTx(tx any) relation.ObjectHashModelInterface {
+	return &ObjectHashGorm{
+		DB: tx.(*gorm.DB),
+	}
+}
+
+func (o *ObjectHashGorm) Take(ctx context.Context, hash string, engine string) (oh *relation.ObjectHashModel, err error) {
+	defer func() {
+		tracelog.SetCtxDebug(ctx, utils.GetFuncName(1), err, "hash", hash, "engine", engine, "objectHash", oh)
+	}()
+	oh = &relation.ObjectHashModel{}
+	return oh, utils.Wrap1(o.DB.Where("hash = ? and engine = ?", hash, engine).Take(oh).Error)
+}
+
+func (o *ObjectHashGorm) Create(ctx context.Context, h []*relation.ObjectHashModel) (err error) {
+	defer func() {
+		tracelog.SetCtxDebug(ctx, utils.GetFuncName(1), err, "objectHash", h)
+	}()
+	return utils.Wrap1(o.DB.Create(h).Error)
+}

pkg/common/db/relation/object_info_model.go

@@ -0,0 +1,45 @@
+package relation
+
+import (
+	"OpenIM/pkg/common/db/table/relation"
+	"OpenIM/pkg/common/tracelog"
+	"OpenIM/pkg/utils"
+	"context"
+	"gorm.io/gorm"
+)
+
+func NewObjectInfo(db *gorm.DB) relation.ObjectInfoModelInterface {
+	return &ObjectInfoGorm{
+		DB: db,
+	}
+}
+
+type ObjectInfoGorm struct {
+	DB *gorm.DB
+}
+
+func (o *ObjectInfoGorm) NewTx(tx any) relation.ObjectInfoModelInterface {
+	return &ObjectInfoGorm{
+		DB: tx.(*gorm.DB),
+	}
+}
+
+func (o *ObjectInfoGorm) SetObject(ctx context.Context, obj *relation.ObjectInfoModel) (err error) {
+	defer func() {
+		tracelog.SetCtxDebug(ctx, utils.GetFuncName(1), err, "objectInfo", obj)
+	}()
+	return utils.Wrap1(o.DB.Transaction(func(tx *gorm.DB) error {
+		if err := tx.Where("name = ?", obj.Name).Delete(&relation.ObjectInfoModel{}).Error; err != nil {
+			return err
+		}
+		return tx.Create(obj).Error
+	}))
+}
+
+func (o *ObjectInfoGorm) Take(ctx context.Context, name string) (info *relation.ObjectInfoModel, err error) {
+	defer func() {
+		tracelog.SetCtxDebug(ctx, utils.GetFuncName(1), err, "name", name, "info", info)
+	}()
+	info = &relation.ObjectInfoModel{}
+	return info, utils.Wrap1(o.DB.Where("name = ?", name).Take(info).Error)
+}

pkg/common/db/relation/object_put_model.go

@@ -0,0 +1,47 @@
+package relation
+
+import (
+	"OpenIM/pkg/common/db/table/relation"
+	"OpenIM/pkg/common/tracelog"
+	"OpenIM/pkg/utils"
+	"context"
+	"gorm.io/gorm"
+)
+
+func NewObjectPut(db *gorm.DB) relation.ObjectPutModelInterface {
+	return &ObjectPutGorm{
+		DB: db,
+	}
+}
+
+type ObjectPutGorm struct {
+	DB *gorm.DB
+}
+
+func (o *ObjectPutGorm) NewTx(tx any) relation.ObjectPutModelInterface {
+	return &ObjectPutGorm{
+		DB: tx.(*gorm.DB),
+	}
+}
+
+func (o *ObjectPutGorm) Create(ctx context.Context, m []*relation.ObjectPutModel) (err error) {
+	defer func() {
+		tracelog.SetCtxDebug(ctx, utils.GetFuncName(1), err, "objectPut", m)
+	}()
+	return utils.Wrap1(o.DB.Create(m).Error)
+}
+
+func (o *ObjectPutGorm) Take(ctx context.Context, putID string) (put *relation.ObjectPutModel, err error) {
+	defer func() {
+		tracelog.SetCtxDebug(ctx, utils.GetFuncName(1), err, "putID", putID, "put", put)
+	}()
+	put = &relation.ObjectPutModel{}
+	return put, utils.Wrap1(o.DB.Where("put_id = ?", putID).Take(put).Error)
+}
+
+func (o *ObjectPutGorm) SetCompleted(ctx context.Context, putID string) (err error) {
+	defer func() {
+		tracelog.SetCtxDebug(ctx, utils.GetFuncName(1), err, "putID", putID)
+	}()
+	return utils.Wrap1(o.DB.Model(&relation.ObjectPutModel{}).Where("put_id = ?", putID).Update("complete", true).Error)
+}

pkg/common/db/table/relation/object_hash.go

@@ -0,0 +1,29 @@
+package relation
+
+import (
+	"context"
+	"time"
+)
+
+const (
+	ObjectHashModelTableName = "object_hash"
+)
+
+type ObjectHashModel struct {
+	Hash       string    `gorm:"column:hash;primary_key;size:32"`
+	Engine     string    `gorm:"column:engine;primary_key;size:16"`
+	Size       int64     `gorm:"column:size"`
+	Bucket     string    `gorm:"column:bucket"`
+	Name       string    `gorm:"column:name"`
+	CreateTime time.Time `gorm:"column:create_time"`
+}
+
+func (ObjectHashModel) TableName() string {
+	return ObjectHashModelTableName
+}
+
+type ObjectHashModelInterface interface {
+	NewTx(tx any) ObjectHashModelInterface
+	Take(ctx context.Context, hash string, engine string) (*ObjectHashModel, error)
+	Create(ctx context.Context, h []*ObjectHashModel) error
+}

pkg/common/db/table/relation/object_info.go

@@ -0,0 +1,27 @@
+package relation
+
+import (
+	"context"
+	"time"
+)
+
+const (
+	ObjectInfoModelTableName = "object_info"
+)
+
+type ObjectInfoModel struct {
+	Name           string     `gorm:"column:name;primary_key"`
+	Hash           string     `gorm:"column:hash"`
+	ExpirationTime *time.Time `gorm:"column:expiration_time"`
+	CreateTime     time.Time  `gorm:"column:create_time"`
+}
+
+func (ObjectInfoModel) TableName() string {
+	return ObjectInfoModelTableName
+}
+
+type ObjectInfoModelInterface interface {
+	NewTx(tx any) ObjectInfoModelInterface
+	SetObject(ctx context.Context, obj *ObjectInfoModel) error
+	Take(ctx context.Context, name string) (*ObjectInfoModel, error)
+}

pkg/common/db/table/relation/object_put.go

@@ -0,0 +1,34 @@
+package relation
+
+import (
+	"context"
+	"time"
+)
+
+const (
+	ObjectPutModelTableName = "object_put"
+)
+
+type ObjectPutModel struct {
+	PutID          string     `gorm:"column:put_id;primary_key"`
+	Hash           string     `gorm:"column:hash"`
+	Path           string     `gorm:"column:path"`
+	Name           string     `gorm:"column:name"`
+	ObjectSize     int64      `gorm:"column:object_size"`
+	FragmentSize   int64      `gorm:"column:fragment_size"`
+	Complete       bool       `gorm:"column:complete"`
+	ExpirationTime *time.Time `gorm:"column:expiration_time"`
+	EffectiveTime  time.Time  `gorm:"column:effective_time"`
+	CreateTime     time.Time  `gorm:"column:create_time"`
+}
+
+func (ObjectPutModel) TableName() string {
+	return ObjectPutModelTableName
+}
+
+type ObjectPutModelInterface interface {
+	NewTx(tx any) ObjectPutModelInterface
+	Create(ctx context.Context, m []*ObjectPutModel) error
+	Take(ctx context.Context, putID string) (*ObjectPutModel, error)
+	SetCompleted(ctx context.Context, putID string) error
+}