feat(minio): add expiry tag and rule (#23)

- **fix(minio): add logger in UploadFile input param**
- **feat(minio): add expiry tag and rule**

Because

- run logging metadata retention

This commit

- add minio expiry tag and rule

minio/minio.go

@@ -19,21 +19,32 @@ import (
 )
 
 type MinioI interface {
-	UploadFile(ctx context.Context, filePath string, fileContent any, fileMimeType string) (url string, objectInfo *miniogo.ObjectInfo, err error)
-	UploadFileBytes(ctx context.Context, logger *zap.Logger, filePath string, fileBytes []byte, fileMimeType string) (url string, objectInfo *miniogo.ObjectInfo, err error)
+	UploadFile(ctx context.Context, logger *zap.Logger, param *UploadFileParam) (url string, objectInfo *miniogo.ObjectInfo, err error)
+	UploadFileBytes(ctx context.Context, logger *zap.Logger, param *UploadFileBytesParam) (url string, objectInfo *miniogo.ObjectInfo, err error)
 	DeleteFile(ctx context.Context, logger *zap.Logger, filePath string) (err error)
 	GetFile(ctx context.Context, logger *zap.Logger, filePath string) ([]byte, error)
 	GetFilesByPaths(ctx context.Context, logger *zap.Logger, filePaths []string) ([]FileContent, error)
 }
 
-const Location = "us-east-1"
+type ExpiryRule struct {
+	Tag            string
+	ExpirationDays int
+}
+
+const (
+	Location = "us-east-1"
+
+	StatusEnabled = "Enabled"
+	ExpiryTag     = "expiry-group"
+)
 
 type minio struct {
-	client *miniogo.Client
-	bucket string
+	client           *miniogo.Client
+	bucket           string
+	expiryRuleConfig map[string]int
 }
 
-func NewMinioClientAndInitBucket(ctx context.Context, cfg *Config, logger *zap.Logger) (MinioI, error) {
+func NewMinioClientAndInitBucket(ctx context.Context, cfg *Config, logger *zap.Logger, expiryRules ...ExpiryRule) (MinioI, error) {
 	logger.Info("Initializing Minio client and bucket...")
 
 	endpoint := net.JoinHostPort(cfg.Host, cfg.Port)
@@ -71,45 +82,91 @@ func NewMinioClientAndInitBucket(ctx context.Context, cfg *Config, logger *zap.L
 	lccfg.Rules = []lifecycle.Rule{
 		{
 			ID:     "expire-bucket-objects",
-			Status: "Enabled",
+			Status: StatusEnabled,
 			Expiration: lifecycle.Expiration{
 				Days: lifecycle.ExpirationDays(30),
 			},
 		},
 	}
+
+	expiryRuleConfig := make(map[string]int)
+	for _, expiryRule := range expiryRules {
+		expiryRuleConfig[expiryRule.Tag] = expiryRule.ExpirationDays
+		lccfg.Rules = append(lccfg.Rules, lifecycle.Rule{
+			ID:     expiryRule.Tag,
+			Status: StatusEnabled,
+			Expiration: lifecycle.Expiration{
+				Days: lifecycle.ExpirationDays(expiryRule.ExpirationDays),
+			},
+			RuleFilter: lifecycle.Filter{
+				Tag: lifecycle.Tag{
+					Key:   ExpiryTag,
+					Value: expiryRule.Tag,
+				},
+			},
+		})
+	}
+
 	err = client.SetBucketLifecycle(ctx, cfg.BucketName, lccfg)
 	if err != nil {
 		logger.Error("setting Bucket lifecycle failed", zap.Error(err))
 		return nil, err
 	}
 
-	return &minio{client: client, bucket: cfg.BucketName}, nil
+	return &minio{client: client, bucket: cfg.BucketName, expiryRuleConfig: expiryRuleConfig}, nil
+}
+
+type UploadFileParam struct {
+	FilePath      string
+	FileContent   any
+	FileMimeType  string
+	ExpiryRuleTag string
+}
+
+type UploadFileBytesParam struct {
+	FilePath      string
+	FileBytes     []byte
+	FileMimeType  string
+	ExpiryRuleTag string
 }
 
-func (m *minio) UploadFile(ctx context.Context, filePath string, fileContent any, fileMimeType string) (url string, objectInfo *miniogo.ObjectInfo, err error) {
-	jsonData, _ := json.Marshal(fileContent)
-	return m.UploadFileBytes(ctx, nil, filePath, jsonData, fileMimeType)
+func (m *minio) UploadFile(ctx context.Context, logger *zap.Logger, param *UploadFileParam) (url string, objectInfo *miniogo.ObjectInfo, err error) {
+	jsonData, _ := json.Marshal(param.FileContent)
+	return m.UploadFileBytes(ctx, logger, &UploadFileBytesParam{
+		FilePath:      param.FilePath,
+		FileBytes:     jsonData,
+		FileMimeType:  param.FileMimeType,
+		ExpiryRuleTag: param.ExpiryRuleTag,
+	})
 }
 
-func (m *minio) UploadFileBytes(ctx context.Context, logger *zap.Logger, filePath string, fileBytes []byte, fileMimeType string) (url string, objectInfo *miniogo.ObjectInfo, err error) {
-	logger.Info("start to upload file to minio", zap.String("filePath", filePath))
-	reader := bytes.NewReader(fileBytes)
+func (m *minio) UploadFileBytes(ctx context.Context, logger *zap.Logger, param *UploadFileBytesParam) (url string, objectInfo *miniogo.ObjectInfo, err error) {
+	logger.Info("start to upload file to minio", zap.String("filePath", param.FilePath))
+	reader := bytes.NewReader(param.FileBytes)
 
 	// Create the file path with folder structure
-	_, err = m.client.PutObject(ctx, m.bucket, filePath, reader, int64(len(fileBytes)), miniogo.PutObjectOptions{ContentType: fileMimeType})
+	_, err = m.client.PutObject(ctx, m.bucket, param.FilePath, reader, int64(len(param.FileBytes)), miniogo.PutObjectOptions{
+		ContentType: param.FileMimeType,
+		UserTags:    map[string]string{ExpiryTag: param.ExpiryRuleTag},
+	})
 	if err != nil {
 		logger.Error("Failed to upload file to MinIO", zap.Error(err))
 		return "", nil, err
 	}
 
 	// Get the object stat (metadata)
-	stat, err := m.client.StatObject(ctx, m.bucket, filePath, miniogo.StatObjectOptions{})
+	stat, err := m.client.StatObject(ctx, m.bucket, param.FilePath, miniogo.StatObjectOptions{})
 	if err != nil {
 		return "", nil, err
 	}
 
 	// Generate the presigned URL
-	presignedURL, err := m.client.PresignedGetObject(ctx, m.bucket, filePath, time.Hour*24*7, nil)
+	expiryDays, ok := m.expiryRuleConfig[param.ExpiryRuleTag]
+	if !ok || expiryDays > 7 { // presignedURL Expires cannot be greater than 7 days.
+		expiryDays = 7
+	}
+	expiryDuration := time.Hour * 24 * time.Duration(expiryDays)
+	presignedURL, err := m.client.PresignedGetObject(ctx, m.bucket, param.FilePath, expiryDuration, nil)
 	if err != nil {
 		return "", nil, err
 	}

minio/minio_test.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/gofrs/uuid"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
 
 	miniox "github.com/instill-ai/x/minio"
 )
@@ -17,13 +18,14 @@ func TestMinio(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
+	log, _ := zap.NewDevelopment()
 	mc, err := miniox.NewMinioClientAndInitBucket(ctx, &miniox.Config{
 		Host:       "localhost",
 		Port:       "19000",
 		RootUser:   "minioadmin",
 		RootPwd:    "minioadmin",
 		BucketName: "instill-ai-model",
-	}, nil)
+	}, log)
 
 	require.NoError(t, err)
 
@@ -35,12 +37,22 @@ func TestMinio(t *testing.T) {
 	data["uid"] = uid.String()
 	jsonBytes, _ := json.Marshal(data)
 
-	url, stat, err := mc.UploadFile(ctx, fileName.String(), data, "application/json")
+	url, stat, err := mc.UploadFile(ctx, log, &miniox.UploadFileParam{
+		FilePath:     fileName.String(),
+		FileContent:  data,
+		FileMimeType: "application/json",
+	})
 	require.NoError(t, err)
 	t.Log("url:", url)
 	t.Log("size:", stat.Size)
 
-	fileBytes, err := mc.GetFile(ctx, nil, fileName.String())
+	fileBytes, err := mc.GetFile(ctx, log, fileName.String())
 	require.NoError(t, err)
 	require.Equal(t, jsonBytes, fileBytes)
+
+	err = mc.DeleteFile(ctx, log, fileName.String())
+	require.NoError(t, err)
+
+	_, err = mc.GetFile(ctx, log, fileName.String())
+	require.Error(t, err)
 }