feat(minio): import updated minio package and add tag on file upload

cmd/main/main.go

@@ -45,7 +45,6 @@ import (
 	"github.com/instill-ai/pipeline-backend/pkg/logger"
 	"github.com/instill-ai/pipeline-backend/pkg/memory"
 	"github.com/instill-ai/pipeline-backend/pkg/middleware"
-	"github.com/instill-ai/pipeline-backend/pkg/minio"
 	"github.com/instill-ai/pipeline-backend/pkg/repository"
 	"github.com/instill-ai/pipeline-backend/pkg/service"
 	"github.com/instill-ai/pipeline-backend/pkg/usage"
@@ -57,6 +56,7 @@ import (
 	customotel "github.com/instill-ai/pipeline-backend/pkg/logger/otel"
 	pipelineworker "github.com/instill-ai/pipeline-backend/pkg/worker"
 	pb "github.com/instill-ai/protogen-go/vdp/pipeline/v1beta"
+	miniox "github.com/instill-ai/x/minio"
 )
 
 const gracefulShutdownWaitPeriod = 15 * time.Second
@@ -251,7 +251,7 @@ func main() {
 	ms := memory.NewMemoryStore()
 
 	// Initialize Minio client
-	minioClient, err := minio.NewMinioClientAndInitBucket(ctx, &config.Config.Minio)
+	minioClient, err := miniox.NewMinioClientAndInitBucket(ctx, &config.Config.Minio, logger, config.MetadataExpiryRules...)
 	if err != nil {
 		logger.Fatal("failed to create minio client", zap.Error(err))
 	}

config/config.go

@@ -15,6 +15,7 @@ import (
 	"github.com/redis/go-redis/v9"
 
 	componentstore "github.com/instill-ai/pipeline-backend/pkg/component/store"
+	miniox "github.com/instill-ai/x/minio"
 )
 
 // Config - Global variable to export
@@ -34,7 +35,7 @@ type AppConfig struct {
 	OpenFGA         OpenFGAConfig         `koanf:"openfga"`
 	InstillCloud    InstillCloudConfig    `koanf:"instillcloud"`
 	ArtifactBackend ArtifactBackendConfig `koanf:"artifactbackend"`
-	Minio           MinioConfig           `koanf:"minio"`
+	Minio           miniox.Config         `koanf:"minio"`
 	AppBackend      AppBackendConfig      `koanf:"appbackend"`
 }
 
@@ -180,16 +181,6 @@ type ArtifactBackendConfig struct {
 	}
 }
 
-// MinioConfig is the minio configuration.
-type MinioConfig struct {
-	Host       string `koanf:"host"`
-	Port       string `koanf:"port"`
-	RootUser   string `koanf:"rootuser"`
-	RootPwd    string `koanf:"rootpwd"`
-	BucketName string `koanf:"bucketname"`
-	Secure     bool   `koanf:"secure"` // Add this line for the Secure option
-}
-
 type AppBackendConfig struct {
 	Host       string `koanf:"host"`
 	PublicPort int    `koanf:"publicport"`
@@ -248,3 +239,14 @@ func ParseConfigFlag() string {
 
 	return *configPath
 }
+
+const (
+	DefaultExpiryTag = "default-expiry"
+)
+
+var MetadataExpiryRules = []miniox.ExpiryRule{
+	{
+		Tag:            DefaultExpiryTag,
+		ExpirationDays: 3,
+	},
+}

go.mod

@@ -39,7 +39,7 @@ require (
 	github.com/influxdata/influxdb-client-go/v2 v2.12.3
 	github.com/instill-ai/protogen-go v0.3.3-alpha.0.20241022025309-9afd9231a821
 	github.com/instill-ai/usage-client v0.2.4-alpha.0.20240123081026-6c78d9a5197a
-	github.com/instill-ai/x v0.5.0-alpha
+	github.com/instill-ai/x v0.5.0-alpha.0.20241025093714-913bdb5a7b0b
 	github.com/itchyny/gojq v0.12.14
 	github.com/jackc/pgx/v5 v5.5.5
 	github.com/jmoiron/sqlx v1.4.0

go.sum

@@ -1279,8 +1279,8 @@ github.com/instill-ai/protogen-go v0.3.3-alpha.0.20241022025309-9afd9231a821 h1:
 github.com/instill-ai/protogen-go v0.3.3-alpha.0.20241022025309-9afd9231a821/go.mod h1:rf0UY7VpEgpaLudYEcjx5rnbuwlBaaLyD4FQmWLtgAY=
 github.com/instill-ai/usage-client v0.2.4-alpha.0.20240123081026-6c78d9a5197a h1:gmy8BcCFDZQan40c/D3f62DwTYtlCwi0VrSax+pKffw=
 github.com/instill-ai/usage-client v0.2.4-alpha.0.20240123081026-6c78d9a5197a/go.mod h1:EpX3Yr661uWULtZf5UnJHfr5rw2PDyX8ku4Kx0UtYFw=
-github.com/instill-ai/x v0.5.0-alpha h1:xIeIvrLzwJYOBmZwOePVFVkKGEcMDqtHmn1cfVVNlIE=
-github.com/instill-ai/x v0.5.0-alpha/go.mod h1:jkVtaq9T2zAFA5N46tlV4K5EEVE7FcOVNbqY4wFWYz8=
+github.com/instill-ai/x v0.5.0-alpha.0.20241025093714-913bdb5a7b0b h1:9QCusI5GbZCmeWKIoqelwlVOVP/aAACNoURCszQBdTA=
+github.com/instill-ai/x v0.5.0-alpha.0.20241025093714-913bdb5a7b0b/go.mod h1:jkVtaq9T2zAFA5N46tlV4K5EEVE7FcOVNbqY4wFWYz8=
 github.com/intel/goresctrl v0.2.0/go.mod h1:+CZdzouYFn5EsxgqAQTEzMfwKwuc0fVdMrT9FCCAVRQ=
 github.com/itchyny/gojq v0.12.14 h1:6k8vVtsrhQSYgSGg827AD+PVVaB1NLXEdX+dda2oZCc=
 github.com/itchyny/gojq v0.12.14/go.mod h1:y1G7oO7XkcR1LPZO59KyoCRy08T3j9vDYRV0GgYSS+s=

pkg/mock/generator.go

@@ -4,4 +4,3 @@ package mock
 //go:generate minimock -g -i github.com/instill-ai/pipeline-backend/pkg/acl.ACLClientInterface -o ./ -s "_mock.gen.go"
 //go:generate minimock -g -i github.com/instill-ai/pipeline-backend/pkg/service.Converter -o ./ -s "_mock.gen.go"
 //go:generate minimock -g -i github.com/instill-ai/protogen-go/core/mgmt/v1beta.MgmtPrivateServiceClient -o ./ -s "_mock.gen.go"
-//go:generate minimock -g -i github.com/instill-ai/pipeline-backend/pkg/minio.MinioI -o ./ -s "_mock.gen.go"