Skip to content

Commit

Permalink
feat(blob): provide the upload object url endpoint (#120)
Browse files Browse the repository at this point in the history
Because

artifact will start to provide the object upload url

This commit

implements the handler
  • Loading branch information
Yougigun authored Oct 21, 2024
1 parent 931b1ca commit e207a2f
Show file tree
Hide file tree
Showing 7 changed files with 172 additions and 17 deletions.
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ require (
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1
github.com/influxdata/influxdb-client-go/v2 v2.12.3
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20241018045010-dcc80f850d9d
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20241021084925-9a2ae77e0adb
github.com/instill-ai/usage-client v0.3.0-alpha.0.20240319060111-4a3a39f2fd61
github.com/instill-ai/x v0.3.0-alpha.0.20231219052200-6230a89e386c
github.com/knadh/koanf v1.5.0
Expand Down Expand Up @@ -105,4 +105,4 @@ require (
gopkg.in/yaml.v3 v3.0.1 // indirect
)

// replace github.com/instill-ai/protogen-go v0.3.3-alpha.0.20241001150423-8d8b9e2fa860 => ./protogen-go
// replace github.com/instill-ai/protogen-go => ./protogen-go
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -344,8 +344,8 @@ github.com/influxdata/influxdb-client-go/v2 v2.12.3 h1:28nRlNMRIV4QbtIUvxhWqaxn0
github.com/influxdata/influxdb-client-go/v2 v2.12.3/go.mod h1:IrrLUbCjjfkmRuaCiGQg4m2GbkaeJDcuWoxiWdQEbA0=
github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7wlPfJLvMCdtV4zPulc4uCPrlywQOmbFOhgQNU=
github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20241018045010-dcc80f850d9d h1:6/5voyjeqeeeYszZ7XjifG2pekRDYdqWD0bgeKz9LHw=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20241018045010-dcc80f850d9d/go.mod h1:rf0UY7VpEgpaLudYEcjx5rnbuwlBaaLyD4FQmWLtgAY=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20241021084925-9a2ae77e0adb h1:z5Z6tuctgQfJuEK3Pd+YbYGfuBcU3a9rfGHcG4Yh7Ow=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20241021084925-9a2ae77e0adb/go.mod h1:rf0UY7VpEgpaLudYEcjx5rnbuwlBaaLyD4FQmWLtgAY=
github.com/instill-ai/usage-client v0.3.0-alpha.0.20240319060111-4a3a39f2fd61 h1:smPTvmXDhn/QC7y/TPXyMTqbbRd0gvzmFgWBChwTfhE=
github.com/instill-ai/usage-client v0.3.0-alpha.0.20240319060111-4a3a39f2fd61/go.mod h1:/TAHs4ybuylk5icuy+MQtHRc4XUnIyXzeNKxX9qDFhw=
github.com/instill-ai/x v0.3.0-alpha.0.20231219052200-6230a89e386c h1:a2RVkpIV2QcrGnSHAou+t/L+vBsaIfFvk5inVg5Uh4s=
Expand Down
60 changes: 60 additions & 0 deletions pkg/handler/object.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package handler

import (
"context"
"fmt"

"go.uber.org/zap"

"github.com/gofrs/uuid"
"github.com/instill-ai/artifact-backend/pkg/customerror"
"github.com/instill-ai/artifact-backend/pkg/logger"
artifactpb "github.com/instill-ai/protogen-go/artifact/artifact/v1alpha"
)

// GetObjectUploadURL returns the upload URL for an object.
func (ph *PublicHandler) GetObjectUploadURL(ctx context.Context, req *artifactpb.GetObjectUploadURLRequest) (*artifactpb.GetObjectUploadURLResponse, error) {
log, _ := logger.GetZapLogger(ctx)
authUID, err := getUserUIDFromContext(ctx)
if err != nil {
err := fmt.Errorf("failed to get user id from header: %v. err: %w", err, customerror.ErrUnauthenticated)
return nil, err
}
creatorUID, err := uuid.FromString(authUID)
if err != nil {
return nil, fmt.Errorf("failed to parse creator uid. err: %w", err)
}

ns, err := ph.service.GetNamespaceByNsID(ctx, req.GetNamespaceId())
if err != nil {
log.Error(
"failed to get namespace ",
zap.Error(err),
zap.String("owner_id(ns_id)", req.GetNamespaceId()),
zap.String("auth_uid", authUID))
return nil, fmt.Errorf("failed to get namespace. err: %w", err)
}
// ACL - check user's permission to upload object in the namespace
err = ph.service.CheckNamespacePermission(ctx, ns)
if err != nil {
log.Error(
"failed to check namespace permission",
zap.Error(err),
zap.String("owner_id(ns_id)", req.GetNamespaceId()),
zap.String("auth_uid", authUID))
return nil, fmt.Errorf("failed to check namespace permission. err: %w", err)
}

// Call the service to get the upload URL
response, err := ph.service.GetUploadURL(ctx, req, ns.NsUID, creatorUID)
if err != nil {
log.Error("failed to get upload URL", zap.Error(err))
return nil, fmt.Errorf("failed to get upload URL. err: %w", err)
}

return response, nil
}

func (ph *PublicHandler) GetObjectDownloadURL(ctx context.Context, req *artifactpb.GetObjectDownloadURLRequest) (*artifactpb.GetObjectDownloadURLResponse, error) {
return nil, nil
}
47 changes: 35 additions & 12 deletions pkg/service/object.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,17 @@ package service

import (
"context"
"fmt"
"time"

"github.com/gofrs/uuid"
"github.com/gogo/status"
"github.com/instill-ai/artifact-backend/pkg/logger"
"github.com/instill-ai/artifact-backend/pkg/minio"
"github.com/instill-ai/artifact-backend/pkg/repository"
"github.com/instill-ai/artifact-backend/pkg/utils"
artifactpb "github.com/instill-ai/protogen-go/artifact/artifact/v1alpha"
"go.uber.org/zap"
"google.golang.org/grpc/codes"
"google.golang.org/protobuf/types/known/timestamppb"
)
Expand All @@ -19,29 +23,33 @@ func (s *Service) GetUploadURL(
ctx context.Context,
req *artifactpb.GetObjectUploadURLRequest,
namespaceUID uuid.UUID,
contentType string,
creatorUID uuid.UUID,
) (*artifactpb.GetObjectUploadURLResponse, error) {
log, _ := logger.GetZapLogger(ctx)
// name cannot be empty
if req.GetObjectName() == "" {
log.Error("name cannot be empty")
return nil, status.Errorf(codes.InvalidArgument, "name cannot be empty")
}

// check expiration_time is valid. if it is lower than 60, we will use 60 as the expiration_time
if req.GetExpirationTime() < 60 {
req.ExpirationTime = 60
if req.GetUrlExpireDays() < 1 {
req.UrlExpireDays = 1
}

// check expiration_time is valid. if it is greater than 7 days, we will use 7 days as the expiration_time
if req.GetExpirationTime() > 7*24*60 {
req.ExpirationTime = 7 * 24 * 60
if req.GetUrlExpireDays() > 7 {
req.UrlExpireDays = 7
}

objectExpireDays := int(req.GetObjectExpireDays())
lastModifiedTime := req.GetLastModifiedTime().AsTime()
contentType := utils.DetermineMimeType(req.GetObjectName())
// create object
object := &repository.Object{
Name: req.GetObjectName(),
NamespaceUID: namespaceUID,
CreatorUID: creatorUID,
ContentType: contentType,
Size: 0, // we will update the size when the object is uploaded. when trying to get download url, we will check the size of the object in minio.
IsUploaded: false, // we will check and update the is_uploaded when trying to get download url.
Expand All @@ -53,6 +61,7 @@ func (s *Service) GetUploadURL(
// create object
createdObject, err := s.Repository.CreateObject(ctx, *object)
if err != nil {
log.Error("failed to create object", zap.Error(err))
return nil, status.Errorf(codes.Internal, "failed to create object: %v", err)
}

Expand All @@ -62,35 +71,39 @@ func (s *Service) GetUploadURL(
createdObject.Destination = minioPath
_, err = s.Repository.UpdateObject(ctx, *createdObject)
if err != nil {
log.Error("failed to update object", zap.Error(err))
return nil, status.Errorf(codes.Internal, "failed to update object: %v", err)
}

// get presigned url for uploading object
presignedURL, err := s.MinIO.MakePresignedURLForUpload(ctx, namespaceUID, createdObject.UID, time.Duration(req.GetExpirationTime())*time.Minute)
presignedURL, err := s.MinIO.MakePresignedURLForUpload(ctx, namespaceUID, createdObject.UID, time.Duration(req.GetUrlExpireDays())*time.Hour*24)
if err != nil {
log.Error("failed to make presigned url for upload", zap.Error(err))
return nil, status.Errorf(codes.Internal, "failed to make presigned url for upload: %v", err)
}

// remove the protocol and host from the presignedURL
path := presignedURL.Path + "?" + presignedURL.RawQuery
presignedURLPathQuery := presignedURL.Path + "?" + presignedURL.RawQuery

// create object_url and update the encoded_url_path
objectURL := &repository.ObjectURL{
NamespaceUID: namespaceUID,
ObjectUID: object.UID,
URLExpireAt: time.Now().UTC().Add(time.Duration(req.GetExpirationTime()) * time.Minute),
MinioURLPath: path,
NamespaceUID: createdObject.NamespaceUID,
ObjectUID: createdObject.UID,
URLExpireAt: time.Now().UTC().Add(time.Duration(req.GetUrlExpireDays()) * time.Hour * 24),
MinioURLPath: presignedURLPathQuery,
EncodedURLPath: "",
Type: repository.ObjectURLTypeUpload,
}

createdObjectURL, err := s.Repository.CreateObjectURL(ctx, *objectURL)
if err != nil {
log.Error("failed to create object url", zap.Error(err))
return nil, status.Errorf(codes.Internal, "failed to create object url: %v", err)
}
createdObjectURL.EncodedURLPath = getEncodedMinioURLPath(createdObjectURL.UID)
_, err = s.Repository.UpdateObjectURL(ctx, *createdObjectURL)
if err != nil {
log.Error("failed to update object url", zap.Error(err))
return nil, status.Errorf(codes.Internal, "failed to update object url: %v", err)
}

Expand All @@ -110,9 +123,11 @@ func getEncodedMinioURLPath(objectURLUUID uuid.UUID) string {

// turn object in db to object in proto
func turnObjectInDBToObjectInProto(object *repository.Object) *artifactpb.Object {
return &artifactpb.Object{
objectInProto := &artifactpb.Object{
Uid: object.UID.String(),
NamespaceUid: object.NamespaceUID.String(),
Name: object.Name,
Creator: object.CreatorUID.String(),
ContentType: object.ContentType,
Size: object.Size,
IsUploaded: object.IsUploaded,
Expand All @@ -121,4 +136,12 @@ func turnObjectInDBToObjectInProto(object *repository.Object) *artifactpb.Object
CreatedTime: timestamppb.New(object.CreateTime),
UpdatedTime: timestamppb.New(object.UpdateTime),
}
if object.LastModifiedTime != nil &&
!object.LastModifiedTime.IsZero() &&
object.LastModifiedTime.Format(time.RFC3339) != "1970-01-01T00:00:00Z" {
lastModifiedTime := timestamppb.New(*object.LastModifiedTime)
fmt.Println("lastModifiedTime.Format(time.RFC3339)", object.LastModifiedTime.Format(time.RFC3339))
objectInProto.LastModifiedTime = lastModifiedTime
}
return objectInProto
}
2 changes: 2 additions & 0 deletions pkg/service/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,15 @@ import (
func (s *Service) CheckNamespacePermission(ctx context.Context, ns *resource.Namespace) error {
// TODO: optimize ACL model
if ns.NsType == "organizations" {
// check if the user is a member of the organization
granted, err := s.ACLClient.CheckPermission(ctx, "organization", ns.NsUID, "member")
if err != nil {
return err
}
if !granted {
return ErrNoPermission
}
// check if the user is the owner of the namespace
} else if ns.NsUID != uuid.FromStringOrNil(resource.GetRequestSingleHeader(ctx, constant.HeaderUserUIDKey)) {
return ErrNoPermission
}
Expand Down
71 changes: 71 additions & 0 deletions pkg/utils/mime.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
package utils

import (
"path/filepath"
"strings"
)

// DetermineMimeType determines the MIME type based on the file extension
func DetermineMimeType(fileName string) string {
ext := strings.ToLower(filepath.Ext(fileName))
switch ext {
case ".pdf":
return "application/pdf"
case ".md":
return "text/markdown"
case ".txt":
return "text/plain"
case ".doc":
return "application/msword"
case ".docx":
return "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
case ".html":
return "text/html"
case ".ppt":
return "application/vnd.ms-powerpoint"
case ".pptx":
return "application/vnd.openxmlformats-officedocument.presentationml.presentation"
case ".xlsx":
return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
case ".xls":
return "application/vnd.ms-excel"
case ".csv":
return "text/csv"
case ".json":
return "application/json"
case ".xml":
return "application/xml"
case ".zip":
return "application/zip"
case ".tar":
return "application/x-tar"
case ".gz":
return "application/gzip"
case ".jpg", ".jpeg":
return "image/jpeg"
case ".png":
return "image/png"
case ".gif":
return "image/gif"
case ".svg":
return "image/svg+xml"
case ".mp3":
return "audio/mpeg"
case ".mp4":
return "video/mp4"
case ".wav":
return "audio/wav"
case ".avi":
return "video/x-msvideo"
case ".mpg", ".mpeg":
return "video/mpeg"
case ".ogg":
return "audio/ogg"
case ".css":
return "text/css"
case ".js":
return "application/javascript"
default:
return "application/octet-stream"
}
}
1 change: 0 additions & 1 deletion pkg/worker/worker.go
Original file line number Diff line number Diff line change
Expand Up @@ -710,7 +710,6 @@ func (wp *fileToEmbWorkerPool) processEmbeddingFile(ctx context.Context, file re
return nil, artifactpb.FileProcessStatus_FILE_PROCESS_STATUS_UNSPECIFIED, err
}


// update the file status to complete status in database
updateMap := map[string]interface{}{
repository.KnowledgeBaseFileColumn.ProcessStatus: artifactpb.FileProcessStatus_name[int32(artifactpb.FileProcessStatus_FILE_PROCESS_STATUS_COMPLETED)],
Expand Down

0 comments on commit e207a2f

Please sign in to comment.