From c506e9901a5defe85fd196e6050173c9e599f56c Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 18 Oct 2024 11:58:32 +0200 Subject: [PATCH] Add media type detection when linking a file (#5191) Co-authored-by: Simon Dumas --- .../plugins/storage/StoragePluginModule.scala | 26 +++- .../delta/plugins/storage/files/Files.scala | 76 ++++------ .../storage/files/FormDataExtractor.scala | 40 +---- .../storage/files/MediaTypeDetector.scala | 37 +++++ .../storage/files/model/FileAttributes.scala | 30 ++-- .../storage/files/model/FileCommand.scala | 32 ++++ .../storage/files/model/FileDescription.scala | 29 ---- .../storages/operations/FileOperations.scala | 11 +- .../storages/operations/LinkFileAction.scala | 56 +++++++ .../operations/StorageFileRejection.scala | 4 +- .../storages/operations/StorageWrite.scala | 15 ++ .../storages/operations/UploadingFile.scala | 4 +- .../storages/operations/disk/package.scala | 3 - .../operations/s3/PutObjectRequest.scala | 24 +-- .../operations/s3/S3FileOperations.scala | 11 +- .../plugins/storage/files/FilesSpec.scala | 23 +-- .../storage/files/FormDataExtractorSpec.scala | 12 +- .../files/MediaTypeDetectorSuite.scala | 43 ++++++ .../files/routes/FilesRoutesSpec.scala | 22 +-- .../operations/LinkFileActionSuite.scala | 140 ++++++++++++++++++ .../operations/s3/S3FileOperationsSuite.scala | 2 +- .../storages/operations/s3/S3Helpers.scala | 6 +- .../nexus/ship/files/FileProcessor.scala | 12 +- .../nexus/ship/files/FileWiring.scala | 35 +++-- .../nexus/ship/storages/StorageWiring.scala | 36 +---- .../bluebrain/nexus/ship/RunShipSuite.scala | 2 +- .../nexus/tests/kg/files/S3StorageSpec.scala | 16 +- 27 files changed, 503 insertions(+), 244 deletions(-) create mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/MediaTypeDetector.scala create mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/LinkFileAction.scala create mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageWrite.scala create mode 100644 delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/MediaTypeDetectorSuite.scala create mode 100644 delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/LinkFileActionSuite.scala diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala index a0fc704894..dd58b7448a 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala @@ -12,11 +12,11 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.contexts.{files => fi import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.{DelegateFilesRoutes, FilesRoutes, LinkFilesRoutes} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.schemas.{files => filesSchemaId} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{FileAttributesUpdateStream, Files} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{FileAttributesUpdateStream, Files, FormDataExtractor, MediaTypeDetector} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.{ShowFileLocation, StorageTypeConfig} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.contexts.{storages => storageCtxId, storagesMetadata => storageMetaCtxId} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model._ -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.FileOperations +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.{FileOperations, LinkFileAction} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.DiskFileOperations import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.RemoteDiskFileOperations import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient @@ -191,6 +191,15 @@ class StoragePluginModule(priority: Int) extends ModuleDef { FileOperations.apply(disk, remoteDisk, s3) } + make[MediaTypeDetector].from { (cfg: StoragePluginConfig) => + new MediaTypeDetector(cfg.files.mediaTypeDetector) + } + + make[LinkFileAction].from { + (fetchStorage: FetchStorage, mediaTypeDetector: MediaTypeDetector, s3FileOps: S3FileOperations) => + LinkFileAction(fetchStorage, mediaTypeDetector, s3FileOps) + } + make[Files].from { ( cfg: StoragePluginConfig, @@ -200,19 +209,20 @@ class StoragePluginModule(priority: Int) extends ModuleDef { clock: Clock[IO], uuidF: UUIDF, as: ActorSystem[Nothing], - fileOps: FileOperations + fileOps: FileOperations, + mediaTypeDetector: MediaTypeDetector, + linkFileAction: LinkFileAction ) => Files( fetchContext, fetchStorage, + FormDataExtractor(mediaTypeDetector)(as.classicSystem), xas, - cfg.files, + cfg.files.eventLog, fileOps, + linkFileAction, clock - )( - uuidF, - as - ) + )(uuidF) } make[FileAttributesUpdateStream].fromEffect { diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala index a6c0c10cb8..93b3bf7fa0 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala @@ -1,7 +1,5 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files -import akka.actor.typed.ActorSystem -import akka.actor.{ActorSystem => ClassicActorSystem} import akka.http.scaladsl.model.ContentTypes.`application/octet-stream` import akka.http.scaladsl.model.Uri import cats.effect.{Clock, IO} @@ -32,6 +30,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.{ApiMappings, ProjectContext} import ch.epfl.bluebrain.nexus.delta.sourcing.ScopedEntityDefinition.Tagger import ch.epfl.bluebrain.nexus.delta.sourcing._ +import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag import ch.epfl.bluebrain.nexus.delta.sourcing.model.{EntityType, ProjectRef, ResourceRef, SuccessElemStream} @@ -47,7 +46,8 @@ final class Files( log: FilesLog, fetchContext: FetchContext, fetchStorage: FetchStorage, - fileOperations: FileOperations + fileOperations: FileOperations, + linkFile: LinkFileAction )(implicit uuidF: UUIDF) { implicit private val kamonComponent: KamonMetricComponent = KamonMetricComponent(entityType.value) @@ -280,17 +280,11 @@ final class Files( tag: Option[UserTag] )(implicit caller: Caller): IO[FileResource] = { for { - projectContext <- fetchContext.onCreate(project) - iri <- id.fold(generateId(projectContext)) { FileId.iriExpander(_, projectContext) } - storageIri <- storageId.traverse(expandStorageIri(_, projectContext)) - (storageRef, storage) <- fetchStorage.onWrite(storageIri, project) - s3Metadata <- fileOperations.link(storage, linkRequest.path) - filename <- IO.fromOption(linkRequest.path.lastSegment)(InvalidFilePath) - attr = FileAttributes.from( - FileDescription(filename, linkRequest.mediaType.orElse(s3Metadata.contentType), linkRequest.metadata), - s3Metadata.metadata - ) - res <- eval(CreateFile(iri, project, storageRef, storage.tpe, attr, caller.subject, tag)) + projectContext <- fetchContext.onCreate(project) + iri <- id.fold(generateId(projectContext)) { FileId.iriExpander(_, projectContext) } + storageIri <- storageId.traverse(expandStorageIri(_, projectContext)) + storageWrite <- linkFile(storageIri, project, linkRequest) + res <- eval(CreateFile(iri, project, storageWrite, caller.subject, tag)) } yield res }.span("linkFile") @@ -302,28 +296,12 @@ final class Files( tag: Option[UserTag] )(implicit caller: Caller): IO[FileResource] = { for { - (iri, pc) <- id.expandIri(fetchContext.onModify) - storageIri <- storageId.traverse(expandStorageIri(_, pc)) - _ <- test(UpdateFile(iri, id.project, testStorageRef, testStorageType, testAttributes, rev, caller.subject, tag)) - (storageRef, storage) <- fetchStorage.onWrite(storageIri, id.project) - s3Metadata <- fileOperations.link(storage, linkRequest.path) - filename <- IO.fromOption(linkRequest.path.lastSegment)(InvalidFilePath) - attr = FileAttributes.from( - FileDescription(filename, linkRequest.mediaType.orElse(s3Metadata.contentType), linkRequest.metadata), - s3Metadata.metadata - ) - res <- eval( - UpdateFile( - iri, - id.project, - storageRef, - storage.tpe, - attr, - rev, - caller.subject, - tag - ) - ) + (iri, pc) <- id.expandIri(fetchContext.onModify) + project = id.project + storageIri <- storageId.traverse(expandStorageIri(_, pc)) + _ <- test(UpdateFile(iri, project, testStorageRef, testStorageType, testAttributes, rev, caller.subject, tag)) + storageWrite <- linkFile(storageIri, project, linkRequest) + res <- eval(UpdateFile(iri, project, storageWrite, rev, caller.subject, tag)) } yield res }.span("updateLinkedFile") @@ -353,7 +331,7 @@ final class Files( _ <- test(UpdateFile(iri, id.project, testStorageRef, testStorageType, testAttributes, rev, caller.subject, tag)) (storageRef, storage) <- fetchStorage.onWrite(storageIri, id.project) metadata <- legacyLinkFile(storage, path, description.filename, iri) - attributes = FileAttributes.from(description, metadata) + attributes = FileAttributes.from(description.filename, description.mediaType, description.metadata, metadata) res <- eval(UpdateFile(iri, id.project, storageRef, storage.tpe, attributes, rev, caller.subject, tag)) } yield res }.span("updateLink") @@ -493,7 +471,8 @@ final class Files( _ <- test(CreateFile(iri, project, testStorageRef, testStorageType, testAttributes, caller.subject, tag)) (storageRef, storage) <- fetchStorage.onWrite(storageIri, project) storageMetadata <- legacyLinkFile(storage, path, description.filename, iri) - fileAttributes = FileAttributes.from(description, storageMetadata) + fileAttributes = + FileAttributes.from(description.filename, description.mediaType, description.metadata, storageMetadata) res <- eval(CreateFile(iri, project, storageRef, storage.tpe, fileAttributes, caller.subject, tag)) } yield res @@ -514,9 +493,8 @@ final class Files( private def saveFileToStorage(iri: Iri, storage: Storage, uploadRequest: FileUploadRequest): IO[FileAttributes] = { for { info <- formDataExtractor(uploadRequest.entity, storage.storageValue.maxFileSize) - description = FileDescription.from(info, uploadRequest.metadata) storageMetadata <- fileOperations.save(storage, info, uploadRequest.contentLength) - } yield FileAttributes.from(description, storageMetadata) + } yield FileAttributes.from(info.filename, info.contentType, uploadRequest.metadata, storageMetadata) }.adaptError { case e: SaveFileRejection => SaveRejection(iri, storage.id, e) } private def generateId(pc: ProjectContext): IO[Iri] = @@ -776,21 +754,21 @@ object Files { def apply( fetchContext: FetchContext, fetchStorage: FetchStorage, + formDataExtractor: FormDataExtractor, xas: Transactors, - config: FilesConfig, + eventLogConfig: EventLogConfig, fileOps: FileOperations, + linkFile: LinkFileAction, clock: Clock[IO] )(implicit - uuidF: UUIDF, - as: ActorSystem[Nothing] - ): Files = { - implicit val classicAs: ClassicActorSystem = as.classicSystem + uuidF: UUIDF + ): Files = new Files( - FormDataExtractor(config.mediaTypeDetector), - ScopedEventLog(definition(clock), config.eventLog, xas), + formDataExtractor, + ScopedEventLog(definition(clock), eventLogConfig, xas), fetchContext, fetchStorage, - fileOps + fileOps, + linkFile ) - } } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala index 8e056fcdfd..986f910972 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala @@ -11,12 +11,9 @@ import akka.stream.scaladsl.{Keep, Sink} import cats.effect.IO import cats.syntax.all._ import ch.epfl.bluebrain.nexus.delta.kernel.error.NotARejection -import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig -import ch.epfl.bluebrain.nexus.delta.kernel.utils.FileUtils import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection.{FileTooLarge, InvalidMultipartFieldName, WrappedAkkaRejection} import scala.concurrent.{ExecutionContext, Future} -import scala.util.Try trait FormDataExtractor { @@ -34,9 +31,9 @@ trait FormDataExtractor { def apply(entity: HttpEntity, maxFileSize: Long): IO[UploadedFileInformation] } -case class UploadedFileInformation( +final case class UploadedFileInformation( filename: String, - suppliedContentType: ContentType, + contentType: Option[ContentType], contents: BodyPartEntity ) @@ -61,9 +58,7 @@ object FormDataExtractor { createStrict = (_, parts) => Multipart.FormData.Strict(parts) ) - def apply( - mediaTypeDetector: MediaTypeDetectorConfig - )(implicit as: ActorSystem): FormDataExtractor = + def apply(mediaTypeDetector: MediaTypeDetector)(implicit as: ActorSystem): FormDataExtractor = new FormDataExtractor { implicit val ec: ExecutionContext = as.getDispatcher @@ -115,40 +110,19 @@ object FormDataExtractor { private def extractFile(part: FormData.BodyPart): Future[Option[UploadedFileInformation]] = part match { case part if part.name == FileFieldName => - val filename = part.filename.filterNot(_.isEmpty).getOrElse(defaultFilename) - val contentType = detectContentType(filename, part.entity.contentType) + val filename = part.filename.filterNot(_.isEmpty).getOrElse(defaultFilename) + val contentTypeFromRequest = part.entity.contentType + val suppliedContentType = Option.when(contentTypeFromRequest != defaultContentType)(contentTypeFromRequest) Future( UploadedFileInformation( filename, - contentType, + mediaTypeDetector(filename, suppliedContentType, Some(contentTypeFromRequest)), part.entity ).some ) case part => part.entity.discardBytes().future.as(None) } - - private def detectContentType(filename: String, contentTypeFromRequest: ContentType) = { - val bodyDefinedContentType = Option.when(contentTypeFromRequest != defaultContentType)(contentTypeFromRequest) - - val extensionOpt = FileUtils.extension(filename) - - def detectFromConfig = for { - extension <- extensionOpt - customMediaType <- mediaTypeDetector.find(extension) - } yield contentType(customMediaType) - - def detectAkkaFromExtension = extensionOpt.flatMap { e => - Try(MediaTypes.forExtension(e)).map(contentType).toOption - } - - bodyDefinedContentType - .orElse(detectFromConfig) - .orElse(detectAkkaFromExtension) - .getOrElse(contentTypeFromRequest) - } - - private def contentType(mediaType: MediaType) = ContentType(mediaType, () => HttpCharsets.`UTF-8`) } } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/MediaTypeDetector.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/MediaTypeDetector.scala new file mode 100644 index 0000000000..bd356df9bb --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/MediaTypeDetector.scala @@ -0,0 +1,37 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files + +import akka.http.scaladsl.model.{ContentType, HttpCharsets, MediaType, MediaTypes} +import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig +import ch.epfl.bluebrain.nexus.delta.kernel.utils.FileUtils + +import scala.util.Try + +/** + * Allows to detect a content type from incoming files from their extensions when the client has not provided one + * + * @param config + * the config with a mapping from the extension to the content type + */ +final class MediaTypeDetector(config: MediaTypeDetectorConfig) { + + def apply(filename: String, provided: Option[ContentType], fallback: Option[ContentType]): Option[ContentType] = { + val extensionOpt = FileUtils.extension(filename) + + def detectFromConfig = for { + extension <- extensionOpt + customMediaType <- config.find(extension) + } yield contentType(customMediaType) + + def detectAkkaFromExtension = extensionOpt.flatMap { e => + Try(MediaTypes.forExtension(e)).map(contentType).toOption + } + + provided + .orElse(detectFromConfig) + .orElse(detectAkkaFromExtension) + .orElse(fallback) + } + + private def contentType(mediaType: MediaType) = ContentType(mediaType, () => HttpCharsets.`UTF-8`) + +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileAttributes.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileAttributes.scala index 464af5eb8e..784e9cbfbb 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileAttributes.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileAttributes.scala @@ -5,9 +5,9 @@ import akka.http.scaladsl.model.{ContentType, Uri} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin import ch.epfl.bluebrain.nexus.delta.sdk.implicits._ import ch.epfl.bluebrain.nexus.delta.sourcing.model.Label -import io.circe.{Decoder, Encoder} import io.circe.generic.extras.Configuration import io.circe.generic.extras.semiauto.deriveConfiguredEncoder +import io.circe.{Decoder, Encoder} import java.util.UUID @@ -44,31 +44,23 @@ final case class FileAttributes( bytes: Long, digest: Digest, origin: FileAttributesOrigin -) extends LimitedFileAttributes - -trait LimitedFileAttributes { - def location: Uri - def path: Path - def filename: String - def mediaType: Option[ContentType] - def keywords: Map[Label, String] - def description: Option[String] - def name: Option[String] - def bytes: Long - def digest: Digest - def origin: FileAttributesOrigin -} +) object FileAttributes { - def from(description: FileDescription, storageMetadata: FileStorageMetadata): FileAttributes = { - val customMetadata = description.metadata.getOrElse(FileCustomMetadata.empty) + def from( + filename: String, + contentType: Option[ContentType], + metadata: Option[FileCustomMetadata], + storageMetadata: FileStorageMetadata + ): FileAttributes = { + val customMetadata = metadata.getOrElse(FileCustomMetadata.empty) FileAttributes( storageMetadata.uuid, storageMetadata.location, storageMetadata.path, - description.filename, - description.mediaType, + filename, + contentType, customMetadata.keywords.getOrElse(Map.empty), customMetadata.description, customMetadata.name, diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileCommand.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileCommand.scala index af5b2646a4..0acca69f2a 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileCommand.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileCommand.scala @@ -2,6 +2,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model import akka.http.scaladsl.model.ContentType import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageType +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageWrite import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag @@ -69,6 +70,17 @@ object FileCommand { override def rev: Int = 0 } + object CreateFile { + def apply( + id: Iri, + project: ProjectRef, + storageWrite: StorageWrite[FileAttributes], + subject: Subject, + tag: Option[UserTag] + ): CreateFile = + CreateFile(id, project, storageWrite.storage, storageWrite.tpe, storageWrite.value, subject, tag) + } + /** * Command to update an existing file * @@ -98,6 +110,26 @@ object FileCommand { tag: Option[UserTag] ) extends FileCommand + object UpdateFile { + def apply( + id: Iri, + project: ProjectRef, + storageWrite: StorageWrite[FileAttributes], + rev: Int, + subject: Subject, + tag: Option[UserTag] + ): UpdateFile = UpdateFile( + id, + project, + storageWrite.storage, + storageWrite.tpe, + storageWrite.value, + rev, + subject, + tag + ) + } + /** * Command to update the custom metadata of a file * diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileDescription.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileDescription.scala index 4eed1b7e70..3fb7dc4181 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileDescription.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileDescription.scala @@ -1,8 +1,6 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model import akka.http.scaladsl.model.ContentType -import cats.implicits.catsSyntaxOptionId -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.UploadedFileInformation import io.circe.Codec import io.circe.generic.extras.Configuration import ch.epfl.bluebrain.nexus.delta.sdk.implicits._ @@ -15,33 +13,6 @@ case class FileDescription( ) object FileDescription { - def from(file: File): FileDescription = { - from(file.attributes) - } - - def from(fileAttributes: FileAttributes): FileDescription = - FileDescription( - fileAttributes.filename, - fileAttributes.mediaType, - FileCustomMetadata( - fileAttributes.name, - fileAttributes.description, - Some(fileAttributes.keywords) - ).some - ) - - def from(info: UploadedFileInformation, metadata: Option[FileCustomMetadata]): FileDescription = { - val md = metadata.getOrElse(FileCustomMetadata.empty) - FileDescription( - info.filename, - Some(info.suppliedContentType), - FileCustomMetadata( - md.name, - md.description, - md.keywords - ).some - ) - } implicit private val config: Configuration = Configuration.default implicit val fileDescriptionCodec: Codec[FileDescription] = deriveConfiguredCodec[FileDescription] diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/FileOperations.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/FileOperations.scala index c1db7dc9d4..efde3f86cb 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/FileOperations.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/FileOperations.scala @@ -6,12 +6,11 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.UploadedFileInformati import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{ComputedFileAttributes, FileAttributes, FileDelegationRequest, FileStorageMetadata} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.{DiskStorage, RemoteDiskStorage, S3Storage} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.{DelegateFileOperation, FetchAttributeRejection, LinkFileRejection, MoveFileRejection} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.{DelegateFileOperation, FetchAttributeRejection, MoveFileRejection} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.UploadingFile.{DiskUploadingFile, RemoteUploadingFile, S3UploadingFile} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.DiskFileOperations import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.RemoteDiskFileOperations import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3.S3FileOperations -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3.S3FileOperations.S3FileMetadata import ch.epfl.bluebrain.nexus.delta.sdk.AkkaSource import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef @@ -28,8 +27,6 @@ trait FileOperations { def legacyLink(storage: Storage, sourcePath: Uri.Path, filename: String): IO[FileStorageMetadata] - def link(storage: Storage, path: Uri.Path): IO[S3FileMetadata] - def fetchAttributes(storage: Storage, attributes: FileAttributes): IO[ComputedFileAttributes] def delegate(storage: Storage, filename: String): IO[FileDelegationRequest.TargetLocation] @@ -71,12 +68,6 @@ object FileOperations { case s => IO.raiseError(FetchAttributeRejection.UnsupportedOperation(s.tpe)) } - override def link(storage: Storage, path: Uri.Path): IO[S3FileMetadata] = - storage match { - case s: S3Storage => s3FileOps.link(s.value.bucket, path) - case s => IO.raiseError(LinkFileRejection.UnsupportedOperation(s.tpe)) - } - override def delegate(storage: Storage, filename: String): IO[FileDelegationRequest.TargetLocation] = storage match { case s: S3Storage => diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/LinkFileAction.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/LinkFileAction.scala new file mode 100644 index 0000000000..197d6994c5 --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/LinkFileAction.scala @@ -0,0 +1,56 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations + +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.MediaTypeDetector +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileLinkRequest} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.FetchStorage +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.S3Storage +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.LinkFileRejection +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3.S3FileOperations +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3.S3FileOperations.S3FileLink +import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri +import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller +import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef + +trait LinkFileAction { + + def apply(storageIri: Option[Iri], project: ProjectRef, request: FileLinkRequest)(implicit + caller: Caller + ): IO[StorageWrite[FileAttributes]] +} + +object LinkFileAction { + + val alwaysFails: LinkFileAction = new LinkFileAction { + override def apply(storageIri: Option[Iri], project: ProjectRef, request: FileLinkRequest)(implicit + caller: Caller + ): IO[StorageWrite[FileAttributes]] = IO.raiseError(LinkFileRejection.Disabled) + } + + def apply( + fetchStorage: FetchStorage, + mediaTypeDetector: MediaTypeDetector, + s3FileOps: S3FileOperations + ): LinkFileAction = apply(fetchStorage, mediaTypeDetector, s3FileOps.link(_, _)) + + def apply( + fetchStorage: FetchStorage, + mediaTypeDetector: MediaTypeDetector, + s3FileLink: S3FileLink + ): LinkFileAction = new LinkFileAction { + override def apply(storageIri: Option[Iri], project: ProjectRef, request: FileLinkRequest)(implicit + caller: Caller + ): IO[StorageWrite[FileAttributes]] = + fetchStorage.onWrite(storageIri, project).flatMap { + case (storageRef, storage: S3Storage) => + s3FileLink(storage.value.bucket, request.path).map { s3Metadata => + val contentType = mediaTypeDetector(s3Metadata.filename, request.mediaType, s3Metadata.contentType) + val attributes = + FileAttributes.from(s3Metadata.filename, contentType, request.metadata, s3Metadata.metadata) + StorageWrite(storageRef, storage.tpe, attributes) + } + case (_, s) => IO.raiseError(LinkFileRejection.UnsupportedOperation(s.tpe)) + } + } + +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageFileRejection.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageFileRejection.scala index fd2344c2b2..ea06d1f0ce 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageFileRejection.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageFileRejection.scala @@ -167,11 +167,13 @@ object StorageFileRejection { object LinkFileRejection { + final case object Disabled extends LinkFileRejection(s"Linking a file is disabled") + final case class InvalidPath(path: Uri.Path) extends LinkFileRejection(s"An S3 path must contain at least the filename. Path was $path") final case class UnsupportedOperation(tpe: StorageType) - extends MoveFileRejection(s"Linking a file in-place is not supported for storages of type '${tpe.iri}'") + extends MoveFileRejection(s"Linking a file is not supported for storages of type '${tpe.iri}'") } sealed abstract class DelegateFileOperation(loggedDetails: String) extends StorageFileRejection(loggedDetails) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageWrite.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageWrite.scala new file mode 100644 index 0000000000..ba78fc15e9 --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageWrite.scala @@ -0,0 +1,15 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations + +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageType +import ch.epfl.bluebrain.nexus.delta.sourcing.model.ResourceRef + +/** + * Result on a write operation on a storage + * @param storage + * the reference of the storage + * @param tpe + * its type + * @param value + * the value returned by the operation + */ +final case class StorageWrite[A](storage: ResourceRef.Revision, tpe: StorageType, value: A) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/UploadingFile.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/UploadingFile.scala index 7641141067..c880142752 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/UploadingFile.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/UploadingFile.scala @@ -49,7 +49,7 @@ object UploadingFile { project: ProjectRef, bucket: String, filename: String, - contentType: ContentType, + contentType: Option[ContentType], contentLength: Long, entity: BodyPartEntity ) extends UploadingFile @@ -70,7 +70,7 @@ object UploadingFile { s.project, s.value.bucket, info.filename, - info.suppliedContentType, + info.contentType, contentLength, info.contents ) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/package.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/package.scala index eb25a99c11..1e929453dd 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/package.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/package.scala @@ -2,7 +2,6 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations import akka.http.scaladsl.model.Uri.Path import cats.effect.IO -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.LimitedFileAttributes import java.net.URI import java.nio.file @@ -10,8 +9,6 @@ import java.nio.file.Paths package object disk { - def absoluteDiskPathFromAttributes(attr: LimitedFileAttributes): IO[file.Path] = absoluteDiskPath(attr.location.path) - def absoluteDiskPath(relative: Path): IO[file.Path] = IO(Paths.get(URI.create(s"file://$relative"))) } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/PutObjectRequest.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/PutObjectRequest.scala index 553753d0a3..b92d27e718 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/PutObjectRequest.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/PutObjectRequest.scala @@ -3,15 +3,21 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3 import akka.http.scaladsl.model.ContentType import software.amazon.awssdk.services.s3.model.{PutObjectRequest => AwsPutObjectRequest} -final case class PutObjectRequest(bucket: String, key: String, contentType: ContentType, contentLength: Long) { +final case class PutObjectRequest(bucket: String, key: String, contentType: Option[ContentType], contentLength: Long) { - def asAws: AwsPutObjectRequest = AwsPutObjectRequest - .builder() - .bucket(bucket) - .checksumAlgorithm(checksumAlgorithm) - .contentType(contentType.value) - .contentLength(contentLength) - .key(key) - .build() + def asAws: AwsPutObjectRequest = { + val request = AwsPutObjectRequest + .builder() + .bucket(bucket) + .checksumAlgorithm(checksumAlgorithm) + .contentLength(contentLength) + .key(key) + + contentType + .fold(request) { ct => + request.contentType(ct.value) + } + .build() + } } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3FileOperations.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3FileOperations.scala index b9d4f1ba97..50d1921b7b 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3FileOperations.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3FileOperations.scala @@ -8,6 +8,7 @@ import cats.syntax.all._ import ch.epfl.bluebrain.nexus.delta.kernel.Logger import ch.epfl.bluebrain.nexus.delta.kernel.utils.{UUIDF, UrlUtils} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection.InvalidFilePath import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileStorageMetadata import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.FetchFileRejection import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.FetchFileRejection.UnexpectedFetchError @@ -15,6 +16,7 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.Uploadi import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3.S3FileOperations.{S3DelegationMetadata, S3FileMetadata} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3.client.S3StorageClient import ch.epfl.bluebrain.nexus.delta.sdk.AkkaSource +import ch.epfl.bluebrain.nexus.delta.sdk.syntax._ import ch.epfl.bluebrain.nexus.delta.sdk.stream.StreamConverter import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef import software.amazon.awssdk.services.s3.model.NoSuchKeyException @@ -31,7 +33,10 @@ trait S3FileOperations { } object S3FileOperations { - final case class S3FileMetadata(contentType: Option[ContentType], metadata: FileStorageMetadata) + + type S3FileLink = (String, Uri.Path) => IO[S3FileMetadata] + + final case class S3FileMetadata(filename: String, contentType: Option[ContentType], metadata: FileStorageMetadata) final case class S3DelegationMetadata(bucket: String, path: Uri) private val log = Logger[S3FileOperations] @@ -84,8 +89,10 @@ object S3FileOperations { uuidf: UUIDF ) = for { - uuid <- uuidf() + uuid <- uuidf() + filename <- IO.fromOption(path.lastSegment)(InvalidFilePath) } yield S3FileMetadata( + filename, resp.contentType, FileStorageMetadata( uuid, diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala index fc73b232e0..f7335a843b 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala @@ -1,7 +1,6 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files -import akka.actor.typed.scaladsl.adapter._ -import akka.actor.{typed, ActorSystem} +import akka.actor.ActorSystem import akka.http.scaladsl.model.ContentTypes.`text/plain(UTF-8)` import akka.http.scaladsl.model.{ContentType, Uri} import akka.testkit.TestKit @@ -13,12 +12,12 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks.FileOperationsM import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.NotComputedDigest import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Storage import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection._ -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileCustomMetadata, FileDescription, FileId, FileUploadRequest} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.remotestorage.RemoteStorageClientFixtures import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejection.StorageNotFound import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageType import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageType.{RemoteDiskStorage => RemoteStorageType} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.{AkkaSourceHelpers, FileOperations} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.{AkkaSourceHelpers, FileOperations, LinkFileAction} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{FetchStorage, StorageFixtures, Storages, StoragesConfig} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv @@ -83,9 +82,8 @@ class FilesSpec(fixture: RemoteStorageClientFixtures) FileDescription(filename, None, Some(FileCustomMetadata(Some(name), Some(description), Some(keywords)))) "The Files operations bundle" when { - implicit val typedSystem: typed.ActorSystem[Nothing] = system.toTyped - implicit val caller: Caller = Caller(bob, Set(bob, Group("mygroup", realm), Authenticated(realm))) - lazy val remoteDiskStorageClient = fixture.init + implicit val caller: Caller = Caller(bob, Set(bob, Group("mygroup", realm), Authenticated(realm))) + lazy val remoteDiskStorageClient = fixture.init val tag = UserTag.unsafe("tag") val otherRead = Permission.unsafe("other/read") @@ -100,11 +98,11 @@ class FilesSpec(fixture: RemoteStorageClientFixtures) val remoteIdIri = nxv + "remote" val remoteId: IdSegment = remoteIdIri - val remoteRev = ResourceRef.Revision(iri"$remoteIdIri?rev=1", remoteIdIri, 1) + val remoteRev = ResourceRef.Revision(remoteIdIri, 1) val diskIdIri = nxv + "disk" val diskId: IdSegment = nxv + "disk" - val diskRev = ResourceRef.Revision(iri"$diskId?rev=1", diskIdIri, 1) + val diskRev = ResourceRef.Revision(diskIdIri, 1) val storageIri = nxv + "other-storage" val storage: IdSegment = nxv + "other-storage" @@ -141,8 +139,11 @@ class FilesSpec(fixture: RemoteStorageClientFixtures) lazy val fetchStorage = FetchStorage(storages, aclCheck) lazy val fileOps: FileOperations = FileOperationsMock.forDiskAndRemoteDisk(remoteDiskStorageClient) - val filesConfig = FilesConfig(eventLogConfig, MediaTypeDetectorConfig.Empty) - lazy val files: Files = Files(fetchContext, fetchStorage, xas, filesConfig, fileOps, clock) + val mediaTypeDetector = new MediaTypeDetector(MediaTypeDetectorConfig.Empty) + val dataExtractor = FormDataExtractor(mediaTypeDetector)(system) + val linkAction = LinkFileAction.alwaysFails + lazy val files: Files = + Files(fetchContext, fetchStorage, dataExtractor, xas, eventLogConfig, fileOps, linkAction, clock) def fileId(file: String): FileId = FileId(file, projectRef) def fileIdIri(iri: Iri): FileId = FileId(iri, projectRef) diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractorSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractorSpec.scala index 5eb32f87c6..679963309a 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractorSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractorSpec.scala @@ -24,7 +24,7 @@ class FormDataExtractorSpec val customMediaType = MediaType.parse("application/custom").rightValue val customContentType = ContentType(customMediaType, () => HttpCharsets.`UTF-8`) val mediaTypeDetector = MediaTypeDetectorConfig(Map("custom" -> customMediaType)) - val extractor = FormDataExtractor(mediaTypeDetector) + val extractor = FormDataExtractor(new MediaTypeDetector(mediaTypeDetector)) def createEntity( bodyPart: String, @@ -71,16 +71,16 @@ class FormDataExtractorSpec extractor(entity, 250).accepted filename shouldEqual "filename" - contentType shouldEqual `application/octet-stream` + contentType.value shouldEqual `application/octet-stream` consume(contents.dataBytes) shouldEqual content } - "be extracted with the custom media type from the config" in { + "be extracted with the custom media type from the detector" in { val entity = createEntity("file", NoContentType, Some("file.custom")) val UploadedFileInformation(filename, contentType, contents) = extractor(entity, 2000).accepted filename shouldEqual "file.custom" - contentType shouldEqual customContentType + contentType.value shouldEqual customContentType consume(contents.dataBytes) shouldEqual content } @@ -89,7 +89,7 @@ class FormDataExtractorSpec val UploadedFileInformation(filename, contentType, contents) = extractor(entity, 250).accepted filename shouldEqual "file.txt" - contentType shouldEqual `text/plain(UTF-8)` + contentType.value shouldEqual `text/plain(UTF-8)` consume(contents.dataBytes) shouldEqual content } @@ -111,7 +111,7 @@ class FormDataExtractorSpec val entity = createEntity("file", `text/plain(UTF-8)`, Some("file.custom")) val UploadedFileInformation(filename, contentType, contents) = extractor(entity, 2000).accepted filename shouldEqual "file.custom" - contentType shouldEqual `text/plain(UTF-8)` + contentType.value shouldEqual `text/plain(UTF-8)` consume(contents.dataBytes) shouldEqual content } diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/MediaTypeDetectorSuite.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/MediaTypeDetectorSuite.scala new file mode 100644 index 0000000000..dcc3c25cdd --- /dev/null +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/MediaTypeDetectorSuite.scala @@ -0,0 +1,43 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files + +import akka.http.scaladsl.model.{ContentType, ContentTypes, HttpCharsets, MediaType} +import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig +import ch.epfl.bluebrain.nexus.testkit.mu.NexusSuite + +class MediaTypeDetectorSuite extends NexusSuite { + + private val customMediaType = MediaType.custom("application/custom", binary = false) + private val customContentType = ContentType(customMediaType, () => HttpCharsets.`UTF-8`) + private val mediaTypeDetector = new MediaTypeDetector(MediaTypeDetectorConfig(Map("custom" -> customMediaType))) + + private val json = Some(ContentTypes.`application/json`) + private val octetStream = Some(ContentTypes.`application/octet-stream`) + + test("Return the default content type for an unknown extension and no input") { + assertEquals( + mediaTypeDetector("file.obj", None, octetStream), + octetStream + ) + } + + test("Return the akka known extension and no input") { + assertEquals( + mediaTypeDetector("file.json", None, octetStream), + json + ) + } + + test("Return the matching content type for an defined extension and no input") { + assertEquals( + mediaTypeDetector("file.custom", None, octetStream), + Some(customContentType) + ) + } + + test("Return the provided input as a priority") { + assertEquals( + mediaTypeDetector("file.custom", json, octetStream), + json + ) + } +} diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala index 183a6cb568..a4f1d94694 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala @@ -1,6 +1,5 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes -import akka.actor.typed import akka.http.scaladsl.model.ContentTypes.{`application/json`, `text/plain(UTF-8)`} import akka.http.scaladsl.model.MediaRanges._ import akka.http.scaladsl.model.MediaTypes.{`multipart/form-data`, `text/html`} @@ -12,16 +11,15 @@ import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks.FileOperationsMock import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.ComputedDigest import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileId} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{contexts => fileContexts, permissions, FileFixtures, Files, FilesConfig} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{contexts => fileContexts, permissions, FileFixtures, Files, FormDataExtractor, MediaTypeDetector} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageType -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.FileOperations +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.{FileOperations, LinkFileAction} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{contexts => storageContexts, permissions => storagesPermissions, FetchStorage, StorageFixtures, Storages, StoragesConfig} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.RdfMediaTypes.`application/ld+json` import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.{contexts, nxv} import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteContextResolution} -import ch.epfl.bluebrain.nexus.delta.sdk.{IndexingAction, NexusHeaders} import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclSimpleCheck import ch.epfl.bluebrain.nexus.delta.sdk.acls.model.AclAddress import ch.epfl.bluebrain.nexus.delta.sdk.directives.DeltaSchemeDirectives @@ -34,6 +32,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.permissions.model.Permission import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContextDummy import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution import ch.epfl.bluebrain.nexus.delta.sdk.utils.BaseRouteSpec +import ch.epfl.bluebrain.nexus.delta.sdk.{IndexingAction, NexusHeaders} import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.{Anonymous, Authenticated, Group, Subject, User} import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Label, ProjectRef, ResourceRef} @@ -54,9 +53,6 @@ class FilesRoutesSpec with FileFixtures with CatsIOValues { - import akka.actor.typed.scaladsl.adapter._ - implicit private val typedSystem: typed.ActorSystem[Nothing] = system.toTyped - // TODO: sort out how we handle this in tests implicit override def rcr: RemoteContextResolution = RemoteContextResolution.fixedIO( @@ -121,10 +117,14 @@ class FilesRoutesSpec lazy val fileOps: FileOperations = FileOperationsMock.disabled lazy val fetchStorage: FetchStorage = FetchStorage(storages, aclCheck) - private val filesConfig = FilesConfig(eventLogConfig, MediaTypeDetectorConfig.Empty) - lazy val files: Files = Files(fetchContext, fetchStorage, xas, filesConfig, fileOps, clock)(uuidF, typedSystem) - private val groupDirectives = DeltaSchemeDirectives(fetchContext) - private lazy val routes = routesWithIdentities(identities) + private val mediaTypeDetector = new MediaTypeDetector(MediaTypeDetectorConfig.Empty) + private val dataExtractor = FormDataExtractor(mediaTypeDetector)(system) + private val linkAction = LinkFileAction.alwaysFails + lazy val files: Files = + Files(fetchContext, fetchStorage, dataExtractor, xas, eventLogConfig, fileOps, linkAction, clock)(uuidF) + private val groupDirectives = DeltaSchemeDirectives(fetchContext) + private lazy val routes = routesWithIdentities(identities) + private def routesWithIdentities(identities: Identities) = Route.seal(FilesRoutes(identities, aclCheck, files, groupDirectives, IndexingAction.noop)) diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/LinkFileActionSuite.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/LinkFileActionSuite.scala new file mode 100644 index 0000000000..6e1220e436 --- /dev/null +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/LinkFileActionSuite.scala @@ -0,0 +1,140 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations + +import akka.http.scaladsl.model._ +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection.InvalidFilePath +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{Digest, FileAttributes, FileLinkRequest, FileStorageMetadata} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{permissions, MediaTypeDetector} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.FetchStorage +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{Storage, StorageType} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.S3Storage +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageValue.S3StorageValue +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3.S3FileOperations.{S3FileLink, S3FileMetadata} +import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode +import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv +import ch.epfl.bluebrain.nexus.delta.sdk.error.ServiceError.AuthorizationFailed +import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller +import ch.epfl.bluebrain.nexus.delta.sdk.syntax._ +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.User +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Label, ProjectRef, ResourceRef} +import ch.epfl.bluebrain.nexus.testkit.mu.NexusSuite +import io.circe.Json + +import java.util.UUID + +class LinkFileActionSuite extends NexusSuite { + + private val realm = Label.unsafe("myrealm") + private val user = User("user", realm) + implicit val caller: Caller = Caller(user, Set.empty) + + private val project = ProjectRef.unsafe("org", "project") + private val storageIri = nxv + "s3-storage" + private val storageRef = ResourceRef.Revision(storageIri, 1) + private val value = S3StorageValue(default = false, "bucket", permissions.read, permissions.write, 100L) + private val s3Storage = S3Storage(storageIri, project, value, Json.Null) + + private val fetchStorage = new FetchStorage { + + override def onRead(id: ResourceRef, project: ProjectRef)(implicit caller: Caller): IO[Storage] = + IO.raiseError(new IllegalStateException("Should not be called")) + + override def onWrite(id: Option[IriOrBNode.Iri], project: ProjectRef)(implicit + caller: Caller + ): IO[(ResourceRef.Revision, Storage)] = + IO.raiseUnless(id.contains(storageIri))(AuthorizationFailed("Fail")) >> IO.pure(storageRef -> s3Storage) + } + + private val mediaTypeDetector = new MediaTypeDetector(MediaTypeDetectorConfig.Empty) + + private val uuid = UUID.randomUUID() + private val contentTypeFromS3 = ContentTypes.`application/octet-stream` + private val fileSize = 100L + private val digest = Digest.NoDigest + + private val s3FileLink: S3FileLink = (_: String, path: Uri.Path) => { + IO.fromOption(path.lastSegment)(InvalidFilePath).map { filename => + S3FileMetadata( + filename, + Some(contentTypeFromS3), + FileStorageMetadata( + uuid, + fileSize, + digest, + FileAttributesOrigin.Link, + Uri(path.toString()), + path + ) + ) + } + } + + private val linkAction = LinkFileAction(fetchStorage, mediaTypeDetector, s3FileLink) + + test("Fail for an unauthorized storage") { + val request = FileLinkRequest(Uri.Path("/path/file.json"), None, None) + linkAction(None, project, request).intercept[AuthorizationFailed] + } + + test("Succeed for a file with media type detection") { + val request = FileLinkRequest(Uri.Path("/path/file.json"), None, None) + val attributes = FileAttributes( + uuid, + Uri(request.path.toString()), + request.path, + "file.json", + Some(ContentTypes.`application/json`), + Map.empty, + None, + None, + fileSize, + digest, + FileAttributesOrigin.Link + ) + val expected = StorageWrite(storageRef, StorageType.S3Storage, attributes) + linkAction(Some(storageIri), project, request).assertEquals(expected) + } + + test("Succeed for a file without media type detection") { + val request = FileLinkRequest(Uri.Path("/path/file.obj"), None, None) + val attributes = FileAttributes( + uuid, + Uri(request.path.toString()), + request.path, + "file.obj", + Some(contentTypeFromS3), + Map.empty, + None, + None, + fileSize, + digest, + FileAttributesOrigin.Link + ) + val expected = StorageWrite(storageRef, StorageType.S3Storage, attributes) + linkAction(Some(storageIri), project, request).assertEquals(expected) + } + + test("Succeed for a file with provided media type") { + val customMediaType = MediaType.custom("application/obj", binary = false) + val customContentType = ContentType(customMediaType, () => HttpCharsets.`UTF-8`) + val request = FileLinkRequest(Uri.Path("/path/file.obj"), Some(customContentType), None) + val attributes = FileAttributes( + uuid, + Uri(request.path.toString()), + request.path, + "file.obj", + Some(customContentType), + Map.empty, + None, + None, + fileSize, + digest, + FileAttributesOrigin.Link + ) + val expected = StorageWrite(storageRef, StorageType.S3Storage, attributes) + linkAction(Some(storageIri), project, request).assertEquals(expected) + } + +} diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3FileOperationsSuite.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3FileOperationsSuite.scala index 2b70562e23..aeb20ace00 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3FileOperationsSuite.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3FileOperationsSuite.scala @@ -66,7 +66,7 @@ class S3FileOperationsSuite val contentLength = content.length.toLong val digest = makeDigest(content) val entity = HttpEntity(content) - val uploading = S3UploadingFile(project, bucket, filename, contentType, contentLength, entity) + val uploading = S3UploadingFile(project, bucket, filename, Some(contentType), contentLength, entity) val location = expectedLocation(project, filename) val expectedMetadata = diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3Helpers.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3Helpers.scala index ac4e637f06..4fc532ab61 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3Helpers.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3Helpers.scala @@ -42,7 +42,7 @@ trait S3Helpers { self: Generators => )(implicit client: S3StorageClient): IO[Unit] = { val bytes = contents.getBytes(StandardCharsets.UTF_8) val key = genString() - val put = PutObjectRequest(bucket, key, ContentTypes.`text/plain(UTF-8)`, bytes.length.toLong) + val put = PutObjectRequest(bucket, key, Some(ContentTypes.`text/plain(UTF-8)`), bytes.length.toLong) client.uploadFile(put, Stream.emit(ByteBuffer.wrap(bytes))) >> test(key) } @@ -51,10 +51,10 @@ trait S3Helpers { self: Generators => )(implicit client: S3StorageClient): IO[Unit] = { val bytes1 = contents1.getBytes(StandardCharsets.UTF_8) val key1 = genString() - val put1 = PutObjectRequest(bucket, key1, ContentTypes.`text/plain(UTF-8)`, bytes1.length.toLong) + val put1 = PutObjectRequest(bucket, key1, Some(ContentTypes.`text/plain(UTF-8)`), bytes1.length.toLong) val bytes2 = contents2.getBytes(StandardCharsets.UTF_8) val key2 = genString() - val put2 = PutObjectRequest(bucket, key2, ContentTypes.`text/plain(UTF-8)`, bytes2.length.toLong) + val put2 = PutObjectRequest(bucket, key2, Some(ContentTypes.`text/plain(UTF-8)`), bytes2.length.toLong) for { _ <- client.uploadFile(put1, Stream.emit(ByteBuffer.wrap(bytes1))) _ <- client.uploadFile(put2, Stream.emit(ByteBuffer.wrap(bytes2))) diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/files/FileProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/files/FileProcessor.scala index 448ad7551a..7d29aad890 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/files/FileProcessor.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/files/FileProcessor.scala @@ -5,7 +5,7 @@ import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.Logger import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig import ch.epfl.bluebrain.nexus.delta.kernel.utils.FileUtils -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.Files +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{Files, MediaTypeDetector} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.Files.definition import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileCommand.CancelEvent import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileEvent._ @@ -13,6 +13,7 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection.{ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.FetchStorage import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.LinkFileAction import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3.client.S3StorageClient import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi @@ -29,6 +30,7 @@ import ch.epfl.bluebrain.nexus.ship.files.FileCopier.FileCopyResult.{FileCopySki import ch.epfl.bluebrain.nexus.ship.files.FileProcessor.{forceMediaType, logger, patchMediaType} import ch.epfl.bluebrain.nexus.ship.files.FileWiring._ import ch.epfl.bluebrain.nexus.ship.storages.StorageWiring +import ch.epfl.bluebrain.nexus.ship.storages.StorageWiring.linkS3FileOperationOnly import io.circe.Decoder class FileProcessor private ( @@ -184,13 +186,19 @@ object FileProcessor { } val fileCopier = FileCopier(s3Client, config.files) + // This part is done during the patchMediaType part which also takes care of setting + // the content type on the S3 object + val mediaTypeDetector = new MediaTypeDetector(MediaTypeDetectorConfig.Empty) + val linkFile = LinkFileAction(fs, mediaTypeDetector, linkS3FileOperationOnly(s3Client)) + val files = new Files( failingFormDataExtractor, ScopedEventLog(definition(clock), config.eventLog, xas), fetchContext, fs, - linkOperationOnly(s3Client) + noFileOperations, + linkFile )(FailingUUID) new FileProcessor(files, projectMapper, fileCopier, clock)(config.files.mediaTypeDetector) diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/files/FileWiring.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/files/FileWiring.scala index 8f0bed6c11..7fde0100b2 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/files/FileWiring.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/files/FileWiring.scala @@ -1,20 +1,35 @@ package ch.epfl.bluebrain.nexus.ship.files -import akka.http.scaladsl.model.HttpEntity +import akka.http.scaladsl.model.{HttpEntity, Uri} import cats.effect.IO -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.FormDataExtractor +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{ComputedFileAttributes, FileAttributes, FileDelegationRequest, FileStorageMetadata} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{FormDataExtractor, UploadedFileInformation} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.FileOperations -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3.client.S3StorageClient -import ch.epfl.bluebrain.nexus.ship.storages.StorageWiring.{failingDiskFileOperations, failingRemoteDiskFileOperations, linkS3FileOperationOnly} +import ch.epfl.bluebrain.nexus.delta.sdk.AkkaSource object FileWiring { - def linkOperationOnly(s3StorageClient: S3StorageClient): FileOperations = - FileOperations.apply( - failingDiskFileOperations, - failingRemoteDiskFileOperations, - linkS3FileOperationOnly(s3StorageClient) - ) + private val noFileOperationError = IO.raiseError(new IllegalArgumentException("FileOperations should not be called")) + + def noFileOperations: FileOperations = new FileOperations { + override def save( + storage: Storage, + info: UploadedFileInformation, + contentLength: Option[Long] + ): IO[FileStorageMetadata] = ??? + + override def fetch(storage: Storage, attributes: FileAttributes): IO[AkkaSource] = noFileOperationError + + override def legacyLink(storage: Storage, sourcePath: Uri.Path, filename: String): IO[FileStorageMetadata] = + noFileOperationError + + override def fetchAttributes(storage: Storage, attributes: FileAttributes): IO[ComputedFileAttributes] = + noFileOperationError + + override def delegate(storage: Storage, filename: String): IO[FileDelegationRequest.TargetLocation] = + noFileOperationError + } def failingFormDataExtractor: FormDataExtractor = (_: HttpEntity, _: Long) => IO.raiseError(new IllegalArgumentException("FormDataExtractor should not be called")) diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/storages/StorageWiring.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/storages/StorageWiring.scala index afee559b9c..d433041840 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/storages/StorageWiring.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/storages/StorageWiring.scala @@ -4,16 +4,13 @@ import akka.http.scaladsl.model.Uri import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF import ch.epfl.bluebrain.nexus.delta.plugins.storage.StorageScopeInitialization -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{ComputedFileAttributes, FileStorageMetadata} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileStorageMetadata import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.Storages import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.S3StorageConfig import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.access.StorageAccess -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.RemoteDiskStorage import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageFields.S3StorageFields import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageValue -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.UploadingFile.{DiskUploadingFile, RemoteUploadingFile, S3UploadingFile} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.DiskFileOperations -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.RemoteDiskFileOperations +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.UploadingFile.S3UploadingFile import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3.S3FileOperations import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3.client.S3StorageClient import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi @@ -21,7 +18,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution import ch.epfl.bluebrain.nexus.delta.sdk.{AkkaSource, Defaults} import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors -import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Label, ProjectRef} +import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef import ch.epfl.bluebrain.nexus.ship.EventClock import ch.epfl.bluebrain.nexus.ship.config.InputConfig @@ -83,33 +80,6 @@ object StorageWiring { ) } - def failingDiskFileOperations: DiskFileOperations = new DiskFileOperations { - override def fetch(path: Uri.Path): IO[AkkaSource] = - IO.raiseError(new IllegalArgumentException("DiskFileOperations should not be called")) - - override def save(uploading: DiskUploadingFile): IO[FileStorageMetadata] = - IO.raiseError(new IllegalArgumentException("DiskFileOperations should not be called")) - } - - def failingRemoteDiskFileOperations: RemoteDiskFileOperations = new RemoteDiskFileOperations { - - override def fetch(folder: Label, path: Uri.Path): IO[AkkaSource] = - IO.raiseError(new IllegalArgumentException("RemoteDiskFileOperations should not be called")) - - override def save(uploading: RemoteUploadingFile): IO[FileStorageMetadata] = - IO.raiseError(new IllegalArgumentException("RemoteDiskFileOperations should not be called")) - - override def legacyLink( - storage: RemoteDiskStorage, - sourcePath: Uri.Path, - filename: String - ): IO[FileStorageMetadata] = - IO.raiseError(new IllegalArgumentException("RemoteDiskFileOperations should not be called")) - - override def fetchAttributes(folder: Label, path: Uri.Path): IO[ComputedFileAttributes] = - IO.raiseError(new IllegalArgumentException("RemoteDiskFileOperations should not be called")) - } - def linkS3FileOperationOnly(s3Client: S3StorageClient): S3FileOperations = new S3FileOperations { override def fetch(bucket: String, path: Uri.Path): IO[AkkaSource] = diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala index 1e343ee517..44fcc436af 100644 --- a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala @@ -61,7 +61,7 @@ class RunShipSuite private def uploadFile(path: String) = { val contentAsBuffer = StandardCharsets.UTF_8.encode(fileContent).asReadOnlyBuffer() - val put = PutObjectRequest(importBucket, path, ContentTypes.`application/octet-stream`, contentLength) + val put = PutObjectRequest(importBucket, path, Some(ContentTypes.`application/octet-stream`), contentLength) s3Client.uploadFile(put, Stream.emit(contentAsBuffer)) } diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/S3StorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/S3StorageSpec.scala index 7168b43cab..5650193c31 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/S3StorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/S3StorageSpec.scala @@ -44,7 +44,6 @@ class S3StorageSpec extends StorageSpec with S3ClientFixtures { override def afterAll(): Unit = { cleanupBucket(bucket).accepted - super.afterAll() } @@ -254,6 +253,21 @@ class S3StorageSpec extends StorageSpec with S3ClientFixtures { } yield assertion } + "succeed with media-type detection" in { + val id = genId() + val path = s"$id/nexus-logo.custom" + val payload = Json.obj("path" := path) + + for { + _ <- uploadLogoFileToS3(bucket, path) + _ <- createFileLink(id, storageId, payload) + assertion <- deltaClient.get[Json](s"/files/$projectRef/$id", Coyote) { (json, response) => + response.status shouldEqual StatusCodes.OK + json should have(mediaTypeField("application/custom")) + } + } yield assertion + } + "be updated" in { val id = genId() val fileContent = genString()