diff --git a/datasafe-business/src/test/java/de/adorsys/datasafe/business/impl/e2e/MultiDFSFunctionalityIT.java b/datasafe-business/src/test/java/de/adorsys/datasafe/business/impl/e2e/MultiDFSFunctionalityIT.java index d0f300e11..556f54867 100644 --- a/datasafe-business/src/test/java/de/adorsys/datasafe/business/impl/e2e/MultiDFSFunctionalityIT.java +++ b/datasafe-business/src/test/java/de/adorsys/datasafe/business/impl/e2e/MultiDFSFunctionalityIT.java @@ -1,8 +1,5 @@ package de.adorsys.datasafe.business.impl.e2e; -import static de.adorsys.datasafe.types.api.shared.DockerUtil.getDockerUri; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertThrows; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.S3ObjectSummary; import dagger.Lazy; @@ -39,6 +36,7 @@ import de.adorsys.datasafe.types.api.types.ReadStorePassword; import de.adorsys.datasafe.types.api.utils.ExecutorServiceUtil; import de.adorsys.datasafe.types.api.utils.ReadKeyPasswordTestFactory; + import java.io.InputStream; import java.io.OutputStream; import java.security.UnrecoverableKeyException; @@ -50,6 +48,7 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; + import lombok.SneakyThrows; import lombok.experimental.Delegate; import lombok.extern.slf4j.Slf4j; @@ -62,6 +61,10 @@ import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; +import static de.adorsys.datasafe.types.api.shared.DockerUtil.getDockerUri; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + /** * This test distributes users' storage access keystore, document encryption keystore, * users' private files into buckets that reside on different buckets. Bootstrap knows only how to @@ -135,6 +138,7 @@ void initDatasafe() { accessKey(CREDENTIALS), secretKey(CREDENTIALS) ), + REGION, CREDENTIALS, EXECUTOR ); @@ -156,6 +160,7 @@ void initDatasafe() { acc.getAccessKey(), acc.getSecretKey() ), + acc.getRegion(), acc.getBucketName(), EXECUTOR ) diff --git a/datasafe-cli/src/main/java/de/adorsys/datasafe/cli/config/DatasafeFactory.java b/datasafe-cli/src/main/java/de/adorsys/datasafe/cli/config/DatasafeFactory.java index c59e94ac9..33c2c905e 100644 --- a/datasafe-cli/src/main/java/de/adorsys/datasafe/cli/config/DatasafeFactory.java +++ b/datasafe-cli/src/main/java/de/adorsys/datasafe/cli/config/DatasafeFactory.java @@ -86,6 +86,7 @@ private static StorageService amazonS3() { acc.getAccessKey(), acc.getSecretKey() ), + acc.getRegion(), // Bucket name is encoded in first path segment acc.getBucketName(), ExecutorServiceUtil.submitterExecutesOnStarvationExecutingService() @@ -134,6 +135,7 @@ private static S3StorageService getStorageService(String accessKey, String secre return new S3StorageService( amazons3, + region, bucket, ExecutorServiceUtil .submitterExecutesOnStarvationExecutingService( diff --git a/datasafe-examples/datasafe-examples-multidfs/src/test/java/de/adorsys/datasafe/examples/business/s3/MultiDfsWithCredentialsExampleIT.java b/datasafe-examples/datasafe-examples-multidfs/src/test/java/de/adorsys/datasafe/examples/business/s3/MultiDfsWithCredentialsExampleIT.java index 668c119c0..a17aff860 100644 --- a/datasafe-examples/datasafe-examples-multidfs/src/test/java/de/adorsys/datasafe/examples/business/s3/MultiDfsWithCredentialsExampleIT.java +++ b/datasafe-examples/datasafe-examples-multidfs/src/test/java/de/adorsys/datasafe/examples/business/s3/MultiDfsWithCredentialsExampleIT.java @@ -1,9 +1,5 @@ package de.adorsys.datasafe.examples.business.s3; -import static de.adorsys.datasafe.examples.business.s3.MinioContainerId.DIRECTORY_BUCKET; -import static de.adorsys.datasafe.examples.business.s3.MinioContainerId.FILES_BUCKET_ONE; -import static de.adorsys.datasafe.examples.business.s3.MinioContainerId.FILES_BUCKET_TWO; -import static org.assertj.core.api.Assertions.assertThat; import com.amazonaws.services.s3.AmazonS3; import dagger.Lazy; import de.adorsys.datasafe.business.impl.service.DaggerDefaultDatasafeServices; @@ -30,6 +26,7 @@ import de.adorsys.datasafe.types.api.resource.StorageIdentifier; import de.adorsys.datasafe.types.api.shared.AwsClientRetry; import de.adorsys.datasafe.types.api.utils.ExecutorServiceUtil; + import java.io.OutputStream; import java.net.URI; import java.nio.charset.StandardCharsets; @@ -39,6 +36,7 @@ import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.regex.Pattern; + import lombok.SneakyThrows; import lombok.experimental.Delegate; import lombok.extern.slf4j.Slf4j; @@ -49,6 +47,11 @@ import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; +import static de.adorsys.datasafe.examples.business.s3.MinioContainerId.DIRECTORY_BUCKET; +import static de.adorsys.datasafe.examples.business.s3.MinioContainerId.FILES_BUCKET_ONE; +import static de.adorsys.datasafe.examples.business.s3.MinioContainerId.FILES_BUCKET_TWO; +import static org.assertj.core.api.Assertions.assertThat; + /** * This example shows how client can register storage system and securely store its access details. * Here, we will use 2 Datasafe class instances - one for securely storing user access credentials @@ -105,6 +108,7 @@ void testMultiUserStorageUserSetup() { // static client that will be used to access `directory` bucket: StorageService directoryStorage = new S3StorageService( directoryClient, + REGION, DIRECTORY_BUCKET.getBucketName(), EXECUTOR ); @@ -133,6 +137,7 @@ void testMultiUserStorageUserSetup() { acc.getAccessKey(), acc.getSecretKey() ), + acc.getRegion(), // Bucket name is encoded in first path segment acc.getBucketName(), EXECUTOR diff --git a/datasafe-examples/datasafe-examples-versioned-s3/src/test/java/de/adorsys/datasafe/examples/business/s3/BaseUserOperationsWithDefaultDatasafeOnVersionedStorageIT.java b/datasafe-examples/datasafe-examples-versioned-s3/src/test/java/de/adorsys/datasafe/examples/business/s3/BaseUserOperationsWithDefaultDatasafeOnVersionedStorageIT.java index 0258e85d6..0dd58dd8e 100644 --- a/datasafe-examples/datasafe-examples-versioned-s3/src/test/java/de/adorsys/datasafe/examples/business/s3/BaseUserOperationsWithDefaultDatasafeOnVersionedStorageIT.java +++ b/datasafe-examples/datasafe-examples-versioned-s3/src/test/java/de/adorsys/datasafe/examples/business/s3/BaseUserOperationsWithDefaultDatasafeOnVersionedStorageIT.java @@ -121,6 +121,7 @@ void init() { .config(new DefaultDFSConfig(cephMappedUrl, "secret"::toCharArray)) .storage(new S3StorageService( cephS3, + "", VERSIONED_BUCKET_NAME, ExecutorServiceUtil.submitterExecutesOnStarvationExecutingService())) .build(); diff --git a/datasafe-rest-impl/src/main/java/de/adorsys/datasafe/rest/impl/config/DatasafeConfig.java b/datasafe-rest-impl/src/main/java/de/adorsys/datasafe/rest/impl/config/DatasafeConfig.java index 12dae475d..ae4dfb826 100644 --- a/datasafe-rest-impl/src/main/java/de/adorsys/datasafe/rest/impl/config/DatasafeConfig.java +++ b/datasafe-rest-impl/src/main/java/de/adorsys/datasafe/rest/impl/config/DatasafeConfig.java @@ -34,12 +34,14 @@ import de.adorsys.datasafe.types.api.context.overrides.OverridesRegistry; import de.adorsys.datasafe.types.api.types.ReadStorePassword; import de.adorsys.datasafe.types.api.utils.ExecutorServiceUtil; + import java.net.URI; import java.nio.file.Paths; import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.regex.Pattern; + import lombok.experimental.Delegate; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; @@ -131,6 +133,7 @@ StorageService clientCredentials(AmazonS3 s3, S3Factory factory, DatasafePropert ExecutorService executorService = ExecutorServiceUtil.submitterExecutesOnStarvationExecutingService(); S3StorageService basicStorage = new S3StorageService( s3, + properties.getAmazonRegion(), properties.getBucketName(), executorService ); @@ -184,6 +187,7 @@ StorageService singleStorageServiceFilesystem(DatasafeProperties properties) { StorageService singleStorageServiceS3(AmazonS3 s3, DatasafeProperties properties) { return new S3StorageService( s3, + properties.getAmazonRegion(), properties.getBucketName(), ExecutorServiceUtil.submitterExecutesOnStarvationExecutingService() ); @@ -202,7 +206,7 @@ StorageService multiStorageService(DatasafeProperties properties) { ) ); - S3StorageService s3StorageService = new S3StorageService(s3(properties), properties.getBucketName(), + S3StorageService s3StorageService = new S3StorageService(s3(properties), properties.getAmazonRegion(), properties.getBucketName(), ExecutorServiceUtil.submitterExecutesOnStarvationExecutingService() ); diff --git a/datasafe-simple-adapter/datasafe-simple-adapter-impl/src/main/java/de/adorsys/datasafe/simple/adapter/impl/SimpleDatasafeServiceImpl.java b/datasafe-simple-adapter/datasafe-simple-adapter-impl/src/main/java/de/adorsys/datasafe/simple/adapter/impl/SimpleDatasafeServiceImpl.java index f2113b056..17b5cc608 100644 --- a/datasafe-simple-adapter/datasafe-simple-adapter-impl/src/main/java/de/adorsys/datasafe/simple/adapter/impl/SimpleDatasafeServiceImpl.java +++ b/datasafe-simple-adapter/datasafe-simple-adapter-impl/src/main/java/de/adorsys/datasafe/simple/adapter/impl/SimpleDatasafeServiceImpl.java @@ -45,12 +45,14 @@ import de.adorsys.datasafe.types.api.types.ReadKeyPassword; import de.adorsys.datasafe.types.api.types.ReadStorePassword; import de.adorsys.datasafe.types.api.utils.ExecutorServiceUtil; + import java.io.InputStream; import java.io.OutputStream; import java.net.URI; import java.nio.file.FileSystems; import java.util.List; import java.util.stream.Collectors; + import lombok.AllArgsConstructor; import lombok.Getter; import lombok.SneakyThrows; @@ -298,6 +300,7 @@ private static SystemRootAndStorageService useAmazonS3(AmazonS3DFSCredentials df } StorageService storageService = new S3StorageService( amazons3, + amazonS3DFSCredentials.getRegion(), amazonS3DFSCredentials.getContainer(), ExecutorServiceUtil .submitterExecutesOnStarvationExecutingService( diff --git a/datasafe-simple-adapter/datasafe-simple-adapter-spring/src/test/java/de/adorsys/datasafe/simple/adapter/spring/InjectionIT.java b/datasafe-simple-adapter/datasafe-simple-adapter-spring/src/test/java/de/adorsys/datasafe/simple/adapter/spring/InjectionIT.java index 75930fc85..a5516904d 100644 --- a/datasafe-simple-adapter/datasafe-simple-adapter-spring/src/test/java/de/adorsys/datasafe/simple/adapter/spring/InjectionIT.java +++ b/datasafe-simple-adapter/datasafe-simple-adapter-spring/src/test/java/de/adorsys/datasafe/simple/adapter/spring/InjectionIT.java @@ -103,7 +103,7 @@ private AbsoluteLocation getPrivateResourceAbsoluteLocation(DFS } if (dfsCredentials instanceof AmazonS3DFSCredentials) { AmazonS3DFSCredentials a = (AmazonS3DFSCredentials) dfsCredentials; - return new AbsoluteLocation<>(BasePrivateResource.forPrivate(new URI(a.getUrl() + "/" + a.getRootBucket()))); + return new AbsoluteLocation<>(BasePrivateResource.forPrivate(new URI(a.getUrl() + "/" + a.getRegion() + "/" + a.getRootBucket()))); } throw new TestException("NYI"); } diff --git a/datasafe-storage/datasafe-storage-impl-s3/src/main/java/de/adorsys/datasafe/storage/impl/s3/S3StorageService.java b/datasafe-storage/datasafe-storage-impl-s3/src/main/java/de/adorsys/datasafe/storage/impl/s3/S3StorageService.java index 59d204569..439e207c9 100644 --- a/datasafe-storage/datasafe-storage-impl-s3/src/main/java/de/adorsys/datasafe/storage/impl/s3/S3StorageService.java +++ b/datasafe-storage/datasafe-storage-impl-s3/src/main/java/de/adorsys/datasafe/storage/impl/s3/S3StorageService.java @@ -45,12 +45,13 @@ public class S3StorageService implements StorageService { /** * @param s3 Connection to S3 - * @param bucketName Bucket to use + * @param region Region to use + * @param bucket Bucket to use * @param executorService Multipart sending threadpool (file chunks are sent in parallel) */ - public S3StorageService(AmazonS3 s3, String bucketName, ExecutorService executorService) { + public S3StorageService(AmazonS3 s3, String region, String bucket, ExecutorService executorService) { this.s3 = s3; - this.router = new StaticBucketRouter(bucketName); + this.router = new StaticBucketRouter(region, bucket); this.executorService = executorService; } diff --git a/datasafe-storage/datasafe-storage-impl-s3/src/main/java/de/adorsys/datasafe/storage/impl/s3/StaticBucketRouter.java b/datasafe-storage/datasafe-storage-impl-s3/src/main/java/de/adorsys/datasafe/storage/impl/s3/StaticBucketRouter.java index 2c43345b5..fa04a8e4c 100644 --- a/datasafe-storage/datasafe-storage-impl-s3/src/main/java/de/adorsys/datasafe/storage/impl/s3/StaticBucketRouter.java +++ b/datasafe-storage/datasafe-storage-impl-s3/src/main/java/de/adorsys/datasafe/storage/impl/s3/StaticBucketRouter.java @@ -8,6 +8,7 @@ @RequiredArgsConstructor public class StaticBucketRouter implements BucketRouter { + private final String region; private final String bucketName; @Override @@ -20,10 +21,16 @@ public String resourceKey(AbsoluteLocation resource) { UnaryOperator trimStartingSlash = str -> str.replaceFirst("^/", ""); String resourcePath = trimStartingSlash.apply(resource.location().getRawPath()); - if (bucketName == null || "".equals(bucketName) || !resourcePath.contains(bucketName)) { - return resourcePath; - } - return trimStartingSlash.apply(resourcePath.substring(resourcePath.indexOf(bucketName) + bucketName.length())); + if (bucketName != null && !bucketName.isEmpty()) { + if (resourcePath.startsWith(bucketName)) { + return trimStartingSlash.apply(resourcePath.substring(resourcePath.indexOf(bucketName) + bucketName.length())); + } + String bucketNameWithRegion = region + "/" + bucketName; + if (resourcePath.startsWith(bucketNameWithRegion)) { + return trimStartingSlash.apply(resourcePath.substring(resourcePath.indexOf(bucketNameWithRegion) + bucketNameWithRegion.length())); + } + } + return resourcePath; } } diff --git a/datasafe-storage/datasafe-storage-impl-s3/src/test/java/de/adorsys/datasafe/storage/impl/s3/S3SystemStorageServiceIT.java b/datasafe-storage/datasafe-storage-impl-s3/src/test/java/de/adorsys/datasafe/storage/impl/s3/S3SystemStorageServiceIT.java index 398bf8735..25974fe2e 100644 --- a/datasafe-storage/datasafe-storage-impl-s3/src/test/java/de/adorsys/datasafe/storage/impl/s3/S3SystemStorageServiceIT.java +++ b/datasafe-storage/datasafe-storage-impl-s3/src/test/java/de/adorsys/datasafe/storage/impl/s3/S3SystemStorageServiceIT.java @@ -16,6 +16,7 @@ import de.adorsys.datasafe.types.api.utils.ExecutorServiceUtil; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.assertj.core.api.AbstractStringAssert; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -27,6 +28,7 @@ import java.io.OutputStream; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; import static de.adorsys.datasafe.types.api.shared.DockerUtil.getDockerUri; import static org.assertj.core.api.Assertions.assertThat; @@ -82,6 +84,7 @@ static void beforeAll() { void init() { this.storageService = new S3StorageService( s3, + "eu-central-1", bucketName, ExecutorServiceUtil.submitterExecutesOnStarvationExecutingService() ); @@ -91,12 +94,31 @@ void init() { void list() { createFileWithMessage(); - assertThat(storageService.list(root)) - .hasSize(1) - .extracting(AbsoluteLocation::location) - .asString().contains(FILE); + // Log root and fileWithMsg URIs for debugging + + Stream> list = storageService.list(root); + List> resultList = list.collect(Collectors.toList()); + + // Check if the size of the list is correct + assertThat(resultList).hasSize(1); + + // Log the returned URI + String uriString = resultList.get(0).location().toASCIIString(); + // log.info("Returned URI in CI/CD: " + uriString); + + // // Add environment-related logging + // log.info("Running in region: " + System.getenv("AWS_REGION")); + // log.info("S3 Bucket Name: " + bucketName); + // log.info("AWS_ACCESS_KEY_ID: " + System.getenv("AWS_ACCESS_KEY_ID")); + // log.info("AWS_SECRET_ACCESS_KEY: " + System.getenv("AWS_SECRET_ACCESS_KEY")); + // log.info("AWS_REGION: " + System.getenv("AWS_REGION")); + // log.info("Minio container started at port: " + minio.getMappedPort(9000)); + // log.info("Minio container is running: " + minio.isRunning()); + + assertThat(uriString).contains(FILE); } + @Test void testListOutOfStandardListFilesLimit() { int numberOfFilesOverLimit = 1010; diff --git a/datasafe-storage/datasafe-storage-impl-s3/src/test/java/de/adorsys/datasafe/storage/impl/s3/StaticBucketRouterTest.java b/datasafe-storage/datasafe-storage-impl-s3/src/test/java/de/adorsys/datasafe/storage/impl/s3/StaticBucketRouterTest.java new file mode 100644 index 000000000..16c71900a --- /dev/null +++ b/datasafe-storage/datasafe-storage-impl-s3/src/test/java/de/adorsys/datasafe/storage/impl/s3/StaticBucketRouterTest.java @@ -0,0 +1,45 @@ +package de.adorsys.datasafe.storage.impl.s3; + +import de.adorsys.datasafe.types.api.resource.AbsoluteLocation; +import de.adorsys.datasafe.types.api.resource.BasePrivateResource; +import de.adorsys.datasafe.types.api.resource.Uri; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +@Slf4j +public class StaticBucketRouterTest { + + private StaticBucketRouter router; + + @BeforeEach + void setup() { + String region = "region"; + String bucketName = "bucket"; + router = new StaticBucketRouter(region, bucketName); + } + + @Test + void resourceKeyTest() { + var root = new AbsoluteLocation<>(BasePrivateResource.forPrivate(new Uri("http://s3-us-west-2.amazonaws.com/bucket/users/myuserid/private/files/bucket/users/otheruser/private/files/somefile.aes"))); + log.info(String.valueOf(root)); + String resourcePath = router.resourceKey(root); + assertThat(resourcePath).hasToString("users/myuserid/private/files/bucket/users/otheruser/private/files/somefile.aes"); + } + + @Test + void noBucketInPath() { + var root = new AbsoluteLocation<>(BasePrivateResource.forPrivate(new Uri("http://bucket.s3-us-west-2.amazonaws.com/users/myuserid/private/files/bucket/users/otheruser/private/files/somefile.aes"))); + String resourcePath = router.resourceKey(root); + assertThat(resourcePath).hasToString("users/myuserid/private/files/bucket/users/otheruser/private/files/somefile.aes"); + } + + @Test + void regionAndBucketInPath() { + var root = new AbsoluteLocation<>(BasePrivateResource.forPrivate(new Uri("s3://host/region/bucket/users/myuserid/private/files/bucket/users/otheruser/private/files/somefile.aes"))); + String resourcePath = router.resourceKey(root); + assertThat(resourcePath).hasToString("users/myuserid/private/files/bucket/users/otheruser/private/files/somefile.aes"); + } +} diff --git a/datasafe-test-storages/src/test/java/de/adorsys/datasafe/teststorage/WithStorageProvider.java b/datasafe-test-storages/src/test/java/de/adorsys/datasafe/teststorage/WithStorageProvider.java index 5c50d8c2b..503632a12 100644 --- a/datasafe-test-storages/src/test/java/de/adorsys/datasafe/teststorage/WithStorageProvider.java +++ b/datasafe-test-storages/src/test/java/de/adorsys/datasafe/teststorage/WithStorageProvider.java @@ -165,83 +165,83 @@ static void shutdown() { @ValueSource protected static Stream allLocalDefaultStorages() { return Stream.of( - fs(), - minio() - /* No CEPH here because it is quite slow*/ + fs(), + minio() + /* No CEPH here because it is quite slow*/ ).filter(Objects::nonNull); } @ValueSource protected static Stream allLocalStorages() { return Stream.of( - fs(), - minio(), - cephVersioned() + fs(), + minio(), + cephVersioned() ).filter(Objects::nonNull); } @ValueSource protected static Stream allDefaultStorages() { return Stream.of( - fs(), - minio(), - s3() + fs(), + minio(), + s3() ).filter(Objects::nonNull); } @ValueSource protected static Stream allStorages() { return Stream.of( - fs(), - minio(), - cephVersioned(), - s3() + fs(), + minio(), + cephVersioned(), + s3() ).filter(Objects::nonNull); } @ValueSource protected static Stream fsOnly() { return Stream.of( - fs() + fs() ).filter(Objects::nonNull); } @ValueSource protected static Stream s3Only() { return Stream.of( - s3() + s3() ).filter(Objects::nonNull); } @ValueSource protected static Stream minioOnly() { return Stream.of( - minio() + minio() ).filter(Objects::nonNull); } protected static StorageDescriptor fs() { return new StorageDescriptor( - StorageDescriptorName.FILESYSTEM, + StorageDescriptorName.FILESYSTEM, () -> new FileSystemStorageService(new Uri(tempDir.toUri())), new Uri(tempDir.toUri()), - null, null, null, - tempDir.toString() + null, null, null, + tempDir.toString() ); } protected static StorageDescriptor minio() { return new StorageDescriptor( - StorageDescriptorName.MINIO, + StorageDescriptorName.MINIO, () -> { minioStorage.get(); - return new S3StorageService(minio, primaryBucket, EXECUTOR_SERVICE); + return new S3StorageService(minio, minioRegion, primaryBucket, EXECUTOR_SERVICE); }, new Uri("s3://" + primaryBucket + "/" + bucketPath + "/"), - minioAccessKeyID, - minioSecretAccessKey, - minioRegion, - primaryBucket + "/" + bucketPath + minioAccessKeyID, + minioSecretAccessKey, + minioRegion, + primaryBucket + "/" + bucketPath ); } @@ -251,16 +251,16 @@ protected static StorageDescriptor cephVersioned() { } return new StorageDescriptor( - StorageDescriptorName.CEPH, + StorageDescriptorName.CEPH, () -> { cephStorage.get(); - return new S3StorageService(ceph, primaryBucket, EXECUTOR_SERVICE); + return new S3StorageService(ceph, cephRegion, primaryBucket, EXECUTOR_SERVICE); }, new Uri("s3://" + primaryBucket + "/" + bucketPath + "/"), - cephAccessKeyID, - cephSecretAccessKey, - cephRegion, - primaryBucket + "/" + bucketPath + cephAccessKeyID, + cephSecretAccessKey, + cephRegion, + primaryBucket + "/" + bucketPath ); } @@ -275,10 +275,10 @@ private static boolean skipCeph() { protected static Function storageServiceByBucket() { if (null == amazonS3) { - return bucketName -> new S3StorageService(minio, bucketName, EXECUTOR_SERVICE); + return bucketName -> new S3StorageService(minio, amazonRegion, bucketName, EXECUTOR_SERVICE); } - return bucketName -> new S3StorageService(amazonS3, bucketName, EXECUTOR_SERVICE); + return bucketName -> new S3StorageService(amazonS3, amazonRegion, bucketName, EXECUTOR_SERVICE); } protected static StorageDescriptor s3() { @@ -287,16 +287,16 @@ protected static StorageDescriptor s3() { } return new StorageDescriptor( - StorageDescriptorName.AMAZON, + StorageDescriptorName.AMAZON, () -> { amazonStorage.get(); - return new S3StorageService(amazonS3, primaryBucket, EXECUTOR_SERVICE); + return new S3StorageService(amazonS3, amazonRegion, primaryBucket, EXECUTOR_SERVICE); }, new Uri("s3://" + primaryBucket + "/" + bucketPath + "/"), - amazonAccessKeyID, - amazonSecretAccessKey, - amazonRegion, - primaryBucket + "/" + bucketPath + amazonAccessKeyID, + amazonSecretAccessKey, + amazonRegion, + primaryBucket + "/" + bucketPath ); } @@ -325,7 +325,7 @@ private static void initS3() { AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder.standard() .withCredentials(new AWSStaticCredentialsProvider( new BasicAWSCredentials(amazonAccessKeyID, amazonSecretAccessKey)) - ); + ); if (buckets.size() > 1) { log.info("Using {} buckets:{}", buckets.size(), buckets); @@ -337,10 +337,10 @@ private static void initS3() { final boolean isRealAmazon = amazonUrl.endsWith(amazonDomain); amazonS3ClientBuilder = amazonS3ClientBuilder - .withClientConfiguration(new ClientConfiguration().withProtocol(Protocol.HTTP)) - .withEndpointConfiguration( - new AwsClientBuilder.EndpointConfiguration(amazonUrl, amazonRegion) - ); + .withClientConfiguration(new ClientConfiguration().withProtocol(Protocol.HTTP)) + .withEndpointConfiguration( + new AwsClientBuilder.EndpointConfiguration(amazonUrl, amazonRegion) + ); if (isRealAmazon) { amazonMappedUrl = amazonProtocol + primaryBucket + "." + amazonDomain; } else { @@ -356,27 +356,27 @@ private static void initS3() { private static void startMinio() { log.info("Starting MINIO"); minioContainer = new GenericContainer("minio/minio") - .withExposedPorts(9000) - .withEnv("MINIO_ACCESS_KEY", minioAccessKeyID) - .withEnv("MINIO_SECRET_KEY", minioSecretAccessKey) - .withCommand("server /data") - .waitingFor(Wait.defaultWaitStrategy()); + .withExposedPorts(9000) + .withEnv("MINIO_ACCESS_KEY", minioAccessKeyID) + .withEnv("MINIO_SECRET_KEY", minioSecretAccessKey) + .withCommand("server /data") + .waitingFor(Wait.defaultWaitStrategy()); minioContainer.start(); Integer mappedPort = minioContainer.getMappedPort(9000); minioMappedUrl = minioUrl + ":" + mappedPort; log.info("Minio mapped URL:" + minioMappedUrl); minio = AmazonS3ClientBuilder.standard() - .withEndpointConfiguration( - new AwsClientBuilder.EndpointConfiguration(minioMappedUrl, minioRegion) - ) - .withCredentials( - new AWSStaticCredentialsProvider( - new BasicAWSCredentials(minioAccessKeyID, minioSecretAccessKey) + .withEndpointConfiguration( + new AwsClientBuilder.EndpointConfiguration(minioMappedUrl, minioRegion) ) - ) - .enablePathStyleAccess() - .build(); + .withCredentials( + new AWSStaticCredentialsProvider( + new BasicAWSCredentials(minioAccessKeyID, minioSecretAccessKey) + ) + ) + .enablePathStyleAccess() + .build(); buckets.forEach(minio::createBucket); @@ -385,45 +385,45 @@ private static void startMinio() { private static void startCeph() { log.info("Starting CEPH"); cephContainer = new GenericContainer("ceph/daemon") - .withExposedPorts(8000) - .withEnv("RGW_FRONTEND_PORT", "8000") - .withEnv("SREE_PORT", "5000") - .withEnv("DEBUG", "verbose") - .withEnv("CEPH_DEMO_UID", "nano") - .withEnv("MON_IP", "127.0.0.1") - .withEnv("CEPH_PUBLIC_NETWORK", "0.0.0.0/0") - .withEnv("CEPH_DAEMON", "demo") - .withEnv("DEMO_DAEMONS", "mon,mgr,osd,rgw") - .withEnv("CEPH_DEMO_ACCESS_KEY", cephAccessKeyID) - .withEnv("CEPH_DEMO_SECRET_KEY", cephSecretAccessKey) - .withCommand("mkdir -p /etc/ceph && mkdir -p /var/lib/ceph && /entrypoint.sh") - .waitingFor(Wait.defaultWaitStrategy().withStartupTimeout(Duration.ofSeconds(180))); + .withExposedPorts(8000) + .withEnv("RGW_FRONTEND_PORT", "8000") + .withEnv("SREE_PORT", "5000") + .withEnv("DEBUG", "verbose") + .withEnv("CEPH_DEMO_UID", "nano") + .withEnv("MON_IP", "127.0.0.1") + .withEnv("CEPH_PUBLIC_NETWORK", "0.0.0.0/0") + .withEnv("CEPH_DAEMON", "demo") + .withEnv("DEMO_DAEMONS", "mon,mgr,osd,rgw") + .withEnv("CEPH_DEMO_ACCESS_KEY", cephAccessKeyID) + .withEnv("CEPH_DEMO_SECRET_KEY", cephSecretAccessKey) + .withCommand("mkdir -p /etc/ceph && mkdir -p /var/lib/ceph && /entrypoint.sh") + .waitingFor(Wait.defaultWaitStrategy().withStartupTimeout(Duration.ofSeconds(180))); cephContainer.start(); Integer mappedPort = cephContainer.getMappedPort(8000); cephMappedUrl = cephUrl + ":" + mappedPort; log.info("Ceph mapped URL:" + cephMappedUrl); ceph = AmazonS3ClientBuilder.standard() - .withEndpointConfiguration( - new AwsClientBuilder.EndpointConfiguration(cephMappedUrl, cephRegion) - ) - .withCredentials( - new AWSStaticCredentialsProvider( - new BasicAWSCredentials(cephAccessKeyID, cephSecretAccessKey) + .withEndpointConfiguration( + new AwsClientBuilder.EndpointConfiguration(cephMappedUrl, cephRegion) + ) + .withCredentials( + new AWSStaticCredentialsProvider( + new BasicAWSCredentials(cephAccessKeyID, cephSecretAccessKey) + ) ) - ) - .enablePathStyleAccess() - .build(); + .enablePathStyleAccess() + .build(); ceph.createBucket(buckets.get(0)); // curiously enough CEPH docs are incorrect, looks like they do support version id: // https://github.com/ceph/ceph/blame/bc065cae7857c352ca36d5f06cdb5107cf72ed41/src/rgw/rgw_rest_s3.cc // so for versioned local tests we can use CEPH ceph.setBucketVersioningConfiguration( - new SetBucketVersioningConfigurationRequest( - primaryBucket, - new BucketVersioningConfiguration(BucketVersioningConfiguration.ENABLED) - ) + new SetBucketVersioningConfigurationRequest( + primaryBucket, + new BucketVersioningConfiguration(BucketVersioningConfiguration.ENABLED) + ) ); }