From 535312f955e50e295a90fa58c670deef3cfda682 Mon Sep 17 00:00:00 2001 From: fh-ms Date: Tue, 13 Aug 2024 12:21:06 +0200 Subject: [PATCH] Ensure correct inventorisation of S3 directories Fixes #250 --- .../store/afs/aws/s3/types/S3Connector.java | 82 ++++++++++++++----- .../blobstore/types/BlobStoreConnector.java | 39 +++++---- 2 files changed, 83 insertions(+), 38 deletions(-) diff --git a/afs/aws/s3/src/main/java/org/eclipse/store/afs/aws/s3/types/S3Connector.java b/afs/aws/s3/src/main/java/org/eclipse/store/afs/aws/s3/types/S3Connector.java index 1f770948..a7621cef 100644 --- a/afs/aws/s3/src/main/java/org/eclipse/store/afs/aws/s3/types/S3Connector.java +++ b/afs/aws/s3/src/main/java/org/eclipse/store/afs/aws/s3/types/S3Connector.java @@ -21,7 +21,9 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Stream; @@ -34,6 +36,7 @@ import software.amazon.awssdk.core.internal.util.Mimetype; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.CommonPrefix; import software.amazon.awssdk.services.s3.model.Delete; import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; @@ -184,16 +187,50 @@ protected Stream childKeys( final BlobStorePath directory ) { - final ListObjectsV2Request request = ListObjectsV2Request - .builder() - .bucket(directory.container()) - .prefix(toChildKeysPrefix(directory)) - .delimiter(BlobStorePath.SEPARATOR) - .build(); - return this.s3.listObjectsV2(request) - .contents() + final Set childKeys = new LinkedHashSet<>(); + final String prefix = toChildKeysPrefix(directory); + String continuationToken = null; + do + { + final ListObjectsV2Request request = ListObjectsV2Request + .builder() + .bucket(directory.container()) + .prefix(prefix) + .delimiter(BlobStorePath.SEPARATOR) + .continuationToken(continuationToken) + .build() + ; + final ListObjectsV2Response response = this.s3.listObjectsV2(request); + // add "directories" + childKeys.addAll( + response + .commonPrefixes() + .stream() + .map(CommonPrefix::prefix) + .collect(toList()) + ); + // add "files" + childKeys.addAll( + response + .contents() + .stream() + .map(S3Object::key) + .collect(toList()) + ); + continuationToken = response.isTruncated() + ? response.nextContinuationToken() + : null + ; + } + while(continuationToken != null); + + return childKeys .stream() - .map(S3Object::key) + /* + * Requested base "directory" will be returned as well if it was created explicitly + * but we don't need it in the child keys listing. + */ + .filter(path -> !path.equals(prefix)) ; } @@ -229,17 +266,18 @@ protected boolean internalDirectoryExists( return true; } - final PutObjectRequest request = PutObjectRequest - .builder() - .bucket(directory.container()) - .key(containerKey) - .build() - ; - this.s3.putObject(request, RequestBody.empty()); - - return true; + final ListObjectsV2Request request = ListObjectsV2Request + .builder() + .bucket(directory.container()) + .prefix(containerKey) + .delimiter(BlobStorePath.SEPARATOR) + .maxKeys(1) + .build() + ; + final List objects = this.s3.listObjectsV2(request).contents(); + return !objects.isEmpty(); } - catch(final NoSuchKeyException e) + catch(final NoSuchBucketException | NoSuchKeyException e) { return false; } @@ -276,8 +314,10 @@ protected boolean internalCreateDirectory( .key(containerKey) .build() ; - final RequestBody body = RequestBody.empty(); - this.s3.putObject(request, body); + this.s3.putObject( + request, + RequestBody.empty() + ); return true; } diff --git a/afs/blobstore/src/main/java/org/eclipse/store/afs/blobstore/types/BlobStoreConnector.java b/afs/blobstore/src/main/java/org/eclipse/store/afs/blobstore/types/BlobStoreConnector.java index 06c58a5a..96e46ca6 100644 --- a/afs/blobstore/src/main/java/org/eclipse/store/afs/blobstore/types/BlobStoreConnector.java +++ b/afs/blobstore/src/main/java/org/eclipse/store/afs/blobstore/types/BlobStoreConnector.java @@ -1,21 +1,5 @@ package org.eclipse.store.afs.blobstore.types; -/*- - * #%L - * EclipseStore Abstract File System Blobstore - * %% - * Copyright (C) 2023 MicroStream Software - * %% - * This program and the accompanying materials are made - * available under the terms of the Eclipse Public License 2.0 - * which is available at https://www.eclipse.org/legal/epl-2.0/ - * - * SPDX-License-Identifier: EPL-2.0 - * #L% - */ - -import org.eclipse.serializer.reference.Reference; - import static java.util.stream.Collectors.toList; import static org.eclipse.serializer.util.X.checkArrayRange; import static org.eclipse.serializer.util.X.notNull; @@ -38,6 +22,22 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +/*- + * #%L + * EclipseStore Abstract File System Blobstore + * %% + * Copyright (C) 2023 MicroStream Software + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + +import org.eclipse.serializer.reference.Reference; + /** * Connector for blob stores which handles the concrete IO operations on a specific connection. @@ -146,7 +146,12 @@ protected static String toChildKeysPrefix( final BlobStorePath directory ) { - return Arrays.stream(directory.pathElements()) + final String[] pathElements = directory.pathElements(); + if(pathElements.length <= 1) + { + return ""; + } + return Arrays.stream(pathElements) .skip(1L) // skip container .collect(Collectors.joining(BlobStorePath.SEPARATOR, "", BlobStorePath.SEPARATOR)) ;