Ensure correct inventorisation of S3 directories

Fixes #250

afs/aws/s3/src/main/java/org/eclipse/store/afs/aws/s3/types/S3Connector.java

@@ -21,7 +21,9 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.regex.Pattern;
 import java.util.stream.Stream;
 
@@ -34,6 +36,7 @@
 import software.amazon.awssdk.core.internal.util.Mimetype;
 import software.amazon.awssdk.core.sync.RequestBody;
 import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.CommonPrefix;
 import software.amazon.awssdk.services.s3.model.Delete;
 import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
 import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse;
@@ -184,16 +187,50 @@ protected Stream<String> childKeys(
 			final BlobStorePath directory
 		)
 		{
-			final ListObjectsV2Request request = ListObjectsV2Request
-                .builder()
-                .bucket(directory.container())
-                .prefix(toChildKeysPrefix(directory))
-                .delimiter(BlobStorePath.SEPARATOR)
-                .build();
-			return this.s3.listObjectsV2(request)
-				.contents()
+			final Set<String> childKeys         = new LinkedHashSet<>();
+			final String      prefix            = toChildKeysPrefix(directory);
+			String            continuationToken = null;
+			do
+			{
+				final ListObjectsV2Request request = ListObjectsV2Request
+	                .builder()
+	                .bucket(directory.container())
+	                .prefix(prefix)
+	                .delimiter(BlobStorePath.SEPARATOR)
+	                .continuationToken(continuationToken)
+	                .build()
+	            ;
+				final ListObjectsV2Response response = this.s3.listObjectsV2(request);
+				// add "directories"
+				childKeys.addAll(
+					response
+						.commonPrefixes()
+						.stream()
+						.map(CommonPrefix::prefix)
+						.collect(toList())
+				);
+				// add "files"
+				childKeys.addAll(
+					response
+						.contents()
+						.stream()
+						.map(S3Object::key)
+						.collect(toList())
+				);
+				continuationToken = response.isTruncated()
+					? response.nextContinuationToken()
+					: null
+				;
+			}
+			while(continuationToken != null);
+
+			return childKeys
 				.stream()
-				.map(S3Object::key)
+				/*
+				 * Requested base "directory" will be returned as well if it was created explicitly
+				 * but we don't need it in the child keys listing.
+				 */
+				.filter(path -> !path.equals(prefix))
 			;
 		}
 
@@ -229,17 +266,18 @@ protected boolean internalDirectoryExists(
 					return true;
 				}
 
-				final PutObjectRequest request = PutObjectRequest
-					.builder()
-					.bucket(directory.container())
-					.key(containerKey)
-					.build()
-				;
-				this.s3.putObject(request, RequestBody.empty());
-
-				return true;
+				final ListObjectsV2Request request = ListObjectsV2Request
+	                .builder()
+	                .bucket(directory.container())
+	                .prefix(containerKey)
+	                .delimiter(BlobStorePath.SEPARATOR)
+	                .maxKeys(1)
+	                .build()
+	            ;
+				final List<S3Object> objects = this.s3.listObjectsV2(request).contents();
+				return !objects.isEmpty();
 			}
-			catch(final NoSuchKeyException e)
+			catch(final NoSuchBucketException | NoSuchKeyException e)
 			{
 				return false;
 			}
@@ -276,8 +314,10 @@ protected boolean internalCreateDirectory(
 				.key(containerKey)
 				.build()
 			;
-			final RequestBody body = RequestBody.empty();
-			this.s3.putObject(request, body);
+			this.s3.putObject(
+				request,
+				RequestBody.empty()
+			);
 
 			return true;
 		}

afs/blobstore/src/main/java/org/eclipse/store/afs/blobstore/types/BlobStoreConnector.java

@@ -1,21 +1,5 @@
 package org.eclipse.store.afs.blobstore.types;
 
-/*-
- * #%L
- * EclipseStore Abstract File System Blobstore
- * %%
- * Copyright (C) 2023 MicroStream Software
- * %%
- * This program and the accompanying materials are made
- * available under the terms of the Eclipse Public License 2.0
- * which is available at https://www.eclipse.org/legal/epl-2.0/
- * 
- * SPDX-License-Identifier: EPL-2.0
- * #L%
- */
-
-import org.eclipse.serializer.reference.Reference;
-
 import static java.util.stream.Collectors.toList;
 import static org.eclipse.serializer.util.X.checkArrayRange;
 import static org.eclipse.serializer.util.X.notNull;
@@ -38,6 +22,22 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+/*-
+ * #%L
+ * EclipseStore Abstract File System Blobstore
+ * %%
+ * Copyright (C) 2023 MicroStream Software
+ * %%
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ * 
+ * SPDX-License-Identifier: EPL-2.0
+ * #L%
+ */
+
+import org.eclipse.serializer.reference.Reference;
+
 
 /**
  * Connector for blob stores which handles the concrete IO operations on a specific connection.
@@ -146,7 +146,12 @@ protected static String toChildKeysPrefix(
 			final BlobStorePath directory
 		)
 		{
-			return Arrays.stream(directory.pathElements())
+			final String[] pathElements = directory.pathElements();
+			if(pathElements.length <= 1)
+			{
+				return "";
+			}
+			return Arrays.stream(pathElements)
 				.skip(1L) // skip container
 				.collect(Collectors.joining(BlobStorePath.SEPARATOR, "", BlobStorePath.SEPARATOR))
 			;