feat(s3Client): add support for ListObjectsV2 action

packages/bun-types/bun.d.ts

@@ -1813,6 +1813,106 @@ declare module "bun" {
    *     const url = bucket.presign("file.pdf");
    *     await bucket.unlink("old.txt");
    */
+  interface S3ListObjectsOptions {
+    /** Limits the response to keys that begin with the specified prefix. */
+    Prefix?: string;
+    /** ContinuationToken indicates to S3 that the list is being continued on this bucket with a token. ContinuationToken is obfuscated and is not a real key. You can use this ContinuationToken for pagination of the list results. */
+    ContinuationToken?: string;
+    /** A delimiter is a character that you use to group keys. */
+    Delimiter?: string;
+    /** Sets the maximum number of keys returned in the response. By default, the action returns up to 1,000 key names. The response might contain fewer keys but will never contain more. */
+    MaxKeys?: number;
+    /** StartAfter is where you want S3 to start listing from. S3 starts listing after this specified key. StartAfter can be any key in the bucket. */
+    StartAfter?: string;
+    /** Encoding type used by S3 to encode the object keys in the response. Responses are encoded only in UTF-8. An object key can contain any Unicode character. However, the XML 1.0 parser can't parse certain characters, such as characters with an ASCII value from 0 to 10. For characters that aren't supported in XML 1.0, you can add this parameter to request that S3 encode the keys in the response. */
+    EncodingType?: "url";
+    /** If you want to return the owner field with each key in the result, then set the FetchOwner field to true. */
+    FetchOwner?: boolean;
+  }
+
+  interface S3ListObjectsResponse {
+    /** All of the keys (up to 1,000) that share the same prefix are grouped together. When counting the total numbers of returns by this API operation, this group of keys is considered as one item.
+     *
+     * A response can contain CommonPrefixes only if you specify a delimiter.
+     *
+     * CommonPrefixes contains all (if there are any) keys between Prefix and the next occurrence of the string specified by a delimiter.
+     *
+     * CommonPrefixes lists keys that act like subdirectories in the directory specified by Prefix.
+     *
+     * For example, if the prefix is notes/ and the delimiter is a slash (/) as in notes/summer/july, the common prefix is notes/summer/. All of the keys that roll up into a common prefix count as a single return when calculating the number of returns. */
+    CommonPrefixes?: { Prefix: string }[];
+    /** Metadata about each object returned. */
+    Contents?: {
+      /** The algorithm that was used to create a checksum of the object. */
+      ChecksumAlgorithm?: "CRC32" | "CRC32C" | "SHA1" | "SHA256" | "CRC64NVME";
+      /** The checksum type that is used to calculate the object’s checksum value. */
+      ChecksumType?: "COMPOSITE" | "FULL_OBJECT";
+      /**
+       * The entity tag is a hash of the object. The ETag reflects changes only to the contents of an object, not its metadata. The ETag may or may not be an MD5 digest of the object data. Whether or not it is depends on how the object was created and how it is encrypted as described below:
+       *
+       * - Objects created by the PUT Object, POST Object, or Copy operation, or through the AWS Management Console, and are encrypted by SSE-S3 or plaintext, have ETags that are an MD5 digest of their object data.
+       * - Objects created by the PUT Object, POST Object, or Copy operation, or through the AWS Management Console, and are encrypted by SSE-C or SSE-KMS, have ETags that are not an MD5 digest of their object data.
+       * - If an object is created by either the Multipart Upload or Part Copy operation, the ETag is not an MD5 digest, regardless of the method of encryption. If an object is larger than 16 MB, the AWS Management Console will upload or copy that object as a Multipart Upload, and therefore the ETag will not be an MD5 digest.
+       *
+       * MD5 is not supported by directory buckets.
+       */
+      ETag?: string;
+      /** The name that you assign to an object. You use the object key to retrieve the object. */
+      Key: string;
+      /** Creation date of the object. */
+      LastModified?: string;
+      /** The owner of the object */
+      Owner?: {
+        /** The ID of the owner. */
+        Id?: string;
+        /** The display name of the owner. */
+        DisplayName?: string;
+      };
+      /** Specifies the restoration status of an object. Objects in certain storage classes must be restored before they can be retrieved. */
+      RestoreStatus?: {
+        /** Specifies whether the object is currently being restored. */
+        IsRestoreInProgress?: boolean;
+        /** Indicates when the restored copy will expire. This value is populated only if the object has already been restored. */
+        RestoreExpiryDate?: string;
+      };
+      /** Size in bytes of the object */
+      Size?: number;
+      /** The class of storage used to store the object. */
+      StorageClass?:
+        | "STANDARD"
+        | "REDUCED_REDUNDANCY"
+        | "GLACIER"
+        | "STANDARD_IA"
+        | "ONEZONE_IA"
+        | "INTELLIGENT_TIERING"
+        | "DEEP_ARCHIVE"
+        | "OUTPOSTS"
+        | "GLACIER_IR"
+        | "SNOW"
+        | "EXPRESS_ONEZONE";
+    }[];
+    /** If ContinuationToken was sent with the request, it is included in the response. You can use the returned ContinuationToken for pagination of the list response.  */
+    ContinuationToken?: string;
+    /** Causes keys that contain the same string between the prefix and the first occurrence of the delimiter to be rolled up into a single result element in the CommonPrefixes collection. These rolled-up keys are not returned elsewhere in the response. Each rolled-up result counts as only one return against the MaxKeys value. */
+    Delimiter?: string;
+    /** Encoding type used by Amazon S3 to encode object key names in the XML response. */
+    EncodingType?: "url";
+    /** Set to false if all of the results were returned. Set to true if more keys are available to return. If the number of results exceeds that specified by MaxKeys, all of the results might not be returned. */
+    IsTruncated?: boolean;
+    /** KeyCount is the number of keys returned with this request. KeyCount will always be less than or equal to the MaxKeys field. For example, if you ask for 50 keys, your result will include 50 keys or fewer. */
+    KeyCount?: number;
+    /** Sets the maximum number of keys returned in the response. By default, the action returns up to 1,000 key names. The response might contain fewer keys but will never contain more. */
+    MaxKeys?: number;
+    /** The bucket name. */
+    Name?: string;
+    /** NextContinuationToken is sent when isTruncated is true, which means there are more keys in the bucket that can be listed. The next list requests to Amazon S3 can be continued with this NextContinuationToken. NextContinuationToken is obfuscated and is not a real key. */
+    NextContinuationToken?: string;
+    /** Keys that begin with the indicated prefix. */
+    Prefix?: string;
+    /** If StartAfter was sent with the request, it is included in the response. */
+    StartAfter?: string;
+  }
+
   type S3Client = {
     /**
      * Create a new instance of an S3 bucket so that credentials can be managed
@@ -1946,6 +2046,14 @@ declare module "bun" {
     unlink(path: string, options?: S3Options): Promise<void>;
     delete: S3Client["unlink"];
 
+    /** Returns some or all (up to 1,000) of the objects in a bucket with each request.
+     *
+     * You can  use the request parameters as selection criteria to return a subset of the objects in a bucket.
+     */
+    listObjects(
+      input?: S3ListObjectsOptions | null,
+      options?: Pick<S3Options, "accessKeyId" | "secretAccessKey" | "sessionToken" | "region" | "bucket" | "endpoint">,
+    ): Promise<S3ListObjectsResponse>;
     /**
      * Get the size of a file in bytes.
      * Uses HEAD request to efficiently get size.

src/bun.js/api/S3Client.classes.ts

@@ -20,6 +20,10 @@ export default [
         fn: "staticUnlink",
         length: 2,
       },
+      listObjects: {
+        fn: "staticListObjects",
+        length: 2,
+      },
       presign: {
         fn: "staticPresign",
         length: 2,
@@ -56,6 +60,10 @@ export default [
         fn: "unlink",
         length: 2,
       },
+      listObjects: {
+        fn: "listObjects",
+        length: 2,
+      },
       presign: {
         fn: "presign",
         length: 2,

src/bun.js/webcore/S3Client.zig

@@ -236,6 +236,18 @@ pub const S3Client = struct {
         });
     }
 
+    pub fn listObjects(ptr: *@This(), globalThis: *JSC.JSGlobalObject, callframe: *JSC.CallFrame) bun.JSError!JSValue {
+        const args = callframe.argumentsAsArray(2);
+
+        const object_keys = args[0];
+        const options = args[1];
+
+        var blob = try S3File.constructS3FileWithS3CredentialsAndOptions(globalThis, .{ .string = bun.PathString.empty }, options, ptr.credentials, ptr.options, null, null);
+
+        defer blob.detach();
+        return blob.store.?.data.s3.listObjects(blob.store.?, globalThis, object_keys, options);
+    }
+
     pub fn unlink(ptr: *@This(), globalThis: *JSC.JSGlobalObject, callframe: *JSC.CallFrame) bun.JSError!JSValue {
         const arguments = callframe.arguments_old(2).slice();
         var args = JSC.Node.ArgumentsSlice.init(globalThis.bunVM(), arguments);
@@ -297,4 +309,18 @@ pub const S3Client = struct {
     pub fn staticStat(globalThis: *JSC.JSGlobalObject, callframe: *JSC.CallFrame) bun.JSError!JSValue {
         return S3File.stat(globalThis, callframe);
     }
+
+    pub fn staticListObjects(globalThis: *JSC.JSGlobalObject, callframe: *JSC.CallFrame) bun.JSError!JSValue {
+        const args = callframe.argumentsAsArray(2);
+        const object_keys = args[0];
+        const options = args[1];
+
+        // get credentials from env
+        const existing_credentials = globalThis.bunVM().transpiler.env.getS3Credentials();
+
+        var blob = try S3File.constructS3FileWithS3Credentials(globalThis, .{ .string = bun.PathString.empty }, options, existing_credentials);
+
+        defer blob.detach();
+        return blob.store.?.data.s3.listObjects(blob.store.?, globalThis, object_keys, options);
+    }
 };

src/bun.js/webcore/blob.zig

@@ -3596,6 +3596,58 @@ pub const Blob = struct {
 
             return value;
         }
+
+        pub fn listObjects(this: *@This(), store: *Store, globalThis: *JSC.JSGlobalObject, listOptions: JSValue, extra_options: ?JSValue) bun.JSError!JSValue {
+            if (!listOptions.isEmptyOrUndefinedOrNull() and !listOptions.isObject()) {
+                return globalThis.throwInvalidArguments("S3Client.listObjects() needs a S3ListObjectsOption as it's first argument", .{});
+            }
+
+            const Wrapper = struct {
+                promise: JSC.JSPromise.Strong,
+                store: *Store,
+
+                pub usingnamespace bun.New(@This());
+
+                pub fn resolve(result: S3.S3ListObjectsResult, self: *@This()) void {
+                    defer self.deinit();
+                    const globalObject = self.promise.globalObject().?;
+                    switch (result) {
+                        .success => |list_result| {
+                            defer list_result.deinit();
+                            self.promise.resolve(globalObject, list_result.toJS(globalObject));
+                        },
+
+                        inline .not_found, .failure => |err| {
+                            self.promise.reject(globalObject, err.toJS(globalObject, self.store.getPath()));
+                        },
+                    }
+                }
+
+                fn deinit(self: *@This()) void {
+                    self.store.deref();
+                    self.promise.deinit();
+                    self.destroy();
+                }
+            };
+
+            const promise = JSC.JSPromise.Strong.init(globalThis);
+            const value = promise.value();
+            const proxy_url = globalThis.bunVM().transpiler.env.getHttpProxy(true, null);
+            const proxy = if (proxy_url) |url| url.href else null;
+            var aws_options = try this.getCredentialsWithOptions(extra_options, globalThis);
+            defer aws_options.deinit();
+
+            const options = S3.getListObjectsOptionsFromJS(globalThis, listOptions) catch bun.outOfMemory();
+
+            S3.listObjects(&aws_options.credentials, options, @ptrCast(&Wrapper.resolve), Wrapper.new(.{
+                .promise = promise,
+                .store = store, // store is needed in case of not found error
+            }), proxy);
+            store.ref();
+
+            return value;
+        }
+
         pub fn initWithReferencedCredentials(pathlike: JSC.Node.PathLike, mime_type: ?http.MimeType, credentials: *S3Credentials) S3Store {
             credentials.ref();
             return .{

src/s3/client.zig

@@ -23,6 +23,10 @@ pub const S3UploadResult = S3SimpleRequest.S3UploadResult;
 pub const S3StatResult = S3SimpleRequest.S3StatResult;
 pub const S3DownloadResult = S3SimpleRequest.S3DownloadResult;
 pub const S3DeleteResult = S3SimpleRequest.S3DeleteResult;
+const S3ListObjects = @import("./list_objects.zig");
+pub const S3ListObjectsResult = S3SimpleRequest.S3ListObjectsResult;
+pub const S3ListObjectsOptions = @import("./list_objects.zig").S3ListObjectsOptions;
+pub const getListObjectsOptionsFromJS = S3ListObjects.getListObjectsOptionsFromJS;
 
 pub fn stat(
     this: *S3Credentials,
@@ -99,6 +103,127 @@ pub fn delete(
     }, .{ .delete = callback }, callback_context);
 }
 
+pub fn listObjects(
+    this: *S3Credentials,
+    listOptions: S3ListObjectsOptions,
+    callback: *const fn (S3ListObjectsResult, *anyopaque) void,
+    callback_context: *anyopaque,
+    proxy_url: ?[]const u8,
+) void {
+    var search_params: bun.ByteList = .{};
+
+    search_params.append(bun.default_allocator, "?") catch bun.outOfMemory();
+
+    if (listOptions.continuation_token) |continuation_token| {
+        var buff: [1024]u8 = undefined;
+        const encoded = S3Credentials.encodeURIComponent(continuation_token, &buff, true) catch bun.outOfMemory();
+        search_params.appendFmt(bun.default_allocator, "continuation-token={s}", .{encoded}) catch bun.outOfMemory();
+    }
+
+    if (listOptions.delimiter) |delimiter| {
+        var buff: [1024]u8 = undefined;
+        const encoded = S3Credentials.encodeURIComponent(delimiter, &buff, true) catch bun.outOfMemory();
+
+        if (listOptions.continuation_token != null) {
+            search_params.appendFmt(bun.default_allocator, "&delimiter={s}", .{encoded}) catch bun.outOfMemory();
+        } else {
+            search_params.appendFmt(bun.default_allocator, "delimiter={s}", .{encoded}) catch bun.outOfMemory();
+        }
+    }
+
+    if (listOptions.encoding_type != null) {
+        if (listOptions.continuation_token != null or listOptions.delimiter != null) {
+            search_params.append(bun.default_allocator, "&encoding-type=url") catch bun.outOfMemory();
+        } else {
+            search_params.append(bun.default_allocator, "encoding-type=url") catch bun.outOfMemory();
+        }
+    }
+
+    if (listOptions.fetch_owner) |fetch_owner| {
+        if (listOptions.continuation_token != null or listOptions.delimiter != null or listOptions.encoding_type != null) {
+            search_params.appendFmt(bun.default_allocator, "&fetch-owner={}", .{fetch_owner}) catch bun.outOfMemory();
+        } else {
+            search_params.appendFmt(bun.default_allocator, "fetch-owner={}", .{fetch_owner}) catch bun.outOfMemory();
+        }
+    }
+
+    if (listOptions.continuation_token != null or listOptions.delimiter != null or listOptions.encoding_type != null or listOptions.fetch_owner != null) {
+        search_params.append(bun.default_allocator, "&list-type=2") catch bun.outOfMemory();
+    } else {
+        search_params.append(bun.default_allocator, "list-type=2") catch bun.outOfMemory();
+    }
+
+    if (listOptions.max_keys) |max_keys| {
+        search_params.appendFmt(bun.default_allocator, "&max-keys={}", .{max_keys}) catch bun.outOfMemory();
+    }
+
+    if (listOptions.prefix) |prefix| {
+        var buff: [1024]u8 = undefined;
+        const encoded = S3Credentials.encodeURIComponent(prefix, &buff, true) catch bun.outOfMemory();
+        search_params.appendFmt(bun.default_allocator, "&prefix={s}", .{encoded}) catch bun.outOfMemory();
+    }
+
+    if (listOptions.start_after) |start_after| {
+        var buff: [1024]u8 = undefined;
+        const encoded = S3Credentials.encodeURIComponent(start_after, &buff, true) catch bun.outOfMemory();
+        search_params.appendFmt(bun.default_allocator, "&start-after={s}", .{encoded}) catch bun.outOfMemory();
+    }
+
+    const result = this.signRequest(.{
+        .path = "",
+        .method = .GET,
+        .search_params = search_params.slice(),
+    }, null) catch |sign_err| {
+        const error_code_and_message = Error.getSignErrorCodeAndMessage(sign_err);
+        callback(.{ .failure = .{ .code = error_code_and_message.code, .message = error_code_and_message.message } }, callback_context);
+
+        return;
+    };
+
+    const headers = JSC.WebCore.Headers.fromPicoHttpHeaders(result.headers(), bun.default_allocator) catch bun.outOfMemory();
+
+    const task = bun.new(S3HttpSimpleTask, .{
+        .http = undefined,
+        .range = null,
+        .sign_result = result,
+        .callback_context = callback_context,
+        .callback = .{ .listObjects = callback },
+        .headers = headers,
+        .vm = JSC.VirtualMachine.get(),
+    });
+
+    task.poll_ref.ref(task.vm);
+
+    const url = bun.URL.parse(result.url);
+    const proxy = proxy_url orelse "";
+
+    task.http = bun.http.AsyncHTTP.init(
+        bun.default_allocator,
+        .GET,
+        url,
+        task.headers.entries,
+        task.headers.buf.items,
+        &task.response_buffer,
+        "",
+        bun.http.HTTPClientResult.Callback.New(
+            *S3HttpSimpleTask,
+            S3HttpSimpleTask.httpCallback,
+        ).init(task),
+        .follow,
+        .{
+            .http_proxy = if (proxy.len > 0) bun.URL.parse(proxy) else null,
+            .verbose = task.vm.getVerboseFetch(),
+            .reject_unauthorized = task.vm.getTLSRejectUnauthorized(),
+        },
+    );
+
+    // queue http request
+    bun.http.HTTPThread.init(&.{});
+    var batch = bun.ThreadPool.Batch{};
+    task.http.schedule(bun.default_allocator, &batch);
+    bun.http.http_thread.schedule(batch);
+}
+
 pub fn upload(
     this: *S3Credentials,
     path: []const u8,