From 35adb0c6c474ddfc4c214366eac6f915968ec143 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 14 Sep 2023 10:10:54 +0100 Subject: [PATCH 1/3] private module use removed --- adlfs/spec.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/adlfs/spec.py b/adlfs/spec.py index 7627ab6d..81e83e8a 100644 --- a/adlfs/spec.py +++ b/adlfs/spec.py @@ -21,11 +21,15 @@ ResourceExistsError, ResourceNotFoundError, ) -from azure.storage.blob import BlobSasPermissions, generate_blob_sas -from azure.storage.blob._models import BlobBlock, BlobProperties, BlobType -from azure.storage.blob._shared.base_client import create_configuration +from azure.storage.blob import ( + BlobBlock, + BlobPrefix, + BlobProperties, + BlobSasPermissions, + BlobType, + generate_blob_sas, +) from azure.storage.blob.aio import BlobServiceClient as AIOBlobServiceClient -from azure.storage.blob.aio._list_blobs_helper import BlobPrefix from fsspec.asyn import AsyncFileSystem, _get_batch_size, get_loop, sync, sync_wrapper from fsspec.spec import AbstractBufferedFile from fsspec.utils import infer_storage_options @@ -61,6 +65,7 @@ "is_current_version", ] _ROOT_PATH = "/" +_DEFAULT_BLOCK_SIZE = 4 * 1024 * 1024 _SOCKET_TIMEOUT_DEFAULT = object() @@ -143,7 +148,7 @@ class AzureBlobFileSystem(AsyncFileSystem): The credentials with which to authenticate. Optional if the account URL already has a SAS token. Can include an instance of TokenCredential class from azure.identity blocksize: int - The block size to use for download/upload operations. Defaults to the value of + The block size to use for download/upload operations. Defaults to hardcoded value of ``BlockBlobService.MAX_BLOCK_SIZE`` client_id: str Client ID to use when authenticating using an AD Service Principal client/secret. @@ -219,7 +224,7 @@ def __init__( sas_token: str = None, request_session=None, socket_timeout=_SOCKET_TIMEOUT_DEFAULT, - blocksize: int = create_configuration(storage_sdk="blob").max_block_size, + blocksize: int = _DEFAULT_BLOCK_SIZE client_id: str = None, client_secret: str = None, tenant_id: str = None, From a82254a6c38a5add78508e9c04a0cd80bfe588a8 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 14 Sep 2023 10:39:47 +0100 Subject: [PATCH 2/3] black ran --- adlfs/spec.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/adlfs/spec.py b/adlfs/spec.py index 81e83e8a..a6409437 100644 --- a/adlfs/spec.py +++ b/adlfs/spec.py @@ -224,7 +224,7 @@ def __init__( sas_token: str = None, request_session=None, socket_timeout=_SOCKET_TIMEOUT_DEFAULT, - blocksize: int = _DEFAULT_BLOCK_SIZE + blocksize: int = _DEFAULT_BLOCK_SIZE, client_id: str = None, client_secret: str = None, tenant_id: str = None, @@ -290,7 +290,6 @@ def __init__( and self.sas_token is None and self.account_key is None ): - ( self.credential, self.sync_credential, @@ -402,7 +401,6 @@ def _get_credential_from_service_principal(self): return (async_credential, sync_credential) def _get_default_azure_credential(self, **kwargs): - """ Create a Credential for authentication using DefaultAzureCredential From 78af9d177bea2408af514475bbbf67fd14fe19d6 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Sat, 16 Sep 2023 16:12:44 -0500 Subject: [PATCH 3/3] Use aio BlobPrefix --- adlfs/spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adlfs/spec.py b/adlfs/spec.py index 16f87bf1..8c3d4cc1 100644 --- a/adlfs/spec.py +++ b/adlfs/spec.py @@ -23,12 +23,12 @@ ) from azure.storage.blob import ( BlobBlock, - BlobPrefix, BlobProperties, BlobSasPermissions, BlobType, generate_blob_sas, ) +from azure.storage.blob.aio import BlobPrefix from azure.storage.blob.aio import BlobServiceClient as AIOBlobServiceClient from fsspec.asyn import AsyncFileSystem, _get_batch_size, get_loop, sync, sync_wrapper from fsspec.spec import AbstractBufferedFile