Skip to content

Commit

Permalink
Performance: Reduce request numbers in exists
Browse files Browse the repository at this point in the history
  • Loading branch information
yanghua committed Sep 16, 2024
1 parent 91c6300 commit cf79efc
Showing 1 changed file with 25 additions and 58 deletions.
83 changes: 25 additions & 58 deletions tosfs/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -1560,10 +1560,31 @@ def exists(self, path: str, **kwargs: Any) -> bool:
# if the path is a bucket
if not key:
return self._exists_bucket(bucket)
elif self.isfile(path):
return self._exists_object(bucket, key, path, version_id)
else:
return self._exists_object(bucket, key.rstrip("/") + "/", path, version_id)

try:
return retryable_func_executor(
lambda: self.tos_client.head_object(bucket, key) or True,
max_retry_num=self.max_retry_num,
)
except TosServerError as e:
if e.status_code == TOS_SERVER_STATUS_CODE_NOT_FOUND:
try:
return retryable_func_executor(
lambda: self.tos_client.head_object(
bucket, key.rstrip("/") + "/"
)
or True,
max_retry_num=self.max_retry_num,
)
except TosServerError as ex:
if e.status_code == TOS_SERVER_STATUS_CODE_NOT_FOUND:
return False
else:
raise ex
else:
raise e
except Exception as ex:
raise TosfsError(f"Tosfs failed with unknown error: {ex}") from ex

def _exists_bucket(self, bucket: str) -> bool:
"""Check if a bucket exists in the TOS.
Expand Down Expand Up @@ -1612,60 +1633,6 @@ def _exists_bucket(self, bucket: str) -> bool:
except Exception as e:
raise TosfsError(f"Tosfs failed with unknown error: {e}") from e

def _exists_object(
self, bucket: str, key: str, path: str, version_id: Optional[str] = None
) -> bool:
"""Check if an object exists in the TOS.
Parameters
----------
bucket : str
The name of the bucket.
key : str
The key of the object.
path : str
The full path of the object.
version_id : str, optional
The version ID of the object (default is None).
Returns
-------
bool
True if the object exists, False otherwise.
Raises
------
tos.exceptions.TosClientError
If there is a client error while checking the object.
tos.exceptions.TosServerError
If there is a server error while checking the object.
TosfsError
If there is an unknown error while checking the object.
Examples
--------
>>> fs = TosFileSystem()
>>> fs._exists_object("mybucket", "myfile", "tos://mybucket/myfile")
True
>>> fs._exists_object("mybucket", "nonexistentfile", "tos://mybucket/nonexistentfile")
False
"""
try:
return retryable_func_executor(
lambda: self.tos_client.head_object(bucket, key) or True,
max_retry_num=self.max_retry_num,
)
except TosClientError as e:
raise e
except TosServerError as e:
if e.status_code == TOS_SERVER_STATUS_CODE_NOT_FOUND:
return False
else:
raise e
except Exception as e:
raise TosfsError(f"Tosfs failed with unknown error: {e}") from e

def _lsbuckets(self) -> List[dict]:
"""List all buckets in the account.
Expand Down

0 comments on commit cf79efc

Please sign in to comment.