From af92587d89488774988e016bc7856c09b44b33a5 Mon Sep 17 00:00:00 2001 From: yanghua Date: Mon, 2 Sep 2024 23:24:30 +0800 Subject: [PATCH] Core: Fix _rm API missed slash issue --- tosfs/core.py | 3 +++ tosfs/tests/conftest.py | 2 +- tosfs/tests/test_tosfs.py | 40 --------------------------------------- 3 files changed, 4 insertions(+), 41 deletions(-) diff --git a/tosfs/core.py b/tosfs/core.py index 710c6e3..59aea2c 100644 --- a/tosfs/core.py +++ b/tosfs/core.py @@ -1311,6 +1311,9 @@ def _listdir( def _rm(self, path: str) -> None: bucket, key, _ = self._split_path(path) + if path.endswith("/") or self.isdir(path): + key = key.rstrip("/") + "/" + try: self.tos_client.delete_object(bucket, key) except tos.exceptions.TosClientError as e: diff --git a/tosfs/tests/conftest.py b/tosfs/tests/conftest.py index acae824..ad7a7bf 100644 --- a/tosfs/tests/conftest.py +++ b/tosfs/tests/conftest.py @@ -53,7 +53,7 @@ def temporary_workspace( tosfs.mkdirs(f"{bucket}/{workspace}/") yield workspace try: - tosfs.rmdir(f"{bucket}/{workspace}/") + tosfs.rm(f"{bucket}/{workspace}/", recursive=True) except Exception as e: logger.error(f"Ignore exception : {e}.") assert not tosfs.exists(f"{bucket}/{workspace}/") diff --git a/tosfs/tests/test_tosfs.py b/tosfs/tests/test_tosfs.py index bac1f88..c693ca5 100644 --- a/tosfs/tests/test_tosfs.py +++ b/tosfs/tests/test_tosfs.py @@ -63,8 +63,6 @@ def test_inner_rm(tosfs: TosFileSystem, bucket: str, temporary_workspace: str) - assert tosfs.ls(f"{bucket}/{temporary_workspace}", detail=False) == [] - tosfs._rm(f"{bucket}/{temporary_workspace}/{file_name}") - def test_info(tosfs: TosFileSystem, bucket: str, temporary_workspace: str) -> None: assert tosfs.info("") == {"name": "", "size": 0, "type": "directory"} @@ -134,7 +132,6 @@ def test_touch(tosfs: TosFileSystem, bucket: str, temporary_workspace: str) -> N assert tosfs.info(f"{bucket}/{temporary_workspace}/{file_name}")["size"] > 0 tosfs.touch(f"{bucket}/{temporary_workspace}/{file_name}", truncate=True) assert tosfs.info(f"{bucket}/{temporary_workspace}/{file_name}")["size"] == 0 - tosfs.rm_file(f"{bucket}/{temporary_workspace}/{file_name}") def test_isdir(tosfs: TosFileSystem, bucket: str, temporary_workspace: str) -> None: @@ -151,8 +148,6 @@ def test_isdir(tosfs: TosFileSystem, bucket: str, temporary_workspace: str) -> N assert not tosfs.isdir(f"{bucket}/{temporary_workspace}/{file_name}") assert not tosfs.isdir(f"{bucket}/{temporary_workspace}/{file_name}/") - tosfs._rm(f"{bucket}/{temporary_workspace}/{file_name}") - def test_isfile(tosfs: TosFileSystem, bucket: str, temporary_workspace: str) -> None: file_name = random_str() @@ -163,8 +158,6 @@ def test_isfile(tosfs: TosFileSystem, bucket: str, temporary_workspace: str) -> assert not tosfs.isfile(f"{bucket}/{temporary_workspace}") assert not tosfs.isfile(f"{bucket}/{temporary_workspace}/") - tosfs._rm(f"{bucket}/{temporary_workspace}/{file_name}") - def test_exists_bucket( tosfs: TosFileSystem, bucket: str, temporary_workspace: str @@ -224,11 +217,6 @@ def test_mkdir(tosfs: TosFileSystem, bucket: str, temporary_workspace: str) -> N assert tosfs.exists(f"{bucket}/{temporary_workspace}/notexist/") assert tosfs.isdir(f"{bucket}/{temporary_workspace}/notexist/") - tosfs.rmdir(f"{bucket}/{temporary_workspace}/notexist/{dir_name}") - tosfs.rmdir(f"{bucket}/{temporary_workspace}/notexist") - tosfs.rmdir(f"{bucket}/{temporary_workspace}/{dir_name}") - tosfs.rmdir(f"{bucket}/{temporary_workspace}") - def test_makedirs(tosfs: TosFileSystem, bucket: str, temporary_workspace: str) -> None: dir_name = random_str() @@ -258,11 +246,6 @@ def test_makedirs(tosfs: TosFileSystem, bucket: str, temporary_workspace: str) - assert tosfs.exists(f"{bucket}/{temporary_workspace}/notexist/") assert tosfs.isdir(f"{bucket}/{temporary_workspace}/notexist/") - tosfs.rmdir(f"{bucket}/{temporary_workspace}/notexist/{dir_name}") - tosfs.rmdir(f"{bucket}/{temporary_workspace}/notexist") - tosfs.rmdir(f"{bucket}/{temporary_workspace}/{dir_name}") - tosfs.rmdir(f"{bucket}/{temporary_workspace}") - def test_put_file(tosfs: TosFileSystem, bucket: str, temporary_workspace: str) -> None: temp_dir = create_temp_dir() @@ -320,7 +303,6 @@ def test_put_file(tosfs: TosFileSystem, bucket: str, temporary_workspace: str) - tosfs.tos_client.get_object(bucket, key).content.read() == b"a" * 1024 * 1024 * 6 ) - tosfs.rm_file(rpath) def test_get_file(tosfs: TosFileSystem, bucket: str, temporary_workspace: str) -> None: @@ -340,8 +322,6 @@ def test_get_file(tosfs: TosFileSystem, bucket: str, temporary_workspace: str) - with pytest.raises(FileNotFoundError): tosfs.get_file(f"{bucket}/{temporary_workspace}/nonexistent", lpath) - tosfs.rm_file(rpath) - def test_walk(tosfs: TosFileSystem, bucket: str, temporary_workspace: str) -> None: with pytest.raises(ValueError, match="Cannot access all of TOS via path ."): @@ -407,14 +387,6 @@ def test_walk(tosfs: TosFileSystem, bucket: str, temporary_workspace: str) -> No assert dir_name in walk_results[2][1] assert walk_results[2][2] == [] - tosfs.rm_file( - f"{bucket}/{temporary_workspace}/{dir_name}/{sub_dir_name}/{sub_file_name}" - ) - tosfs.rm_file(f"{bucket}/{temporary_workspace}/{dir_name}/{file_name}") - tosfs.rmdir(f"{bucket}/{temporary_workspace}/{dir_name}/{sub_dir_name}") - tosfs.rmdir(f"{bucket}/{temporary_workspace}/{dir_name}") - tosfs.rmdir(f"{bucket}/{temporary_workspace}") - def test_find(tosfs: TosFileSystem, bucket: str, temporary_workspace: str) -> None: with pytest.raises(ValueError, match="Cannot access all of TOS via path ."): @@ -510,8 +482,6 @@ def test_file_write( content ) - tosfs.rm_file(f"{bucket}/{temporary_workspace}/{file_name}") - def test_file_write_encdec( tosfs: TosFileSystem, bucket: str, temporary_workspace: str @@ -554,8 +524,6 @@ def test_file_write_encdec( ) assert response.read().decode("ibm500") == content - tosfs.rm_file(f"{bucket}/{temporary_workspace}/{file_name}") - def test_file_write_mpu( tosfs: TosFileSystem, bucket: str, temporary_workspace: str @@ -574,8 +542,6 @@ def test_file_write_mpu( content ) - tosfs.rm_file(f"{bucket}/{temporary_workspace}/{file_name}") - def test_file_read(tosfs: TosFileSystem, bucket: str, temporary_workspace: str) -> None: file_name = random_str() @@ -589,8 +555,6 @@ def test_file_read(tosfs: TosFileSystem, bucket: str, temporary_workspace: str) with tosfs.open(f"{bucket}/{temporary_workspace}/{file_name}", "rb") as f: assert f.read().decode() == content - tosfs.rm_file(f"{bucket}/{temporary_workspace}/{file_name}") - def test_file_read_encdec( tosfs: TosFileSystem, bucket: str, temporary_workspace: str @@ -629,8 +593,6 @@ def test_file_read_encdec( ) as f: assert f.read() == content - tosfs.rm_file(f"{bucket}/{temporary_workspace}/{file_name}") - def test_file_readlines( tosfs: TosFileSystem, bucket: str, temporary_workspace: str @@ -645,5 +607,3 @@ def test_file_readlines( with tosfs.open(f"{bucket}/{temporary_workspace}/{file_name}", "rb") as f: assert f.readlines() == [b"hello\n", b"world"] - - tosfs.rm_file(f"{bucket}/{temporary_workspace}/{file_name}")