Fsspec: Add fsspec compatibility test cases (#54)

* Add fsspec compatibility test cases * Refactor test code * Fix test_copy bug * Fix du issue * Fix walk and find test issues * Fix fmt and lint issues * Revert Makefile and actions
fsspec · Sep 10, 2024 · 7d0e2f3 · 7d0e2f3
1 parent c6cd336
commit 7d0e2f3
Show file tree

Hide file tree

Showing 3 changed files with 736 additions and 14 deletions.
diff --git a/tosfs/core.py b/tosfs/core.py
@@ -896,6 +896,10 @@ def cp_file(
         bucket, key, vers = self._split_path(path1)
 
         info = self.info(path1, bucket, key, version_id=vers)
+        if info["type"] == "directory":
+            logger.warning("Do not support copy directory %s.", path1)
+            return
+
         size = info["size"]
 
         _, _, parts_suffix = info.get("ETag", "").strip('"').partition("-")
@@ -1133,22 +1137,25 @@ def _find_file_dir(
                 out = [self.info(path)]
             except FileNotFoundError:
                 out = []
-        dirs = []
-        for o in out:
-            par = self._parent(o["name"])
-            if len(path) <= len(par):
-                d = {
-                    "Key": self._split_path(par)[1].rstrip("/"),
-                    "Size": 0,
-                    "name": par.rstrip("/"),
-                    "type": "directory",
-                }
-                dirs.append(d)
+        dirs = {
+            self._parent(o["name"]): {
+                "Key": self._parent(o["name"]).rstrip("/"),
+                "Size": 0,
+                "name": self._parent(o["name"]).rstrip("/"),
+                "type": "directory",
+            }
+            for o in out
+            if len(path) <= len(self._parent(o["name"]))
+        }
+
         if withdirs:
-            out = sorted(out + dirs, key=lambda x: x["name"])
+            for dir_info in dirs.values():
+                if dir_info not in out:
+                    out.append(dir_info)
         else:
             out = [o for o in out if o["type"] == "file"]
-        return out
+
+        return sorted(out, key=lambda x: x["name"])
 
     def _open_remote_file(
         self,

diff --git a/tosfs/tests/conftest.py b/tosfs/tests/conftest.py
@@ -13,9 +13,11 @@
 # limitations under the License.
 
 import os
-from typing import Generator
+from typing import Any, Generator
 
+import fsspec
 import pytest
+from fsspec.registry import known_implementations
 from tos import EnvCredentialsProvider
 
 from tosfs.core import TosFileSystem, logger
@@ -40,6 +42,19 @@ def tosfs(_tosfs_env_prepare: None) -> TosFileSystem:
     return tosfs
 
 
+@pytest.fixture(scope="module")
+def fsspecfs(_tosfs_env_prepare: None) -> Any:
+    known_implementations["tos"] = {"class": "tosfs.core.TosFileSystem"}
+
+    fsspecfs, _ = fsspec.core.url_to_fs(
+        "tos://",
+        endpoint_url=os.environ.get("TOS_ENDPOINT"),
+        region=os.environ.get("TOS_REGION"),
+        credentials_provider=EnvCredentialsProvider(),
+    )
+    return fsspecfs
+
+
 @pytest.fixture(scope="module")
 def bucket() -> str:
     return os.environ.get("TOS_BUCKET", "proton-ci")