Skip to content

Commit

Permalink
Bug: Fix rm api and walk api bug
Browse files Browse the repository at this point in the history
  • Loading branch information
yanghua committed Sep 25, 2024
1 parent 8a68a8c commit 80f71f3
Show file tree
Hide file tree
Showing 3 changed files with 56 additions and 12 deletions.
16 changes: 6 additions & 10 deletions poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ readme = "README.md"

[tool.poetry.dependencies]
python = "^3.9"
fsspec = ">=2023.5.0"
fsspec = "==2023.5.0"
tos = ">=2.7.0"

[tool.poetry.group.dev.dependencies]
Expand Down
50 changes: 49 additions & 1 deletion tosfs/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -1091,7 +1091,7 @@ def find(
"Can not specify 'prefix' option alongside 'maxdepth' options."
)
if maxdepth:
return super().find(
return self._fsspec_find(
bucket + "/" + key,
maxdepth=maxdepth,
withdirs=withdirs,
Expand All @@ -1106,6 +1106,54 @@ def find(
else:
return [o["name"] for o in out]

def _fsspec_find( # noqa #
self,
path: str,
maxdepth: Optional[int] = None,
withdirs: bool = False,
detail: bool = False,
**kwargs: Any, # type: ignore
) -> Any:
"""List all files below path.
Copied from fsspec(2024.9.0) to fix fsspec(2023.5.0.)
Like posix ``find`` command without conditions
Parameters
----------
path : str
maxdepth: int or None
If not None, the maximum number of levels to descend
withdirs: bool
Whether to include directory paths in the output. This is True
when used by glob, but users usually only want files.
kwargs are passed to ``ls``.
"""
# TODO: allow equivalent of -name parameter
path = self._strip_protocol(path)
out = {}

# Add the root directory if withdirs is requested
# This is needed for posix glob compliance
if withdirs and path != "" and self.isdir(path):
out[path] = self.info(path)

for _, dirs, files in self._fsspec_walk(path, maxdepth, detail=True, **kwargs):
if withdirs:
files.update(dirs)
out.update({info["name"]: info for name, info in files.items()})
if not out and self.isfile(path):
# walk works on directories, but find should also return [path]
# when path happens to be a file
out[path] = {}
names = sorted(out)
if not detail:
return names
else:
return {name: out[name] for name in names}

def expand_path(
self,
path: Union[str, List[str]],
Expand Down

0 comments on commit 80f71f3

Please sign in to comment.