From 4d1ec47c94349f96124bdadb0164ef07775429da Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Fri, 15 Nov 2024 21:23:30 +0100 Subject: [PATCH] Apply ruff/pyupgrade rules (UP) (#1717) Co-authored-by: Martin Durant --- fsspec/asyn.py | 8 +++----- fsspec/caching.py | 9 ++------- fsspec/implementations/asyn_wrapper.py | 3 ++- fsspec/implementations/reference.py | 4 ++-- .../implementations/tests/test_asyn_wrapper.py | 4 +++- fsspec/implementations/tests/test_smb.py | 2 +- fsspec/implementations/tests/test_zip.py | 2 +- fsspec/spec.py | 18 ++++++++---------- fsspec/tests/abstract/__init__.py | 2 +- fsspec/tests/test_async.py | 2 +- fsspec/tests/test_fuse.py | 3 +-- fsspec/tests/test_utils.py | 4 +--- fsspec/utils.py | 3 ++- pyproject.toml | 8 ++++++++ 14 files changed, 36 insertions(+), 36 deletions(-) diff --git a/fsspec/asyn.py b/fsspec/asyn.py index de41839ea..aea097b46 100644 --- a/fsspec/asyn.py +++ b/fsspec/asyn.py @@ -816,11 +816,9 @@ async def _glob(self, path, maxdepth=None, **kwargs): p: info for p, info in sorted(allpaths.items()) if pattern.match( - ( - p + "/" - if append_slash_to_dirname and info["type"] == "directory" - else p - ) + p + "/" + if append_slash_to_dirname and info["type"] == "directory" + else p ) } diff --git a/fsspec/caching.py b/fsspec/caching.py index bc74ad241..34231c881 100644 --- a/fsspec/caching.py +++ b/fsspec/caching.py @@ -7,9 +7,9 @@ import os import threading import warnings +from concurrent.futures import Future, ThreadPoolExecutor from itertools import groupby from operator import itemgetter -from concurrent.futures import Future, ThreadPoolExecutor from typing import ( TYPE_CHECKING, Any, @@ -87,12 +87,7 @@ def _log_stats(self) -> str: if self.hit_count == 0 and self.miss_count == 0: # a cache that does nothing, this is for logs only return "" - return " , %s: %d hits, %d misses, %d total requested bytes" % ( - self.name, - self.hit_count, - self.miss_count, - self.total_requested_bytes, - ) + return f" , {self.name}: {self.hit_count} hits, {self.miss_count} misses, {self.total_requested_bytes} total requested bytes" def __repr__(self) -> str: # TODO: use rich for better formatting diff --git a/fsspec/implementations/asyn_wrapper.py b/fsspec/implementations/asyn_wrapper.py index 9ba7811ce..0d462652b 100644 --- a/fsspec/implementations/asyn_wrapper.py +++ b/fsspec/implementations/asyn_wrapper.py @@ -1,6 +1,7 @@ import asyncio -import inspect import functools +import inspect + from fsspec.asyn import AsyncFileSystem diff --git a/fsspec/implementations/reference.py b/fsspec/implementations/reference.py index d8df6ee52..2df8b0e0d 100644 --- a/fsspec/implementations/reference.py +++ b/fsspec/implementations/reference.py @@ -176,7 +176,7 @@ def open_refs(field, record): try: df = self.pd.read_parquet(data, engine=self.engine) refs = {c: df[c].to_numpy() for c in df.columns} - except IOError: + except OSError: refs = None return refs @@ -431,7 +431,7 @@ def write(self, field, record, base_url=None, storage_options=None): if len(partition) < self.record_size: try: original = self.open_refs(field, record) - except IOError: + except OSError: pass if original: diff --git a/fsspec/implementations/tests/test_asyn_wrapper.py b/fsspec/implementations/tests/test_asyn_wrapper.py index d29202003..82e9f3d4c 100644 --- a/fsspec/implementations/tests/test_asyn_wrapper.py +++ b/fsspec/implementations/tests/test_asyn_wrapper.py @@ -1,10 +1,12 @@ import asyncio -import pytest import os +import pytest + import fsspec from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper from fsspec.implementations.local import LocalFileSystem + from .test_local import csv_files, filetexts diff --git a/fsspec/implementations/tests/test_smb.py b/fsspec/implementations/tests/test_smb.py index a83e3cc91..083b5e7ae 100644 --- a/fsspec/implementations/tests/test_smb.py +++ b/fsspec/implementations/tests/test_smb.py @@ -29,7 +29,7 @@ def stop_docker(container): - cmd = shlex.split('docker ps -a -q --filter "name=%s"' % container) + cmd = shlex.split(f'docker ps -a -q --filter "name={container}"') cid = subprocess.check_output(cmd).strip().decode() if cid: subprocess.call(["docker", "rm", "-f", "-v", cid]) diff --git a/fsspec/implementations/tests/test_zip.py b/fsspec/implementations/tests/test_zip.py index 814273761..59559c0b9 100644 --- a/fsspec/implementations/tests/test_zip.py +++ b/fsspec/implementations/tests/test_zip.py @@ -105,7 +105,7 @@ def test_zip_glob_star(m): outfiles = fs.glob("*") assert len(outfiles) == 1 - fn = f"{os.path.dirname(os.path.abspath((__file__)))}/out.zip" + fn = f"{os.path.dirname(os.path.abspath(__file__))}/out.zip" fs = fsspec.filesystem("zip", fo=fn, mode="r") outfiles = fs.glob("*") assert len(outfiles) == 1 diff --git a/fsspec/spec.py b/fsspec/spec.py index 9659f2e98..a0bbeb758 100644 --- a/fsspec/spec.py +++ b/fsspec/spec.py @@ -10,7 +10,7 @@ from errno import ESPIPE from glob import has_magic from hashlib import sha256 -from typing import Any, ClassVar, Dict, Tuple +from typing import Any, ClassVar from .callbacks import DEFAULT_CALLBACK from .config import apply_config, conf @@ -117,8 +117,8 @@ class AbstractFileSystem(metaclass=_Cached): _extra_tokenize_attributes = () # Set by _Cached metaclass - storage_args: Tuple[Any, ...] - storage_options: Dict[str, Any] + storage_args: tuple[Any, ...] + storage_options: dict[str, Any] def __init__(self, *args, **storage_options): """Create and configure file-system instance @@ -615,11 +615,9 @@ def glob(self, path, maxdepth=None, **kwargs): p: info for p, info in sorted(allpaths.items()) if pattern.match( - ( - p + "/" - if append_slash_to_dirname and info["type"] == "directory" - else p - ) + p + "/" + if append_slash_to_dirname and info["type"] == "directory" + else p ) } @@ -1442,7 +1440,7 @@ def from_json(blob: str) -> AbstractFileSystem: return json.loads(blob, cls=FilesystemJSONDecoder) - def to_dict(self, *, include_password: bool = True) -> Dict[str, Any]: + def to_dict(self, *, include_password: bool = True) -> dict[str, Any]: """ JSON-serializable dictionary representation of this filesystem instance. @@ -1483,7 +1481,7 @@ def to_dict(self, *, include_password: bool = True) -> Dict[str, Any]: ) @staticmethod - def from_dict(dct: Dict[str, Any]) -> AbstractFileSystem: + def from_dict(dct: dict[str, Any]) -> AbstractFileSystem: """ Recreate a filesystem instance from dictionary representation. diff --git a/fsspec/tests/abstract/__init__.py b/fsspec/tests/abstract/__init__.py index 44181420f..2df70a2e3 100644 --- a/fsspec/tests/abstract/__init__.py +++ b/fsspec/tests/abstract/__init__.py @@ -225,7 +225,7 @@ def _10_files_with_hashed_names(self, some_fs, some_join, some_path): for i in range(10): hashed_i = md5(str(i).encode("utf-8")).hexdigest() path = some_join(source, f"{hashed_i}.txt") - some_fs.pipe(path=path, value=f"{i}".encode("utf-8")) + some_fs.pipe(path=path, value=f"{i}".encode()) return source diff --git a/fsspec/tests/test_async.py b/fsspec/tests/test_async.py index aa3c9bd4f..4282996ad 100644 --- a/fsspec/tests/test_async.py +++ b/fsspec/tests/test_async.py @@ -208,7 +208,7 @@ async def get_data(self): async def test_async_streamed_file_write(): test_fs = DummyAsyncFS() streamed_file = await test_fs.open_async("misc/foo.txt", mode="wb") - inp_data = "foo-bar".encode("utf8") * streamed_file.blocksize * 2 + inp_data = b"foo-bar" * streamed_file.blocksize * 2 await streamed_file.write(inp_data) assert streamed_file.loc == len(inp_data) await streamed_file.close() diff --git a/fsspec/tests/test_fuse.py b/fsspec/tests/test_fuse.py index 5222d2fd9..ef3005367 100644 --- a/fsspec/tests/test_fuse.py +++ b/fsspec/tests/test_fuse.py @@ -121,8 +121,7 @@ def test_chmod(mount_local): cp = subprocess.run( ["cp", str(mount_dir / "text"), str(mount_dir / "new")], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + capture_output=True, check=False, ) diff --git a/fsspec/tests/test_utils.py b/fsspec/tests/test_utils.py index 1eeee912b..c10554613 100644 --- a/fsspec/tests/test_utils.py +++ b/fsspec/tests/test_utils.py @@ -45,9 +45,7 @@ def test_read_block(): def test_read_block_split_before(): """Test start/middle/end cases of split_before.""" - d = ( - "#header" + "".join(">foo{i}\nFOOBAR{i}\n".format(i=i) for i in range(100000)) - ).encode() + d = ("#header" + "".join(f">foo{i}\nFOOBAR{i}\n" for i in range(100000))).encode() # Read single record at beginning. # All reads include beginning of file and read through termination of diff --git a/fsspec/utils.py b/fsspec/utils.py index faa63937f..3c624be9c 100644 --- a/fsspec/utils.py +++ b/fsspec/utils.py @@ -82,7 +82,8 @@ def infer_storage_options( # https://msdn.microsoft.com/en-us/library/jj710207.aspx windows_path = re.match(r"^/([a-zA-Z])[:|]([\\/].*)$", path) if windows_path: - path = "%s:%s" % windows_path.groups() + drive, path = windows_path.groups() + path = f"{drive}:{path}" if protocol in ["http", "https"]: # for HTTP, we don't want to parse, as requests will anyway diff --git a/pyproject.toml b/pyproject.toml index 75a70d43f..ef707f50d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -184,6 +184,8 @@ select = [ "RUF024", "SIM", "SLOT", + "SIM101", + "UP", ] ignore = [ # Loop control variable `loop` not used within loop body @@ -208,6 +210,12 @@ ignore = [ # Fix these codes later "G004", "PERF203", + "UP007", + "UP011", + "UP015", + "UP018", + # deprecated + "UP027", "SIM102", "SIM105", "SIM108",