From fb2dd31a7eb358d9399f21f1455d7cf47cc0138b Mon Sep 17 00:00:00 2001 From: liormizrahi Date: Sat, 9 Nov 2024 17:14:14 +0200 Subject: [PATCH 1/9] WIP --- .github/workflows/testing.yml | 2 +- .gitignore | 34 +-------------- Pipfile | 1 + pyproject.toml | 60 ++++++++++++++++++++++++++ s3path/accessor.py | 3 ++ s3path/current_version.py | 68 +++++++++++++++++++----------- setup.py | 25 +++++------ tests/test_path_operations.py | 2 + tests/test_pure_path_operations.py | 2 + 9 files changed, 125 insertions(+), 72 deletions(-) create mode 100644 pyproject.toml diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 6b43e22..63bbbff 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8, 3.9, "3.10", 3.11, 3.12] + python-version: [3.9, "3.10", 3.11, 3.12, 3.13] steps: - uses: actions/checkout@v2 diff --git a/.gitignore b/.gitignore index 6361508..0a750f6 100644 --- a/.gitignore +++ b/.gitignore @@ -3,9 +3,6 @@ __pycache__/ *.py[cod] *$py.class -# C extensions -*.so - # Distribution / packaging .Python build/ @@ -47,40 +44,15 @@ coverage.xml .hypothesis/ .pytest_cache/ -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - # Sphinx documentation docs/_build/ # PyBuilder target/ -# Jupyter Notebook -.ipynb_checkpoints - # pyenv .python-version -# celery beat schedule file -celerybeat-schedule - -# SageMath parsed files -*.sage.py - # Environments .env .venv @@ -97,13 +69,11 @@ venv.bak/ # Rope project settings .ropeproject -# mkdocs documentation -/site - # mypy .mypy_cache/ # PyCharm .idea/ -pyproject.toml +# Pipfile +Pipfile.lock \ No newline at end of file diff --git a/Pipfile b/Pipfile index ff0d577..f98efda 100644 --- a/Pipfile +++ b/Pipfile @@ -2,6 +2,7 @@ url = "https://pypi.org/simple" verify_ssl = true name = "pypi" +python_version = "3.13" [packages] moto = "*" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..9642adb --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,60 @@ +[project] +name = 's3path' +dynamic = ['version', 'description'] +requires-python = '>=3.9' +authors = [{name = 'Lior Mizrahi', email = 'li.mizr@gmail.com'}] +maintainers = [{name = 'Lior Mizrahi', email = 'li.mizr@gmail.com'}] +readme = {file = 'README.txt', content-type = 'text/x-rst'} +license = {file = 'LICENSE', text = 'Apache 2.0'} +dependencies = [ + 'boto3>=1.16.35', + 'smart-open>=5.1.0', +] +classifiers = [ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'Natural Language :: English', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', +] + +[project.urls] +Homepage = 'https://github.com/liormizr/s3path' +Documentation = 'https://github.com/liormizr/s3path?tab=readme-ov-file' +Repository = 'https://github.com/liormizr/s3path.git' +Issues = 'https://github.com/liormizr/s3path/issues' +Releases = 'https://github.com/liormizr/s3path/releases' + +[project.optional-dependencies] +dev = [ + 'moto', + 'pytest', + 'sphinx', + 'twine', + 'pytest-cov', + 'ipython', + 'ipdb', + 's3path' +] + +[build-system] +requires = ['setuptools>=61.2'] +build-backend = 'setuptools.build_meta' + +[tool.distutils.bdist_rpm] +doc-files = 'LICENSE README.rst' + +[tool.setuptools] +packages = ['s3path'] +license-files = ['LICENSE'] +include-package-data = false + +[tool.setuptools.dynamic] +version = {attr = 's3path.__version__'} +description = {file = 'REAMDE.rst', content-type = 'text/x-rst'} diff --git a/s3path/accessor.py b/s3path/accessor.py index fc7f22d..5b46f66 100644 --- a/s3path/accessor.py +++ b/s3path/accessor.py @@ -1,5 +1,6 @@ import sys import importlib.util +from warnings import warn from os import stat_result from threading import Lock from itertools import chain @@ -493,6 +494,8 @@ def set_configuration(self, path, *, resource=None, arguments=None, glob_new_alg if resource is not None: self.resources[path_name] = resource if glob_new_algorithm is not None: + warn(f'glob_new_algorithm Configuration is Deprecated, ' + f'in the new version we use only the new algorithm for Globing', category=DeprecationWarning) self.general_options[path_name] = {'glob_new_algorithm': glob_new_algorithm} self.get_configuration.cache_clear() diff --git a/s3path/current_version.py b/s3path/current_version.py index b77f83e..80a8922 100644 --- a/s3path/current_version.py +++ b/s3path/current_version.py @@ -41,22 +41,14 @@ def register_configuration_parameter( glob_new_algorithm=glob_new_algorithm) - -class _S3Flavour: - def __getattr__(self, name): - return getattr(posixpath, name) - - -flavour = _S3Flavour() - - class PureS3Path(PurePath): """ PurePath subclass for AWS S3 service. S3 is not a file-system but we can look at it like a POSIX system. """ - _flavour = flavour + _flavour = posixpath + parser = posixpath __slots__ = () def __init__(self, *args): @@ -70,7 +62,25 @@ def __init__(self, *args): new_parts.remove(part) self._raw_paths = new_parts - self._load_parts() + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + # self._load_parts() + # if sys.version_info >= (3, 13): + # self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + # else: + # self._load_parts() + + def _load_parts(self): + paths = self._raw_paths + if len(paths) == 0: + path = '' + elif len(paths) == 1: + path = paths[0] + else: + path = self._flavour.join(*paths) + drv, root, tail = self._parse_path(path) + self._drv = drv + self._root = root + self._tail_cached = tail @classmethod def from_uri(cls, uri: str): @@ -277,6 +287,10 @@ def is_mount(self) -> Literal[False]: class S3Path(_PathNotSupportedMixin, Path, PureS3Path): + @classmethod + def from_uri(cls, uri): + return cls(PureS3Path.from_uri(uri)) + def stat(self, *, follow_symlinks: bool = True) -> accessor.StatResult: """ Returns information about this path (similarly to boto3's ObjectSummary). @@ -425,13 +439,13 @@ def exists(self) -> bool: return True return accessor.exists(self) - def iterdir(self): # todo: -> Generator[S3Path, None, None]: + def iterdir(self): """ When the path points to a Bucket or a key prefix, yield path objects of the directory contents """ self._absolute_path_validation() for name in accessor.listdir(self): - yield self._make_child_relpath(name) + yield self / name def open( self, @@ -452,29 +466,33 @@ def open( errors=errors, newline=newline) - def glob(self, pattern: str): # todo: -> Generator[S3Path, None, None]: + def glob(self, pattern: str, *, case_sensitive=None, recurse_symlinks=False): """ Glob the given relative pattern in the Bucket / key prefix represented by this path, yielding all matching files (of any kind) """ self._absolute_path_validation() - general_options = accessor.configuration_map.get_general_options(self) - glob_new_algorithm = general_options['glob_new_algorithm'] - if not glob_new_algorithm: - yield from super().glob(pattern) - return + if case_sensitive is False or recurse_symlinks is True: + raise ValueError('Glob is case-sensitive and no symbolic links are allowed') + + # general_options = accessor.configuration_map.get_general_options(self) + # glob_new_algorithm = general_options['glob_new_algorithm'] + # import ipdb; ipdb.set_trace() + # if not glob_new_algorithm: + # yield from super().glob(pattern) + # return yield from self._glob(pattern) - def rglob(self, pattern: str): # todo: -> Generator[S3Path, None, None]: + def rglob(self, pattern: str, *, case_sensitive=None, recurse_symlinks=False): """ This is like calling S3Path.glob with "**/" added in front of the given relative pattern """ self._absolute_path_validation() - general_options = accessor.configuration_map.get_general_options(self) - glob_new_algorithm = general_options['glob_new_algorithm'] - if not glob_new_algorithm: - yield from super().rglob(pattern) - return + # general_options = accessor.configuration_map.get_general_options(self) + # glob_new_algorithm = general_options['glob_new_algorithm'] + # if not glob_new_algorithm: + # yield from super().rglob(pattern) + # return yield from self._rglob(pattern) def get_presigned_url(self, expire_in: Union[timedelta, int] = 3600) -> str: diff --git a/setup.py b/setup.py index dc8e5e6..9891de3 100644 --- a/setup.py +++ b/setup.py @@ -4,32 +4,29 @@ with open("README.rst", "r") as fh: long_description = fh.read() setup( - name='s3path', - version='0.5.8', - url='https://github.com/liormizr/s3path', - author='Lior Mizrahi', - author_email='li.mizr@gmail.com', - packages=['s3path'], - install_requires=[ - 'boto3>=1.16.35', - 'smart-open>=5.1.0', - ], - license='Apache 2.0', + name='s3path', # V + version='0.5.8', # V + url='https://github.com/liormizr/s3path', # V + author='Lior Mizrahi', # V + author_email='li.mizr@gmail.com', # V + packages=['s3path'], # V + install_requires=['boto3>=1.16.35','smart-open>=5.1.0',], # V + license='Apache 2.0', # V long_description=long_description, long_description_content_type='text/x-rst', - python_requires='>=3.8', + python_requires='>=3.9', # V include_package_data=True, - classifiers=[ + classifiers=[ # V 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', 'Natural Language :: English', 'License :: OSI Approved :: Apache Software License', 'Operating System :: OS Independent', 'Programming Language :: Python', - 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', ], ) diff --git a/tests/test_path_operations.py b/tests/test_path_operations.py index 5f84b71..7e393c6 100644 --- a/tests/test_path_operations.py +++ b/tests/test_path_operations.py @@ -6,6 +6,7 @@ from tempfile import NamedTemporaryFile import boto3 +import ipdb import requests from botocore.exceptions import ClientError import pytest @@ -165,6 +166,7 @@ def test_glob_nested_folders_issue_no_120(s3_mock): def test_glob_old_algo(s3_mock, enable_old_glob): + # import ipdb; ipdb.set_trace() test_glob(s3_mock) diff --git a/tests/test_pure_path_operations.py b/tests/test_pure_path_operations.py index a96a720..062e0af 100644 --- a/tests/test_pure_path_operations.py +++ b/tests/test_pure_path_operations.py @@ -1,9 +1,11 @@ import os +import sys import pytest from pathlib import Path, PurePosixPath, PureWindowsPath from s3path import PureS3Path +@pytest.mark.skipif(sys.version_info > (3, 12), reason="Not supported on python3.13 or higher") def test_paths_of_a_different_flavour(): with pytest.raises(TypeError): PureS3Path('/bucket/key') < PurePosixPath('/bucket/key') From 602e9c1ea03fb0672e7809affd67ed0f327b127f Mon Sep 17 00:00:00 2001 From: liormizrahi Date: Sat, 9 Nov 2024 18:07:24 +0200 Subject: [PATCH 2/9] remove old glob version for python 3.12 and up --- pyproject.toml => _pyproject.toml | 0 s3path/current_version.py | 64 ++++++++++++++----------------- tests/test_path_operations.py | 31 ++------------- 3 files changed, 33 insertions(+), 62 deletions(-) rename pyproject.toml => _pyproject.toml (100%) diff --git a/pyproject.toml b/_pyproject.toml similarity index 100% rename from pyproject.toml rename to _pyproject.toml diff --git a/s3path/current_version.py b/s3path/current_version.py index 2e61aa2..5d1d0bc 100644 --- a/s3path/current_version.py +++ b/s3path/current_version.py @@ -34,8 +34,6 @@ def register_configuration_parameter( raise TypeError(f'parameters argument have to be a dict type. got {type(path)}') if parameters is None and resource is None and glob_new_algorithm is None: raise ValueError('user have to specify parameters or resource arguments') - if glob_new_algorithm is False and sys.version_info >= (3, 13): - raise ValueError('old glob algorithm can only be used by python versions below 3.13') accessor.configuration_map.set_configuration( path, resource=resource, @@ -463,20 +461,45 @@ def glob(self, pattern: str, *, case_sensitive=None, recurse_symlinks=False): """ Glob the given relative pattern in the Bucket / key prefix represented by this path, yielding all matching files (of any kind) + + The glob method is using a new Algorithm that better fit S3 API """ self._absolute_path_validation() if case_sensitive is False or recurse_symlinks is True: raise ValueError('Glob is case-sensitive and no symbolic links are allowed') - yield from self._glob(pattern) + sys.audit("pathlib.Path.glob", self, pattern) + if not pattern: + raise ValueError(f'Unacceptable pattern: {pattern}') + drv, root, pattern_parts = self._parse_path(pattern) + if drv or root: + raise NotImplementedError("Non-relative patterns are unsupported") + for part in pattern_parts: + if part != '**' and '**' in part: + raise ValueError("Invalid pattern: '**' can only be an entire path component") + selector = _Selector(self, pattern=pattern) + yield from selector.select() def rglob(self, pattern: str, *, case_sensitive=None, recurse_symlinks=False): """ This is like calling S3Path.glob with "**/" added in front of the given relative pattern + + The rglob method is using a new Algorithm that better fit S3 API """ self._absolute_path_validation() - yield from self._rglob(pattern) + sys.audit("pathlib.Path.rglob", self, pattern) + if not pattern: + raise ValueError(f'Unacceptable pattern: {pattern}') + drv, root, pattern_parts = self._parse_path(pattern) + if drv or root: + raise NotImplementedError("Non-relative patterns are unsupported") + for part in pattern_parts: + if part != '**' and '**' in part: + raise ValueError("Invalid pattern: '**' can only be an entire path component") + pattern = f'**{self._flavour.sep}{pattern}' + selector = _Selector(self, pattern=pattern) + yield from selector.select() def get_presigned_url(self, expire_in: Union[timedelta, int] = 3600) -> str: """ @@ -555,35 +578,6 @@ def _scandir(self): """ return accessor.scandir(self) - def _glob(self, pattern): - """ Glob with new Algorithm that better fit S3 API """ - sys.audit("pathlib.Path.glob", self, pattern) - if not pattern: - raise ValueError(f'Unacceptable pattern: {pattern}') - drv, root, pattern_parts = self._parse_path(pattern) - if drv or root: - raise NotImplementedError("Non-relative patterns are unsupported") - for part in pattern_parts: - if part != '**' and '**' in part: - raise ValueError("Invalid pattern: '**' can only be an entire path component") - selector = _Selector(self, pattern=pattern) - yield from selector.select() - - def _rglob(self, pattern): - """ RGlob with new Algorithm that better fit S3 API """ - sys.audit("pathlib.Path.rglob", self, pattern) - if not pattern: - raise ValueError(f'Unacceptable pattern: {pattern}') - drv, root, pattern_parts = self._parse_path(pattern) - if drv or root: - raise NotImplementedError("Non-relative patterns are unsupported") - for part in pattern_parts: - if part != '**' and '**' in part: - raise ValueError("Invalid pattern: '**' can only be an entire path component") - pattern = f'**{self._flavour.sep}{pattern}' - selector = _Selector(self, pattern=pattern) - yield from selector.select() - class PureVersionedS3Path(PureS3Path): """ @@ -671,8 +665,8 @@ class VersionedS3Path(PureVersionedS3Path, S3Path): << VersionedS3Path('//', version_id='') """ - # def __init__(self, *args, version_id): - # super().__init__(*args) + def __init__(self, *args, version_id): + super().__init__(*args) def _is_wildcard_pattern(pat): diff --git a/tests/test_path_operations.py b/tests/test_path_operations.py index 484014b..28b6d43 100644 --- a/tests/test_path_operations.py +++ b/tests/test_path_operations.py @@ -6,7 +6,6 @@ from tempfile import NamedTemporaryFile import boto3 -import ipdb import requests from botocore.exceptions import ClientError import pytest @@ -165,15 +164,9 @@ def test_glob_nested_folders_issue_no_120(s3_mock): assert list(path.glob("further/*")) == [S3Path('/my-bucket/s3path-test/nested/further/test.txt')] -@pytest.mark.skipif(sys.version_info >= (3, 13), reason="requires python3.12 or lower") def test_glob_old_algo(s3_mock, enable_old_glob): - # import ipdb; ipdb.set_trace() - test_glob(s3_mock) - - -@pytest.mark.skipif(sys.version_info >= (3, 13), reason="requires python3.12 or lower") -def test_glob_nested_folders_issue_no_115_old_algo(s3_mock, enable_old_glob): - test_glob_nested_folders_issue_no_115(s3_mock) + with pytest.deprecated_call(): + test_glob(s3_mock) def test_glob_issue_160(s3_mock): @@ -249,21 +242,6 @@ def test_glob_nested_folders_issue_no_179(s3_mock): S3Path('/my-bucket/s3path/nested/further/andfurther')] -@pytest.mark.skipif(sys.version_info >= (3, 13), reason="requires python3.12 or lower") -def test_glob_issue_160_old_algo(s3_mock, enable_old_glob): - test_glob_issue_160(s3_mock) - - -@pytest.mark.skipif(sys.version_info >= (3, 13), reason="requires python3.12 or lower") -def test_glob_issue_160_weird_behavior_old_algo(s3_mock, enable_old_glob): - test_glob_issue_160_weird_behavior(s3_mock) - - -@pytest.mark.skipif(sys.version_info >= (3, 13), reason="requires python3.12 or lower") -def test_glob_nested_folders_issue_no_179_old_algo(s3_mock, enable_old_glob): - test_glob_nested_folders_issue_no_179(s3_mock) - - def test_rglob(s3_mock): s3 = boto3.resource('s3') s3.create_bucket(Bucket='test-bucket') @@ -292,7 +270,6 @@ def test_rglob(s3_mock): S3Path('/test-bucket/test_pathlib.py')] -@pytest.mark.skipif(sys.version_info >= (3, 13), reason="requires python3.12 or lower") def test_rglob_old_algo(s3_mock, enable_old_glob): test_rglob(s3_mock) @@ -321,9 +298,9 @@ def test_accessor_scandir(s3_mock): S3Path('/test-bucket/test_pathlib.py')] -@pytest.mark.skipif(sys.version_info >= (3, 13), reason="requires python3.12 or lower") def test_accessor_scandir_old_algo(s3_mock, enable_old_glob): - test_accessor_scandir(s3_mock) + with pytest.deprecated_call(): + test_accessor_scandir(s3_mock) def test_is_dir(s3_mock): From 719a45387849955f1a7e51c32834f552dcd5121e Mon Sep 17 00:00:00 2001 From: liormizrahi Date: Sat, 9 Nov 2024 18:20:49 +0200 Subject: [PATCH 3/9] check if needed drv, root initial creation --- s3path/current_version.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/s3path/current_version.py b/s3path/current_version.py index 5d1d0bc..2cc2e39 100644 --- a/s3path/current_version.py +++ b/s3path/current_version.py @@ -62,20 +62,6 @@ def __init__(self, *args): new_parts.remove(part) self._raw_paths = new_parts - self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) - - def _load_parts(self): - paths = self._raw_paths - if len(paths) == 0: - path = '' - elif len(paths) == 1: - path = paths[0] - else: - path = self._flavour.join(*paths) - drv, root, tail = self._parse_path(path) - self._drv = drv - self._root = root - self._tail_cached = tail @classmethod def from_uri(cls, uri: str): From a2cdbb8d41cd6f627eb809a326a1521ae7f78859 Mon Sep 17 00:00:00 2001 From: liormizrahi Date: Sat, 9 Nov 2024 18:56:28 +0200 Subject: [PATCH 4/9] WIP --- s3path/current_version.py | 6 +++++- tests/test_path_operations.py | 17 ++++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/s3path/current_version.py b/s3path/current_version.py index 2cc2e39..0f5cc3b 100644 --- a/s3path/current_version.py +++ b/s3path/current_version.py @@ -47,7 +47,7 @@ class PureS3Path(PurePath): S3 is not a file-system but we can look at it like a POSIX system. """ - _flavour = posixpath + _flavour = posixpath # not relevant after Python version 3.13 parser = posixpath __slots__ = () @@ -62,6 +62,10 @@ def __init__(self, *args): new_parts.remove(part) self._raw_paths = new_parts + if sys.version_info >= (3, 13): + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + else: + self._load_parts() @classmethod def from_uri(cls, uri: str): diff --git a/tests/test_path_operations.py b/tests/test_path_operations.py index 28b6d43..68e269d 100644 --- a/tests/test_path_operations.py +++ b/tests/test_path_operations.py @@ -165,7 +165,10 @@ def test_glob_nested_folders_issue_no_120(s3_mock): def test_glob_old_algo(s3_mock, enable_old_glob): - with pytest.deprecated_call(): + if sys.version_info > (3, 12): + with pytest.deprecated_call(): + test_glob(s3_mock) + else: test_glob(s3_mock) @@ -271,7 +274,11 @@ def test_rglob(s3_mock): def test_rglob_old_algo(s3_mock, enable_old_glob): - test_rglob(s3_mock) + if sys.version_info > (3, 12): + with pytest.deprecated_call(): + test_rglob(s3_mock) + else: + test_rglob(s3_mock) def test_accessor_scandir(s3_mock): @@ -299,7 +306,10 @@ def test_accessor_scandir(s3_mock): def test_accessor_scandir_old_algo(s3_mock, enable_old_glob): - with pytest.deprecated_call(): + if sys.version_info > (3, 12): + with pytest.deprecated_call(): + test_accessor_scandir(s3_mock) + else: test_accessor_scandir(s3_mock) @@ -828,6 +838,7 @@ def test_unlink(s3_mock): S3Path("/test-bucket/fake_folder").unlink(missing_ok=True) S3Path("/fake-bucket/").unlink(missing_ok=True) + def test_absolute(s3_mock): s3 = boto3.resource('s3') s3.create_bucket(Bucket='test-bucket') From 5b7b0313d8a21f00da5505e5d313a131b6c84fd4 Mon Sep 17 00:00:00 2001 From: liormizrahi Date: Sat, 9 Nov 2024 19:00:51 +0200 Subject: [PATCH 5/9] WIP --- tests/test_s3path_configuration.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/test_s3path_configuration.py b/tests/test_s3path_configuration.py index b69184b..f042862 100644 --- a/tests/test_s3path_configuration.py +++ b/tests/test_s3path_configuration.py @@ -142,9 +142,3 @@ def test_issue_123(): new_resource, _ = accessor.configuration_map.get_configuration(path) assert new_resource is s3 assert new_resource is not old_resource - - -@pytest.mark.skipif(sys.version_info < (3, 13), reason="requires python3.13 or higher") -def test_register_configuration_parameter_old_algo(): - with pytest.raises(ValueError): - register_configuration_parameter(PureS3Path('/'), glob_new_algorithm=False) \ No newline at end of file From f63e731cb0e2f4899419bad23324e196953c2f2a Mon Sep 17 00:00:00 2001 From: liormizrahi Date: Sat, 9 Nov 2024 19:03:23 +0200 Subject: [PATCH 6/9] WIP --- tests/test_pure_path_operations.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_pure_path_operations.py b/tests/test_pure_path_operations.py index 062e0af..1894b2a 100644 --- a/tests/test_pure_path_operations.py +++ b/tests/test_pure_path_operations.py @@ -5,7 +5,6 @@ from s3path import PureS3Path -@pytest.mark.skipif(sys.version_info > (3, 12), reason="Not supported on python3.13 or higher") def test_paths_of_a_different_flavour(): with pytest.raises(TypeError): PureS3Path('/bucket/key') < PurePosixPath('/bucket/key') From 7c46c19b0bef4944b017f81b0dd8f77919fb8a8e Mon Sep 17 00:00:00 2001 From: liormizrahi Date: Sat, 9 Nov 2024 19:16:15 +0200 Subject: [PATCH 7/9] cleaned up _flavour atter --- s3path/current_version.py | 46 ++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/s3path/current_version.py b/s3path/current_version.py index 0f5cc3b..6b3c036 100644 --- a/s3path/current_version.py +++ b/s3path/current_version.py @@ -41,14 +41,20 @@ def register_configuration_parameter( glob_new_algorithm=glob_new_algorithm) +class _S3Parser: + def __getattr__(self, name): + return getattr(posixpath, name) + + class PureS3Path(PurePath): """ PurePath subclass for AWS S3 service. S3 is not a file-system but we can look at it like a POSIX system. """ - _flavour = posixpath # not relevant after Python version 3.13 - parser = posixpath + + parser = _flavour = _S3Parser() # _flavour is not relevant after Python version 3.13 + __slots__ = () def __init__(self, *args): @@ -90,7 +96,7 @@ def from_bucket_key(cls, bucket: str, key: str): >> PureS3Path.from_bucket_key(bucket='', key='') << PureS3Path('//') """ - bucket = cls(cls._flavour.sep, bucket) + bucket = cls(cls.parser.sep, bucket) if len(bucket.parts) != 2: raise ValueError(f'bucket argument contains more then one path element: {bucket}') key = cls(key) @@ -122,7 +128,7 @@ def key(self) -> str: The AWS S3 Key name, or '' """ self._absolute_path_validation() - key = self._flavour.sep.join(self.parts[2:]) + key = self.parser.sep.join(self.parts[2:]) return key def as_uri(self) -> str: @@ -379,7 +385,7 @@ def mkdir(self, mode: int = 0o777, parents: bool = False, exist_ok: bool = False raise FileNotFoundError(f'No bucket in {type(self)} {self}') if self.key and not parents: raise FileNotFoundError(f'Only bucket path can be created, got {self}') - if type(self)(self._flavour.sep, self.bucket).exists(): + if type(self)(self.parser.sep, self.bucket).exists(): raise FileExistsError(f'Bucket {self.bucket} already exists') accessor.mkdir(self, mode) except OSError: @@ -487,7 +493,7 @@ def rglob(self, pattern: str, *, case_sensitive=None, recurse_symlinks=False): for part in pattern_parts: if part != '**' and '**' in part: raise ValueError("Invalid pattern: '**' can only be an entire path component") - pattern = f'**{self._flavour.sep}{pattern}' + pattern = f'**{self.parser.sep}{pattern}' selector = _Selector(self, pattern=pattern) yield from selector.select() @@ -675,14 +681,14 @@ def __init__(self, path, *, pattern): def select(self): for target in self._deep_cached_dir_scan(): - target = f'{self._path._flavour.sep}{self._path.bucket}{target}' + target = f'{self._path.parser.sep}{self._path.bucket}{target}' if self.match(target): yield type(self._path)(target) def _prefix_splitter(self, pattern): if not _is_wildcard_pattern(pattern): if self._path.key: - return f'{self._path.key}{self._path._flavour.sep}{pattern}', '' + return f'{self._path.key}{self._path.parser.sep}{pattern}', '' return pattern, '' *_, pattern_parts = self._path._parse_path(pattern) @@ -690,21 +696,21 @@ def _prefix_splitter(self, pattern): for index, part in enumerate(pattern_parts): if _is_wildcard_pattern(part): break - prefix += f'{part}{self._path._flavour.sep}' + prefix += f'{part}{self._path.parser.sep}' if pattern.startswith(prefix): pattern = pattern.replace(prefix, '', 1) key_prefix = self._path.key if key_prefix: - prefix = self._path._flavour.sep.join((key_prefix, prefix)) + prefix = self._path.parser.sep.join((key_prefix, prefix)) return prefix, pattern def _calculate_pattern_level(self, pattern): if '**' in pattern: return None if self._prefix: - pattern = f'{self._prefix}{self._path._flavour.sep}{pattern}' + pattern = f'{self._prefix}{self._path.parser.sep}{pattern}' *_, pattern_parts = self._path._parse_path(pattern) return len(pattern_parts) @@ -719,23 +725,23 @@ def _calculate_full_or_just_folder(self, pattern): def _deep_cached_dir_scan(self): cache = set() - prefix_sep_count = self._prefix.count(self._path._flavour.sep) + prefix_sep_count = self._prefix.count(self._path.parser.sep) for key in accessor.iter_keys(self._path, prefix=self._prefix, full_keys=self._full_keys): - key_sep_count = key.count(self._path._flavour.sep) + 1 - key_parts = key.rsplit(self._path._flavour.sep, maxsplit=key_sep_count - prefix_sep_count) + key_sep_count = key.count(self._path.parser.sep) + 1 + key_parts = key.rsplit(self._path.parser.sep, maxsplit=key_sep_count - prefix_sep_count) target_path_parts = key_parts[:self._target_level] target_path = '' for part in target_path_parts: if not part: continue - target_path += f'{self._path._flavour.sep}{part}' + target_path += f'{self._path.parser.sep}{part}' if target_path in cache: continue yield target_path cache.add(target_path) def _compile_pattern_parts(self, prefix, pattern, bucket): - pattern = self._path._flavour.sep.join(( + pattern = self._path.parser.sep.join(( '', bucket, prefix, @@ -745,14 +751,14 @@ def _compile_pattern_parts(self, prefix, pattern, bucket): new_regex_pattern = '' for part in pattern_parts: - if part == self._path._flavour.sep: + if part == self._path.parser.sep: continue if '**' in part: - new_regex_pattern += f'{self._path._flavour.sep}*(?s:{part.replace("**", ".*")})' + new_regex_pattern += f'{self._path.parser.sep}*(?s:{part.replace("**", ".*")})' continue if '*' == part: - new_regex_pattern += f'{self._path._flavour.sep}(?s:[^/]+)' + new_regex_pattern += f'{self._path.parser.sep}(?s:[^/]+)' continue - new_regex_pattern += f'{self._path._flavour.sep}{fnmatch.translate(part)[:-2]}' + new_regex_pattern += f'{self._path.parser.sep}{fnmatch.translate(part)[:-2]}' new_regex_pattern += r'/*\Z' return re.compile(new_regex_pattern).fullmatch From c1768d94b9897f818b8eae594805831f65475a3f Mon Sep 17 00:00:00 2001 From: liormizrahi Date: Sat, 9 Nov 2024 19:39:30 +0200 Subject: [PATCH 8/9] add doc changes for glob --- docs/advance.rst | 1 + docs/interface.rst | 11 +++++++++-- s3path/__init__.py | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/advance.rst b/docs/advance.rst index e481b11..a20fb78 100644 --- a/docs/advance.rst +++ b/docs/advance.rst @@ -123,6 +123,7 @@ To enable the old (pathlib common) Algorithm you can configure it like this: >>> path = PureS3Path('/') >>> register_configuration_parameter(path, glob_new_algorithm=False) +**Note: from version 0.6.0 glob implementation will work only with the new algorithm, there for the glob_new_algorithm arg is in depreciation cycle** .. _pathlib : https://docs.python.org/3/library/pathlib.html .. _boto3 : https://github.com/boto/boto3 diff --git a/docs/interface.rst b/docs/interface.rst index a452b1d..699eb61 100644 --- a/docs/interface.rst +++ b/docs/interface.rst @@ -159,7 +159,8 @@ In other words, it enables recursive globbing: S3Path('/pypi-proxy/boto3/index.html'), S3Path('/pypi-proxy/botocore/index.html')] -New in version 0.4.0: + +In version 0.4.0: New Algorithm that better suited to s3 API. Especially for recursive searches. @@ -169,6 +170,9 @@ To enable the old (pathlib common) Algorithm you can configure it like this: register_configuration_parameter(path, glob_new_algorithm=False) +New version 0.6.0: +glob implementation will work only with the new algorithm, there for the glob_new_algorithm arg is in depreciation cycle + For more configuration details please see this `Advanced S3Path configuration`_ **NOTE:** Using the "**" pattern in large Buckets may consume an inordinate amount of time in the old algorithm. @@ -325,10 +329,13 @@ This is like calling S3Path.glob_ with ``"**/"`` added in front of the given rel S3Path('/pypi-proxy/index.html'), S3Path('/pypi-proxy/botocore/index.html')] -New in version 0.4.0: +Version 0.4.0: New Algorithm that better suited to s3 API. Especially for recursive searches. +New version 0.6.0: +glob implementation will work only with the new algorithm, there for the glob_new_algorithm arg is in depreciation cycle + S3Path.rmdir() ^^^^^^^^^^^^^^ diff --git a/s3path/__init__.py b/s3path/__init__.py index 7877e26..9661d94 100644 --- a/s3path/__init__.py +++ b/s3path/__init__.py @@ -5,7 +5,7 @@ from pathlib import Path from . import accessor -__version__ = '0.5.8' +__version__ = '0.6.0' __all__ = ( 'Path', 'register_configuration_parameter', From 42d48feb5762cbff80381ee31b306ce6362866d6 Mon Sep 17 00:00:00 2001 From: liormizrahi Date: Sat, 9 Nov 2024 19:49:22 +0200 Subject: [PATCH 9/9] cleanup --- _pyproject.toml | 60 --------------------------------------- s3path/current_version.py | 4 +-- setup.py | 20 ++++++------- 3 files changed, 12 insertions(+), 72 deletions(-) delete mode 100644 _pyproject.toml diff --git a/_pyproject.toml b/_pyproject.toml deleted file mode 100644 index 9642adb..0000000 --- a/_pyproject.toml +++ /dev/null @@ -1,60 +0,0 @@ -[project] -name = 's3path' -dynamic = ['version', 'description'] -requires-python = '>=3.9' -authors = [{name = 'Lior Mizrahi', email = 'li.mizr@gmail.com'}] -maintainers = [{name = 'Lior Mizrahi', email = 'li.mizr@gmail.com'}] -readme = {file = 'README.txt', content-type = 'text/x-rst'} -license = {file = 'LICENSE', text = 'Apache 2.0'} -dependencies = [ - 'boto3>=1.16.35', - 'smart-open>=5.1.0', -] -classifiers = [ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Developers', - 'Natural Language :: English', - 'License :: OSI Approved :: Apache Software License', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: 3.13', -] - -[project.urls] -Homepage = 'https://github.com/liormizr/s3path' -Documentation = 'https://github.com/liormizr/s3path?tab=readme-ov-file' -Repository = 'https://github.com/liormizr/s3path.git' -Issues = 'https://github.com/liormizr/s3path/issues' -Releases = 'https://github.com/liormizr/s3path/releases' - -[project.optional-dependencies] -dev = [ - 'moto', - 'pytest', - 'sphinx', - 'twine', - 'pytest-cov', - 'ipython', - 'ipdb', - 's3path' -] - -[build-system] -requires = ['setuptools>=61.2'] -build-backend = 'setuptools.build_meta' - -[tool.distutils.bdist_rpm] -doc-files = 'LICENSE README.rst' - -[tool.setuptools] -packages = ['s3path'] -license-files = ['LICENSE'] -include-package-data = false - -[tool.setuptools.dynamic] -version = {attr = 's3path.__version__'} -description = {file = 'REAMDE.rst', content-type = 'text/x-rst'} diff --git a/s3path/current_version.py b/s3path/current_version.py index 6b3c036..5d05634 100644 --- a/s3path/current_version.py +++ b/s3path/current_version.py @@ -313,7 +313,7 @@ def owner(self) -> str: raise KeyError('file not found') return accessor.owner(self) - def rename(self, target): # todo: Union[str, S3Path]) -> S3Path: + def rename(self, target): """ Renames this file or Bucket / key prefix / key to the given target. If target exists and is a file, it will be replaced silently if the user has permission. @@ -327,7 +327,7 @@ def rename(self, target): # todo: Union[str, S3Path]) -> S3Path: accessor.rename(self, target) return type(self)(target) - def replace(self, target): # todo: Union[str, S3Path]) -> S3Path: + def replace(self, target): """ Renames this Bucket / key prefix / key to the given target. If target points to an existing Bucket / key prefix / key, it will be unconditionally replaced. diff --git a/setup.py b/setup.py index 9891de3..114befb 100644 --- a/setup.py +++ b/setup.py @@ -4,19 +4,19 @@ with open("README.rst", "r") as fh: long_description = fh.read() setup( - name='s3path', # V - version='0.5.8', # V - url='https://github.com/liormizr/s3path', # V - author='Lior Mizrahi', # V - author_email='li.mizr@gmail.com', # V - packages=['s3path'], # V - install_requires=['boto3>=1.16.35','smart-open>=5.1.0',], # V - license='Apache 2.0', # V + name='s3path', + version='0.5.8', + url='https://github.com/liormizr/s3path', + author='Lior Mizrahi', + author_email='li.mizr@gmail.com', + packages=['s3path'], + install_requires=['boto3>=1.16.35','smart-open>=5.1.0',], + license='Apache 2.0', long_description=long_description, long_description_content_type='text/x-rst', - python_requires='>=3.9', # V + python_requires='>=3.9', include_package_data=True, - classifiers=[ # V + classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', 'Natural Language :: English',