Skip to content

Commit

Permalink
remove old glob version for python 3.12 and up
Browse files Browse the repository at this point in the history
  • Loading branch information
liormizr committed Nov 9, 2024
1 parent 0be7cd9 commit 602e9c1
Show file tree
Hide file tree
Showing 3 changed files with 33 additions and 62 deletions.
File renamed without changes.
64 changes: 29 additions & 35 deletions s3path/current_version.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,6 @@ def register_configuration_parameter(
raise TypeError(f'parameters argument have to be a dict type. got {type(path)}')
if parameters is None and resource is None and glob_new_algorithm is None:
raise ValueError('user have to specify parameters or resource arguments')
if glob_new_algorithm is False and sys.version_info >= (3, 13):
raise ValueError('old glob algorithm can only be used by python versions below 3.13')
accessor.configuration_map.set_configuration(
path,
resource=resource,
Expand Down Expand Up @@ -463,20 +461,45 @@ def glob(self, pattern: str, *, case_sensitive=None, recurse_symlinks=False):
"""
Glob the given relative pattern in the Bucket / key prefix represented by this path,
yielding all matching files (of any kind)
The glob method is using a new Algorithm that better fit S3 API
"""
self._absolute_path_validation()
if case_sensitive is False or recurse_symlinks is True:
raise ValueError('Glob is case-sensitive and no symbolic links are allowed')

yield from self._glob(pattern)
sys.audit("pathlib.Path.glob", self, pattern)
if not pattern:
raise ValueError(f'Unacceptable pattern: {pattern}')
drv, root, pattern_parts = self._parse_path(pattern)
if drv or root:
raise NotImplementedError("Non-relative patterns are unsupported")
for part in pattern_parts:
if part != '**' and '**' in part:
raise ValueError("Invalid pattern: '**' can only be an entire path component")
selector = _Selector(self, pattern=pattern)
yield from selector.select()

def rglob(self, pattern: str, *, case_sensitive=None, recurse_symlinks=False):
"""
This is like calling S3Path.glob with "**/" added in front of the given relative pattern
The rglob method is using a new Algorithm that better fit S3 API
"""
self._absolute_path_validation()

yield from self._rglob(pattern)
sys.audit("pathlib.Path.rglob", self, pattern)
if not pattern:
raise ValueError(f'Unacceptable pattern: {pattern}')
drv, root, pattern_parts = self._parse_path(pattern)
if drv or root:
raise NotImplementedError("Non-relative patterns are unsupported")
for part in pattern_parts:
if part != '**' and '**' in part:
raise ValueError("Invalid pattern: '**' can only be an entire path component")
pattern = f'**{self._flavour.sep}{pattern}'
selector = _Selector(self, pattern=pattern)
yield from selector.select()

def get_presigned_url(self, expire_in: Union[timedelta, int] = 3600) -> str:
"""
Expand Down Expand Up @@ -555,35 +578,6 @@ def _scandir(self):
"""
return accessor.scandir(self)

def _glob(self, pattern):
""" Glob with new Algorithm that better fit S3 API """
sys.audit("pathlib.Path.glob", self, pattern)
if not pattern:
raise ValueError(f'Unacceptable pattern: {pattern}')
drv, root, pattern_parts = self._parse_path(pattern)
if drv or root:
raise NotImplementedError("Non-relative patterns are unsupported")
for part in pattern_parts:
if part != '**' and '**' in part:
raise ValueError("Invalid pattern: '**' can only be an entire path component")
selector = _Selector(self, pattern=pattern)
yield from selector.select()

def _rglob(self, pattern):
""" RGlob with new Algorithm that better fit S3 API """
sys.audit("pathlib.Path.rglob", self, pattern)
if not pattern:
raise ValueError(f'Unacceptable pattern: {pattern}')
drv, root, pattern_parts = self._parse_path(pattern)
if drv or root:
raise NotImplementedError("Non-relative patterns are unsupported")
for part in pattern_parts:
if part != '**' and '**' in part:
raise ValueError("Invalid pattern: '**' can only be an entire path component")
pattern = f'**{self._flavour.sep}{pattern}'
selector = _Selector(self, pattern=pattern)
yield from selector.select()


class PureVersionedS3Path(PureS3Path):
"""
Expand Down Expand Up @@ -671,8 +665,8 @@ class VersionedS3Path(PureVersionedS3Path, S3Path):
<< VersionedS3Path('/<bucket>/<key>', version_id='<version_id>')
"""

# def __init__(self, *args, version_id):
# super().__init__(*args)
def __init__(self, *args, version_id):
super().__init__(*args)


def _is_wildcard_pattern(pat):
Expand Down
31 changes: 4 additions & 27 deletions tests/test_path_operations.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
from tempfile import NamedTemporaryFile

import boto3
import ipdb
import requests
from botocore.exceptions import ClientError
import pytest
Expand Down Expand Up @@ -165,15 +164,9 @@ def test_glob_nested_folders_issue_no_120(s3_mock):
assert list(path.glob("further/*")) == [S3Path('/my-bucket/s3path-test/nested/further/test.txt')]


@pytest.mark.skipif(sys.version_info >= (3, 13), reason="requires python3.12 or lower")
def test_glob_old_algo(s3_mock, enable_old_glob):
# import ipdb; ipdb.set_trace()
test_glob(s3_mock)


@pytest.mark.skipif(sys.version_info >= (3, 13), reason="requires python3.12 or lower")
def test_glob_nested_folders_issue_no_115_old_algo(s3_mock, enable_old_glob):
test_glob_nested_folders_issue_no_115(s3_mock)
with pytest.deprecated_call():
test_glob(s3_mock)


def test_glob_issue_160(s3_mock):
Expand Down Expand Up @@ -249,21 +242,6 @@ def test_glob_nested_folders_issue_no_179(s3_mock):
S3Path('/my-bucket/s3path/nested/further/andfurther')]


@pytest.mark.skipif(sys.version_info >= (3, 13), reason="requires python3.12 or lower")
def test_glob_issue_160_old_algo(s3_mock, enable_old_glob):
test_glob_issue_160(s3_mock)


@pytest.mark.skipif(sys.version_info >= (3, 13), reason="requires python3.12 or lower")
def test_glob_issue_160_weird_behavior_old_algo(s3_mock, enable_old_glob):
test_glob_issue_160_weird_behavior(s3_mock)


@pytest.mark.skipif(sys.version_info >= (3, 13), reason="requires python3.12 or lower")
def test_glob_nested_folders_issue_no_179_old_algo(s3_mock, enable_old_glob):
test_glob_nested_folders_issue_no_179(s3_mock)


def test_rglob(s3_mock):
s3 = boto3.resource('s3')
s3.create_bucket(Bucket='test-bucket')
Expand Down Expand Up @@ -292,7 +270,6 @@ def test_rglob(s3_mock):
S3Path('/test-bucket/test_pathlib.py')]


@pytest.mark.skipif(sys.version_info >= (3, 13), reason="requires python3.12 or lower")
def test_rglob_old_algo(s3_mock, enable_old_glob):
test_rglob(s3_mock)

Expand Down Expand Up @@ -321,9 +298,9 @@ def test_accessor_scandir(s3_mock):
S3Path('/test-bucket/test_pathlib.py')]


@pytest.mark.skipif(sys.version_info >= (3, 13), reason="requires python3.12 or lower")
def test_accessor_scandir_old_algo(s3_mock, enable_old_glob):
test_accessor_scandir(s3_mock)
with pytest.deprecated_call():
test_accessor_scandir(s3_mock)


def test_is_dir(s3_mock):
Expand Down

0 comments on commit 602e9c1

Please sign in to comment.