Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Sync to source master #735

Open
wants to merge 14 commits into
base: master
Choose a base branch
from
Prev Previous commit
Next Next commit
Python 10 compatibility
Milhan KIM committed May 2, 2022
commit 70ebc59fce6f27b3adedd7da8cf760c3f509df07
13 changes: 3 additions & 10 deletions pottery/base.py
Original file line number Diff line number Diff line change
@@ -27,7 +27,7 @@
import os
import uuid
import warnings
from typing import Any
from typing import Any, Final
from typing import AnyStr
from typing import ClassVar
from typing import ContextManager
@@ -42,21 +42,13 @@
from redis import RedisError
from redis.asyncio import Redis as AIORedis # type: ignore
from redis.client import Pipeline
# TODO: When we drop support for Python 3.7, change the following imports to:
# from typing import Final
# from typing import Protocol
# from typing import final
from typing_extensions import Final
from typing_extensions import Protocol
from typing_extensions import final

from .annotations import JSONTypes
from .exceptions import InefficientAccessWarning
from .exceptions import QuorumIsImpossible
from .exceptions import RandomKeyError
from .monkey import logger


_default_url: Final[str] = os.environ.get('REDIS_URL', 'redis://localhost:6379/0')
_default_redis: Final[Redis] = Redis.from_url(_default_url, socket_timeout=1)

@@ -80,7 +72,7 @@ def random_key(*,
key = random_key(
redis=redis,
prefix=prefix,
num_tries=num_tries-1,
num_tries=num_tries - 1,
)
return key

@@ -174,6 +166,7 @@ def key(self) -> str: # pragma: no cover

class _Clearable:
'Mixin class that implements clearing (emptying) a Redis-backed collection.'

def clear(self: _HasRedisClientAndKey) -> None:
'Remove the elements in a Redis-backed container. O(n)'
self.redis.unlink(self.key) # Available since Redis 4.0.0
10 changes: 5 additions & 5 deletions pottery/bloom.py
Original file line number Diff line number Diff line change
@@ -22,17 +22,14 @@
import math
import uuid
import warnings
from typing import Any
from typing import Any, final
from typing import Callable
from typing import Generator
from typing import Iterable
from typing import Set
from typing import cast

import mmh3
# TODO: When we drop support for Python 3.7, change the following import to:
# from typing import final
from typing_extensions import final

from .annotations import F
from .annotations import JSONTypes
@@ -45,6 +42,7 @@
# https://docs.python.org/3/library/functools.html#functools.cached_property
def _store_on_self(*, attr: str) -> Callable[[F], F]:
"Decorator to store/cache a method's return value as an attribute on self."

def decorator(func: F) -> F:
@functools.wraps(func)
def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any:
@@ -54,7 +52,9 @@ def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any:
value = func(self, *args, **kwargs)
setattr(self, attr, value)
return value

return cast(F, wrapper)

return decorator


@@ -121,7 +121,7 @@ def size(self) -> int:
size = (
-self.num_elements
* math.log(self.false_positives)
/ math.log(2)**2
/ math.log(2) ** 2
)
size = math.ceil(size)
return size
15 changes: 7 additions & 8 deletions pottery/cache.py
Original file line number Diff line number Diff line change
@@ -23,7 +23,7 @@
import collections
import functools
import itertools
from typing import Any
from typing import Any, Final
from typing import Callable
from typing import ClassVar
from typing import Collection
@@ -38,25 +38,21 @@

from redis import Redis
from redis.exceptions import WatchError
# TODO: When we drop support for Python 3.7, change the following import to:
# from typing import Final
from typing_extensions import Final

from .annotations import JSONTypes
from .base import _default_redis
from .base import logger
from .base import random_key
from .dict import RedisDict


F = TypeVar('F', bound=Callable[..., JSONTypes])

UpdateMap = Mapping[JSONTypes, Union[JSONTypes, object]]
UpdateItem = Tuple[JSONTypes, Union[JSONTypes, object]]
UpdateIter = Iterable[UpdateItem]
UpdateArg = Union[UpdateMap, UpdateIter]

_DEFAULT_TIMEOUT: Final[int] = 60 # seconds
_DEFAULT_TIMEOUT: Final[int] = 60 # seconds


class CacheInfo(NamedTuple):
@@ -185,6 +181,7 @@ def cache_clear() -> None:
wrapper.cache_info = cache_info # type: ignore
wrapper.cache_clear = cache_clear # type: ignore
return cast(F, wrapper)

return decorator


@@ -193,8 +190,10 @@ def _set_expiration(func: F) -> F:
def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any:
value = func(self, *args, **kwargs)
if self._timeout:
self._cache.redis.expire(self._cache.key, self._timeout) # Available since Redis 1.0.0
self._cache.redis.expire(self._cache.key,
self._timeout) # Available since Redis 1.0.0
return value

return cast(F, wrapper)


@@ -320,7 +319,7 @@ def __retry(self, callable: Callable[[], Any], *, try_num: int = 0) -> Any:
return callable()
except WatchError: # pragma: no cover
if try_num < self._num_tries - 1:
return self.__retry(callable, try_num=try_num+1)
return self.__retry(callable, try_num=try_num + 1)
raise

@_set_expiration
10 changes: 5 additions & 5 deletions pottery/counter.py
Original file line number Diff line number Diff line change
@@ -26,20 +26,18 @@
import collections
import contextlib
import itertools
from typing import Callable
from typing import Callable, Counter
from typing import Iterable
from typing import List
from typing import Tuple
from typing import Union
from typing import cast

from redis.client import Pipeline
from typing_extensions import Counter

from .annotations import JSONTypes
from .dict import RedisDict


InitIter = Iterable[JSONTypes]
InitArg = Union[InitIter, Counter]

@@ -124,7 +122,8 @@ def to_counter(self) -> Counter[JSONTypes]:
def __math_op(self,
other: Counter[JSONTypes],
*,
method: Callable[[Counter[JSONTypes], Counter[JSONTypes]], Counter[JSONTypes]],
method: Callable[
[Counter[JSONTypes], Counter[JSONTypes]], Counter[JSONTypes]],
) -> Counter[JSONTypes]:
with self._watch(other):
counter = self.__to_counter()
@@ -197,7 +196,8 @@ def __imath_op(self,
# Available since Redis 2.0.0:
pipeline.hset(self.key, mapping=encoded_to_set) # type: ignore
if encoded_to_del:
pipeline.hdel(self.key, *encoded_to_del) # Available since Redis 2.0.0
pipeline.hdel(self.key,
*encoded_to_del) # Available since Redis 2.0.0
return self

def __iadd__(self, other: Counter[JSONTypes]) -> Counter[JSONTypes]:
4 changes: 1 addition & 3 deletions pottery/executor.py
Original file line number Diff line number Diff line change
@@ -22,11 +22,9 @@

import concurrent.futures
from types import TracebackType
from typing import Type
from typing import Type, Literal
from typing import overload

from typing_extensions import Literal


class BailOutExecutor(concurrent.futures.ThreadPoolExecutor):
'''ThreadPoolExecutor subclass that doesn't wait for futures on .__exit__().
33 changes: 18 additions & 15 deletions pottery/list.py
Original file line number Diff line number Diff line change
@@ -28,7 +28,7 @@
import itertools
import uuid
import warnings
from typing import Any
from typing import Any, final
from typing import Callable
from typing import Iterable
from typing import List
@@ -37,9 +37,6 @@
from redis import Redis
from redis import ResponseError
from redis.client import Pipeline
# TODO: When we drop support for Python 3.7, change the following import to:
# from typing import final
from typing_extensions import final

from .annotations import F
from .annotations import JSONTypes
@@ -55,6 +52,7 @@ def wrapper(*args: Any, **kwargs: Any) -> Any:
return func(*args, **kwargs)
except ResponseError as error:
raise IndexError('list assignment index out of range') from error

return wrapper


@@ -125,9 +123,9 @@ def __getitem__(self, index: slice | int) -> Any:
)
indices = self.__slice_to_indices(index)
if indices.step >= 0:
start, stop = indices.start, indices.stop-1
start, stop = indices.start, indices.stop - 1
else:
start, stop = indices.stop+1, indices.start
start, stop = indices.stop + 1, indices.start
pipeline.multi() # Available since Redis 1.2.0
pipeline.lrange(self.key, start, stop) # Available since Redis 1.0.0
encoded_values = pipeline.execute()[0] # Available since Redis 1.2.0
@@ -138,12 +136,13 @@ def __getitem__(self, index: slice | int) -> Any:
else:
index = self.__slice_to_indices(index).start
len_ = cast(int, pipeline.llen(self.key)) # Available since Redis 1.0.0
if index not in {-1, 0, len_-1}:
if index not in {-1, 0, len_ - 1}:
warnings.warn(
cast(str, InefficientAccessWarning.__doc__),
InefficientAccessWarning,
)
encoded_value = pipeline.lindex(self.key, index) # Available since Redis 1.0.0
encoded_value = pipeline.lindex(self.key,
index) # Available since Redis 1.0.0
if encoded_value is None:
raise IndexError('list index out of range')
value = self._decode(cast(bytes, encoded_value))
@@ -164,7 +163,8 @@ def __setitem__(self, index: slice | int, value: JSONTypes) -> None: # type: ig
indices = self.__slice_to_indices(index)
pipeline.multi() # Available since Redis 1.2.0
for index, encoded_value in zip(indices, encoded_values):
pipeline.lset(self.key, index, encoded_value) # Available since Redis 1.0.0
pipeline.lset(self.key, index,
encoded_value) # Available since Redis 1.0.0
indices, num = indices[len(encoded_values):], 0
for index in indices:
pipeline.lset(self.key, index, 0) # Available since Redis 1.0.0
@@ -174,14 +174,15 @@ def __setitem__(self, index: slice | int, value: JSONTypes) -> None: # type: ig
else:
index = self.__slice_to_indices(index).start
len_ = cast(int, pipeline.llen(self.key)) # Available since Redis 1.0.0
if index not in {-1, 0, len_-1}:
if index not in {-1, 0, len_ - 1}:
warnings.warn(
cast(str, InefficientAccessWarning.__doc__),
InefficientAccessWarning,
)
pipeline.multi() # Available since Redis 1.2.0
encoded_value = self._encode(value)
pipeline.lset(self.key, index, encoded_value) # Available since Redis 1.0.0
pipeline.lset(self.key, index,
encoded_value) # Available since Redis 1.0.0

@_raise_on_error
def __delitem__(self, index: slice | int) -> None: # type: ignore
@@ -227,7 +228,8 @@ def _insert(self,
pipeline: Pipeline,
) -> None:
encoded_value = self._encode(value)
current_length = cast(int, pipeline.llen(self.key)) # Available since Redis 1.0.0
current_length = cast(int,
pipeline.llen(self.key)) # Available since Redis 1.0.0
if 0 < index < current_length:
# Python's list API requires us to insert an element before the
# given *index.* Redis supports only inserting an element before a
@@ -245,7 +247,7 @@ def _insert(self,
pipeline.multi() # Available since Redis 1.2.0
pipeline.lset(self.key, index, uuid4) # Available since Redis 1.0.0
pipeline.linsert(self.key, 'BEFORE', uuid4, encoded_value)
pipeline.lset(self.key, index+1, pivot) # Available since Redis 1.0.0
pipeline.lset(self.key, index + 1, pivot) # Available since Redis 1.0.0
else:
pipeline.multi() # Available since Redis 1.2.0
push_method = pipeline.lpush if index <= 0 else pipeline.rpush # Available since Redis 1.0.0
@@ -263,7 +265,8 @@ def sort(self, *, key: str | None = None, reverse: bool = False) -> None:
cast(str, InefficientAccessWarning.__doc__),
InefficientAccessWarning,
)
self.redis.sort(self.key, desc=reverse, store=self.key) # Available since Redis 1.0.0
self.redis.sort(self.key, desc=reverse,
store=self.key) # Available since Redis 1.0.0

def __eq__(self, other: Any) -> bool:
if type(other) not in {self.__class__, self._ALLOWED_TO_EQUAL}:
@@ -330,7 +333,7 @@ def pop(self, index: int | None = None) -> JSONTypes:
len_ = len(self)
if index and index >= len_:
raise IndexError('pop index out of range')
elif index in {0, None, len_-1, -1}:
elif index in {0, None, len_ - 1, -1}:
pop_method = 'lpop' if index == 0 else 'rpop'
pipeline.multi() # Available since Redis 1.2.0
getattr(pipeline, pop_method)(self.key) # Available since Redis 1.0.0
13 changes: 6 additions & 7 deletions pottery/monkey.py
Original file line number Diff line number Diff line change
@@ -16,27 +16,22 @@
# --------------------------------------------------------------------------- #
'Monkey patches.'


# TODO: When we drop support for Python 3.9, remove the following import. We
# only need it for X | Y union type annotations as of 2022-01-29.
from __future__ import annotations

import logging

# TODO: When we drop support for Python 3.7, change the following import to:
# from typing import Final
from typing_extensions import Final

from typing import Final

logger: Final[logging.Logger] = logging.getLogger('pottery')
logger.addHandler(logging.NullHandler())


import functools # isort: skip
import json # isort: skip
from typing import Any # isort: skip
from typing import Callable # isort: skip


class PotteryEncoder(json.JSONEncoder):
'Custom JSON encoder that can serialize Pottery containers.'

@@ -49,16 +44,20 @@ def default(self, o: Any) -> Any:
return o.to_list() # type: ignore
return super().default(o)


def _decorate_dumps(func: Callable[..., str]) -> Callable[..., str]:
'Decorate json.dumps() to use PotteryEncoder by default.'

@functools.wraps(func)
def wrapper(*args: Any,
cls: type[json.JSONEncoder] = PotteryEncoder,
**kwargs: Any,
) -> str:
return func(*args, cls=cls, **kwargs)

return wrapper


json.dumps = _decorate_dumps(json.dumps)

logger.info(
Loading