Merge branch 'unstable'

.travis.yml

@@ -5,12 +5,14 @@ python:
   - "3.3"
   - "3.4"
   - "3.5"
+  - "3.6-dev"
   - "nightly"
 services:
   - redis-server
 install:
   - "if [[ $REDIS_VERSION == '3.0' ]]; then REDIS_VERSION=3.0 make redis-install; fi"
   - "if [[ $REDIS_VERSION == '3.2' ]]; then REDIS_VERSION=3.2 make redis-install; fi"
+  - "if [[ $REDIS_VERSION == '4.0' ]]; then REDIS_VERSION=4.0 make redis-install; fi"
   - pip install -r dev-requirements.txt
   - pip install -e .
   - "if [[ $HIREDIS == '1' ]]; then pip install hiredis; fi"
@@ -23,6 +25,10 @@ env:
   - HIREDIS=0 REDIS_VERSION=3.2
   # Redis 3.2 and HIREDIS
   - HIREDIS=1 REDIS_VERSION=3.2
+  # Redis 4.0
+  - HIREDIS=0 REDIS_VERSION=4.0
+  # Redis 4.0 and HIREDIS
+  - HIREDIS=1 REDIS_VERSION=4.0
 script: 
   - make start
   - coverage erase

README.md

@@ -42,7 +42,6 @@ Small sample script that shows how to get started with RedisCluster. It can also
 >>> # Requires at least one node for cluster discovery. Multiple nodes is recommended.
 >>> startup_nodes = [{"host": "127.0.0.1", "port": "7000"}]
 
->>> # Note: decode_responses must be set to True when used with python3
 >>> rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
 
 >>> rc.set("foo", "bar")
@@ -55,8 +54,8 @@ True
 
 ## License & Authors
 
-Copyright (c) 2013-2016 Johan Andersson
+Copyright (c) 2013-2017 Johan Andersson
 
 MIT (See docs/License.txt file)
 
-The license should be the same as redis-py (https://github.com/andymccurdy/redis-py)
+The license should be the same as redis-py (https://github.com/andymccurdy/redis-py)

docs/authors.rst

@@ -22,3 +22,5 @@ Authors who contributed code or testing:
  - monklof - https://github.com/monklof
  - dutradda - https://github.com/dutradda
  - AngusP - https://github.com/AngusP
+ - Doug Kent - https://github.com/dkent
+ - VascoVisser - https://github.com/VascoVisser

docs/index.rst

@@ -44,7 +44,7 @@ Small sample script that shows how to get started with RedisCluster. It can also
     >>> # Requires at least one node for cluster discovery. Multiple nodes is recommended.
     >>> startup_nodes = [{"host": "127.0.0.1", "port": "7000"}]
 
-    >>> # Note: decode_responses must be set to True when used with python3
+    >>> # Note: See note on Python 3 for decode_responses behaviour
     >>> rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
 
     >>> rc.set("foo", "bar")
@@ -53,6 +53,9 @@ Small sample script that shows how to get started with RedisCluster. It can also
     'bar'
 
 
+.. note:: Python 3
+
+    Since Python 3 changed to Unicode strings from Python 2's ASCII, the return type of *most* commands will be binary strings, unless the class is instantiated with the option ``decode_responses=True``. In this case, the responses will be Python 3 strings (Unicode). For the init argument `decode_responses`, when set to False, redis-py-cluster will not attempt to decode the responses it receives. In Python 3, this means the responses will be of type `bytes`. In Python 2, they will be native strings (`str`). If `decode_responses` is set to True, for Python 3 responses will be `str`, for Python 2 they will be `unicode`.
 
 Dependencies & supported python versions
 ----------------------------------------
@@ -67,14 +70,15 @@ Dependencies & supported python versions
 Supported python versions
 -------------------------
 
-- 2.7.x
-- 3.3.x
-- 3.4.1+
-- 3.5.x
+- 2.7
+- 3.3
+- 3.4.1+ (See note)
+- 3.5
+- 3.6
 
 Experimental:
 
-- Up to 3.6.0a0
+- 3.7-dev
 
 
 .. note:: Python 3.4.0

docs/release-notes.rst

@@ -1,11 +1,20 @@
 Release Notes
 =============
 
-Unstable
---------
+1.3.4 (Mar 5, 2017)
+-------------------
 
+    * Package is now built as a wheel and source package when releases is built.
+    * Fixed issues with some key types in `NodeManager.keyslot()`.
     * Add support for `PUBSUB` subcommands `CHANNELS`, `NUMSUB [arg] [args...]` and `NUMPAT`.
     * Add method `set_result_callback(command, callback)` allowing the default reply callbacks to be changed, in the same way `set_response_callback(command, callback)` inherited from Redis-Py does for responses.
+    * Node manager now honors defined max_connections variable so connections that is emited from that class uses the same variable.
+    * Fixed a bug in cluster detection when running on python 3.x and decode_responses=False was used.
+      Data back from redis for cluster structure is now converted no matter what the data you want to set/get later is using.
+    * Add SSLClusterConnection for connecting over TLS/SSL to Redis Cluster
+    * Add new option to make the nodemanager to follow the cluster when nodes move around by avoiding to query the original list of startup nodes that was provided
+      when the client object was first created. This could make the client handle drifting clusters on for example AWS easier but there is a higher risk of the client talking to
+      the wrong group of nodes during split-brain event if the cluster is not consistent. This feature is EXPERIMENTAL and use it with care.
 
 1.3.3 (Dec 15, 2016)
 --------------------

rediscluster/__init__.py

@@ -16,7 +16,7 @@
 setattr(redis, "StrictClusterPipeline", StrictClusterPipeline)
 
 # Major, Minor, Fix version
-__version__ = (1, 3, 3)
+__version__ = (1, 3, 4)
 
 if sys.version_info[0:3] == (3, 4, 0):
     raise RuntimeError("CRITICAL: rediscluster do not work with python 3.4.0. Please use 3.4.1 or higher.")

rediscluster/client.py

@@ -53,7 +53,7 @@ class StrictRedisCluster(StrictRedis):
             "ECHO", "CONFIG GET", "CONFIG SET", "SLOWLOG GET", "CLIENT KILL", "INFO",
             "BGREWRITEAOF", "BGSAVE", "CLIENT LIST", "CLIENT GETNAME", "CONFIG RESETSTAT",
             "CONFIG REWRITE", "DBSIZE", "LASTSAVE", "PING", "SAVE", "SLOWLOG LEN", "SLOWLOG RESET",
-            "TIME", "KEYS", "CLUSTER INFO", "PUBSUB CHANNELS",  
+            "TIME", "KEYS", "CLUSTER INFO", "PUBSUB CHANNELS",
             "PUBSUB NUMSUB", "PUBSUB NUMPAT",
         ], 'all-nodes'),
         string_keys_to_dict([
@@ -126,7 +126,7 @@ class StrictRedisCluster(StrictRedis):
     }
 
     def __init__(self, host=None, port=None, startup_nodes=None, max_connections=32, max_connections_per_node=False, init_slot_cache=True,
-                 readonly_mode=False, reinitialize_steps=None, skip_full_coverage_check=False, **kwargs):
+                 readonly_mode=False, reinitialize_steps=None, skip_full_coverage_check=False, nodemanager_follow_cluster=False, **kwargs):
         """
         :startup_nodes:
             List of nodes that initial bootstrapping can be done from
@@ -141,6 +141,10 @@ def __init__(self, host=None, port=None, startup_nodes=None, max_connections=32,
         :skip_full_coverage_check:
             Skips the check of cluster-require-full-coverage config, useful for clusters
             without the CONFIG command (like aws)
+        :nodemanager_follow_cluster:
+            The node manager will during initialization try the last set of nodes that
+            it was operating on. This will allow the client to drift along side the cluster
+            if the cluster nodes move around alot.
         :**kwargs:
             Extra arguments that will be sent into StrictRedis instance when created
             (See Official redis-py doc for supported kwargs
@@ -173,6 +177,7 @@ def __init__(self, host=None, port=None, startup_nodes=None, max_connections=32,
                 reinitialize_steps=reinitialize_steps,
                 max_connections_per_node=max_connections_per_node,
                 skip_full_coverage_check=skip_full_coverage_check,
+                nodemanager_follow_cluster=nodemanager_follow_cluster,
                 **kwargs
             )
 
@@ -753,31 +758,28 @@ def renamenx(self, src, dst):
 
         return False
 
-
     def pubsub_channels(self, pattern='*', aggregate=True):
         """
         Return a list of channels that have at least one subscriber.
         Aggregate toggles merging of response.
         """
         return self.execute_command('PUBSUB CHANNELS', pattern, aggregate=aggregate)
 
-
     def pubsub_numpat(self, aggregate=True):
         """
         Returns the number of subscriptions to patterns.
         Aggregate toggles merging of response.
         """
         return self.execute_command('PUBSUB NUMPAT', aggregate=aggregate)
 
-
     def pubsub_numsub(self, *args, **kwargs):
         """
         Return a list of (channel, number of subscribers) tuples
         for each channel given in ``*args``.
-        
+
         ``aggregate`` keyword argument toggles merging of response.
         """
-        options = { 'aggregate': kwargs.get('aggregate', True) }
+        options = {'aggregate': kwargs.get('aggregate', True)}
         return self.execute_command('PUBSUB NUMSUB', *args, **options)
 
     ####

rediscluster/connection.py

@@ -17,7 +17,7 @@
 # 3rd party imports
 from redis._compat import nativestr
 from redis.client import dict_merge
-from redis.connection import ConnectionPool, Connection, DefaultParser
+from redis.connection import ConnectionPool, Connection, DefaultParser, SSLConnection
 from redis.exceptions import ConnectionError
 
 
@@ -57,6 +57,35 @@ def on_connect(self):
                 raise ConnectionError('READONLY command failed')
 
 
+class SSLClusterConnection(SSLConnection):
+    """
+    Manages TCP communication over TLS/SSL to and from a Redis cluster
+    Usage:
+        pool = ClusterConnectionPool(connection_class=SSLClusterConnection, ...)
+        client = StrictRedisCluster(connection_pool=pool)
+    """
+    description_format = "SSLClusterConnection<host=%(host)s,port=%(port)s,db=%(db)s>"
+
+    def __init__(self, **kwargs):
+        self.readonly = kwargs.pop('readonly', False)
+        kwargs['parser_class'] = ClusterParser
+        kwargs.pop('ssl', None)  # Needs to be removed to avoid exception in redis Connection init
+        super(SSLClusterConnection, self).__init__(**kwargs)
+
+    def on_connect(self):
+        '''
+        Initialize the connection, authenticate and select a database and send READONLY if it is
+        set during object initialization.
+        '''
+        super(SSLClusterConnection, self).on_connect()
+
+        if self.readonly:
+            self.send_command('READONLY')
+
+            if nativestr(self.read_response()) != 'OK':
+                raise ConnectionError('READONLY command failed')
+
+
 class UnixDomainSocketConnection(Connection):
     """
     """
@@ -71,11 +100,15 @@ class ClusterConnectionPool(ConnectionPool):
 
     def __init__(self, startup_nodes=None, init_slot_cache=True, connection_class=ClusterConnection,
                  max_connections=None, max_connections_per_node=False, reinitialize_steps=None,
-                 skip_full_coverage_check=False, **connection_kwargs):
+                 skip_full_coverage_check=False, nodemanager_follow_cluster=False, **connection_kwargs):
         """
         :skip_full_coverage_check:
             Skips the check of cluster-require-full-coverage config, useful for clusters
             without the CONFIG command (like aws)
+        :nodemanager_follow_cluster:
+            The node manager will during initialization try the last set of nodes that
+            it was operating on. This will allow the client to drift along side the cluster
+            if the cluster nodes move around alot.
         """
         super(ClusterConnectionPool, self).__init__(connection_class=connection_class, max_connections=max_connections)
 
@@ -92,8 +125,18 @@ def __init__(self, startup_nodes=None, init_slot_cache=True, connection_class=Cl
         self.max_connections = max_connections or 2 ** 31
         self.max_connections_per_node = max_connections_per_node
 
-        self.nodes = NodeManager(startup_nodes, reinitialize_steps=reinitialize_steps,
-                                 skip_full_coverage_check=skip_full_coverage_check, **connection_kwargs)
+        if connection_class == SSLClusterConnection:
+            connection_kwargs['ssl'] = True  # needed in StrictRedis init
+
+        self.nodes = NodeManager(
+            startup_nodes,
+            reinitialize_steps=reinitialize_steps,
+            skip_full_coverage_check=skip_full_coverage_check,
+            max_connections=self.max_connections,
+            nodemanager_follow_cluster=nodemanager_follow_cluster,
+            **connection_kwargs
+        )
+
         if init_slot_cache:
             self.nodes.initialize()
 
@@ -297,7 +340,7 @@ class ClusterReadOnlyConnectionPool(ClusterConnectionPool):
     """
 
     def __init__(self, startup_nodes=None, init_slot_cache=True, connection_class=ClusterConnection,
-                 max_connections=None, **connection_kwargs):
+                 max_connections=None, nodemanager_follow_cluster=False, **connection_kwargs):
         """
         """
         super(ClusterReadOnlyConnectionPool, self).__init__(
@@ -306,6 +349,7 @@ def __init__(self, startup_nodes=None, init_slot_cache=True, connection_class=Cl
             connection_class=connection_class,
             max_connections=max_connections,
             readonly=True,
+            nodemanager_follow_cluster=nodemanager_follow_cluster,
             **connection_kwargs)
 
     def get_node_by_slot(self, slot):

rediscluster/crc.py

@@ -43,7 +43,7 @@ def _crc16_py3(data):
     """
     """
     crc = 0
-    for byte in data.encode("utf-8"):
+    for byte in data:
         crc = ((crc << 8) & 0xff00) ^ x_mode_m_crc16_lookup[((crc >> 8) & 0xff) ^ byte]
     return crc & 0xffff
 
@@ -52,7 +52,7 @@ def _crc16_py2(data):
     """
     """
     crc = 0
-    for byte in data.encode("utf-8"):
+    for byte in data:
         crc = ((crc << 8) & 0xff00) ^ x_mode_m_crc16_lookup[((crc >> 8) & 0xff) ^ ord(byte)]
     return crc & 0xffff