diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..0953592 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,21 @@ +version: "2" + +build: + os: "ubuntu-latest" + tools: + python: "3.12" + +jobs: + post_create_environment: + - pip install poetry + - poetry config virtualenvs.create false + + post_install: + -poetry install --with docs + +sphinx: + configuration: docs/source/conf.py + +formats: + - pdf + - epub \ No newline at end of file diff --git a/CHANGES.rst b/CHANGES.rst new file mode 100644 index 0000000..3fc7430 --- /dev/null +++ b/CHANGES.rst @@ -0,0 +1,14 @@ +Version 0.0.15 +-------------- + +Released 2024/09/07 + +- Added this documentation! + +- changed ``VALKEY_CLIENT_CLASS`` to ``BASE_CLIENT_CLASS`` for clarity + +- changed ``CLIENT_KWARGS`` to ``BASE_CLIENT_KWARGS`` for clarity + +- added some docstring to compressor classes + +- fixed some messages generated by autocomplete \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..747ffb7 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/source/changes.rst b/docs/source/changes.rst new file mode 100644 index 0000000..799b439 --- /dev/null +++ b/docs/source/changes.rst @@ -0,0 +1,5 @@ +======= +Changes +======= + +.. include:: ../../CHANGES.rst \ No newline at end of file diff --git a/docs/source/commands/commands.rst b/docs/source/commands/commands.rst new file mode 100644 index 0000000..d3800a2 --- /dev/null +++ b/docs/source/commands/commands.rst @@ -0,0 +1,9 @@ +======== +Commands +======== + +.. toctree:: + :maxdepth: 2 + + connection_pool_commands + valkey_native_commands \ No newline at end of file diff --git a/docs/source/commands/connection_pool_commands.rst b/docs/source/commands/connection_pool_commands.rst new file mode 100644 index 0000000..480258b --- /dev/null +++ b/docs/source/commands/connection_pool_commands.rst @@ -0,0 +1,16 @@ +========================== +Access the connection pool +========================== + +you can get the connection pool using this code: + +.. code-block:: python + + from django_valkey import get_valkey_connection + + r = get_valkey_connection("default") # use the name defined in ``CACHES`` settings + connection_pool = r.connection_pool + print(f"created connections so far: {connection_pool._created_connections}") + + +this will verify how many connections the pool has opened. diff --git a/docs/source/commands/valkey_native_commands.rst b/docs/source/commands/valkey_native_commands.rst new file mode 100644 index 0000000..98ae30f --- /dev/null +++ b/docs/source/commands/valkey_native_commands.rst @@ -0,0 +1,189 @@ +=============== +valkey commands +=============== + +if you need to use valkey operations, you can access the client as follows: + +.. code-block:: pycon + + >>> from django.core.cache import cache + >>> cache.set("key", "value1", nx=True) + True + >>> cache.set("key", "value2", nx=True) + False + >>> cache.get("key") + "value1" + +the list of supported commands is very long, if needed you can check the methods at ``django_valkey.cache.ValkeyCache``` + +Infinite timeout +**************** + +django-valkey comes with infinite timeouts supported out of the box. And it +behaves in the same way as django backend contract specifies: + +- ``timeout=0`` expires the value immediately. +- ``timeout=None`` infinite timeout. + +.. code-block:: python + + cache.set("key", "value", timeout=None) + + +Scan and Delete in bulk +*********************** + +when you need to search for keys that have similar patterns, or delete them, you can use the helper methods that come with django-valkey: + +.. code-block:: pycon + + >>> from django.core.cache import cache + >>> cache.keys("foo_*") + ["foo_1", "foo_2"] + +if you are looking for a very large amount of data, this is **not** suitable; instead use ``iter_keys``. +this will return a generator that you can iterate over more efficiently. + +.. code-block:: python + + >>> from django.core.cache import cache + >>> cache.iter_keys("foo_*") + >> next(cache.iter_keys("foo_*)) + 'foo_1' + >>> foos = cache.iter_keys("foo_*") + >>> for i in foos: + print(i) + 'foo_1' + 'foo_2' + +to delete keys, you should use ``delete_pattern`` which has the same glob pattern syntax as ``keys`` and returns the number of deleted keys. + +.. code-block:: pycon + + >>> from django.core.cache import cache + >>> cache.delete_pattern("foo_*") + 2 + +To achieve the best performance while deleting many keys, you should set ``DJANGO_VALKEY_SCAN_ITERSIZE`` to a relatively +high number (e.g., 100_000) by default in Django settings or pass it directly to the ``delete_pattern``. + +.. code-block:: pycon + + >>> from django.core.cache import cache + >>> cache.delete_pattern("foo_*", itersize=100_000) + +Get ttl (time-to-live) from key +******************************* + +with valkey you can access to ttl of any sorted key, to do so, django-valky exposes the ``ttl`` method. + +the ttl method returns: + +- `0` if key does not exists (or already expired). +- ``None`` for keys that exist but does not have expiration. +- the ttl value for any volatile key (any key that has expiration). + +.. code-block:: pycon + + >>> from django.core.cache import cache + >>> cache.set("foo", "value", timeout=25) + >>> cache.ttl("foo") + 25 + >>> cache.ttl("not-exists") + 0 + +you can also access the ttl of any sorted key in milliseconds, use the ``pttl`` method to do so: + +.. code-block:: pycon + + >>> from django.core.cache import cache + >>> cache.set("foo", "value", timeout=25) + >>> cache.pttl("foo") + 25000 + >>> cache.pttl("non-existent") + 0 + +Expire & Persist +**************** + +in addition to the ``ttl`` and ``pttl`` methods, you can use the ``persist`` method so the key would have infinite timout: + +.. code-block:: pycon + + >>> cache.set("foo", "bar", timeout=22) + >>> cache.ttl("foo") + 22 + >>> cache.persist("foo") + True + >>> cache.ttl("foo") + None + +you can also use ``expire`` to set a new timeout on the key: + +.. code-block:: pycon + + >>> cache.set("foo", "bar", timeout=22) + >>> cache.expire("foo", timeout=5) + True + >>> cache.ttl("foo") + 5 + +The ``pexpire`` method can be used to set new timeout in millisecond precision: + + +.. code-block:: pycon + + >>> cache.set("foo", "bar", timeout=22) + >>> cache.pexpire("foo", timeout=5505) + True + >>> cache.pttl("foo") + 5505 + +The ``expire_at`` method can be used to make the key expire at a specific moment in time: + +.. code-block:: pycon + + >>> cache.set("foo", "bar", timeout=22) + >>> cache.expire_at("foo", datetime.now() + timedelta(hours=1)) + True + >>> cache.ttl("foo") + 3600 + +The ``pexpire_at`` method can be used to make the key expire at a specific moment in time, with milliseconds precision: + +.. code-block:: pycon + + >>> cache.set("foo", "bar", timeout=22) + >>> cache.pexpire_at("foo", datetime.now() + timedelta(milliseconds=900, hours=1)) + True + >>> cache.ttl("foo") + 3601 + >>> cache.pttl("foo") + 3600900 + +Locks +***** + +django-valkey also supports locks. +valkey has distributed named locks which are identical to ``threading.Lock`` so you can useit as replacement. + +.. code-block:: python + + with cache.lock("somekey"): + do_something()) + +Access Raw client +***************** + +if the commands provided by django-valkey backend is not enough, or you want to use them in a different way, you can access the underlying client as follows: + +.. code-block:: pycon + + >>> from django-valkey import get_valkey_connection + >>> con = get_valkey_connection("default") + >>> con + + +**Warning**: not all clients support this feature: +ShardClient will raise an exception if tried to be used like this. \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..c250991 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,33 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "django-valkey" +copyright = "2024, amirreza" +author = "amirreza" +release = "0.0.15" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [] + +templates_path = ["_templates"] +exclude_patterns = [] + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +import sphinx_pdj_theme + +html_theme = "sphinx_pdj_theme" +html_static_path = [sphinx_pdj_theme.get_html_theme_path()] + +html_sidebars = { + "**": ["globaltoc.html", "sourcelink.html", "searchbox.html"], +} diff --git a/docs/source/configure/advanced_configurations.rst b/docs/source/configure/advanced_configurations.rst new file mode 100644 index 0000000..c51f2fb --- /dev/null +++ b/docs/source/configure/advanced_configurations.rst @@ -0,0 +1,474 @@ +======================= +Advanced configurations +======================= + +Configure the database URL +########################## + +as we discussed in :doc:`basic_configurations` you can configure your database URL like this: + +.. code-block:: python + + CACHES = { + "default": { + "BACKEND": "django_valkey.cache.ValkeyCache", + "LOCATION": "valkey://127.0.0.1:6379", + "OPTIONS": {...} + } + } + +or if you are using ACL, you can do something like this: + +.. code-block:: python + + CACHES = { + "default": { + ... + "LOCATION": "valkey://django@localhost:6379/0", + "OPTIONS": { + "PASSWORD": "mysecret" + } + } + } + +Now, lets look at other ways to configure this: + +* valkey://[[username]:[password]]@localhost:6379 +* valkeys://[[username]:[password]@localhost:6379 +* unix://[[username]:[password]@/path/to/socket.sock + +These three URL schemes are supported: +* ``valkey://``: creates a normal TCP socket connection +* ``valkeys://``: creates a SSL wrapped TCP socket connection +* ``unix://``: creates a Unix Domain Socket connection + +Specify a database number: +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +you can specify a database number in your URL like this: +* A ``db`` querystring option, e.g. ``valkey://localhost:6379?db=0`` +* if using the ``valkey://`` scheme, the path argumentof the URL, e.g. ``valkey://localhost:6379/0`` + + +Configure as session backend +############################ + +django can by default use any cache backend as a session backend and you benefit from that by using django-valkey as backend for session storage without installing any additional backends: +just add these settings to your settings.py + +.. code-block:: python + + SESSION_ENGINE = "django.contrib.sessions.backends.cache" + SESSION_CACHE_ALIAS = "default + +Configure the client +#################### + +by default django-valkey uses ``django_valkey.client.default.DefaultClient`` to work with the underlying API +you can, however, plug in another client, either one that we provide or one of your own + +the DefaultClient is used by default, but if you want to be explicit you can set it like this: + +.. code-block:: python + + CACHE = { + "default": { + "BACKEND": "django_valkey.cache.ValkeyCache", + "LOCATION": [ + "valkey://127.0.0.1:6379", + ], + "OPTIONS": { + "CLIENT_CLASS": "django_valkey.client.DefaultClient" + } + } + } + +Use Sentinel client +^^^^^^^^^^^^^^^^^^^ + +In order to facilitate using `Valkey Sentinels: `_, django-valkey comes with a built-in sentinel client and a connection factory + +since this is a big topic, you can find detailed explanation in :doc:`sentinel_configurations` +but for a simple configuration this will work: + +.. code-block:: python + + DJANGO_VALKEY_CONNECTION_FACTORY = "django_valkey.pool.SentinelConnectionFactory" + + SENTINELS = [ + ('sentinel-1', 26379), + ] + + CACHES = { + "default": { + "BACKEND": "django_valkey.cache.ValkeyCache", + "LOCATION": "valkey://service_name/db", + "OPTIONS": { + "CLIENT_CLASS": "django_valkey.client.SentinelClient", + "SENTINELS": SENTINELS, + + # optional + "SENTINEL_KWARGS": {} + } + } + } + +Use Shard client +^^^^^^^^^^^^^^^^ + +this pluggable client implements client-side sharding. to use it, change you cache settings to look like this: +*WARNING*: sharded client is experimental + +.. code-block:: python + + CACHE = { + "default": { + "BACKEND": "django_valkey.cache.ValkeyCache", + "LOCATION": [ + "valkey://127.0.0.1:6379/1", + "valkey://127.0.0.1:6379/2", + ], + "OPTIONS": { + "CLIENT_CLASS": "django_valkey.client.ShardClient" + } + } + } + +Use Herd client +^^^^^^^^^^^^^^^ + +This pluggable client help dealing with the thundering herd problem. you can read more about it on: `Wikipedia `_ +to use this client change your configs to look like this: + +.. code-block:: python + + CACHES = { + "default": { + ... + "OPTIONS": { + "CLIENT_CLASS": "django_valkey.client.HerdClient", + } + } + } + # optional: + CACHE_HERD_TIMEOUT = 60 # default is 60 + + +Configure the serializer +######################## + +by default django-valkey uses python's pickle library to serialize data. +you can stick to pickle, use one of the alternative serializes we provide, or write your own and plug it in. + +django-valkey's pickle serializer uses pickle.DEFAULT_PROTOCOL as the default protocol version, but if you want to change it you can do it like this: + +.. code-block:: python + + CACHES = { + "default": { + # ... + "OPTIONS": { + "PICKLE_VERSION": 5 + } + } + } + +*note*: the pickle version shouldn't be higher that ``pickle.HIGHEST_PROTOCOL`` + +Use Json serializer +^^^^^^^^^^^^^^^^^^^ + +if you want to use the json serializer instead of pickle, add it to the configuration like this: + +.. code-block:: python + + CACHES = { + "default": { + ... + "OPTIONS": { + "SERIALIZER": "django_valkey.serializer.json.JSONSerializer", + ... + } + } + } + +and you're good to go + +Use Msgpack serializer +^^^^^^^^^^^^^^^^^^^^^^ + +to use the msgpack serializer you should first install the msgpack package as explained in :ref:`msgpack` +then configure your settings like this: + +.. code-block:: python + + CACHES = { + "default": { + ... + "OPTIONS": { + "SERIALIZER": "django_valkey.serializer.msgpack.MSGPackSerializer", + ... + } + } + } + +and done + +Fun fact +^^^^^^^^ +you can serialize every type in the python built-ins, and probably non built-ins, but you have to check which serializer supports that type. + +Pluggable Compressors +##################### + +by default django-valkey uses the ``django_valkey.compressors.identity.IdentityCompressor`` class as compressor, however you should *note* that this class doesn't compress anything; +it only returns the same value it's been passed to, but why do we have it then? +the reason is that this class works as a placeholder, so when we want to use a compressor, we can swap the classes. + +django valkey comes with a number of built-in compressors (some of them need a 3rd-party package to be installed) +as of now we have these compressors available: + +* :ref:`brotli` +* :ref:`bz2` +* :ref:`gzip` +* :ref:`lz4` +* :ref:`lzma` +* :ref:`zlib` +* :ref:`zstd` + +and you can easily write your own compressor and use that instead if you want. + +since the list is long we'll look into compressor configs in :doc:`compressors` + +Pluggable parsers +################# + +valkey-py (the valkey client used by django-valkey) comes with a pure python parser that works well for most common tasks, but if you want some performance boost you can use libvalkey. + +libvalkey is a Valkey client written in C and it has it's own parser that can be used with django-valkey. + +the only thing you need to do is install libvalkey: + +.. code-block:: console + + pip install django-valkey[libvalkey] + +and valkey-py will take care of te rest + +Use a custom parser +^^^^^^^^^^^^^^^^^^^ + +if you want to use your own parser just add it to the ``OPTIONS`` like so: + +.. code-block:: python + + CACHES = { + "default": { + ... + "OPTIONS": { + "PARSER_CLASS": "path.to.parser", + } + } + } + +Pluggable Base Client +##################### + +django valkey uses the Valkey client ``valkey.client.Valkey`` as a base client by default. +But It is possible to use an alternative client. + +You can customize the client used by django-valkey by setting ``BASE_CLIENT_CLASS`` in you settings. +optionally you can provide arguments to be passed to this class by setting ``BASE_CLIENT_KWARGS``. + +.. code-block:: python + + CACHES = { + "default": { + "OPTIONS": { + "BASE_CLIENT_CLASS": "path.to.client", + "BASE_CLIENT_KWARGS": {"something": True}, + } + } + } + +Connection Factory +################## + +django valkey has two connection factories built-in, ``django-valkey.pool.ConnectionFactory`` and ``django_valkey.pool.SentinelConnectionFactory``. +if you need to use another one, you can configure it globally by setting ``DJANGO_VALKEY_CONNECTION_FACTORY`` or per server by setting ``CONNECTION_FACTORY`` in ``OPTIONS`` +it could look like this: + +.. code-block:: python + + DJANGO_VALKEY_CONNECTION_FACTORY = "path.to.my.factory" + + # or: + + CACHES = { + "default": { + ... + "OPTIONS": { + "CONNECTION_FACTORY": "path.to.it", + } + }, + "another_service": { + ... + "OPTIONS": { + "CONNECTION_FACTORY": "path.to.another", + } + } + } + +a connection factory could look like this: + +.. code-block:: python + + class ConnectionFactory(object): + def get_connection_pool(self, params: dict): + # Given connection parameters in the `params` argument, return new + # connection pool. It should be overwritten if you want do + # something before/after creating the connection pool, or return + # your own connection pool. + pass + + def get_connection(self, params: dict): + # Given connection parameters in the `params` argument, return a + # new connection. It should be overwritten if you want to do + # something before/after creating a new connection. The default + # implementation uses `get_connection_pool` to obtain a pool and + # create a new connection in the newly obtained pool. + pass + + def get_or_create_connection_pool(self, params: dict): + # This is a high layer on top of `get_connection_pool` for + # implementing a cache of created connection pools. It should be + # overwritten if you want change the default behavior. + pass + + def make_connection_params(self, url: str) -> dict: + # The responsibility of this method is to convert basic connection + # parameters and other settings to fully connection pool ready + # connection parameters. + pass + + def connect(self, url: str): + # This is really a public API and entry point for this factory + # class. This encapsulates the main logic of creating the + # previously mentioned `params` using `make_connection_params` and + # creating a new connection using the `get_connection` method. + pass + +Connection pools +################ + +Behind the scenes, django-valkey uses the underlying valkey-py connection pool +implementation, and exposes a simple way to configure it. Alternatively, you +can directly customize a connection/connection pool creation for a backend. + +The default valkey-py behavior is to not close connections, recycling them when +possible. + +Configure default connection pool +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The default connection pool is simple. For example, you can customize the +maximum number of connections in the pool by setting ``CONNECTION_POOL_KWARGS`` +in the ``CACHES`` setting: + +.. code-block:: python + + CACHES = { + "default": { + "BACKEND": "django_valkey.cache.ValkeyCache", + # ... + "OPTIONS": { + "CONNECTION_POOL_KWARGS": {"max_connections": 100} + } + } + } + +Since the default connection pool passes all keyword arguments it doesn't use +to its connections, you can also customize the connections that the pool makes +by adding those options to ``CONNECTION_POOL_KWARGS``: + +.. code-block:: python + + CACHES = { + "default": { + ... + "OPTIONS": { + "CONNECTION_POOL_KWARGS": {"max_connection": 100, "retry_on_timeout": True} + } + } + } + +you can check :doc:`../commands/connection_pool_commands` to see how you can access the connection pool directly and see information about it + +Use your own connection pool +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +to use your own connection pool, set ``CONNECTION_POOL_CLASS`` in your backends ``OPTIONS`` +it could look like this: + +.. code-block:: python + + CACHES = { + "default": { + ... + "OPTIONS": { + "CONNECTION_POOL_CLASS": "path.to.mypool", + } + } + } + +for simplicity you can subclass the connection pool provided by valkey-py package: + +.. code-block:: python + + from valkey.connection import ConnectionPool + + class MyOwnPool(ConnectionPool): + pass + +Closing connection +################## +by default django-valkey keeps the connection to valkey server after a ``close()`` call. +you can change this behaviour for all cache servers (globally) by ``DJANGO_VALKEY_CLOSE_CONNECTION = True`` in the django settings +or by setting ``"CLOSE_CONNECTION": True`` (at cache level) in the ``OPTIONS`` for each configured cache server. + +.. code-block:: python + + DJANGO_VALKEY_CLOSE_CONNECTION = True + + # or: + + CACHE = { + "default": { + ... + "OPTIONS": { + "CLOSE_CONNECTION": True, + } + } + } + + +SSL/TLS self-signed certificate +############################### + +In case you encounter a Valkey server offering a TLS connection using a +self-signed certificate you may disable certification verification with the +following: + +.. code-block:: python + + CACHES = { + "default": { + "BACKEND": "django_valkey.cache.ValkeyCache", + "LOCATION": "valkeys://127.0.0.1:6379/1", + "OPTIONS": { + "CLIENT_CLASS": "django_valkey.client.DefaultClient", + "CONNECTION_POOL_KWARGS": {"ssl_cert_reqs": None} + } + } + } + diff --git a/docs/source/configure/basic_configurations.rst b/docs/source/configure/basic_configurations.rst new file mode 100644 index 0000000..d93ee09 --- /dev/null +++ b/docs/source/configure/basic_configurations.rst @@ -0,0 +1,123 @@ +=================== +Basic configuration +=================== + +if you haven't installed django-valkey yet, head out to :doc:`../installation`. + + +Configure as cache backend +########################## + +to start using django-valkey, change your django cache setting to something like this: + +.. code-block:: python + + CACHES = { + "default": { + "BACKEND": "django_valkey.cache.ValkeyCache", + "LOCATION": "valkey://127.0.0.1:6379", + "OPTIONS": {...} + } + } + +django-valkey uses the valkey-py native URL notation for connection strings, it allows better interoperability and has a connection string in more "standard" way. will explore this more in :doc:`advanced_configurations`. + +when using `Valkey's ACLs `_ you will need to add the username and password to the URL. +the login for the user ``django`` would look like this: + +.. code-block:: python + + CACHES = { + "default": { + ... + "LOCATION": "valkey://django:mysecret@localhost:6379/0", + ... + } + } + + +you can also provide the password in the ``OPTIONS`` dictionary +this is specially useful if you have a password that is not URL safe +but *notice* that if a password is provided by the URL, it won't be overriden by the password in ``OPTIONS``. + +.. code-block:: python + + CACHES = { + "default": { + "BACKEND": "django-valkey.cache.ValkeyCache", + "LOCATION": "valkey://django@localhost:6379/0", + "OPTIONS": { + "PASSWORD": "mysecret" + } + } + } + +**Note:** you probably should read the password from environment variables + + +Memcached exception behavior +############################ + +In some situations, when Valkey is only used for cache, you do not want +exceptions when Valkey is down. This is default behavior in the memcached +backend and it can be emulated in django-valkey. + +For setup memcached like behaviour (ignore connection exceptions), you should +set ``IGNORE_EXCEPTIONS`` settings on your cache configuration: + +.. code-block:: python + + CACHES = { + "default": { + # ... + "OPTIONS": { + "IGNORE_EXCEPTIONS": True, + } + } + } + +Also, if you want to apply the same settings to all configured caches, you can set the global flag in +your settings: + +.. code-block:: python + + DJANGO_VALKEY_IGNORE_EXCEPTIONS = True + +Log exceptions when ignored +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +when ignoring exceptions with ``IGNORE_EXCEPTIONS`` or ``DJANGO_VALKEY_IGNORE_EXCEPTION``, you may optionally log exceptions by setting the global variable ``DJANGO_VALKEY_LOG_EXCEPTION`` in your settings: + +.. code-block:: python + + DJANGO_VALKEY_LOG_IGNORED_EXCEPTION = True + +If you wish to specify a logger in which the exceptions are outputted, set the global variable ``DJANGO_VALKEY_LOGGER`` to the string name or path of the desired logger. +the default value is ``__name__`` if no logger was specified + +.. code-block:: python + + DJANGO_VALKEY_LOGGER = "some.logger" + + +Socket timeout +############## + +Socket timeout can be set using ``SOCKET_TIMEOUT`` and +``SOCKET_CONNECT_TIMEOUT`` options: + +.. code-block:: python + + CACHES = { + "default": { + # ... + "OPTIONS": { + "SOCKET_CONNECT_TIMEOUT": 5, # seconds + "SOCKET_TIMEOUT": 5, # seconds + } + } + } + +``SOCKET_CONNECT_TIMEOUT`` is the timeout for the connection to be established +and ``SOCKET_TIMEOUT`` is the timeout for read and write operations after the +connection is established. diff --git a/docs/source/configure/compressors.rst b/docs/source/configure/compressors.rst new file mode 100644 index 0000000..f943e18 --- /dev/null +++ b/docs/source/configure/compressors.rst @@ -0,0 +1,242 @@ +Compressor support +################## + +the default compressor class is ``django_valkey.compressors.identity.IdentityCompressor`` + +this class doesn't compress the data, it only returns the data as is, but it works as a swappable placeholder if we want to compress data. + +compressors have two common global settings and some of them have a number of their own. +the common settings are: +``CACHE_COMPRESS_LEVEL`` which tells the compression tool how much compression to perform. +``CACHE_COMPRESS_MIN_LENGTH`` which tells django-valkey how big a data should be to be compressed. + +the library specific settings are listed in their respective sections, you should look at their documentations for more info. + +.. _brotli: + +Brotli compression +****************** + +to use the brotli compressor you need to install the ``brotli`` package first +you can do that with: + +.. code-block:: console + + pip install django-valkey[brotli] + +or simply + +.. code-block:: console + + pip install brotli + + +to configure the compressor you should edit you settings files to look like this: + +.. code-block:: python + + + CACHES = { + "default": { + # ... + "OPTIONS": { + "COMPRESSOR": "django_valkey.compressors.brotli.BrotliCompressor", + } + } + } + # optionally you can set compression parameters: + CACHE_COMPRESS_LEVEL = 11 # defaults to 11 + CACHE_COMPRESS_MIN_LENGTH = 15 # defaults to 15 + COMPRESS_BROTLI_LGWIN = 22 # defaults to 22 + COMPRESS_BROTLI_LGBLOCK = 0 # defaults to 0 + COMPRESS_BROTLI_MODE = "GENERIC" # defaults to "GENERIC" other options are: ("GENERIC", "TEXT", "FONT") + +*NOTE* the values shown here are only examples and *not* best practice or anything. + +you can read more about this compressor at their `documentations `_ (and source code, for this matter) + +.. _bz2: + +Bz2 compression +*************** + +the bz2 compression library comes in python's standard library, so if you have a normal python installation just configure it and it's done: + +.. code-block:: python + + CACHES = { + "default": { + # ... + "OPTIONS": { + "COMPRESSOR": "django_valkey.compressors.bz2.Bz2Compressor", + } + } + } + # optionally you can set compression parameters: + CACHE_COMPRESS_LEVEL = 9 # defaults to 9 + CACHE_COMPRESS_MIN_LEVEL = 15 # defaults to 15 + +*NOTE* the values shown here are only examples and *not* best practice or anything. + +.. _gzip: + +Gzip compression +**************** + +the gzip compression library also comes with python's standard library, so configure it like this and you are done: + +.. code-block:: python + + CACHES = { + "default": { + # ... + "OPTIONS": { + "COMPRESSOR": "django_valkey.compressors.gzip.GzipCompressor", + } + } + } + # optionally you can set compression parameters: + CACHE_COMPRESS_LEVEL = 9 # defaults to 9 + CACHE_COMPRESS_MIN_LEVEL = 15 # defaults to 15 + +*NOTE* the values shown here are only examples and *not* best practice or anything. + +.. _lz4: + +Lz4 compression +*************** + +to use the lz4 compression you need to install the lz4 package first: + +.. code-block:: console + + pip install django-valkey[lz4] + +or simply + +.. code-block:: console + + pip install lz4 + +then you can configure it like this: + +.. code-block:: python + + CACHES = { + "default": { + # ... + "OPTIONS": { + "COMPRESSOR": "django_valkey.compressors.lz4.Lz4Compressor", + } + } + } + # optionally you can set compression parameters: + CACHE_COMPRESS_LEVEL = 0 # defaults to 0 + CACHE_COMPRESS_MIN_LEVEL = 15 # defaults to 15 + + COMPRESS_LZ4_BLOCK_SIZE = 0 # defaults to 0 + COMPRESS_LZ4_CONTENT_CHECKSUM = 0 # defaults to 0 + COMPRESS_LZ4_BLOCK_LINKED = True # defaults to True + COMPRESS_LZ4_STORE_SIZE = True # defaults to True + +*NOTE* the values shown here are only examples and *not* best practice or anything. + + +.. _lzma: + +Lzma compression +**************** + +lzma compression library also comes with python's standard library + +.. code-block:: python + + CACHES = { + "default": { + # ... + "OPTIONS": { + "COMPRESSOR": "django_valkey.compressors.lzma.LzmaCompressor", + } + } + } + # optionally you can set compression parameters: + CACHE_COMPRESS_LEVEL = 9 # defaults to 4 + CACHE_COMPRESS_MIN_LEVEL = 15 # defaults to 15 + + COMPRESS_LZMA_FORMAT = 1 # defaults to 1 + COMPRESS_LZMA_CHECK = -1 # defaults to -1 + COMPRESS_LZMA_FILTERS = None # defaults to None + + # optional decompression parameters + DECOMPRESS_LZMA_MEMLIMIT = None # defaults to None (if you want to change this, make sure you read lzma docs about it's dangers) + DECOMPRESS_LZMA_FORMAT = 0 # defaults to 4 + DECOMPERSS_LZMA_FILTERS = None # defaults to None + +*NOTE* the values shown here are only examples and *not* best practice or anything. + +.. _zlib: + +Zlib compression +**************** + +zlib compression library also comes with python's standard library + +.. code-block:: python + + CACHES = { + "default": { + # ... + "OPTIONS": { + "COMPRESSOR": "django_valkey.compressors.zlib.ZlibCompressor", + } + } + } + # optionally you can set compression parameters: + CACHE_COMPRESS_LEVEL = 9 # defaults to 6 + CACHE_COMPRESS_MIN_LEVEL = 15 # defaults to 15 + + compress_zlib_wbits = 15 # defaults to 15 (NOTE: only available in python 3.11 and newer + +*NOTE* the values shown here are only examples and *not* best practice or anything. + + +.. _zstd: + +Zstd compression +**************** + +to use zstd compression you need to have the pyzstd library installed + +.. code-block:: console + + pip install django-valkey[pyzstd] + +or simply + +.. code-block:: console + + pip install pyzstd + +then you can configure it as such: + +.. code-block:: python + + CACHES = { + "default": { + # ... + "OPTIONS": { + "COMPRESSOR": "django_valkey.compressors.zstd.ZStdCompressor", + } + } + } + # optionally you can set compression parameters: + CACHE_COMPRESS_LEVEL = 1 # defaults to 1 + CACHE_COMPRESS_MIN_LEVEL = 15 # defaults to 15 + + # the below settings are all defaulted to None + COMPRESS_ZSTD_OPTIONS = {...} # if you set this, `CACHE_COMPRESS_LEVEL` will be ignored. + DECOMPRESS_ZSTD_OPTIONS = {...} # note: if you don't set this, the above one will be used. + COMPRESS_ZSTD_DICT = {...} + DECOMPRESS_ZSTD_DICT = {...} # note: if you don't set this, the above one will be used. + +*NOTE* the values shown here are only examples and *not* best practice or anything. \ No newline at end of file diff --git a/docs/source/configure/configurations.rst b/docs/source/configure/configurations.rst new file mode 100644 index 0000000..8ed9edf --- /dev/null +++ b/docs/source/configure/configurations.rst @@ -0,0 +1,11 @@ +============== +Configurations +============== + +.. toctree:: + :maxdepth: 2 + + basic_configurations + advanced_configurations + sentinel_configurations + compressors \ No newline at end of file diff --git a/docs/source/configure/sentinel_configurations.rst b/docs/source/configure/sentinel_configurations.rst new file mode 100644 index 0000000..da6aed2 --- /dev/null +++ b/docs/source/configure/sentinel_configurations.rst @@ -0,0 +1,122 @@ +====================== +Sentinel configuration +====================== + +a sentinel configuration has these parts: + +1. ``DJANGO_VALKEY_CONNECTION_FACTORY``: you can use the default ConnectionFactory or SentinelConnectionFactory + SentinelConnectionFactory inherits from ConnectionFactory but adds checks to see if configuration is correct, also adds features to make configuration more robust. + +2. ``CACHES["default"]["OPTIONS"]["CONNECTION_FACTORY"]``: does what the above option does, but only in the scope of the cache server it was defined in. + +3. ``CACHES["default"]["OPTIONS"]["CLIENT_CLASS"]``: setting the client class to SentinelClient will add some checks to ensure proper configs and makes working with primary and replica pools easier + you can get by just using the DefaultClient but using SentinelClient is recommended. +4. ``CACHES["default"]["OPTIONS"]["CONNECTION_POOL_CLASS"]``: if you have configured the above settings to use Sentinel friendly options you don't have to set this, otherwise you might want to set this to ``valkey.sentinel.SentinelConnectionPool``. + +5. ``CACHES["default"]["OPTIONS"]["SENTINELS"]``: a list of (host, port) providing the sentinel's connection information. + +6. ``CACHES["default"]["OPTIONS"]["SENTINEL_KWARGS"]``: a dictionary of arguments sent down to the underlying Sentinel client + +the below code is a bit long but comprehensive example of different ways to configure a sentinel backend. +*Note* that depending on how you configured your backend, you might need to adjust the ``LOCATION`` to fit other configs + +.. code-block:: python + + + DJANGO_VALKEY_CONNECTION_FACTORY = "django_valkey.pool.SentinelConnectionFactory" + + # These sentinels are shared between all the examples, and are passed + # directly to valkey Sentinel. These can also be defined inline. + SENTINELS = [ + ('sentinel-1', 26379), + ('sentinel-2', 26379), + ('sentinel-3', 26379), + ] + + CACHES = { + "default": { + ... + "LOCATION": "valkey://service_name/db", # note you should pass in valkey service name, not address + "OPTIONS": { + # While the default client will work, this will check you + # have configured things correctly, and also create a + # primary and replica pool for the service specified by + # LOCATION rather than requiring two URLs. + "CLIENT_CLASS": "django_valkey.client.SentinelClient", + + # these are passed directly to valkey sentinel + "SENTINELS": SENTINELS, + + # optional + "SENTINEL_KWARGS": {}, + + # you can override the connection pool (optional) + # (it is originally defined in connection factory) + "CONNECTION_POOL_CLASS": "valkey.sentinel.SentinelConnectionPool", + }, + }, + + # a minimal example using the SentinelClient + "minimal": { + "BACKEND": "django_valkey.cache.ValkeyCache", + "LOCATION": "valkey://minimal_service_name/db", + + "OPTIONS": { + "CLIENT_CLASS": "django_valkey.client.SentinelClient", + "SENTINELS": SENTINELS, + }, + }, + + # a minimal example using the DefaultClient + "other": { + "BACKEND": "django_valkey.cache.ValkeyCache", + "LOCATION": [ + # The DefaultClient is [primary, replicas], but with the + # SentinelConnectionPool it only requires "is_master=1" for primary and "is_master=0" for replicas. + "valkey://other_service_name/db?is_master=1", + "valkey://other_service_name/db?is_master=0", + ] + "OPTIONS": {"SENTINELS": SENTINELS}, + }, + + # a minimal example only using replicas in read only mode + # (and the DefaultClient). + "readonly": { + "BACKEND": "django_valkey.cache.ValkeyCache", + "LOCATION": "valkey://readonly_service_name/db?is_master=0", + "OPTIONS": {"SENTINELS": SENTINELS}, + }, + } + + + +Use sentinel and normal servers together +**************************************** +it is also possible to set some caches as sentinels ans some as not: + +.. code-block:: python + + SENTINELS = [ + ('sentinel-1', 26379), + ('sentinel-2', 26379), + ('sentinel-3', 26379), + ] + CACHES = { + "sentinel": { + "BACKEND": "django_valkey.cache.ValkeyCache", + "LOCATION": "valkey://service_name/db", + "OPTIONS": { + "CLIENT_CLASS": "django_valkey.client.SentinelClient", + "SENTINELS": SENTINELS, + "CONNECTION_POOL_CLASS": "valkey.sentinel.SentinelConnectionPool", + "CONNECTION_FACTORY": "django_valkey.pool.SentinelConnectionFactory", + }, + }, + "default": { + "BACKEND": "django_valkey.cache.ValkeyCache", + "LOCATION": "valkey://127.0.0.1:6379/1", + "OPTIONS": { + "CLIENT_CLASS": "django_valkey.client.DefaultClient", + }, + }, + } diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..5dfc4df --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,72 @@ +.. django-valkey documentation master file, created by + sphinx-quickstart on Fri Sep 6 23:14:07 2024. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +============= +django-valkey +============= + +`valkey `_ is an open source (BSD) high-performance key/value database that supports a variety of workloads such as *caching*, message queues, and can act as a primary database. + +django-valkey is a customizable valkey backend to be used as a caching database in django. +this project was initially a fork of the wonderful ``django-redis`` project. + +django-valkey Features +###################### + +#. Uses native valkey-py url notation connection strings +#. Pluggable clients: + #. Default Client + #. Herd Client + #. Sentinel Client + #. Sharded Client + #. or just plug in your own client +#. Pluggable serializers: + #. Pickle Serializer + #. Json Serializer + #. msgpack serializer + #. or plug in your own serializer +#. Pluggable compression: + #. brotli compression + #. bz2 compression (bzip2) + #. gzip compression + #. lz4 compression + #. lzma compression + #. zlib compression + #. zstd compression + #. plug in your own +#. Pluggable parsers + #. Valkey's default parser + #. plug in your own +#. Pluggable connection pool + #. Valkey's default connection pool + #. plug in your own +#. Comprehensive test suite +#. Supports infinite timeouts +#. Facilities for raw access to Valkey client/connection pool +#. Highly configurable (really, just look around) +#. Unix sockets supported by default + + +Requirements +############ + +- `Python`_ 3.10+ +- `Django`_ 3.2.9+ +- `valkey-py`_ 6.0.0+ (probably works on older versions too) +- `Valkey server`_ 7.2.6+ (probably works with older versions too) + +.. _Python: https://www.python.org/downloads/ +.. _Django: https://www.djangoproject.com/download/ +.. _valkey-py: https://pypi.org/project/valkey/ +.. _Valkey server: https://valkey.io/download + +.. toctree:: + :maxdepth: 3 + :caption: Index: + + installation + configure/configurations + commands/commands + changes diff --git a/docs/source/installation.rst b/docs/source/installation.rst new file mode 100644 index 0000000..a730da5 --- /dev/null +++ b/docs/source/installation.rst @@ -0,0 +1,65 @@ +================== +Installation guide +================== + + +Basic Installation: +################### + +.. code-block:: console + + pip install django-valkey + +| +| + +Install with C-bindings for maximum performance: +################################################ + +.. code-block:: console + + pip install django-valkey[libvalkey] + +| +| + +Install with 3rd-party serializers +################################## + + +.. _msgpack: + +Install with msgpack serializer: +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: console + + pip install django-valkey[msgpack] + +| +| + + +Install with 3rd party compression libraries: +############################################# + +lz4 library: +^^^^^^^^^^^^ + +.. code-block:: console + + pip install django-valkey[lz4] + +pyzstd library: +^^^^^^^^^^^^^^^ + +.. code-block:: + + pip install django-valkey[pyzstd] + +brotli library: +^^^^^^^^^^^^^^^ + +.. code-block:: console + + pip install django-valkey[brotli] \ No newline at end of file