changed CI to use valkey image directly

.github/workflows/ci.yml

@@ -1,62 +1,64 @@
 name: CI
 
 on: [push, pull_request]
+permissions:
+  contents: read
 
 jobs:
-  test:
+  populate-cache:
     runs-on: ubuntu-latest
+    timeout-minutes: 60
+    name: Update docker cache
+    steps:
+      - uses: actions/checkout@v4
+      - name: Cache docker images
+        id: custom-cache
+        uses: actions/cache@v4
+        with:
+          path: ./custom-cache/
+          key: custom-cache
+      - if: ${{ steps.custom-cache.outputs.cache-hit != 'true' || github.event_name == 'schedule' }}
+        name: Update cache
+        run: |
+          mkdir -p ./custom-cache/
+          docker compose --profile all build
+          docker pull valkey/valkey:latest
+          docker save django-valkey-cluster:latest valkey/valkey:latest -o ./custom-cache/all.tar
 
+  test:
+    runs-on: ubuntu-latest
+    needs: [populate-cache]
+    timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:
         python-version:
           - '3.10'
           - '3.11'
           - '3.12'
-          #          - '3.13'
+#          - '3.13'
         django-version:
           - '4.2'
           - '5.0'
           - '5.1'
-        valkey-version:
-          - 'latest'
-          - '7.2'
-
-    services:
-      valkey:
-        image: valkey/valkey:${{ matrix.valkey-version }}
-        ports:
-          - 6379:6379
-        options: >-
-          --health-cmd "valkey-cli ping"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-          --volume /tmp:/tmp
-        env:
-          VALKEY_EXTRA_FLAGS: '--save "" --unixsocket /tmp/valkey.sock --unixsocketperm 777'
-
-      sentinel:
-        image: bitnami/valkey-sentinel:${{ matrix.valkey-version }}
-        ports:
-          - 26379:26379
-        options: >-
-          --health-cmd "valkey-cli -p 26379 ping"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-          --volume /tmp:/tmp
-        env:
-          VALKEY_SENTINEL_QUORUM: "1"
-          VALKEY_SENTINEL_AOF_ENABLED: "no"
-          VALKEY_PRIMARY_SET: "mymaster"
+    env:
+      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
     steps:
       - uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Cache docker images
+        id: custom-cache
+        uses: actions/cache@v4
+        with:
+          path: ./custom-cache/
+          fail-on-cache-miss: true
+          key: custom-cache
+      - name: Use Cache
+        run: docker image load -i ./custom-cache/all.tar
 
 #      - name: Cache
 #        id: cached-poetry
@@ -84,13 +86,9 @@ jobs:
 
       - name: tests
         run: |
-          VALKEY_PRIMARY=$(tests/start_valkey.sh)
-          VALKEY_SENTINEL=$(tests/start_valkey.sh --sentinel)
-          CONTAINERS="$VALKEY_PRIMARY $VALKEY_SENTINEL"
-          trap "docker stop $CONTAINERS && docker rm $CONTAINERS" EXIT
-          tests/wait_for_valkey.sh $VALKEY_PRIMARY 6379
-          tests/wait_for_valkey.sh $VALKEY_SENTINEL 26379
-
+          poetry run invoke devenv
+          chmod +x ./util/wait-for-it.sh
+          ./util/wait-for-it.sh localhost:16379
           
           poetry run pytest tests/*.py --ds=tests.settings.sqlite
           poetry run pytest tests/*.py --ds=tests.settings.sqlite_herd
@@ -110,4 +108,3 @@ jobs:
 
         env:
           DJANGO: ${{ matrix.django-version }}
-          VALKEY: ${{ matrix.valkey-version }}

compose.yaml

@@ -0,0 +1,61 @@
+
+
+---
+
+services:
+
+  valkey:
+    image: valkey/valkey:latest
+    container_name: valkey-standalone
+    ports:
+      - 6379:6379
+    entrypoint: "/usr/local/bin/docker-entrypoint.sh --enable-debug-command yes --enable-module-command yes --unixsocket /run/valkey.sock --unixsocketperm 777"
+    profiles:
+      - standalone
+      - sentinel
+      - replica
+      - all
+
+  replica:
+    image: valkey/valkey:latest
+    container_name: valkey-replica
+    depends_on:
+      - valkey
+    entrypoint: "/usr/local/bin/docker-entrypoint.sh --replicaof valkey 6379"
+    ports:
+      - 6380:6379
+    profiles:
+      - replica
+      - all
+
+  cluster:
+    container_name: valkey-cluster
+    build:
+      context: .
+      dockerfile: dockers/Dockerfile.cluster
+    ports:
+      - 16379:16379
+      - 16380:16380
+      - 16381:16381
+      - 16382:16382
+      - 16383:16383
+      - 16384:16384
+    volumes:
+      - "./dockers/cluster.valkey.conf:/valkey.conf:ro"
+    profiles:
+      - cluster
+      - all
+
+  sentinel:
+    image: valkey/valkey:latest
+    container_name: valkey-sentinel
+    depends_on:
+      - valkey
+    entrypoint: "/usr/local/bin/valkey-sentinel /valkey.conf --port 26379"
+    ports:
+      - 26379:26379
+    volumes:
+      - "./dockers/sentinel.conf:/valkey.conf"
+    profiles:
+      - sentinel
+      - all

dockers/Dockerfile.cluster

@@ -0,0 +1,6 @@
+FROM valkey/valkey:latest as rss
+
+COPY dockers/create_cluster.sh /create_cluster.sh
+RUN chmod a+x /create_cluster.sh
+
+ENTRYPOINT [ "/create_cluster.sh"]

dockers/cluster.valkey.conf

@@ -0,0 +1,2 @@
+protected-mode no
+enable-debug-command yes

dockers/create_cluster.sh

@@ -0,0 +1,47 @@
+#! /bin/bash
+
+mkdir -p /nodes
+touch /nodes/nodemap
+if [ -z ${START_PORT} ]; then
+    START_PORT=16379
+fi
+if [ -z ${END_PORT} ]; then
+    END_PORT=16384
+fi
+if [ ! -z "$3" ]; then
+    START_PORT=$2
+    START_PORT=$3
+fi
+echo "STARTING: ${START_PORT}"
+echo "ENDING: ${END_PORT}"
+
+for PORT in `seq ${START_PORT} ${END_PORT}`; do
+  mkdir -p /nodes/$PORT
+  if [[ -e /valkey.conf ]]; then
+    cp /valkey.conf /nodes/$PORT/valkey.conf
+  else
+    touch /nodes/$PORT/valkey.conf
+  fi
+  cat << EOF >> /nodes/$PORT/valkey.conf
+port ${PORT}
+cluster-enabled yes
+daemonize yes
+logfile /valkey.log
+dir /nodes/$PORT
+EOF
+
+  set -x
+  /usr/local/bin/valkey-server /nodes/$PORT/valkey.conf
+  sleep 1
+  if [ $? -ne 0 ]; then
+    echo "Valkey failed to start, exiting."
+    continue
+  fi
+  echo 127.0.0.1:$PORT >> /nodes/nodemap
+done
+if [ -z "${VALKEY_PASSWORD}" ]; then
+    echo yes | /usr/local/bin/valkey-cli --cluster create `seq -f 127.0.0.1:%g ${START_PORT} ${END_PORT}` --cluster-replicas 1
+else
+    echo yes | /usr/local/bin/valkey-cli -a ${VALKEY_PASSWORD} --cluster create `seq -f 127.0.0.1:%g ${START_PORT} ${END_PORT}` --cluster-replicas 1
+fi
+tail -f /valkey.log

dockers/sentinel.conf

@@ -0,0 +1,5 @@
+sentinel resolve-hostnames yes
+sentinel monitor mymaster valkey 6379 1
+sentinel down-after-milliseconds mymaster 5000
+sentinel failover-timeout mymaster 60000
+sentinel parallel-syncs mymaster 1