From d2abb22cbbba87a6ed6d08eecb068f258a3df42a Mon Sep 17 00:00:00 2001 From: Aaron Lane <10655063+aaron-lane@users.noreply.github.com> Date: Sun, 21 Jan 2024 18:50:48 -0500 Subject: [PATCH] Use Redis for Celery Result Backend (#348) * Disable Celery worker mingling * Enable Redis result backend * Setup Redis in dev container * Ensure tasks are invoked correctly in tests * Add Redis to commit tests * Remove unnecessary Redis memory optimization * Use REDIS_URL environment variable * Pin to Redis 7.0 * Pin to PostgreSQL 14.10 * Pin to Redis 7 --- .devcontainer/devcontainer.json | 7 ++++++- .env.sample | 2 +- .github/workflows/deployment.yml | 10 ++++++++-- Makefile | 2 +- Procfile | 2 +- app/worker/tasks/importers/test__init__.py | 3 +-- app/worker/tasks/test__init__.py | 8 ++++---- app/worker/tasks/test_exporter.py | 4 ++-- reboot/celeryconfig.py | 4 ++-- requirements.txt | 4 ++-- 10 files changed, 28 insertions(+), 18 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index d136a7ac..22eaf696 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -9,7 +9,12 @@ // Features to add to the dev container. More info: https://containers.dev/features. "features": { - "ghcr.io/devcontainers-contrib/features/postgres-asdf:1": {} + "ghcr.io/devcontainers-contrib/features/postgres-asdf:1": { + "version": "14.10" + }, + "ghcr.io/itsmechlark/features/redis-server:1": { + "version": "7" + } }, // Use portAttributes to map port numbers to default attributes. diff --git a/.env.sample b/.env.sample index 247580b0..56ecb6f2 100644 --- a/.env.sample +++ b/.env.sample @@ -1,6 +1,5 @@ ADMIN=NAME,email@host.ca ALLOWED_HOSTS=rebootcanadadb.herokuapp.com,rebootcanada.herokuapp.com,localhost,0.0.0.0,127.0.0.1 -CELERY_RESULT_BACKEND="rpc://" CLOUDAMQP_APIKEY= CLOUDAMQP_URL= CSRF_TRUSTED_ORIGINS=.preview.app.github.dev @@ -14,6 +13,7 @@ EMAIL_HOST=smtp.gmail.com EMAIL_HOST_DISPLAY_NAME= EMAIL_HOST_USER= EMAIL_HOST_PASSWORD= +REDIS_URL="redis://localhost:6379/0" # SECRET_KEY: Check out https://www.miniwebtool.com/django-secret-key-generator/ for generating new key SECRET_KEY=abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+) SECURE_SSL_REDIRECT=False diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index e0351cd0..f05b9514 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -52,7 +52,6 @@ jobs: env: ADMIN: NAME,email@host.ca ALLOWED_HOSTS: localhost,0.0.0.0,127.0.0.1 - CELERY_RESULT_BACKEND: "rpc://" CSRF_TRUSTED_ORIGINS: "" DB_NAME: reboot DB_USER: root @@ -62,12 +61,13 @@ jobs: EMAIL_HOST: smtp.gmail.com EMAIL_HOST_USER: "" EMAIL_HOST_PASSWORD: "" + REDIS_URL: "redis://localhost:6379/0" SECRET_KEY: ${{ github.run_id }}-${{ github.run_attempt }} SECURE_SSL_REDIRECT: False services: postgres: - image: postgres:13 + image: postgres:14.10 env: POSTGRES_USER: ${{ env.DB_USER }} POSTGRES_PASSWORD: ${{ env.DB_PASSWORD }} @@ -76,6 +76,12 @@ jobs: - 5432:5432 options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + redis: + image: redis:7 + ports: + - 6379:6379 + options: --entrypoint redis-server --health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5 + steps: - name: Checkout the Git repository uses: actions/checkout@v3 diff --git a/Makefile b/Makefile index aa089b2e..9fd41dbb 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,7 @@ migrate: .PHONY: celery celery: - celery worker -A reboot --without-gossip --without-heartbeat + celery worker -A reboot --without-heartbeat --without-gossip --without-mingle .PHONY: clean clean: diff --git a/Procfile b/Procfile index 723c0efd..9ac1b955 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,2 @@ -worker: celery worker -A reboot --without-gossip --without-heartbeat +worker: celery worker -A reboot --without-heartbeat --without-gossip --without-mingle web: gunicorn reboot.wsgi --log-level info diff --git a/app/worker/tasks/importers/test__init__.py b/app/worker/tasks/importers/test__init__.py index 0a64d5e8..e25a34ac 100644 --- a/app/worker/tasks/importers/test__init__.py +++ b/app/worker/tasks/importers/test__init__.py @@ -95,8 +95,7 @@ def test_historical_data_importer(self) -> None: }) csvvalue = csvfile.getvalue().splitlines() - importers.historical_data_importer( - csvpath=csvvalue) + importers.historical_data_importer.apply(args=[csvvalue]) got_donor = Donor.objects.get(donor_name="Example Danger Donor") diff --git a/app/worker/tasks/test__init__.py b/app/worker/tasks/test__init__.py index ce95b09b..40d0408e 100644 --- a/app/worker/tasks/test__init__.py +++ b/app/worker/tasks/test__init__.py @@ -20,8 +20,8 @@ def test_receiptor_single_file(self) -> None: queryset_json = serializers.serialize(format="json", queryset=queryset) total_count = Donation.objects.count() - response = tasks.receiptor( - queryset=queryset_json, total_count=total_count) + result = tasks.receiptor.apply(args=[queryset_json, total_count]) + response = result.get() self.assertEqual(first=response.status_code, second=200) self.assertEqual( @@ -34,8 +34,8 @@ def test_receiptor_multiple_files(self) -> None: queryset_json = serializers.serialize(format="json", queryset=queryset) total_count = Donation.objects.count() - response = tasks.receiptor( - queryset=queryset_json, total_count=total_count) + result = tasks.receiptor.apply(args=[queryset_json, total_count]) + response = result.get() self.assertEqual(first=response.status_code, second=200) self.assertEqual( diff --git a/app/worker/tasks/test_exporter.py b/app/worker/tasks/test_exporter.py index 073099b1..9d4cad0e 100644 --- a/app/worker/tasks/test_exporter.py +++ b/app/worker/tasks/test_exporter.py @@ -28,8 +28,8 @@ def test_exporter(self) -> None: qs = serialize(format="json", queryset=queryset) total_count = len(queryset) - response = exporter(file_name=file_name, qs=qs, - total_count=total_count) + result = exporter.apply(args=[file_name, qs, total_count]) + response = result.get() content_type = response["Content-Type"] self.assertEqual(first=response.status_code, second=200) diff --git a/reboot/celeryconfig.py b/reboot/celeryconfig.py index 287f4c0f..290d77b9 100644 --- a/reboot/celeryconfig.py +++ b/reboot/celeryconfig.py @@ -1,6 +1,6 @@ import ssl -from decouple import config +from decouple import config broker_url = config('CLOUDAMQP_URL', default='amqp://guest@localhost//') broker_connection_timeout = 30 @@ -10,7 +10,7 @@ worker_prefetch_multiplier = 1 worker_concurrency = 10 accept_content = ['json', 'pickle'] -result_backend = config("CELERY_RESULT_BACKEND", default=broker_url) +result_backend = config("REDIS_URL") task_serializer = 'pickle' result_serializer = 'pickle' diff --git a/requirements.txt b/requirements.txt index 1971e069..b0c11aab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ # Commented requirements indicate the latest patches of the earliest releases that support Python 3.9. # https://pypi.org/project/celery/5.1.2/ -# celery==5.1.2 -celery==4.4.7 +# celery[redis]==5.1.2 +celery[redis]==4.4.7 # https://pypi.org/project/Django/2.2.28/ Django==2.2.28 # https://pypi.org/project/dj-database-url/1.0.0/