Add Basis of Deployment Pipeline Workflow; Update to Python 3.9 (#345)

* Fix container startup This change ensures that everything is ready to run `make test` * Delete obsolete Travis config * Delete obsolete pytest config * Upgrade to Python 3.9 * Fix open file warnings in DonationMailGenerator * Add GitHub Actions extension to dev container * Separate dev and prod requirements * Add lint target to run autopep8 * Add initial deployment pipeline workflow * Add lint-check, lint-fix * Add workflow dispatch trigger to deployment workflow * Fix rabbitmq container options * Collapse rabbitmq volumes * Use absolute path for volumes * Install RabbitMQ certs out of workspace * Fix configuration volume pathname * Use sudo to copy RabbitMQ files * Set RabbitMQ container name * Set environment variables for steps * Enable debug * Define CSRF_TRUSTED_ORIGINS * Define missing variables * Add optional DB_PASSWORD for local * Set DB_PASSWORD * Use random SECRET_KEY * Refactor install, env, codespace targets * Enable pip caching * Add autopep8 to requirements-dev * Use consistent line length for autopep8, ruff * Fix autopep8 issues * Add ruff to requirements-dev * Differentiate between formatting and linting * Fix ruff issues * Expose RabbitMQ TLS port * Set CLOUDAMQP_URL * Expose RabbitMQ TCP and TLS * Revert "Set CLOUDAMQP_URL" This reverts commit 55b17c0. * Synchronize PostgreSQL settings * Drop unnecessary variables * Setup RabbitMQ without container * Include RabbitMQ config file * Don't detach server * Copy certs to config directory * Use Actions-specific RabbitMQ conf * Revert "Use Actions-specific RabbitMQ conf" This reverts commit 51049de. * Use local config file * Stop automatically started RabbitMQ instance * Revert "Don't detach server" This reverts commit eeb3ec4. * Use rabbitmq-github-actions * Use configuration file for pep8 * Restore unique autopep8 settings * Set autopep8 aggressive = 3 * Revert "Set autopep8 aggressive = 3" This reverts commit 62f9cf4. * Revert to line length 79 * Fix setup syntax error * Extract email bodies to static files * Disable debug * Read email bodies as text, not binary
CodeTheChangeUBC · Dec 24, 2023 · eaa6e53 · eaa6e53
1 parent ea14fbd
commit eaa6e53
Show file tree

Hide file tree

Showing 40 changed files with 490 additions and 162 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/devcontainers/python:0-3.7
+FROM mcr.microsoft.com/devcontainers/python:3.9
 
 RUN apt-get update
 RUN	apt-get --yes install rabbitmq-server

diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -24,9 +24,7 @@
     }
   },
 
-  // Use 'postCreateCommand' to run commands after the container is created.
-  "postCreateCommand": "make codespace",
-  "postStartCommand": "make env && make celery",
+  "postStartCommand": "make codespace",
 
   // Configure tool-specific properties.
   "customizations": {
@@ -37,7 +35,8 @@
         "ms-python.autopep8",
         "charliermarsh.ruff",
         "ms-azuretools.vscode-docker",
-        "ms-vscode.makefile-tools"
+        "ms-vscode.makefile-tools",
+        "github.vscode-github-actions"
       ],
       "settings": {
         "python.venvPath": "${workspaceFolder}"

diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml
@@ -0,0 +1,72 @@
+name: Deployment
+
+on:
+  - push
+  - workflow_dispatch
+
+permissions:
+  contents: read
+
+jobs:
+  commit:
+    runs-on: ubuntu-latest
+
+    env:
+      ADMIN: NAME,[email protected]
+      ALLOWED_HOSTS: localhost,0.0.0.0,127.0.0.1
+      CELERY_RESULT_BACKEND: "rpc://"
+      CSRF_TRUSTED_ORIGINS: ""
+      DB_NAME: reboot
+      DB_USER: root
+      DB_PASSWORD: ${{ github.run_id }}-${{ github.run_attempt }}
+      DEBUG: False
+      DJANGO_DATABASE: local
+      EMAIL_HOST: smtp.gmail.com
+      EMAIL_HOST_USER: ""
+      EMAIL_HOST_PASSWORD: ""
+      SECRET_KEY: ${{ github.run_id }}-${{ github.run_attempt }}
+      SECURE_SSL_REDIRECT: False
+
+    services:
+      postgres:
+        image: postgres:13
+        env:
+          POSTGRES_USER: ${{ env.DB_USER }}
+          POSTGRES_PASSWORD: ${{ env.DB_PASSWORD }}
+          POSTGRES_DB: ${{ env.DB_NAME}}
+        ports:
+          - 5432:5432
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+
+    steps:
+    - name: Checkout the Git repository
+      uses: actions/checkout@v3
+    - name: Set up RabbitMQ 3.12
+      uses: namoshek/rabbitmq-github-action@v1
+      with:
+        version: "3.12"
+        ports: "5671:5671 5672:5672"
+        certificates: ${{ github.workspace }}/dev/ssl/cert
+        config: ${{ github.workspace }}/rabbitmq-actions.conf
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.9"
+        cache: "pip"
+        cache-dependency-path: |
+          ./requirements-dev.txt
+          ./requirements.txt
+    - name: Install dependencies
+      run: |
+        make install
+    - name: Format with autopep8
+      run: |
+        make format-check
+    - name: Lint with ruff
+      run: |
+        make lint-check
+    - name: Test with unittest
+      run: |
+        make celery &
+        make test
+        make coverage
diff --git a/.gitignore b/.gitignore
@@ -117,3 +117,6 @@ venv.bak/
 
 # DB stuff
 db.sqlite3
+
+# Dev Container
+nohup.out
diff --git a/.travis.yml b/.travis.yml
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -7,14 +7,19 @@
     "editor.defaultFormatter": "ms-python.autopep8",
     "editor.formatOnSave": true
   },
+  "autopep8.args": [
+    "--aggressive",
+    "--aggressive",
+    "--max-line-length",
+    "79"
+  ],
   "python.analysis.autoImportCompletions": true,
   "python.analysis.packageIndexDepths":[
     {
       "name": "django",
       "depth": 5
     },
   ],
-  "python.formatting.provider": "none",
   "python.testing.unittestArgs": [
     "--verbose",
     "--start-directory",
@@ -26,7 +31,7 @@
   ],
   "python.testing.pytestEnabled": false,
   "python.testing.unittestEnabled": true,
-  "ruff.args": [
+  "ruff.lint.args": [
     "--line-length",
     "79"
   ]

diff --git a/Makefile b/Makefile
@@ -18,19 +18,13 @@ heroku:
 
 .PHONY: install
 install:
-	sh scripts/start_db.sh
-	sh scripts/create_db.sh
 	python3 -m venv venv
-	make post-install
-
-.PHONY: post-install
-post-install:
 	pip install -U pip
 	pip install -r requirements.txt
+	pip install -r requirements-dev.txt
 	make migrate
 	make groups
 	make static
-	make stopenv
 
 .PHONY: static
 static:
@@ -43,6 +37,7 @@ server:
 .PHONY: env
 env:
 	sh scripts/start_db.sh
+	sh scripts/create_db.sh
 ifeq ($(USER),vscode)
 	sudo rabbitmq-server -detached
 else
@@ -84,7 +79,9 @@ codespace:
 	sudo cp ./rabbitmq-devcontainer.conf /etc/rabbitmq/rabbitmq.conf
 	make .env
 	make .git/hooks/pre-commit
+	make env
 	make install
+	nohup bash -c 'make celery &'
 
 .PHONY: test
 test:
@@ -94,6 +91,22 @@ test:
 coverage:
 	coverage report --sort=miss --show-missing
 
+.PHONY: format-check
+format-check:
+	autopep8 --diff --recursive --aggressive --aggressive --exit-code ./app ./reboot
+
+.PHONY: format-fix
+format-fix:
+	autopep8 --in-place --recursive --aggressive --aggressive ./app ./reboot
+
+.PHONY: lint-check
+lint-check:
+	ruff check --unsafe-fixes ./app ./reboot
+
+.PHONY: lint-fix
+lint-fix:
+	ruff check --fix --unsafe-fixes ./app ./reboot
+
 .env:
 	cp .env.sample .env
 

diff --git a/app/admin.py b/app/admin.py
@@ -1,19 +1,22 @@
 # -*- coding: utf-8 -*-
-from django.contrib import messages
-from django.contrib import admin
+from django.contrib import admin, messages
 from django.db import models
-from django.http import HttpResponseRedirect, HttpRequest
 from django.forms import Textarea
+from django.http import HttpRequest, HttpResponseRedirect
 from django.utils import timezone as tz
 from rangefilter.filter import DateRangeFilter
 
 from app.constants.str import (
-    PERMISSION_DENIED, UNVERIFIED_DONATION, RECEIPTED_DONATION,
-    UNEVALUATED_DONATION, EMPTY_DONATION, ITEM_NOT_RECEIVED)
+    EMPTY_DONATION,
+    ITEM_NOT_RECEIVED,
+    PERMISSION_DENIED,
+    RECEIPTED_DONATION,
+    UNEVALUATED_DONATION,
+    UNVERIFIED_DONATION,
+)
 from app.enums import ItemStatusEnum
-from app.models import (
-    Donor, Donation, Item, ItemDevice, ItemDeviceType)
-from app.filters import DonorBusinessFilter, DonationStatusFilter
+from app.filters import DonationStatusFilter, DonorBusinessFilter
+from app.models import Donation, Donor, Item, ItemDevice, ItemDeviceType
 from app.views.views import download_receipt, export_csv
 from app.widgets.CustomForeignKeyRawIdWidget import CustomForeignKeyRawIdWidget
 
@@ -157,7 +160,8 @@ class DonationAdmin(ResourceAdmin):
                     'pledge_date',
                     'tax_receipt_created_at',
                     'pick_up',
-                    'total_value', # total value of donation
+                    # total value of donation
+                    'total_value',
                     'verified',
                     'item_count',
                     'donate_date',

diff --git a/app/management/commands/creategroups.py b/app/management/commands/creategroups.py
@@ -2,9 +2,9 @@
 Create permission groups
 Create permissions to models for a set of groups
 """
-from django.core.management.base import BaseCommand
 from django.contrib.auth.models import Group, Permission
-from app.models import Item
+from django.core.management.base import BaseCommand
+
 from app.constants.perm_list import FRONTLINE, MANAGEMENT
 
 
@@ -13,7 +13,7 @@ def create_group(name, perms):
     for perm in perms:
         print(f'Adding permission with codname={perm}')
         permission = Permission.objects.get(codename=perm)
-        if not permission in group.permissions.all():
+        if permission not in group.permissions.all():
             group.permissions.add(permission)
 
 

diff --git a/app/migrations/0002_auto_20190730_0312.py b/app/migrations/0002_auto_20190730_0312.py
@@ -3,7 +3,6 @@
 from __future__ import unicode_literals
 
 from django.db import migrations, models
-import django.utils.timezone
 
 
 class Migration(migrations.Migration):

diff --git a/app/migrations/0003_auto_20190730_0316.py b/app/migrations/0003_auto_20190730_0316.py
@@ -14,7 +14,11 @@ class Migration(migrations.Migration):
     operations = [
         migrations.AlterModelOptions(
             name='item',
-            options={'permissions': (
-                ('update_status', 'Can update item status'), ('update_value', 'Can update item value'))},
+            options={
+                'permissions': (
+                    ('update_status',
+                     'Can update item status'),
+                    ('update_value',
+                     'Can update item value'))},
         ),
     ]
diff --git a/app/migrations/0004_auto_20190731_0416.py b/app/migrations/0004_auto_20190731_0416.py
@@ -2,9 +2,10 @@
 # Generated by Django 1.11.22 on 2019-07-31 08:16
 from __future__ import unicode_literals
 
-import app.enums.source_enum
-from django.db import migrations, models
 import django.utils.timezone
+from django.db import migrations, models
+
+import app.enums.source_enum
 
 
 class Migration(migrations.Migration):
@@ -21,19 +22,34 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='donation',
             name='source',
-            field=models.CharField(choices=[('DIRECT_MANUAL_INPUT', 'Direct Manual Input'), ('ONLINE_FORM', 'On-line Form'), ('WEBSITE_IMPORT', 'Website File Import'), ('HISTORICAL_DATA',
-                                                                                                                                                                         'Historical Data'), ('THIRD_PARTY_DATA', '3rd Party Data')], default=app.enums.source_enum.SourceEnum('Direct Manual Input'), max_length=255, verbose_name='Source'),
+            field=models.CharField(
+                choices=[
+                    ('DIRECT_MANUAL_INPUT',
+                     'Direct Manual Input'),
+                    ('ONLINE_FORM',
+                     'On-line Form'),
+                    ('WEBSITE_IMPORT',
+                     'Website File Import'),
+                    ('HISTORICAL_DATA',
+                     'Historical Data'),
+                    ('THIRD_PARTY_DATA',
+                     '3rd Party Data')],
+                default=app.enums.source_enum.SourceEnum('Direct Manual Input'),
+                max_length=255,
+                verbose_name='Source'),
         ),
         migrations.AlterField(
             model_name='donation',
             name='pledge_date',
             field=models.DateField(
-                default=django.utils.timezone.now, verbose_name='Pledge Date'),
+                default=django.utils.timezone.now,
+                verbose_name='Pledge Date'),
         ),
         migrations.AlterField(
             model_name='donor',
             name='want_receipt',
             field=models.BooleanField(
-                default=True, verbose_name='Want Tax Receipt?'),
+                default=True,
+                verbose_name='Want Tax Receipt?'),
         ),
     ]
diff --git a/app/migrations/0005_auto_20190802_0933.py b/app/migrations/0005_auto_20190802_0933.py
@@ -15,13 +15,35 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='donation',
             name='source',
-            field=models.CharField(choices=[('DIRECT_MANUAL_INPUT', 'Direct Manual Input'), ('ONLINE_FORM', 'On-line Form'), ('WEBSITE_IMPORT', 'Website File Import'),
-                                            ('HISTORICAL_DATA', 'Historical Data'), ('THIRD_PARTY_DATA', '3rd Party Data')], default='DIRECT_MANUAL_INPUT', max_length=255, verbose_name='Source'),
+            field=models.CharField(
+                choices=[
+                    ('DIRECT_MANUAL_INPUT',
+                     'Direct Manual Input'),
+                    ('ONLINE_FORM',
+                     'On-line Form'),
+                    ('WEBSITE_IMPORT',
+                     'Website File Import'),
+                    ('HISTORICAL_DATA',
+                     'Historical Data'),
+                    ('THIRD_PARTY_DATA',
+                     '3rd Party Data')],
+                default='DIRECT_MANUAL_INPUT',
+                max_length=255,
+                verbose_name='Source'),
         ),
         migrations.AlterField(
             model_name='donor',
             name='want_receipt',
-            field=models.CharField(choices=[('EMAIL', 'Email'), ('MAIL', 'Mail'), (
-                'REFUSED', 'Refused')], default='EMAIL', max_length=255, verbose_name='Tax Receipt Via'),
+            field=models.CharField(
+                choices=[
+                    ('EMAIL',
+                     'Email'),
+                    ('MAIL',
+                     'Mail'),
+                    ('REFUSED',
+                     'Refused')],
+                default='EMAIL',
+                max_length=255,
+                verbose_name='Tax Receipt Via'),
         ),
     ]