Merge branch 'main' into ans_wagtail_charts_tweaks

.dockerignore

@@ -17,10 +17,12 @@
 !dump-data.sh
 !Dockerfile
 !frontend.sh
+!index.sh
 !initial-data.sh
 !jest.config.js
 !package.json
 !refresh-data.sh
+!test.sql.gz
 !yarn.lock
 
 # ...but DO ignore these others

.github/workflows/backend.yml

@@ -2,12 +2,6 @@ name: backend
 
 on:
   workflow_dispatch:
-    paths:
-      - '**.py'
-      - 'requirements/**.txt'
-      - '**.html'
-      - '**.mo'
-      - '**.po'
   pull_request:
     paths:
       - '**.py'
@@ -16,12 +10,6 @@ on:
       - '**.mo'
       - '**.po'
   merge_group:
-    paths:
-      - '**.py'
-      - 'requirements/**.txt'
-      - '**.html'
-      - '**.mo'
-      - '**.po'
 
 jobs:
   backend:

.github/workflows/build-artifact.yml

@@ -7,7 +7,6 @@ on:
 
 jobs:
   Build:
-    if: "! github.event.pull_request.head.repo.fork"
     runs-on: ubuntu-latest
     env:
       ARTIFACT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
@@ -20,25 +19,53 @@ jobs:
         with:
             node-version: '20.x'
 
+      - name: Install Act dependencies (if needed)
+        if: ${{ env.ACT }}
+        run: corepack enable
+
       - name: Install dependencies
         run: yarn
 
       - name: Build frontend
         run: yarn run build
 
-      - name: Run the build process with Docker
-        uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185
+      - name: Login to GHCR
+        uses: docker/login-action@v3
         with:
-            registry: ghcr.io
-            username: ${{ github.actor }}
-            password: ${{ secrets.GITHUB_TOKEN }}
-            image: ghcr.io/${{ github.repository }}-builder:main
-            options: -v ${{ github.workspace }}:/cfgov
-            run: |
-              source scl_source enable rh-python38
-              ./call_create.sh
-
-      - name: Upload arifact
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Pull image to build artifact
+        run: docker pull ghcr.io/cfpb/cfgov-artifact-builder:latest
+
+      - name: Build artifact
+        run: |
+          docker run \
+          -v `pwd`:/cfgov \
+          ghcr.io/cfpb/cfgov-artifact-builder:latest
+
+      - name: Pull image to test artifact
+        run: docker pull ghcr.io/cfpb/cfgov-artifact-tester:latest
+
+      - name: Run built artifact as a Docker container
+        run: |
+          docker run \
+          --name cfgov \
+          -d \
+          -v `pwd`:/cfgov:ro \
+          -p 8000:80 \
+          --env-file docker/centos7/test.env \
+          ghcr.io/cfpb/cfgov-artifact-tester:latest
+
+      - name: Wait for artifact container to be ready
+        run: |
+          timeout 180s bash -c \
+            'until [ "$(docker inspect -f {{.State.Health.Status}} cfgov)" == "healthy" ]; do sleep 2; done'
+
+      # Don't upload artifacts to S3 if PR is against a fork.
+      - name: Upload artifact to S3
+        if: "! github.event.pull_request.head.repo.fork"
         uses: keithweaver/[email protected]
         with:
           command: cp

.github/workflows/builder.yml

@@ -1,14 +1,11 @@
-name: Create and publish a Docker image that can be used to build a deployable zipfile
+name: Maintain deployable zipfile Docker images
 
 on:
   push:
     branches: ['main']
     paths:
-      - 'docker/builder/**'
-
-env:
-  REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}-builder
+      - 'docker/centos7/**'
+  workflow_dispatch:
 
 jobs:
   build-and-push-image:
@@ -25,20 +22,26 @@ jobs:
       - name: Log in to the Container registry
         uses: docker/login-action@v3
         with:
-          registry: ${{ env.REGISTRY }}
+          registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Extract metadata (tags, labels) for Docker
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+      - name: Build deployable zipfile builder image
+        run: |
+          docker build docker/centos7 \
+          --target=cfgov-artifact-builder \
+          -t ghcr.io/cfpb/cfgov-artifact-builder:latest
 
-      - name: Build and push Docker image
-        uses: docker/build-push-action@v5
-        with:
-          context: ./docker/builder/
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
+      - name: Build deployable zipfile tester image
+        run: |
+          docker build docker/centos7 \
+          --target=cfgov-artifact-tester \
+          -t ghcr.io/cfpb/cfgov-artifact-tester:latest
+
+      - name: Push deployable zipfile builder image
+        run: |
+          docker push ghcr.io/cfpb/cfgov-artifact-builder:latest
+
+      - name: Push deployable zipfile tester image
+        run: |
+          docker push ghcr.io/cfpb/cfgov-artifact-tester:latest

.github/workflows/filter-backend.yml

@@ -8,12 +8,6 @@ on:
       - '**.mo'
       - '**.po'
   merge_group:
-    paths-ignore:
-      - '**.py'
-      - 'requirements/**.txt'
-      - '**.html'
-      - '**.mo'
-      - '**.po'
 
 jobs:
   backend:

.github/workflows/helm.yml

@@ -0,0 +1,21 @@
+name: Helm testing
+on:
+  pull_request:
+    paths:
+      - helm/**
+
+jobs:
+  helm:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Install Helm
+        uses: azure/setup-helm@v4
+
+      - name: Install Helm dependencies
+        run: helm dependency update helm
+
+      - name: Lint Helm chart
+        run: helm lint --strict -f helm/values.local.yaml helm

.gitignore

@@ -145,6 +145,10 @@ cfgov/regulations3k/jinja2/regulations3k/regulations3k-service-worker.js.map
 cfgov/regulations3k/jinja2/regulations3k/workbox-*.js
 cfgov/regulations3k/jinja2/regulations3k/workbox-*.js.map
 
+# Helm Charts #
+helm/apache/
+helm/charts/
+
 # Apache #
 ##########
 cfgov/apache/logs

.prettierignore

@@ -6,6 +6,7 @@ cfgov/retirement_api/data/
 cfgov/static_built/
 **/fixtures/
 collectstatic/
+helm/
 
 # Ignore all HTML files:
 *.html

Dockerfile

@@ -150,6 +150,10 @@ ENV ALLOWED_HOSTS '["*"]'
 # Copy the application code over
 COPY cfgov ./cfgov/
 COPY static.in ./static.in/
+COPY refresh-data.sh .
+COPY initial-data.sh .
+COPY index.sh .
+COPY test.sql.gz .
 
 # Copy our static build over from node-builder
 COPY --from=node-builder ${APP_HOME} ${APP_HOME}
@@ -170,4 +174,4 @@ RUN chown -R ${USERNAME}:${USERNAME} ${APP_HOME}
 USER $USERNAME
 
 # Run Gunicorn
-CMD gunicorn cfgov.wsgi:application -b :8000
+CMD gunicorn --reload cfgov.wsgi:application -b :8000

cfgov/apache/conf.modules.d/00-base.conf

@@ -58,7 +58,7 @@ LoadModule slotmem_shm_module modules/mod_slotmem_shm.so
 LoadModule socache_dbm_module modules/mod_socache_dbm.so
 LoadModule socache_memcache_module modules/mod_socache_memcache.so
 LoadModule socache_shmcb_module modules/mod_socache_shmcb.so
-LoadModule status_module modules/mod_status.so
+# LoadModule status_module modules/mod_status.so
 LoadModule substitute_module modules/mod_substitute.so
 LoadModule suexec_module modules/mod_suexec.so
 LoadModule unique_id_module modules/mod_unique_id.so

cfgov/apache/conf/httpd.conf

@@ -31,7 +31,6 @@ PassEnv APACHE_UPLOADS_F_ALIAS
 PassEnv APACHE_HTTPS_FORWARDED_HOST
 PassEnv APACHE_PORT
 PassEnv LIMIT_REQUEST_BODY
-PassEnv APACHE_STATUS_ALLOW_FROM
 
 # These values are needed for running in containerized Apache
 PassEnv APACHE_USER

cfgov/apache/dockerfile

@@ -0,0 +1,24 @@
+FROM httpd:2.4-alpine
+
+# Copy the Apache configuration file
+COPY ./conf /usr/local/apache2/conf
+COPY ./conf.d /usr/local/apache2/conf.d
+COPY ./conf.modules.d /usr/local/apache2/conf.modules.d
+
+# Add Apache ENV variables
+ENV APACHE_PORT="80"
+ENV APACHE_USER="www-data"
+ENV APACHE_GROUP="www-data"
+ENV APACHE_SERVER_ROOT="/usr/local/apache2/"
+ENV APACHE_UPLOADS_F_ALIAS="/src/consumerfinance.gov/cfgov/f/"
+ENV STATIC_PATH="/tmp"
+ENV ERROR_LOG="/proc/self/fd/2"
+ENV ACCESS_LOG="/proc/self/fd/1"
+ENV LIMIT_REQUEST_BODY="0"
+ENV APACHE_STATUS_ALLOW_FROM="127.0.0.1"
+ENV APACHE_PROCESS_COUNT="4"
+ENV CFGOV_APPLICATION_HOST="localhost"
+
+EXPOSE 80
+
+CMD ["httpd-foreground"]