Merge branch 'main' into feat/1454-smp-success-on-parent-step-5

bcgov · Aug 23, 2024 · 1d7b05a · 1d7b05a
2 parents b071399 + 8c04a32
commit 1d7b05a
Show file tree

Hide file tree

Showing 301 changed files with 14,686 additions and 5,621 deletions.
diff --git a/.github/workflows/.deploy.yml b/.github/workflows/.deploy.yml
@@ -79,13 +79,16 @@ jobs:
           overwrite: true
           parameters:
             -p ZONE=${{ inputs.target }}
+            -p DB_PASSWORD='${{ secrets.DB_PASSWORD }}'
             -p FORESTCLIENTAPI_KEY='${{ secrets.FORESTCLIENTAPI_KEY }}'
             -p ORACLE_PASSWORD='${{ secrets.ORACLE_PASSWORD }}'
             -p ORACLE_SERVICE='${{ vars.ORACLE_SERVICE }}'
             -p ORACLE_USER='${{ vars.ORACLE_USER }}'
             -p ORACLE_SYNC_USER='${{ vars.ORACLE_SYNC_USER }}'
             -p ORACLE_SYNC_PASSWORD='${{ secrets.ORACLE_SYNC_PASSWORD }}'
             -p ORACLE_CERT_SECRET='${{ secrets.ORACLE_CERT_SECRET }}'
+            -p ORACLE_HOST='${{ vars.ORACLE_HOST }}'
+            -p VITE_USER_POOLS_WEB_CLIENT_ID=${{ secrets.VITE_USER_POOLS_WEB_CLIENT_ID }}
 
       - name: Database
         if: steps.triggers.outputs.core == 'true' || steps.triggers.outputs.sync == 'true'
@@ -94,18 +97,18 @@ jobs:
           oc_namespace: ${{ vars.OC_NAMESPACE }}
           oc_server: ${{ vars.OC_SERVER }}
           oc_token: ${{ secrets.OC_TOKEN }}
-          file: database/openshift.deploy.yml
+          file: common/openshift.database.yml
           overwrite: false
           parameters:
-            -p TAG=${{ inputs.tag }}
             -p ZONE=${{ inputs.target }}
-            -p DB_PASSWORD='${{ secrets.DB_PASSWORD }}'
             ${{ github.event_name == 'pull_request' && '-p DB_PVC_SIZE=192Mi' || '' }}
+            ${{ github.event_name == 'pull_request' && '-p MEMORY_REQUEST=100Mi' || '' }}
+            ${{ github.event_name == 'pull_request' && '-p MEMORY_LIMIT=200Mi' || '' }}
 
   deploy:
     name: Deploy
-    if: needs.init.outputs.deploy_core == 'true'
     environment: ${{ inputs.environment }}
+    if: needs.init.outputs.deploy_core == 'true'
     needs: [init]
     runs-on: ubuntu-22.04
     timeout-minutes: 10
@@ -126,7 +129,6 @@ jobs:
               -p FAM_MODDED_ZONE=${{ needs.init.outputs.fam-modded-zone }}
               -p VITE_SPAR_BUILD_VERSION=snapshot-${{ inputs.target || github.event.number }}
               -p VITE_USER_POOLS_ID=${{ vars.VITE_USER_POOLS_ID }}
-              -p VITE_USER_POOLS_WEB_CLIENT_ID=${{ vars.VITE_USER_POOLS_WEB_CLIENT_ID }}
           - name: oracle-api
             file: oracle-api/openshift.deploy.yml
             overwrite: true
@@ -148,7 +150,7 @@ jobs:
             -p TAG=${{ inputs.tag }}
             -p ZONE=${{ inputs.target }}
             ${{ github.event_name == 'pull_request' && '-p MIN_REPLICAS=1' || '' }}
-            ${{ github.event_name == 'pull_request' && '-p MAX_REPLICAS=2' || '' }}
+            ${{ github.event_name == 'pull_request' && '-p MAX_REPLICAS=1' || '' }}
             ${{ matrix.parameters }}
           verification_path: ${{ matrix.verification_path }}
           verification_retry_attempts: 5
@@ -157,6 +159,7 @@ jobs:
   # ETL testing will only run on Pull Requests if the sync/ directory is modified
   sync:
     name: Deploy (sync)
+    environment: ${{ inputs.environment }}
     if: needs.init.outputs.deploy_sync == 'true'
     needs: [init]
     runs-on: ubuntu-latest
@@ -173,7 +176,8 @@ jobs:
           parameters:
             -p TAG=${{ inputs.tag }}
             -p ZONE=${{ inputs.target }}
-            -p TEST_MODE=true
+            ${{ github.event_name == 'pull_request' && '-p TEST_MODE=true' || '' }}
+
 
       - name: Override OpenShift version
         if: github.event_name == 'pull_request'
@@ -187,25 +191,4 @@ jobs:
 
       - name: Run sync ETL
         if: github.event_name == 'pull_request'
-        run: |
-          # Run and verify job
-
-          # Login
-          oc login --token=${{ secrets.oc_token }} --server=${{ vars.oc_server }}
-          oc project ${{ secrets.oc_namespace }} #Safeguard!
-
-          # Exit on errors or unset variables
-          set -eu
-
-          # Create job
-          CRONJOB=nr-spar-${{ inputs.target }}-sync
-          RUN_JOB=${CRONJOB}--$(date +"%Y-%m-%d--%H-%M-%S")
-          oc create job ${RUN_JOB} --from=cronjob/${CRONJOB}
-
-          # Follow
-          oc wait --for=condition=ready pod --selector=job-name=${RUN_JOB} --timeout=1m
-          oc logs -l job-name=${RUN_JOB} --tail=50 --follow
-
-          # Verify successful completion
-          oc wait --for jsonpath='{.status.phase}'=Succeeded pod --selector=job-name=${RUN_JOB} --timeout=1m
-          echo "Job successful!"
+        run: ./sync/oc_run.sh ${{ inputs.tag }} ${{ secrets.oc_token }}
diff --git a/.github/workflows/.tests.yml b/.github/workflows/.tests.yml
@@ -26,7 +26,7 @@ jobs:
       VITE_SERVER_URL: https://${{ github.event.repository.name }}-${{ inputs.target }}-backend.apps.silver.devops.gov.bc.ca
       VITE_ORACLE_SERVER_URL: https://nr-spar-${{ inputs.target }}-oracle-api.apps.silver.devops.gov.bc.ca
       VITE_USER_POOLS_ID: ${{ vars.VITE_USER_POOLS_ID }}
-      VITE_USER_POOLS_WEB_CLIENT_ID: ${{ vars.VITE_USER_POOLS_WEB_CLIENT_ID }}
+      VITE_USER_POOLS_WEB_CLIENT_ID: ${{ secrets.VITE_USER_POOLS_WEB_CLIENT_ID }}
       VITE_ZONE: TEST
     runs-on: ubuntu-22.04
     steps:

diff --git a/.github/workflows/demo.yml b/.github/workflows/demo.yml
@@ -30,7 +30,7 @@ jobs:
 
           # Login
           oc login --token=${{ secrets.oc_token }} --server=${{ vars.oc_server }}
-          oc project ${{ secrets.oc_namespace }} #Safeguard!
+          oc project ${{ vars.oc_namespace }} #Safeguard!
 
           # Delete and replace route
           oc delete route/${{ env.REPO }}-${{ env.DEST }} --ignore-not-found=true

diff --git a/.github/workflows/job-sync.yml b/.github/workflows/job-sync.yml
@@ -9,10 +9,10 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
-  sync:
-    name: Sync
-    runs-on: ubuntu-latest
+  sync-test:
     environment: test
+    name: Sync (TEST)
+    runs-on: ubuntu-latest
     steps:
       - name: Override OpenShift version
         env:
@@ -23,26 +23,24 @@ jobs:
           oc version
         working-directory: /usr/local/bin/
 
-      - name: ETL Sync
-        run: |
-          # Run and verify job
-
-          # Login
-          oc login --token=${{ secrets.oc_token }} --server=${{ vars.oc_server }}
-          oc project ${{ secrets.oc_namespace }} #Safeguard!
+      - uses: actions/checkout@v4
+      - name: ETL (TEST)
+        run: ./sync/oc_run.sh test ${{ secrets.oc_token }}
 
-          # Exit on errors or unset variables
-          set -eu
-
-          # Create job
-          CRONJOB=nr-spar-test-sync
-          RUN_JOB=${CRONJOB}--$(date +"%Y-%m-%d--%H-%M-%S")
-          oc create job ${RUN_JOB} --from=cronjob/${CRONJOB}
-
-          # Follow
-          oc wait --for=condition=ready pod --selector=job-name=${RUN_JOB} --timeout=1m
-          oc logs -l job-name=${RUN_JOB} --tail=50 --follow
+  sync-prod:
+    environment: prod
+    name: Sync (PROD)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Override OpenShift version
+        env:
+          OC: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable-4.13/openshift-client-linux.tar.gz
+        run: |
+          # Download and extract with retry, continuing on error
+          (wget ${{ env.OC }} -qcO - | tar -xzvf - oc)|| !! || true
+          oc version
+        working-directory: /usr/local/bin/
 
-          # Verify successful completion
-          oc wait --for jsonpath='{.status.phase}'=Succeeded pod --selector=job-name=${RUN_JOB} --timeout=1m
-          echo "Job successful!"
+      - uses: actions/checkout@v4
+      - name: ETL (PROD)
+        run: ./sync/oc_run.sh prod ${{ secrets.oc_token }}
diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml
@@ -36,12 +36,35 @@ jobs:
       tag: ${{ needs.init.outputs.pr }}
       target: test
 
-  deploy-prod:
-    name: PROD
+  promote:
+    name: Promote Images
+    env:
+      target: ${{ needs.init.outputs.pr }}
+      tag: prod
     needs: [init, deploys]
-    secrets: inherit
-    uses: ./.github/workflows/.deploy.yml
-    with:
-      environment: prod
-      tag: ${{ needs.init.outputs.pr }}
-      target: prod
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        package: [backend, frontend, oracle-api, sync]
+    steps:
+      - uses: shrink/actions-docker-registry-tag@v4
+        with:
+          registry: ghcr.io
+          repository: ${{ github.repository }}/${{ matrix.package }}
+          target: ${{ env.target }}
+          tags: ${{ env.tag }}
+
+      - run: |
+          # Verify tagging
+          INSPECT="docker manifest inspect ghcr.io/${{ github.repository }}/${{ matrix.package }}"
+          TARGET=$(${INSPECT}:${{ env.target }} | jq -r '.manifests[] | select(.platform.architecture=="amd64") | .digest')
+          TAG=$(${INSPECT}:${{ env.tag }} | jq -r '.manifests[] | select(.platform.architecture=="amd64") | .digest')
+          echo "TARGET: ${TARGET}"
+          echo "TAG: ${TAG}"
+          if [ "${TARGET}" != "${TAG}" ]; then
+            echo "ERROR: Tagging failed!"
+            echo "RETRY=true" >> $GITHUB_ENV
+          else
+            echo "ERROR: Tagging success!"
+            echo "RETRY=false" >> $GITHUB_ENV
+          fi
diff --git a/.github/workflows/pr-close.yml b/.github/workflows/pr-close.yml
@@ -18,5 +18,5 @@ jobs:
       oc_token: ${{ secrets.OC_TOKEN }}
     with:
       cleanup: label
-      packages: database backend frontend oracle-api sync
+      packages: backend frontend oracle-api sync
 
diff --git a/.github/workflows/pr-open.yml b/.github/workflows/pr-open.yml
@@ -18,11 +18,14 @@ jobs:
       packages: write
     strategy:
       matrix:
-        package: [database, common, backend, frontend, oracle-api, sync]
+        package: [backend, frontend, oracle-api, sync]
     steps:
       - uses: bcgov-nr/[email protected]
         id: build
         with:
+          build_args: |
+            BUILD_NUMBER=${{ github.event.number }}
+            BUILDKIT_INLINE_CACHE=1
           package: ${{ matrix.package }}
           tag: ${{ github.event.number }}
           tag_fallback: latest

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -0,0 +1,25 @@
+name: PROD
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'The tag set to deploy; e.g. latest or PR number'
+        required: false
+
+concurrency:
+  # Do not interrupt previous workflows
+  group: ${{ github.workflow }}
+  cancel-in-progress: false
+
+jobs:
+  deploy-prod:
+    name: PROD
+    secrets: inherit
+    uses: ./.github/workflows/.deploy.yml
+    with:
+      environment: prod
+      tag: ${{ inputs.tag || 'prod' }}
+      target: prod
diff --git a/backend/.mvn/wrapper/maven-wrapper.properties b/backend/.mvn/wrapper/maven-wrapper.properties
@@ -15,4 +15,4 @@
 # specific language governing permissions and limitations
 # under the License.
 wrapperVersion=3.3.1
-distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.8/apache-maven-3.9.8-bin.zip
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.9/apache-maven-3.9.9-bin.zip
diff --git a/backend/Dockerfile b/backend/Dockerfile
@@ -15,6 +15,10 @@ RUN ./mvnw -B package -Pnative -DskipTests
 FROM gcr.io/distroless/java-base:nonroot AS deploy
 ARG PORT=8090
 
+# Receive build number as argument, retain as environment variable
+ARG BUILD_NUMBER
+ENV BUILD_NUMBER=${BUILD_NUMBER}
+
 # Copy
 WORKDIR /app
 COPY --from=build /app/target/nr-spar-backend ./nr-spar-backend

diff --git a/backend/openshift.deploy.yml b/backend/openshift.deploy.yml
@@ -35,13 +35,13 @@ parameters:
   - name: FORESTCLIENTAPI_ADDRESS
     value: "https://nr-forest-client-api-prod.api.gov.bc.ca/api"
   - name: CPU_REQUEST
-    value: 100m
+    value: 25m
   - name: CPU_LIMIT
-    value: 300m
+    value: 100m
   - name: MEMORY_REQUEST
-    value: 100Mi
+    value: 150Mi
   - name: MEMORY_LIMIT
-    value: 500Mi
+    value: 450Mi
   - name: MIN_REPLICAS
     description: The minimum amount of replicas for the horizontal pod autoscaler.
     value: "3"
@@ -64,6 +64,10 @@ parameters:
   - name: AWS_COGNITO_ISSUER_URI
     description: AWS Cognito JWT Server URI
     required: true
+  - name: RANDOM_EXPRESSION
+    description: Random expression to make sure deployments update
+    from: "[a-zA-Z0-9]{32}"
+    generate: expression
 objects:
   - apiVersion: apps/v1
     kind: Deployment
@@ -72,7 +76,7 @@ objects:
         app: ${NAME}-${ZONE}
       name: ${NAME}-${ZONE}-${COMPONENT}
     spec:
-      replicas: 1
+      replicas: ${{MIN_REPLICAS}}
       selector:
         matchLabels:
           deployment: ${NAME}-${ZONE}-${COMPONENT}
@@ -133,6 +137,8 @@ objects:
                   value: ${DB_POOL_MAX_LIFETIME}
                 - name: AWS_COGNITO_ISSUER_URI
                   value: ${AWS_COGNITO_ISSUER_URI}
+                - name: RANDOM_EXPRESSION
+                  value: ${RANDOM_EXPRESSION}
               resources:
                 requests:
                   cpu: ${CPU_REQUEST}
@@ -204,3 +210,9 @@ objects:
             target:
               type: Utilization
               averageUtilization: 80
+        - type: Resource
+          resource:
+            name: memory
+            target:
+              type: Utilization
+              averageUtilization: 80