workflows for Disaster Recovery

DFE-Digital · Jan 31, 2025 · 6a0b50e · 6a0b50e
1 parent 3aae8ec
commit 6a0b50e
Show file tree

Hide file tree

Showing 3 changed files with 215 additions and 3 deletions.
diff --git a/.github/workflows/database-restore.yml b/.github/workflows/database-restore.yml
@@ -5,15 +5,41 @@ on:
     - cron: "0 1 * * *"
   workflow_dispatch:
     inputs:
+      backup-sanitise:
+        required: true
+        type: boolean
+        default: true
       overwriteThisMorningsBackup:
         required: true
         type: boolean
         default: false
+      environment:
+        description: Environment to backup
+        required: true
+        default: qa
+        type: choice
+        options:
+        - qa
+        - production
+      backup-file:
+        description: |
+          Backup file name (without extension). Default is ptt_[env]_adhoc_YYYY-MM-DD. Set it explicitly when backing up a point-in-time (PTR) server. (Optional)
+        required: false
+        type: string
+        default: default
+      db-server:
+        description: |
+          Name of the database server. Default is the live server. When backing up a point-in-time (PTR) server, use the full name of the PTR server. (Optional)
+
+env:
+  SERVICE_NAME: publish
+  SERVICE_SHORT: ptt
+  TF_VARS_PATH: terraform/aks/workspace_variables
 
 jobs:
-  backup:
+  backup_sanitise:
     name: Sanitise Production Database Backup
-    if: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.overwriteThisMorningsBackup == 'true') }}
+    if: ${{ github.event.inputs.backup-sanitise == 'true' && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.overwriteThisMorningsBackup == 'true')) }}
     runs-on: ubuntu-latest
     services:
       postgres:
@@ -110,7 +136,7 @@ jobs:
           rm ${SANITISED_FILE_NAME}.sql.gz
 
   restore:
-    needs: [backup]
+    needs: [backup_sanitise]
     runs-on: ubuntu-latest
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -129,3 +155,51 @@ jobs:
           azure-credentials: ${{ secrets[format('AZURE_CREDENTIALS_{0}', matrix.environment)] }}
           prod-credentials: ${{ secrets.AZURE_CREDENTIALS_PRODUCTION }}
           environment: ${{ matrix.environment }}
+
+  backup:
+    name: Backup Database
+    if: ${{ github.event.inputs.backup-sanitise == 'false' }}
+    runs-on: ubuntu-latest
+    environment:
+      name: ${{ inputs.environment || 'production' }}
+    env:
+      DEPLOY_ENV: ${{ inputs.environment || 'production'  }}
+      BACKUP_FILE: ${{ inputs.backup-file || 'schedule'  }}
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - uses: azure/login@v2
+      with:
+        creds:  ${{ secrets[format('AZURE_CREDENTIALS_{0}', github.event.inputs.environment)] }}
+
+    - name: Set environment variables
+      run: |
+        source global_config/${DEPLOY_ENV}.sh
+        tf_vars_file=${TF_VARS_PATH}/${DEPLOY_ENV}.tfvars.json
+        echo "CLUSTER=$(jq -r '.cluster' ${tf_vars_file})" >> $GITHUB_ENV
+        echo "RESOURCE_GROUP_NAME=${RESOURCE_NAME_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-rg" >> $GITHUB_ENV
+        echo "STORAGE_ACCOUNT_NAME=${RESOURCE_NAME_PREFIX}${SERVICE_SHORT}dbbkp${CONFIG_SHORT}sa" >> $GITHUB_ENV
+        TODAY=$(date +"%F")
+        echo "DB_SERVER=${RESOURCE_NAME_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-pg" >> $GITHUB_ENV
+        if [ "${BACKUP_FILE}" == "schedule" ]; then
+          BACKUP_FILE=${SERVICE_SHORT}_${CONFIG_SHORT}_${TODAY}
+        elif [ "${BACKUP_FILE}" == "default" ]; then
+          BACKUP_FILE=${SERVICE_SHORT}_${CONFIG_SHORT}_adhoc_${TODAY}
+        else
+          BACKUP_FILE=${BACKUP_FILE}
+        fi
+        echo "BACKUP_FILE=${BACKUP_FILE}" >> $GITHUB_ENV
+        echo "KEYVAULT_NAME=${RESOURCE_NAME_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-inf-kv" >> $GITHUB_ENV
+
+    - name: Backup ${{ env.DEPLOY_ENV }} postgres
+      uses: DFE-Digital/github-actions/backup-postgres@master
+      with:
+        storage-account: ${{ env.STORAGE_ACCOUNT_NAME }}
+        resource-group: ${{ env.RESOURCE_GROUP_NAME }}
+        app-name: publish-${{ env.DEPLOY_ENV }}
+        cluster: ${{ env.CLUSTER }}
+        azure-credentials:  ${{ secrets[format('AZURE_CREDENTIALS_{0}', github.event.inputs.environment)] }}
+        backup-file: ${{ env.BACKUP_FILE }}.sql
+        db-server-name: ${{ inputs.db-server }}
+        slack-webhook: ${{ secrets.SLACK_WEBHOOK }}
diff --git a/.github/workflows/postgres-ptr.yml b/.github/workflows/postgres-ptr.yml
@@ -0,0 +1,69 @@
+name: Restore database from point in time to new database server
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: Environment to restore
+        required: true
+        default: qa
+        type: choice
+        options:
+        - qa
+        - production
+      confirm-production:
+        description: Must be set to true if restoring production
+        required: true
+        default: 'false'
+        type: choice
+        options:
+        - 'false'
+        - 'true'
+      restore-time:
+        description: Restore point in time in UTC. e.g. 2024-07-24T06:00:00
+        type: string
+        required: true
+      new-db-server:
+        description: Name of the new database server. Default is <original-server-name>-ptr.
+        type: string
+
+env:
+  SERVICE_SHORT: ptt
+  TF_VARS_PATH: terraform/aks/workspace_variables
+
+jobs:
+  ptr-restore:
+    name: PTR Restore AKS Database
+    if: ${{ inputs.environment != 'production' || (inputs.environment == 'production' && github.event.inputs.confirm-production == 'true' )  }}
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    concurrency: deploy_${{ inputs.environment }}
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set environment variables
+      run: |
+        source global_config/${{ inputs.environment }}.sh
+        tf_vars_file=${TF_VARS_PATH}/${{ inputs.environment }}.tfvars.json
+        echo "CLUSTER=$(jq -r '.cluster' ${tf_vars_file})" >> $GITHUB_ENV
+        echo "RESOURCE_GROUP_NAME=${RESOURCE_NAME_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-rg" >> $GITHUB_ENV
+
+        DB_SERVER="${RESOURCE_NAME_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-pg"
+        if [[ -n "${{ inputs.new-db-server }}" ]]; then
+          NEW_DB_SERVER="${{ inputs.new-db-server }}"
+        else
+          NEW_DB_SERVER="${DB_SERVER}-ptr"
+        fi
+        echo "DB_SERVER=${DB_SERVER}" >> $GITHUB_ENV
+        echo "NEW_DB_SERVER=${NEW_DB_SERVER}" >> $GITHUB_ENV
+
+    - name: Restore ${{ inputs.environment }} postgres
+      uses: DFE-Digital/github-actions/ptr-postgres@master
+      with:
+        resource-group: ${{ env.RESOURCE_GROUP_NAME }}
+        source-server: ${{ env.DB_SERVER }}
+        new-server: ${{ env.NEW_DB_SERVER }}
+        restore-time: ${{ inputs.restore-time }}
+        cluster: ${{ env.CLUSTER }}
+        azure-credentials: ${{ secrets[format('AZURE_CREDENTIALS_{0}', github.event.inputs.environment)] }}
diff --git a/.github/workflows/postgres-restore.yml b/.github/workflows/postgres-restore.yml
@@ -0,0 +1,69 @@
+name: Restore database from Azure storage
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: Environment to restore
+        required: true
+        default: qa
+        type: choice
+        options:
+        - qa
+        - production
+      confirm-production:
+        description: Must be set to true if restoring production
+        required: true
+        default: 'false'
+        type: choice
+        options:
+        - 'false'
+        - 'true'
+      backup-file:
+        description: Name of the backup file in Azure storage. e.g. ptt_prod_2024-08-09.sql.gz. The default value is today's scheduled backup.
+        type: string
+        required: false
+
+env:
+  SERVICE_NAME: publish
+  SERVICE_SHORT: ptt
+  TF_VARS_PATH: terraform/aks/workspace_variables
+
+jobs:
+  restore:
+    name: Restore AKS Database
+    if: ${{ inputs.environment != 'production' || (inputs.environment == 'production' && inputs.confirm-production == 'true' )  }}
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    concurrency: deploy_${{ inputs.environment }}
+
+    steps:
+    - uses: actions/checkout@v4
+      name: Checkout
+
+    - name: Set environment variables
+      run: |
+        source global_config/${{ inputs.environment }}.sh
+        tf_vars_file=${{ env.TF_VARS_PATH }}/${{ inputs.environment }}.tfvars.json
+        echo "CLUSTER=$(jq -r '.cluster' ${tf_vars_file})" >> $GITHUB_ENV
+        echo "RESOURCE_GROUP_NAME=${RESOURCE_NAME_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-rg" >> $GITHUB_ENV
+        echo "STORAGE_ACCOUNT_NAME=${RESOURCE_NAME_PREFIX}${SERVICE_SHORT}dbbkp${CONFIG_SHORT}sa" >> $GITHUB_ENV
+        echo "DB_SERVER=${RESOURCE_NAME_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-pg" >> $GITHUB_ENV
+        TODAY=$(date +"%F")
+        echo "BACKUP_FILE=${SERVICE_SHORT}_${CONFIG_SHORT}_${TODAY}.sql" >> $GITHUB_ENV
+        if [ "${{ inputs.backup-file }}" != "" ]; then
+          BACKUP_FILE=${{ inputs.backup-file }}
+        else
+          BACKUP_FILE=${SERVICE_SHORT}_${CONFIG_SHORT}_${TODAY}.sql.gz
+        fi
+        echo "BACKUP_FILE=$BACKUP_FILE" >> $GITHUB_ENV
+
+    - name: Restore ${{ inputs.environment }} postgres
+      uses: DFE-Digital/github-actions/restore-postgres-backup@master
+      with:
+        storage-account: ${{ env.STORAGE_ACCOUNT_NAME }}
+        resource-group: ${{ env.RESOURCE_GROUP_NAME }}
+        app-name: ${{ env.SERVICE_NAME }}-${{ inputs.environment }}
+        cluster: ${{ env.CLUSTER }}
+        azure-credentials: ${{ secrets[format('AZURE_CREDENTIALS_{0}', github.event.inputs.environment)] }}
+        backup-file: ${{ env.BACKUP_FILE }}