From d55487657dbd128258960b5436fb5529a93cfaa7 Mon Sep 17 00:00:00 2001 From: Andrea Grillo <andrea.grillo@pagopa.it> Date: Thu, 25 Jan 2024 09:35:42 +0100 Subject: [PATCH] [EC-101] PR and Release pipelines for TF configurations (#112) --- .github/workflows/call_code_review_infra.yml | 68 +++++++++ .github/workflows/call_release_infra.yml | 130 ++++++++++++++++++ .github/workflows/call_release_ms.yml | 2 +- .github/workflows/pr_container_apps_infra.yml | 51 +++++++ .github/workflows/pr_functions_infra.yml | 45 ++++++ .github/workflows/pr_mongo_infra.yml | 37 +++++ .../release_container_apps_infra.yml | 46 +++++++ .github/workflows/release_functions_infra.yml | 40 ++++++ .github/workflows/release_mongo_infra.yml | 40 ++++++ .github/workflows/release_ms.yml | 1 + .identity/env/dev/terraform.tfvars | 3 +- .identity/env/prod/terraform.tfvars | 3 +- .identity/env/uat/terraform.tfvars | 3 +- .../onboarding-functions/.terraform.lock.hcl | 24 +++- .../onboarding-functions/functions.tf | 4 +- 15 files changed, 490 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/call_code_review_infra.yml create mode 100644 .github/workflows/call_release_infra.yml create mode 100644 .github/workflows/pr_container_apps_infra.yml create mode 100644 .github/workflows/pr_functions_infra.yml create mode 100644 .github/workflows/pr_mongo_infra.yml create mode 100644 .github/workflows/release_container_apps_infra.yml create mode 100644 .github/workflows/release_functions_infra.yml create mode 100644 .github/workflows/release_mongo_infra.yml diff --git a/.github/workflows/call_code_review_infra.yml b/.github/workflows/call_code_review_infra.yml new file mode 100644 index 000000000..35c027cda --- /dev/null +++ b/.github/workflows/call_code_review_infra.yml @@ -0,0 +1,68 @@ +on: + workflow_call: + inputs: + environment: + type: string + required: true + dir: + type: string + required: true + env_vars: + type: string + required: false + description: List of environment variables to set up, given in env=value format. + +env: + ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + ARM_USE_OIDC: true + ARM_USE_AZUREAD: true + ARM_STORAGE_USE_AZUREAD: true + +jobs: + tf_plan: + name: 'Terraform Plan' + runs-on: ubuntu-20.04 + environment: ${{ inputs.environment }}-ci + permissions: + id-token: write + contents: read + env: + ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID_CI }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + steps: + + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + name: Checkout + + - name: Set Environment Variables + if: ${{ inputs.env_vars }} + run: | + for i in "${{ inputs.env_vars }}" + do + printf "%s\n" $i >> $GITHUB_ENV + done + + - name: Azure Login + uses: azure/login@cb79c773a3cfa27f31f25eb3f677781210c9ce3d # v1.6.1 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID_CI }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Set Terraform Version + id: set-terraform-version + run: | + echo "terraform_version=$(cat .terraform-version)" >> $GITHUB_OUTPUT + + - uses: hashicorp/setup-terraform@a1502cd9e758c50496cc9ac5308c4843bcd56d36 # v3.0.0 + name: Setup Terraform + with: + terraform_version: ${{ steps.set-terraform-version.outputs.terraform_version }} + + - name: Terraform Plan + working-directory: ${{ inputs.dir }} + shell: bash + run: | + bash ./terraform.sh plan ${{ inputs.environment }} diff --git a/.github/workflows/call_release_infra.yml b/.github/workflows/call_release_infra.yml new file mode 100644 index 000000000..177c6fb82 --- /dev/null +++ b/.github/workflows/call_release_infra.yml @@ -0,0 +1,130 @@ +on: + workflow_call: + inputs: + environment: + type: string + required: true + dir: + type: string + required: true + env_vars: + type: string + required: false + description: List of environment variables to set up, given in env=value format. + +env: + ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + ARM_USE_OIDC: true + ARM_USE_AZUREAD: true + ARM_STORAGE_USE_AZUREAD: true + +jobs: + + tf_plan: + name: 'Terraform Plan' + runs-on: ubuntu-20.04 + environment: ${{ inputs.environment }}-ci + permissions: + id-token: write + contents: read + outputs: + terraform_version: ${{ steps.set-terraform-version.outputs.terraform_version }} + + steps: + + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + name: Checkout + + - name: Set Environment Variables + if: ${{ inputs.env_vars }} + run: | + for i in "${{ inputs.env_vars }}" + do + printf "%s\n" $i >> $GITHUB_ENV + done + + - name: Set Terraform Version + id: set-terraform-version + run: | + echo "terraform_version=$(cat .terraform-version)" >> $GITHUB_OUTPUT + + - uses: hashicorp/setup-terraform@a1502cd9e758c50496cc9ac5308c4843bcd56d36 # v3.0.0 + name: Setup Terraform + with: + terraform_version: ${{ steps.set-terraform-version.outputs.terraform_version }} + + - name: Terraform Plan + uses: pagopa/terraform-preapply-azure-action@54ded8cda3437c3f6a9f46baf69cb321ce82f5cd + with: + client_id: ${{ secrets.AZURE_CLIENT_ID_CI }} + tenant_id: ${{ secrets.AZURE_TENANT_ID }} + subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + dir: ${{ inputs.dir }} + azure_environment: ${{ inputs.environment }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: "Upload Terraform Plan as Artifact" + uses: actions/upload-artifact@694cdabd8bdb0f10b2cea11669e1bf5453eed0a6 # v4.2.0 + with: + name: tfplan + path: ${{ inputs.dir }}/tfplan-${{ inputs.environment }}-${{ github.sha }} + if-no-files-found: error + retention-days: 14 + + apply: + name: 'Terraform Apply' + runs-on: ubuntu-20.04 + needs: [tf_plan] + environment: ${{ inputs.environment }}-cd + permissions: + id-token: write + contents: read + env: + ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID_CD }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + steps: + + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + name: Checkout + + - name: Azure Login + uses: azure/login@cb79c773a3cfa27f31f25eb3f677781210c9ce3d # v1.6.1 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID_CD }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Download Terraform Plan as Artifact + uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1 + with: + name: tfplan + path: ${{ inputs.dir }} + + - name: Retrieve Terraform Modules from Cache + id: cache-terraform-modules + uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 + with: + path: ${{ inputs.dir }}/.terraform + key: terraform-${{ inputs.dir }}-${{ github.sha }} + restore-keys: | + terraform-${{ inputs.dir }} + + - uses: hashicorp/setup-terraform@a1502cd9e758c50496cc9ac5308c4843bcd56d36 # v3.0.0 + name: Setup Terraform + with: + terraform_version: ${{ needs.tf_plan.outputs.terraform_version }} + + - name: Terraform Init + shell: bash + working-directory: ${{ inputs.dir }} + run: | + bash ./terraform.sh init ${{ inputs.environment }} + + - name: Terraform Apply + shell: bash + working-directory: ${{ inputs.dir }} + run: | + terraform apply -lock-timeout=3000s -auto-approve -input=false tfplan-${{ inputs.environment }}-${{ github.sha }} diff --git a/.github/workflows/call_release_ms.yml b/.github/workflows/call_release_ms.yml index 668a7d982..e9da8d39e 100644 --- a/.github/workflows/call_release_ms.yml +++ b/.github/workflows/call_release_ms.yml @@ -118,7 +118,7 @@ jobs: release: name: 'Release' - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 needs: [build, tf_plan] environment: ${{ inputs.environment }}-cd permissions: diff --git a/.github/workflows/pr_container_apps_infra.yml b/.github/workflows/pr_container_apps_infra.yml new file mode 100644 index 000000000..07d5e83a8 --- /dev/null +++ b/.github/workflows/pr_container_apps_infra.yml @@ -0,0 +1,51 @@ +name: Code Review container app infra + +on: + workflow_dispatch: + + pull_request: + branches: + - main + - releases/* + types: + - opened + - edited + - synchronize + - reopened + - ready_for_review + paths: + - './infra/container_apps/onboarding-ms/**' + - './.github/workflows/pr_container_apps_infra.yml' + - './.github/workflows/call_code_review_infra.yml' + +jobs: + code_review_dev: + uses: ./.github/workflows/call_code_review_infra.yml + name: '[DEV] Container Apps Infra Code Review' + if: ${{ !startsWith(github.ref_name, 'releases/') }} + secrets: inherit + with: + environment: dev + dir: ./infra/container_apps/onboarding-ms + env_vars: | + TF_VAR_image_tag=sha-$(git rev-parse --short ${{ github.sha }}) + + code_review_uat: + uses: ./.github/workflows/call_code_review_infra.yml + name: '[UAT] Container Apps Infra Code Review' + secrets: inherit + with: + environment: uat + dir: ./infra/container_apps/onboarding-ms + env_vars: | + TF_VAR_image_tag=sha-$(git rev-parse --short ${{ github.sha }}) + + code_review_prod: + uses: ./.github/workflows/call_code_review_infra.yml + name: '[PROD] Container Apps Infra Code Review' + secrets: inherit + with: + environment: prod + dir: ./infra/container_apps/onboarding-ms + env_vars: | + TF_VAR_image_tag=sha-$(git rev-parse --short ${{ github.sha }}) diff --git a/.github/workflows/pr_functions_infra.yml b/.github/workflows/pr_functions_infra.yml new file mode 100644 index 000000000..7f1e518f5 --- /dev/null +++ b/.github/workflows/pr_functions_infra.yml @@ -0,0 +1,45 @@ +name: Code Review functions infra + +on: + workflow_dispatch: + + pull_request: + branches: + - main + - releases/* + types: + - opened + - edited + - synchronize + - reopened + - ready_for_review + paths: + - './infra/functions/onboarding-functions/**' + - './.github/workflows/pr_functions_infra.yml' + - './.github/workflows/call_code_review_infra.yml' + +jobs: + code_review_dev: + uses: ./.github/workflows/call_code_review_infra.yml + name: '[DEV] Functions Infra Code Review' + if: ${{ !startsWith(github.ref_name, 'releases/') }} + secrets: inherit + with: + environment: dev + dir: ./infra/functions/onboarding-functions + + code_review_uat: + uses: ./.github/workflows/call_code_review_infra.yml + name: '[UAT] Functions Infra Code Review' + secrets: inherit + with: + environment: uat + dir: ./infra/functions/onboarding-functions + + code_review_prod: + uses: ./.github/workflows/call_code_review_infra.yml + name: '[PROD] Functions Infra Code Review' + secrets: inherit + with: + environment: prod + dir: ./infra/functions/onboarding-functions diff --git a/.github/workflows/pr_mongo_infra.yml b/.github/workflows/pr_mongo_infra.yml new file mode 100644 index 000000000..dda48bb49 --- /dev/null +++ b/.github/workflows/pr_mongo_infra.yml @@ -0,0 +1,37 @@ +name: Code Review mongo infra + +on: + workflow_dispatch: + + pull_request: + branches: + - main + - releases/* + types: + - opened + - edited + - synchronize + - reopened + - ready_for_review + paths: + - './infra/mongo/**' + - './.github/workflows/pr_mongo_infra.yml' + - './.github/workflows/call_code_review_infra.yml' + +jobs: + code_review_dev: + uses: ./.github/workflows/call_code_review_infra.yml + name: '[DEV] Mongo Infra Code Review' + if: ${{ !startsWith(github.ref_name, 'releases/') }} + secrets: inherit + with: + environment: dev + dir: ./infra/mongo + + code_review_uat: + uses: ./.github/workflows/call_code_review_infra.yml + name: '[UAT] Mongo Infra Code Review' + secrets: inherit + with: + environment: uat + dir: ./infra/mongo diff --git a/.github/workflows/release_container_apps_infra.yml b/.github/workflows/release_container_apps_infra.yml new file mode 100644 index 000000000..b3adb39c1 --- /dev/null +++ b/.github/workflows/release_container_apps_infra.yml @@ -0,0 +1,46 @@ +name: Deploy Container Apps infra + +on: + push: + branches: + - main + - releases/* + paths: + - './infra/container_apps/onboarding-ms/**' + + workflow_dispatch: + +jobs: + + release_dev: + uses: ./.github/workflows/call_release_infra.yml + name: '[Dev] Container Apps Infra Release' + if: github.ref_name == 'main' + secrets: inherit + with: + environment: dev + dir: ./infra/container_apps/onboarding-ms + env_vars: | + TF_VAR_image_tag=sha-$(git rev-parse --short ${{ github.sha }}) + + release_uat: + uses: ./.github/workflows/call_release_infra.yml + name: '[UAT] Container Apps Infra Release' + if: startsWith(github.ref_name, 'releases/') + secrets: inherit + with: + environment: uat + dir: ./infra/container_apps/onboarding-ms + env_vars: | + TF_VAR_image_tag=sha-$(git rev-parse --short ${{ github.sha }}) + + release_prod: + uses: ./.github/workflows/call_release_infra.yml + name: '[Prod] Container Apps Infra Release' + if: startsWith(github.ref_name, 'releases/') + secrets: inherit + with: + environment: prod + dir: ./infra/container_apps/onboarding-ms + env_vars: | + TF_VAR_image_tag=sha-$(git rev-parse --short ${{ github.sha }}) \ No newline at end of file diff --git a/.github/workflows/release_functions_infra.yml b/.github/workflows/release_functions_infra.yml new file mode 100644 index 000000000..47063a14f --- /dev/null +++ b/.github/workflows/release_functions_infra.yml @@ -0,0 +1,40 @@ +name: Deploy functions infra + +on: + push: + branches: + - main + - releases/* + paths: + - './infra/functions/onboarding-functions/**' + + workflow_dispatch: + +jobs: + + release_dev: + uses: ./.github/workflows/call_release_infra.yml + name: '[Dev] Functions Infra Release' + if: github.ref_name == 'main' + secrets: inherit + with: + environment: dev + dir: ./infra/functions/onboarding-functions + + release_uat: + uses: ./.github/workflows/call_release_infra.yml + name: '[UAT] Functions Infra Release' + if: startsWith(github.ref_name, 'releases/') + secrets: inherit + with: + environment: uat + dir: ./infra/functions/onboarding-functions + + release_prod: + uses: ./.github/workflows/call_release_infra.yml + name: '[Prod] Functions Infra Release' + if: startsWith(github.ref_name, 'releases/') + secrets: inherit + with: + environment: prod + dir: ./infra/functions/onboarding-functions diff --git a/.github/workflows/release_mongo_infra.yml b/.github/workflows/release_mongo_infra.yml new file mode 100644 index 000000000..2d300c295 --- /dev/null +++ b/.github/workflows/release_mongo_infra.yml @@ -0,0 +1,40 @@ +name: Deploy mongo infra + +on: + push: + branches: + - main + - releases/* + paths: + - './infra/mongo' + + workflow_dispatch: + +jobs: + + release_dev: + uses: ./.github/workflows/call_release_infra.yml + name: '[Dev] Mongo Infra Release' + if: github.ref_name == 'main' + secrets: inherit + with: + environment: dev + dir: ./infra/mongo + + release_uat: + uses: ./.github/workflows/call_release_infra.yml + name: '[UAT] Mongo Infra Release' + if: startsWith(github.ref_name, 'releases/') + secrets: inherit + with: + environment: uat + dir: ./infra/mongo + + release_prod: + uses: ./.github/workflows/call_release_infra.yml + name: '[Prod] Mongo Infra Release' + if: startsWith(github.ref_name, 'releases/') + secrets: inherit + with: + environment: prod + dir: ./infra/mongo \ No newline at end of file diff --git a/.github/workflows/release_ms.yml b/.github/workflows/release_ms.yml index 619641c65..a62ea9a20 100644 --- a/.github/workflows/release_ms.yml +++ b/.github/workflows/release_ms.yml @@ -7,6 +7,7 @@ on: - releases/* paths: - "apps/onboarding-ms/**" + - "infra/container_apps/onboarding-ms/**" - "apps/pom.xml" - "pom.xml" diff --git a/.identity/env/dev/terraform.tfvars b/.identity/env/dev/terraform.tfvars index 4c8f75ba1..f312eb62d 100644 --- a/.identity/env/dev/terraform.tfvars +++ b/.identity/env/dev/terraform.tfvars @@ -29,7 +29,8 @@ cd_github_federations = [ environment_ci_roles = { subscription = [ "Reader", - "Key Vault Secrets User" + "PagoPA IaC Reader", + "Reader and Data Access" ] resource_groups = { "terraform-state-rg" = [ diff --git a/.identity/env/prod/terraform.tfvars b/.identity/env/prod/terraform.tfvars index ee46633dd..0d704152b 100644 --- a/.identity/env/prod/terraform.tfvars +++ b/.identity/env/prod/terraform.tfvars @@ -29,7 +29,8 @@ cd_github_federations = [ environment_ci_roles = { subscription = [ "Reader", - "Key Vault Secrets User" + "PagoPA IaC Reader", + "Reader and Data Access" ] resource_groups = { "terraform-state-rg" = [ diff --git a/.identity/env/uat/terraform.tfvars b/.identity/env/uat/terraform.tfvars index c96743206..c4d0d6c30 100644 --- a/.identity/env/uat/terraform.tfvars +++ b/.identity/env/uat/terraform.tfvars @@ -29,7 +29,8 @@ cd_github_federations = [ environment_ci_roles = { subscription = [ "Reader", - "Key Vault Secrets User" + "PagoPA IaC Reader", + "Reader and Data Access" ] resource_groups = { "terraform-state-rg" = [ diff --git a/infra/functions/onboarding-functions/.terraform.lock.hcl b/infra/functions/onboarding-functions/.terraform.lock.hcl index 12a5d184a..96e82fa4f 100644 --- a/infra/functions/onboarding-functions/.terraform.lock.hcl +++ b/infra/functions/onboarding-functions/.terraform.lock.hcl @@ -26,7 +26,7 @@ provider "registry.terraform.io/hashicorp/azuread" { provider "registry.terraform.io/hashicorp/azurerm" { version = "3.71.0" - constraints = ">= 3.30.0, <= 3.71.0, <= 3.84.0, <= 3.85.0, <= 3.86.0" + constraints = ">= 3.30.0, <= 3.84.0, <= 3.85.0, <= 3.86.0" hashes = [ "h1:QI0iaPNi0qAOIbXptd4ZObi0D5X1jojom5774GtEspA=", "h1:nTc6DFS9euNgUkNylQ/AxNYN9Ln1dyL+WVIBNcict7Y=", @@ -47,6 +47,28 @@ provider "registry.terraform.io/hashicorp/azurerm" { ] } +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.2" + hashes = [ + "h1:IMVAUHKoydFrlPrl9OzasDnw/8ntZFerCC9iXw1rXQY=", + "h1:m467k2tZ9cdFFgHW7LPBK2GLPH43LC6wc3ppxr8yvoE=", + "h1:vWAsYRd7MjYr3adj8BVKRohVfHpWQdvkIwUQ2Jf5FVM=", + "h1:zT1ZbegaAYHwQa+QwIFugArWikRJI9dqohj8xb0GY88=", + "zh:3248aae6a2198f3ec8394218d05bd5e42be59f43a3a7c0b71c66ec0df08b69e7", + "zh:32b1aaa1c3013d33c245493f4a65465eab9436b454d250102729321a44c8ab9a", + "zh:38eff7e470acb48f66380a73a5c7cdd76cc9b9c9ba9a7249c7991488abe22fe3", + "zh:4c2f1faee67af104f5f9e711c4574ff4d298afaa8a420680b0cb55d7bbc65606", + "zh:544b33b757c0b954dbb87db83a5ad921edd61f02f1dc86c6186a5ea86465b546", + "zh:696cf785090e1e8cf1587499516b0494f47413b43cb99877ad97f5d0de3dc539", + "zh:6e301f34757b5d265ae44467d95306d61bef5e41930be1365f5a8dcf80f59452", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:913a929070c819e59e94bb37a2a253c228f83921136ff4a7aa1a178c7cce5422", + "zh:aa9015926cd152425dbf86d1abdbc74bfe0e1ba3d26b3db35051d7b9ca9f72ae", + "zh:bb04798b016e1e1d49bcc76d62c53b56c88c63d6f2dfe38821afef17c416a0e1", + "zh:c23084e1b23577de22603cff752e59128d83cfecc2e6819edadd8cf7a10af11e", + ] +} + provider "registry.terraform.io/integrations/github" { version = "5.18.3" constraints = "5.18.3" diff --git a/infra/functions/onboarding-functions/functions.tf b/infra/functions/onboarding-functions/functions.tf index a502cfc6c..79d24a427 100644 --- a/infra/functions/onboarding-functions/functions.tf +++ b/infra/functions/onboarding-functions/functions.tf @@ -7,7 +7,7 @@ resource "azurerm_resource_group" "onboarding_fn_rg" { module "onboarding_fn_snet" { count = var.cidr_subnet_selc_onboarding_fn != null ? 1 : 0 - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//subnet?ref=v7.38.0" + source = "github.com/pagopa/terraform-azurerm-v3.git//subnet?ref=v7.47.2" name = format("%s-onboarding-fn-snet", local.project) resource_group_name = data.azurerm_resource_group.rg_vnet.name virtual_network_name = data.azurerm_virtual_network.vnet.name @@ -23,7 +23,7 @@ module "onboarding_fn_snet" { } module "selc_onboarding_fn" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//function_app?ref=v7.38.0" + source = "github.com/pagopa/terraform-azurerm-v3.git//function_app?ref=v7.47.2" name = format("%s-onboarding-fn", local.project) location = azurerm_resource_group.onboarding_fn_rg.location