From 328b900c3b4fd105804029b2c1e770783d3f4ba6 Mon Sep 17 00:00:00 2001 From: Sacha Narinx Date: Wed, 14 Aug 2024 17:56:53 +0400 Subject: [PATCH] New policy build workflow (#1725) Co-authored-by: Jack Tracey <41163455+jtracey93@users.noreply.github.com> --- .../PolicyPesterTestHelper.psm1 | 99 ++++++++++++++ .../Test-BuildPolicies.Tests.ps1 | 62 +++++++++ .github/workflows/check-policy-build.yml | 51 +++++++ .github/workflows/update-portal.yml | 125 ------------------ docs/wiki/ALZ-Contribution-Guide.md | 8 +- docs/wiki/Whats-new.md | 6 + 6 files changed, 225 insertions(+), 126 deletions(-) create mode 100644 .github/actions-pester/Test-BuildPolicies.Tests.ps1 create mode 100644 .github/workflows/check-policy-build.yml delete mode 100644 .github/workflows/update-portal.yml diff --git a/.github/actions-pester/PolicyPesterTestHelper.psm1 b/.github/actions-pester/PolicyPesterTestHelper.psm1 index 133dbac558..dff9b8f264 100644 --- a/.github/actions-pester/PolicyPesterTestHelper.psm1 +++ b/.github/actions-pester/PolicyPesterTestHelper.psm1 @@ -32,3 +32,102 @@ function Get-PolicyFiles return $_ } } + +function Remove-JSONMetadata { + + [CmdletBinding()] + param ( + [Parameter(Mandatory = $true)] + [hashtable] $TemplateObject + ) + $TemplateObject.Remove('metadata') + + # Differantiate case: With user defined types (resources property is hashtable) vs without user defined types (resources property is array) + if ($TemplateObject.resources.GetType().BaseType.Name -eq 'Hashtable') { + # Case: Hashtable + $resourceIdentifiers = $TemplateObject.resources.Keys + for ($index = 0; $index -lt $resourceIdentifiers.Count; $index++) { + if ($TemplateObject.resources[$resourceIdentifiers[$index]].type -eq 'Microsoft.Resources/deployments' -and $TemplateObject.resources[$resourceIdentifiers[$index]].properties.template.GetType().BaseType.Name -eq 'Hashtable') { + $TemplateObject.resources[$resourceIdentifiers[$index]] = Remove-JSONMetadata -TemplateObject $TemplateObject.resources[$resourceIdentifiers[$index]].properties.template + } + } + } else { + # Case: Array + for ($index = 0; $index -lt $TemplateObject.resources.Count; $index++) { + if ($TemplateObject.resources[$index].type -eq 'Microsoft.Resources/deployments' -and $TemplateObject.resources[$index].properties.template.GetType().BaseType.Name -eq 'Hashtable') { + $TemplateObject.resources[$index] = Remove-JSONMetadata -TemplateObject $TemplateObject.resources[$index].properties.template + } + } + } + + return $TemplateObject +} + +function ConvertTo-OrderedHashtable { + + [CmdletBinding()] + param ( + [Parameter(Mandatory = $true)] + [string] $JSONInputObject # Must be string to workaround auto-conversion + ) + + $JSONObject = ConvertFrom-Json $JSONInputObject -AsHashtable -Depth 99 -NoEnumerate + $orderedLevel = [ordered]@{} + + if (-not ($JSONObject.GetType().BaseType.Name -eq 'Hashtable')) { + return $JSONObject # E.g. in primitive data types [1,2,3] + } + + foreach ($currentLevelKey in ($JSONObject.Keys | Sort-Object -Culture 'en-US')) { + + if ($null -eq $JSONObject[$currentLevelKey]) { + # Handle case in which the value is 'null' and hence has no type + $orderedLevel[$currentLevelKey] = $null + continue + } + + switch ($JSONObject[$currentLevelKey].GetType().BaseType.Name) { + { $PSItem -in @('Hashtable') } { + $orderedLevel[$currentLevelKey] = ConvertTo-OrderedHashtable -JSONInputObject ($JSONObject[$currentLevelKey] | ConvertTo-Json -Depth 99) + } + 'Array' { + $arrayOutput = @() + + # Case: Array of arrays + $arrayElements = $JSONObject[$currentLevelKey] | Where-Object { $_.GetType().BaseType.Name -eq 'Array' } + foreach ($array in $arrayElements) { + if ($array.Count -gt 1) { + # Only sort for arrays with more than one item. Otherwise single-item arrays are casted + $array = $array | Sort-Object -Culture 'en-US' + } + $arrayOutput += , (ConvertTo-OrderedHashtable -JSONInputObject ($array | ConvertTo-Json -Depth 99)) + } + + # Case: Array of objects + $hashTableElements = $JSONObject[$currentLevelKey] | Where-Object { $_.GetType().BaseType.Name -eq 'Hashtable' } + foreach ($hashTable in $hashTableElements) { + $arrayOutput += , (ConvertTo-OrderedHashtable -JSONInputObject ($hashTable | ConvertTo-Json -Depth 99)) + } + + # Case: Primitive data types + $primitiveElements = $JSONObject[$currentLevelKey] | Where-Object { $_.GetType().BaseType.Name -notin @('Array', 'Hashtable') } | ConvertTo-Json -Depth 99 | ConvertFrom-Json -AsHashtable -NoEnumerate -Depth 99 + if ($primitiveElements.Count -gt 1) { + $primitiveElements = $primitiveElements | Sort-Object -Culture 'en-US' + } + $arrayOutput += $primitiveElements + + if ($array.Count -gt 1) { + # Only sort for arrays with more than one item. Otherwise single-item arrays are casted + $arrayOutput = $arrayOutput | Sort-Object -Culture 'en-US' + } + $orderedLevel[$currentLevelKey] = $arrayOutput + } + Default { + # string/int/etc. + $orderedLevel[$currentLevelKey] = $JSONObject[$currentLevelKey] + } + } + } + + return $orderedLevel +} diff --git a/.github/actions-pester/Test-BuildPolicies.Tests.ps1 b/.github/actions-pester/Test-BuildPolicies.Tests.ps1 new file mode 100644 index 0000000000..eb50026ebf --- /dev/null +++ b/.github/actions-pester/Test-BuildPolicies.Tests.ps1 @@ -0,0 +1,62 @@ +Describe 'UnitTest-BuildPolicies' { + + BeforeAll { + Import-Module -Name $PSScriptRoot\PolicyPesterTestHelper.psm1 -Force -Verbose + + New-Item -Name "buildout" -Type Directory + + # Build the PR policies, initiatives, and role definitions to a temp folder + bicep build ./src/templates/policies.bicep --outfile ./buildout/policies.json + bicep build ./src/templates/initiatives.bicep --outfile ./buildout/initiatives.json + bicep build ./src/templates/roles.bicep --outfile ./buildout/customRoleDefinitions.json + } + + Context "Check Policy Builds" { + + It "Check policies build done" { + $prFile = "./eslzArm/managementGroupTemplates/policyDefinitions/policies.json" + $buildFile = "./buildout/policies.json" + + $buildJson = Remove-JSONMetadata -TemplateObject (Get-Content $buildFile -Raw | ConvertFrom-Json -Depth 99 -AsHashtable) + $buildJson = ConvertTo-OrderedHashtable -JSONInputObject (ConvertTo-Json $buildJson -Depth 99) + + $prJson = Remove-JSONMetadata -TemplateObject (Get-Content $prFile -Raw | ConvertFrom-Json -Depth 99 -AsHashtable) + $prJson = ConvertTo-OrderedHashtable -JSONInputObject (ConvertTo-Json $prJson -Depth 99) + + # Compare files we built to the PR files + (ConvertTo-Json $buildJson -Depth 99) | Should -Be (ConvertTo-Json $prJson -Depth 99) -Because "the [policies.json] should be based on the latest [policies.bicep] file. Please run [` bicep build ./src/templates/policies.bicep --outfile ./eslzArm/managementGroupTemplates/policyDefinitions/policies.json `] using the latest Bicep CLI version." + } + + It "Check initiatives build done" { + $PRfile = "./eslzArm/managementGroupTemplates/policyDefinitions/initiatives.json" + $buildFile = "./buildout/initiatives.json" + + $buildJson = Remove-JSONMetadata -TemplateObject (Get-Content $buildFile -Raw | ConvertFrom-Json -Depth 99 -AsHashtable) + $buildJson = ConvertTo-OrderedHashtable -JSONInputObject (ConvertTo-Json $buildJson -Depth 99) + + $prJson = Remove-JSONMetadata -TemplateObject (Get-Content $prFile -Raw | ConvertFrom-Json -Depth 99 -AsHashtable) + $prJson = ConvertTo-OrderedHashtable -JSONInputObject (ConvertTo-Json $prJson -Depth 99) + + # Compare files we built to the PR files + (ConvertTo-Json $buildJson -Depth 99) | Should -Be (ConvertTo-Json $prJson -Depth 99) -Because "the [initiatives.json] should be based on the latest [initiatives.bicep] file. Please run [` bicep build ./src/templates/initiatives.bicep --outfile ./eslzArm/managementGroupTemplates/policyDefinitions/initiatives.json `] using the latest Bicep CLI version." + } + + It "Check role definitions build done" { + $PRfile = "./eslzArm/managementGroupTemplates/roleDefinitions/customRoleDefinitions.json" + $buildFile = "./buildout/customRoleDefinitions.json" + + $buildJson = Remove-JSONMetadata -TemplateObject (Get-Content $buildFile -Raw | ConvertFrom-Json -Depth 99 -AsHashtable) + $buildJson = ConvertTo-OrderedHashtable -JSONInputObject (ConvertTo-Json $buildJson -Depth 99) + + $prJson = Remove-JSONMetadata -TemplateObject (Get-Content $prFile -Raw | ConvertFrom-Json -Depth 99 -AsHashtable) + $prJson = ConvertTo-OrderedHashtable -JSONInputObject (ConvertTo-Json $prJson -Depth 99) + + # Compare files we built to the PR files + (ConvertTo-Json $buildJson -Depth 99) | Should -Be (ConvertTo-Json $prJson -Depth 99) -Because "the [customRoleDefinitions.json] should be based on the latest [customRoleDefinitions.bicep] file. Please run [` bicep build ./src/templates/roles.bicep --outfile ./eslzArm/managementGroupTemplates/roleDefinitions/customRoleDefinitions.json `] using the latest Bicep CLI version." + } + } + + AfterAll { + # These are not the droids you are looking for... + } +} \ No newline at end of file diff --git a/.github/workflows/check-policy-build.yml b/.github/workflows/check-policy-build.yml new file mode 100644 index 0000000000..322607cf9c --- /dev/null +++ b/.github/workflows/check-policy-build.yml @@ -0,0 +1,51 @@ +--- + name: Check Policy Build + + ########################################## + # Start the job on PR for all branches # + ########################################## + + # yamllint disable-line rule:truthy + on: + pull_request: + types: + - opened + - reopened + - synchronize + - ready_for_review + paths: + - "eslzArm/**.json" + - "src/Alz.Tools/**" + - "src/**.json" + - "src/**.bicep" + + ############### + # Set the Job # + ############### + + jobs: + check-policy: + name: Check Policy Build + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Check build + shell: pwsh + run: | + Import-Module Pester -Force + $pesterConfiguration = @{ + Run = @{ + Container = New-PesterContainer -Path "./.github/actions-pester/Test-BuildPolicies.Tests.ps1" + PassThru = $true + } + Output = @{ + Verbosity = 'Detailed' + } + } + $result = Invoke-Pester -Configuration $pesterConfiguration + exit $result.FailedCount \ No newline at end of file diff --git a/.github/workflows/update-portal.yml b/.github/workflows/update-portal.yml deleted file mode 100644 index 116bb686fd..0000000000 --- a/.github/workflows/update-portal.yml +++ /dev/null @@ -1,125 +0,0 @@ ---- -name: Update Portal Experience - -########################################## -# Start the job on push for all branches # -########################################## - -# yamllint disable-line rule:truthy -on: - pull_request_target: - types: - - opened - - reopened - - synchronize - - ready_for_review - paths: - - "eslzArm/**.json" - - "src/Alz.Tools/**" - - "src/**.json" - - "src/**.bicep" - -env: - github_user_name: "github-actions" - github_email: "41898282+github-actions[bot]@users.noreply.github.com" - github_commit_message: "Auto-update Portal experience" - github_pr_number: ${{ github.event.number }} - github_pr_repo: ${{ github.event.pull_request.head.repo.full_name }} - -permissions: - contents: write - -############### -# Set the Job # -############### - -jobs: - update-portal: - name: Update Portal Experience - runs-on: ubuntu-latest - if: | - ( - github.event.pull_request.head.repo.full_name == 'Azure/Enterprise-Scale' - ) - || - ( - github.event.pull_request.head.repo.full_name != 'Azure/Enterprise-Scale' - && - contains(github.event.pull_request.labels.*.name, 'PR: Safe to test :test_tube:') - ) - || - ( - github.event_name == 'workflow_dispatch' - ) - || - ( - github.event_name == 'merge_group' - ) - - steps: - - name: Check out repository - uses: actions/checkout@v3 - - - name: Show env - run: env | sort - - - name: Check out PR - run: | - echo "==> Check out PR..." - gh pr checkout "$github_pr_number" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Configure local git - run: | - echo "git user name : $github_user_name" - git config --global user.name "$github_user_name" - echo "git user email : $github_email" - git config --global user.email "$github_email" - - - name: Update policies - run: bicep build ./src/templates/policies.bicep --outfile ./eslzArm/managementGroupTemplates/policyDefinitions/policies.json - - - name: Update policy set definitions (initiatives) - run: bicep build ./src/templates/initiatives.bicep --outfile ./eslzArm/managementGroupTemplates/policyDefinitions/initiatives.json - - - name: Update roles - run: bicep build ./src/templates/roles.bicep --outfile ./eslzArm/managementGroupTemplates/roleDefinitions/customRoleDefinitions.json - - - name: Check git status - run: | - echo "==> Check git status..." - git status --short --branch - - - name: Stage changes - run: | - echo "==> Stage changes..." - mapfile -t STATUS_LOG < <(git status --short | grep eslzArm/) - if [ ${#STATUS_LOG[@]} -gt 0 ]; then - echo "Found changes to the following files:" - printf "%s\n" "${STATUS_LOG[@]}" - git add --all ./eslzArm - else - echo "No changes to add." - fi - - - name: Push changes - run: | - echo "==> Check git diff..." - mapfile -t GIT_DIFF < <(git diff --cached) - printf "%s\n" "${GIT_DIFF[@]}" - - if [ ${#GIT_DIFF[@]} -gt 0 ]; then - - echo "==> Commit changes..." - git commit --message "$github_commit_message [$GITHUB_ACTOR/${GITHUB_SHA::8}]" - - echo "==> Push changes..." - echo "Pushing changes to: $github_pr_repo" - git push "https://$GITHUB_TOKEN@github.com/$github_pr_repo.git" "HEAD:$GITHUB_HEAD_REF" - - else - echo "No changes found." - fi - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/docs/wiki/ALZ-Contribution-Guide.md b/docs/wiki/ALZ-Contribution-Guide.md index f412ffcaab..d3e00ad8a6 100644 --- a/docs/wiki/ALZ-Contribution-Guide.md +++ b/docs/wiki/ALZ-Contribution-Guide.md @@ -183,7 +183,13 @@ For a policy set definition, additional code should be added inside of the `load `loadTextContent('../resources/Microsoft.Authorization/policySetDefinitions/Deploy-Sql-Security.json')` -The policy definition files will be compiled into a `policies.json` file from the `policy.bicep` file which was amended. +~~The policy definition files will be compiled into a `policies.json` file from the `policy.bicep` file which was amended.~~ + +> Due to security compliance requirements, we've made core changes that mean we no longer automatically build the policies, initiatives and roles templates after changes in the `src` folder are committed. This means that you as a contributor must run the bicep build commands to generate the required outputs as part of your pull request. Depending on the files you've updated these are the commands: +> +> - `bicep build ./src/templates/policies.bicep --outfile ./eslzArm/managementGroupTemplates/policyDefinitions/policies.json` +> - `bicep build ./src/templates/initiatives.bicep --outfile ./eslzArm/managementGroupTemplates/policyDefinitions/initiatives.json` +> - `bicep build ./src/templates/roles.bicep --outfile ./eslzArm/managementGroupTemplates/roleDefinitions/customRoleDefinitions.json` Once the policy work has been completed, a pull request should be submitted to the repository: diff --git a/docs/wiki/Whats-new.md b/docs/wiki/Whats-new.md index b3e3f31d95..9b8d8b28d6 100644 --- a/docs/wiki/Whats-new.md +++ b/docs/wiki/Whats-new.md @@ -50,6 +50,12 @@ Here's what's changed in Enterprise Scale/Azure Landing Zones: ### August 2024 +> NOTE TO CONTRIBUTORS: Due to security compliance requirements, we've made core changes that mean we no longer automatically build the policies, initiatives and roles templates after changes in the `src` folder are committed. This means that you as a contributor must run the bicep build commands to generate the required outputs as part of your pull request. Depending on the files you've updated these are the commands (assuming you have bicep installed): +> +> - `bicep build ./src/templates/policies.bicep --outfile ./eslzArm/managementGroupTemplates/policyDefinitions/policies.json` +> - `bicep build ./src/templates/initiatives.bicep --outfile ./eslzArm/managementGroupTemplates/policyDefinitions/initiatives.json` +> - `bicep build ./src/templates/roles.bicep --outfile ./eslzArm/managementGroupTemplates/roleDefinitions/customRoleDefinitions.json` + #### Other - Cleaned up the Log Analytics "solutions" in portal ARM template, as these are no longer required and deployed by ALZ.