From 2dbef424d7b9588e445076cf93d6161dfbc20671 Mon Sep 17 00:00:00 2001 From: jcrichlake <145698165+jcrichlake@users.noreply.github.com> Date: Wed, 13 Mar 2024 09:32:21 -0400 Subject: [PATCH] Privatize the Service (#931) * WIP vnet work * Fixing names * Fixing names 2 * Move network stuff to net.tf * Try creating some subnets * The database should use the new database subnet * The app should use the new app subnet * Only do zone balancing in the higher environments where we have at least 3 containers * Use dev vnet range * Give the app subnet ability to use Azure internal routing * Assign the route table to the new subnets * Assign the security group to the new subnets * delegate the app subnet to serverFarms * Add HTTP and Postgres security rule * Remove extra security rule * separate out HTTPS and HTTP security rule * Update security rule priorities * Comment out db migration so deploy can continue * Updating network terraform to remove routes and security groups * Adding security group and rules * Formatting * Fixing name for security group * Fixing lists * Fixing tags * Testing re-adding flyway * Nope still broken * Adding app security group and rules * Fixing dupe name issue * Fixing outbound * Adding ip logging temp * Removing virtual network link as test * Removing admin * Specify IP restrictions for denying all external traffic * advanced tool site also uses main site stuff * Adding 1 admin * Removing admin * Adding Ad and 1 service_delegation action * Adding NSG * Adding NSG * Removing old public firewall rule * Adding 1 admin * Use a smaller NSG * Adding dependency for db * Add a route table to direct entra traffic to the Internet * Bring back api admin for db * link the DNS zone with the vnet * Reduce size of db NSG * Rename some of the db nsg rules * Rename and slim down the app nsg rules * Remove commented out blocks * Rename database route table * Rename the db route table again * Remove the NSG auth rule * Terraform fmt * Re-adding migrations * Re-adding migrations * Adding deployer * Try creating a VPN * Try VPNing into Azure to do database migrations * v2.0.2 of the OVPN github action * Put the configuration back on the VPN * Different path to the OVPN file * ping the database * debug dig * Install liquibase manually without GitHub action that ignores the VPN * change working directory to the top level * Clean-up and fail the GitHub action if VPN is not connected * Update DB rollback GitHub action * dig debug on rollback * debug in wrong location * Ue correct secrets * Rename secrets to be more uniform * Adding vpn subnets and dns resolver * Renaming cert * Giving ip to inbound endpoint * Setting dns resolver ip to dynamic * Adding subnets dynamically * Adding dev secrets * Specify VPN root certificate per environment * Fixing internal * Add empty VPN root certificate to the internal and pr environments * Use generic secret names that will be per-environment thanks to GitHub * Marking vpn as not required * Mark the VPN secrets as optional for the Terraform deploy * Do not do VPN with rollback of the internal environment * Updating terraform * Updating terraform to remove conditions * Updating terraform to remove conditions * Removing even more conditions * Don't do internal networking for app in internal and pr environments * Better dynamic site_config * Do not use a root certificate for the VPN if it is not specified * Up the dynamic store * Make the vnet resource when deploying to the PR environment * use a shared local variable for when the environment is in a CDC domain * Only do the VPN when not internal and not pr when deploying * Apply VPN secrets to the stg and prd environments deployment * Remove IP logging * Terraform format * Update terraform comment * Started writing documentation on interacting with the app service * Point to the VPN documentation * Have the security group associations wait for the VPN because these updates seem to be mutually exclusive * Terraform format * Change documentation for when a Flexion domain-hosted environment * Update link to VPN Notion documentation * Change the dependency of the VPN and subnet stuff --------- Co-authored-by: halprin --- .github/workflows/cicd.yml | 4 + .github/workflows/db_rollback.yml | 38 +- .github/workflows/dev-deploy.yml | 4 + .github/workflows/prod-deploy.yml | 4 + .../workflows/terraform-deploy_reusable.yml | 46 ++- .secrets.baseline | 10 +- README.md | 35 +- docs/database.md | 18 +- operations/environments/dev/main.tf | 5 +- operations/environments/pr/main.tf | 10 +- operations/environments/prd/main.tf | 5 +- operations/environments/stg/main.tf | 5 +- operations/template/app.tf | 30 +- operations/template/db.tf | 5 + operations/template/key.tf | 2 - operations/template/main.tf | 1 + operations/template/net.tf | 327 ++++++++++++++++++ operations/template/variables.tf | 6 + operations/template/vpn.tf | 68 ++++ operations/vpn/dev.ovpn | 48 +++ 20 files changed, 632 insertions(+), 39 deletions(-) create mode 100644 operations/template/net.tf create mode 100644 operations/template/vpn.tf create mode 100644 operations/vpn/dev.ovpn diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 4e75f233d..5487572ce 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -25,6 +25,10 @@ jobs: AZURE_CLIENT_ID: ${{ secrets.AZURE_CDC_CLIENT_ID }} AZURE_TENANT_ID: ${{ secrets.AZURE_CDC_TENANT_ID }} AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_CDC_DMZ_C1_SUBSCRIPTION_ID }} + VPN_TLS_KEY: ${{ secrets.VPN_TLS_KEY }} + VPN_CA_CERTIFICATE: ${{ secrets.VPN_CA_CERTIFICATE }} + VPN_GITHUB_CERTIFICATE: ${{ secrets.VPN_GITHUB_CERTIFICATE}} + VPN_GITHUB_SECRET_KEY: ${{ secrets.VPN_GITHUB_SECRET_KEY }} staging-deploy: name: Staging Application Deploy diff --git a/.github/workflows/db_rollback.yml b/.github/workflows/db_rollback.yml index 8a0581f3a..f8b463758 100644 --- a/.github/workflows/db_rollback.yml +++ b/.github/workflows/db_rollback.yml @@ -64,11 +64,35 @@ jobs: echo "DATABASE_HOSTNAME=$DATABASE_HOSTNAME" >> "$GITHUB_ENV" echo "DATABASE_PASSWORD=$DATABASE_PASSWORD" >> "$GITHUB_ENV" - - name: Run Db Rollback - uses: liquibase-github-actions/rollback-count@v4.26.0 + - name: Connect to VPN + uses: josiahsiegel/action-connect-ovpn@v2.0.2 + id: connect_vpn + if: github.event.inputs.environment != 'internal' with: - changelogFile: ${{ github.event.inputs.rollbackFile }} - count: ${{ github.event.inputs.rollbackCount }} - url: "jdbc:postgresql://${{ env.DATABASE_HOSTNAME }}:5432/postgres" - username: cdcti-github - password: ${{ env.DATABASE_PASSWORD }} + PING_URL: ${{ env.DATABASE_HOSTNAME }} + FILE_OVPN: ./operations/vpn/${{ github.event.inputs.environment }}.ovpn + TLS_KEY: ${{ secrets.VPN_TLS_KEY }} + env: + CA_CRT: ${{ secrets.VPN_CA_CERTIFICATE }} + USER_CRT: ${{ secrets.VPN_GITHUB_CERTIFICATE }} + USER_KEY: ${{ secrets.VPN_GITHUB_SECRET_KEY }} + + - name: Fail if VPN isn't Connected + if: github.event.inputs.environment != 'internal' && steps.connect_vpn.outputs.STATUS != 'true' + run: | + echo 'VPN connected: ${{ steps.connect_vpn.outputs.STATUS }}' + exit 1 + + - name: Install Liquibase + run: | + wget -O- https://repo.liquibase.com/liquibase.asc | gpg --dearmor > liquibase-keyring.gpg && \cat liquibase-keyring.gpg | sudo tee /usr/share/keyrings/liquibase-keyring.gpg > /dev/null && \echo 'deb [arch=amd64 signed-by=/usr/share/keyrings/liquibase-keyring.gpg] https://repo.liquibase.com stable main' | sudo tee /etc/apt/sources.list.d/liquibase.list + sudo apt-get update + sudo apt-get install liquibase + liquibase -v + + - name: Run Db migration + run: liquibase rollback-count --changelog-file ${{ github.event.inputs.rollbackFile }} --count ${{ github.event.inputs.rollbackCount }} --url 'jdbc:postgresql://${{ env.DATABASE_HOSTNAME }}:5432/postgres' --username cdcti-github --password '${{ env.DATABASE_PASSWORD }}' + + - name: Disconnect VPN + if: github.event.inputs.environment != 'internal' && always() + run: sudo killall openvpn diff --git a/.github/workflows/dev-deploy.yml b/.github/workflows/dev-deploy.yml index 5f1318c3b..28fff8a23 100644 --- a/.github/workflows/dev-deploy.yml +++ b/.github/workflows/dev-deploy.yml @@ -18,6 +18,10 @@ jobs: AZURE_CLIENT_ID: ${{ secrets.AZURE_CDC_CLIENT_ID }} AZURE_TENANT_ID: ${{ secrets.AZURE_CDC_TENANT_ID }} AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_CDC_DMZ_C1_SUBSCRIPTION_ID }} + VPN_TLS_KEY: ${{ secrets.VPN_TLS_KEY }} + VPN_CA_CERTIFICATE: ${{ secrets.VPN_CA_CERTIFICATE }} + VPN_GITHUB_CERTIFICATE: ${{ secrets.VPN_GITHUB_CERTIFICATE}} + VPN_GITHUB_SECRET_KEY: ${{ secrets.VPN_GITHUB_SECRET_KEY }} dev-deploy: name: Dev Application Deploy diff --git a/.github/workflows/prod-deploy.yml b/.github/workflows/prod-deploy.yml index b0bc3a012..df379e1df 100644 --- a/.github/workflows/prod-deploy.yml +++ b/.github/workflows/prod-deploy.yml @@ -22,6 +22,10 @@ jobs: AZURE_CLIENT_ID: ${{ secrets.AZURE_CDC_CLIENT_ID }} AZURE_TENANT_ID: ${{ secrets.AZURE_CDC_TENANT_ID }} AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_CDC_DMZ_C1_SUBSCRIPTION_ID }} + VPN_TLS_KEY: ${{ secrets.VPN_TLS_KEY }} + VPN_CA_CERTIFICATE: ${{ secrets.VPN_CA_CERTIFICATE }} + VPN_GITHUB_CERTIFICATE: ${{ secrets.VPN_GITHUB_CERTIFICATE}} + VPN_GITHUB_SECRET_KEY: ${{ secrets.VPN_GITHUB_SECRET_KEY }} prod-deploy: name: Prod Application Deploy diff --git a/.github/workflows/terraform-deploy_reusable.yml b/.github/workflows/terraform-deploy_reusable.yml index e9ca6495b..8c7471936 100644 --- a/.github/workflows/terraform-deploy_reusable.yml +++ b/.github/workflows/terraform-deploy_reusable.yml @@ -24,6 +24,14 @@ on: required: true AZURE_SUBSCRIPTION_ID: required: true + VPN_TLS_KEY: + required: false + VPN_CA_CERTIFICATE: + required: false + VPN_GITHUB_CERTIFICATE: + required: false + VPN_GITHUB_SECRET_KEY: + required: false outputs: REGISTRY: description: The container registry @@ -85,13 +93,39 @@ jobs: echo "DATABASE_HOSTNAME=$DATABASE_HOSTNAME" >> "$GITHUB_ENV" echo "DATABASE_PASSWORD=$DATABASE_PASSWORD" >> "$GITHUB_ENV" - - name: Run Db migration - uses: liquibase-github-actions/update@v4.26.0 + - name: Connect to VPN + uses: josiahsiegel/action-connect-ovpn@v2.0.2 + id: connect_vpn + if: inputs.ENVIRONMENT != 'internal' && inputs.ENVIRONMENT != 'pr' with: - changelogFile: ./etor/databaseMigrations/root.yml - url: "jdbc:postgresql://${{ env.DATABASE_HOSTNAME }}:5432/postgres" - username: cdcti-github - password: ${{ env.DATABASE_PASSWORD }} + PING_URL: ${{ env.DATABASE_HOSTNAME }} + FILE_OVPN: ./operations/vpn/${{ inputs.ENVIRONMENT }}.ovpn + TLS_KEY: ${{ secrets.VPN_TLS_KEY }} + env: + CA_CRT: ${{ secrets.VPN_CA_CERTIFICATE }} + USER_CRT: ${{ secrets.VPN_GITHUB_CERTIFICATE }} + USER_KEY: ${{ secrets.VPN_GITHUB_SECRET_KEY }} + + - name: Fail if VPN isn't Connected + if: inputs.ENVIRONMENT != 'internal' && inputs.ENVIRONMENT != 'pr' && steps.connect_vpn.outputs.STATUS != 'true' + run: | + echo 'VPN connected: ${{ steps.connect_vpn.outputs.STATUS }}' + exit 1 + + - name: Install Liquibase + run: | + wget -O- https://repo.liquibase.com/liquibase.asc | gpg --dearmor > liquibase-keyring.gpg && \cat liquibase-keyring.gpg | sudo tee /usr/share/keyrings/liquibase-keyring.gpg > /dev/null && \echo 'deb [arch=amd64 signed-by=/usr/share/keyrings/liquibase-keyring.gpg] https://repo.liquibase.com stable main' | sudo tee /etc/apt/sources.list.d/liquibase.list + sudo apt-get update + sudo apt-get install liquibase + liquibase -v + + - name: Run Db migration + working-directory: ./ + run: liquibase update --changelog-file ./etor/databaseMigrations/root.yml --url 'jdbc:postgresql://${{ env.DATABASE_HOSTNAME }}:5432/postgres' --username cdcti-github --password '${{ env.DATABASE_PASSWORD }}' + + - name: Disconnect VPN + if: inputs.ENVIRONMENT != 'internal' && inputs.ENVIRONMENT != 'pr' && always() + run: sudo killall openvpn - id: export-terraform-output name: Export Terraform Output diff --git a/.secrets.baseline b/.secrets.baseline index 97e303e35..2bcc73f34 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -138,7 +138,7 @@ "filename": "README.md", "hashed_secret": "367e3228ed16bf72b36de9a4134ee8b825cafacb", "is_verified": false, - "line_number": 285, + "line_number": 316, "is_secret": false }, { @@ -146,7 +146,7 @@ "filename": "README.md", "hashed_secret": "40bd7d88eae0468b048e62e1056ac390970b2b51", "is_verified": false, - "line_number": 290, + "line_number": 321, "is_secret": false }, { @@ -154,7 +154,7 @@ "filename": "README.md", "hashed_secret": "0d46754ae17642645ca041edaac9a1c1569f5edc", "is_verified": false, - "line_number": 295, + "line_number": 326, "is_secret": false } ], @@ -244,7 +244,7 @@ "filename": "operations/template/db.tf", "hashed_secret": "7cb6efb98ba5972a9b5090dc2e517fe14d12cb04", "is_verified": false, - "line_number": 16, + "line_number": 18, "is_secret": false } ], @@ -269,5 +269,5 @@ } ] }, - "generated_at": "2024-02-16T15:55:01Z" + "generated_at": "2024-03-11T15:51:37Z" } diff --git a/README.md b/README.md index af72baf4c..3e5984b4d 100644 --- a/README.md +++ b/README.md @@ -174,13 +174,14 @@ is also meant to be the Wild West. Dev deploys similarly to the Internal enviro ##### Staging -The Staging environment is production-like and meant to be stable. It deploys to a non-CDC Azure Entra domain and +The Staging environment is production-like and meant to be stable. It deploys to a CDC Azure Entra domain and subscription. Deployments occur when a commit is made to the `main` branch. `main` is a protected branch and requires PR reviews before merge. ##### Prod -The Prod environment does not exist yet. +The Production environment is the real deal. It deploys to a CDC Azure Entra domain and subscription. Deployments +occur when a release is published. #### Initial Azure and GitHub Configuration @@ -210,6 +211,36 @@ Entra domains and subscriptions. [Internal environment deployment](./.github/workflows/internal-deploy.yml). Make sure you set the `AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, and `AZURE_SUBSCRIPTION_ID` based on the secrets created previously. +#### Interacting with Deployed Environments + +The PR and Internal environment is available on the public Internet and able to be interacted with directly. + +The Dev, Staging, and Prod environment are deployed inside a Vnet and require special steps to interact with these. + +##### Application + +The application basically has a firewall in place. You need to add (and remove when you're done) your IP address to the +firewall allow list. + +1. Log into CyberArk and then into Azure with your -SU account. +2. Navigate to the environment's app service. +3. Click on Networking in the left pane. +4. Click on the "Enabled with access restrictions" link under "Inbound traffic configuration". +5. Add a new rule to allow your _public_ IP address. Provide an appropriate name with your name. The priority will + need a lower number than the existing denies. It will look like your IP address with a `/32` appended. E.g. + `192.168.0.1/32`. +6. Click "Save". + +You will now be able to interact with that environment's application. Don't forget to remove your rule and save when +you are done. + +##### Database + +You will need to connect to the VPN for the given environment first, and then you can interact with the database. +Notion contains the +[instructions for connecting to the VPN](https://www.notion.so/flexion-cdc-ti/Azure-VPN-pieces-d814ddcb87b1467f93ccf473e9cdb69c?pvs=4). +After connecting, you can follow the [database documentation](docs/database.md) to gain access. + ### Pre-Commit Hooks We use [`pre-commit`](https://pre-commit.com) to run [some hooks](./.pre-commit-config.yaml) on every commit. These diff --git a/docs/database.md b/docs/database.md index e3b2abcd2..98894bd29 100644 --- a/docs/database.md +++ b/docs/database.md @@ -17,17 +17,19 @@ Choose a Postgres client, [pgAdmin](https://www.pgadmin.org/) is the most full f 6. Upon creating the table for the first time you may have to instruct your client to refresh (In pgAdmin right-click on the left hand menu and select Refresh) ## Connecting to an Azure Hosted Database -1. Install the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) -2. Open the azure environment you are trying to access from within your browser and navigate to the database -3. Inside of the Azure database page select the Networking option from the left hand nav -4. Click the link that says `Add current client IP address` and then save the page +1. For a CDC domain-hosted environment, connect to the VPN. See the + [VPN documentation](https://www.notion.so/flexion-cdc-ti/Azure-VPN-pieces-d814ddcb87b1467f93ccf473e9cdb69c?pvs=4). +2. Install the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) +3. Open the Azure environment you are trying to access from within your browser and navigate to the database +4. Inside of the Azure database page select the Networking option from the left hand nav +5. For a Flexion domain-hosted environment, click the link that says `Add current client IP address` and then save the page 1. NOTE: You should only add your local IP address on a temporary basis, you should remove it after the verification is complete -5. On the left hand navigation select Authentication and select the `Add Microsoft Entra Admins` link to add your user to the list. Select Okay and then save the underlying page +6. On the left hand navigation select Authentication and select the `Add Microsoft Entra Admins` link to add your user to the list. Select Okay and then save the underlying page 1. NOTE: This permission should only be added temporarily and removed after you are finished with verification -6. Enter new connection settings from Azure into your db client of choice +7. Enter new connection settings from Azure into your db client of choice 1. Password will come from step 8 of these instructions and can be left blank -7. Run `az login` inside of your local terminal -8. Run `az account get-access-token --resource https://ossrdbms-aad.database.windows.net` to get a temporary password +8. Run `az login` inside of your local terminal +9. Run `az account get-access-token --resource https://ossrdbms-aad.database.windows.net` to get a temporary password ## Modifying the database schema To modify the schema there are a few locations in the code we need to update. diff --git a/operations/environments/dev/main.tf b/operations/environments/dev/main.tf index 9f74e4e4b..3b43f2fe4 100644 --- a/operations/environments/dev/main.tf +++ b/operations/environments/dev/main.tf @@ -27,6 +27,7 @@ provider "azurerm" { module "template" { source = "../../template/" - environment = "dev" - deployer_id = "f5feabe7-5d37-40ba-94f2-e5c0760b4561" //github app registration in CDC Azure Entra + environment = "dev" + deployer_id = "f5feabe7-5d37-40ba-94f2-e5c0760b4561" //github app registration in CDC Azure Entra + vpn_root_certificate = "MIIC5jCCAc6gAwIBAgIIWvb3sLkOQtcwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAxMGVlBOIENBMB4XDTI0MDMwODE1MjM0OFoXDTI3MDMwODE1MjM0OFowETEPMA0GA1UEAxMGVlBOIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt1v2bAATE7IOJkqUrbwQw6X99fi3Ywf1bv0uZ0gGDjG10H+PB2BUzZ94RNcB4Oezi6t+/WAQUkhRozemFkegSkfKHEehAT6nu6OBXKt2rH/oJtpKR791ab9H9aQ6e5LO9OZ237QL6XikhGG7HXqG9ndYnhBYPy2/pd8VV6ZwqMR3PkfBJaC4tKy4d8dim+PMpT5rqPGbsf9H+dydvG6JOKZiHb3/yqi6fqoise1yY64aDwFC9MbEbtgXpvmBFsei2PA/XH5FqE6F/kyCg7mO5TSYYEqx0PCTPmICAT4iw5ELMyAhVKL2OpMjqw5YAYr/TGqlfyEYpBBQMvC3K9OmUwIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQU5idI+AFsHn8BjvNBE5ShE+aFor0wDQYJKoZIhvcNAQELBQADggEBAJkDtuHj4QGyXtooiM7xfHZ/lGdDZvF+KAVfFKAlsIO8y1NS2iAeNT6MmampwzWzXIUMk9vxvALUoh2MJkWP5CX2e3vDj2lGpbhK5//rfWDin1/jj28+KZzSVsk4i/EkdBWW7eCKU401rafVOjSLmM5mfDTAHNrFxzQWJF5WL7TxrQw7chrnpy4v0V7/y4h+QsQja8LXx9keEdB2BQSjndAqxB9dblFALantpuEOM2pS3GCaC2REXSnKsgSEQoVL07MSndpCpdv5bsEkppM5LBC6gL66a43Lho3kSCm4ZU51mjJtNwadeBXpHjkJ1yiBA7CG/Roa+THAiV+VMP75g3E=" # pragma: allowlist secret } diff --git a/operations/environments/pr/main.tf b/operations/environments/pr/main.tf index 1dd4081bb..a20e24e21 100644 --- a/operations/environments/pr/main.tf +++ b/operations/environments/pr/main.tf @@ -28,11 +28,19 @@ resource "azurerm_resource_group" "group" { //create the PR resource group becau location = "East US" } +resource "azurerm_virtual_network" "vnet" { //create the PR Vnet because it has a dynamic name that cannot be always pre-created + name = "csels-rsti-pr${var.pr_number}-moderate-app-vnet" + location = azurerm_resource_group.group.location + resource_group_name = azurerm_resource_group.group.name + + address_space = ["10.0.0.0/25"] +} + module "template" { source = "../../template/" environment = "pr${var.pr_number}" deployer_id = "d59c2c86-de5e-41b7-a752-0869a73f5a60" //github app registration in Flexion Azure Entra - depends_on = [azurerm_resource_group.group] + depends_on = [azurerm_resource_group.group, azurerm_virtual_network.vnet] } diff --git a/operations/environments/prd/main.tf b/operations/environments/prd/main.tf index f26865daf..3bf0afd2a 100644 --- a/operations/environments/prd/main.tf +++ b/operations/environments/prd/main.tf @@ -27,6 +27,7 @@ provider "azurerm" { module "template" { source = "../../template/" - environment = "prd" - deployer_id = "f5feabe7-5d37-40ba-94f2-e5c0760b4561" //github app registration in CDC Azure Entra + environment = "prd" + deployer_id = "f5feabe7-5d37-40ba-94f2-e5c0760b4561" //github app registration in CDC Azure Entra + vpn_root_certificate = "MIIC5jCCAc6gAwIBAgIIeHnOQDhz00AwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAxMGVlBOIENBMB4XDTI0MDMwODE1NDA1M1oXDTI3MDMwODE1NDA1M1owETEPMA0GA1UEAxMGVlBOIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAkgeXZ6ReEQ5HAqlXULUUdVfCMtMPmlTeCFFkhD9i5E5lRg78PyJqczHMzCB6l83O/PrLWXjT3/s/R58cfeHJg/SndGwt/2uKhj1kNW7Ivc8kF0pgSL3lDR+NSj5OPda45EY30ZlTjgygmb9MjfCT2BmgjGcfUbgm0jzgDZsk7bLUUJkL38DJP+v2M6sDxyxMjoY9gJ1Kq5Fg81serJlZHaACShuuhgiKqH3+hwvIPluK8Y40FWfiKpGRjdkAXGTmB+afMeA4L1amyticIPzzOytIHFIDMOKgJRL62UQe+alzubXkYbDtEgDCOwF8k5TRiu9MUwID34CLkp2VWnLnUwIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUHyrypPmh+KVb2sspeGsxboG1hQwwDQYJKoZIhvcNAQELBQADggEBAGmfFRLgqLQxedGHeXQoajHzhCvk+62lDR1xy0s2mklA3eRxzOyaXRPgmM6lbGBm6LdLxo5nxGgfD4h2vOBZl4MXOFLryLm97QtDZ34YkxGn+tugUAXpWBB/EJIynib1Ywyg6Kv6g3oYjf2bc8Ae9bOWGR0FtOGn8TvmSzKLXoUwQd0u9DEA774YtpvPxHxw69uyf8x2nekpyWNyFbR6DWJEA9M+BHeR0oGEGoc5FH6zTgstbdeNVou3NNQlRKlWD26vWeCeQvbKDK5+KuOPjjDTimGdx1GfA9z/ai/pX+K/NKvvC4JXQdW7jYYu3QFglP70esT9mBCxVQbXd49oD9M=" # pragma: allowlist secret } diff --git a/operations/environments/stg/main.tf b/operations/environments/stg/main.tf index 83cb26b98..64fac59fa 100644 --- a/operations/environments/stg/main.tf +++ b/operations/environments/stg/main.tf @@ -27,6 +27,7 @@ provider "azurerm" { module "template" { source = "../../template/" - environment = "stg" - deployer_id = "f5feabe7-5d37-40ba-94f2-e5c0760b4561" //github app registration in CDC Azure Entra + environment = "stg" + deployer_id = "f5feabe7-5d37-40ba-94f2-e5c0760b4561" //github app registration in CDC Azure Entra + vpn_root_certificate = "MIIC5jCCAc6gAwIBAgIIUC720RvICDQwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAxMGVlBOIENBMB4XDTI0MDMwODE1MzY1MloXDTI3MDMwODE1MzY1MlowETEPMA0GA1UEAxMGVlBOIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtn+H6PbB2y/JmoTTNuxljVY7I02BblPmnzwzQhEPAZjoMVQTsEvS5rr/ILLFFF5FdTcyPYJpD5Tvd+w1v62xV4QhZSFpSSyfRvsi6uzOLOyDOhVN++GWAjKyTvaOO654JwX/qj7nHSYQLQFtnf9OkixZazO8o0snXpGCSYKgxhBox6+XyZpjjwoFt+wMrNalrAOWCtAp/pgIB7xyStcWyGEi7vACiV+7rzI2Kxh+PfaltS4wU1vWN7jN2GxMbVG3539ybiT4fpoGuDWjZ7t7tp1LgQa1n7tlvNR0W01pdt7U/fPL9ynfyuP8Wph8eetW9THYtJkBTNk7KyhE+z36TwIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQU85kmRn9Cnq9LjFeKjwrMUhgo3xowDQYJKoZIhvcNAQELBQADggEBAHjzfciR/TqJojJ3xd2AmMQev5Aw6Wf9gFfhv0eb9bmqyeJ23bYhOvqWxIxb01TBp5CNhWgWuUE68cQpEqafu9JOITDk9GtQ9m6/4sHOhzqM11beGqKlomQuT+I/M/gS0pUcr//W7riTkOQQI6DHKgpoGoRXpk9/V5GrwQauZjy1hRyRpVlg4xDgJJqRr5PKUErtA07DYck+AblJW4msglfyM2HTvvMLNdsmiZmjFdU1osT0WT/W9nY+RGadAo47x6qknpFoDoVtIQ3XNH3C5Scl1bGphfQdmEjNVhg7a8gSWat7n1OjFiz3OvTqy5MsssmRz4WlOM5+xOhiT2OambA=" # pragma: allowlist secret } diff --git a/operations/template/app.tf b/operations/template/app.tf index c3e2edbb9..6352c6f15 100644 --- a/operations/template/app.tf +++ b/operations/template/app.tf @@ -14,7 +14,7 @@ resource "azurerm_service_plan" "plan" { location = data.azurerm_resource_group.group.location os_type = "Linux" sku_name = local.higher_environment_level ? "P1v3" : "P0v3" - zone_balancing_enabled = true + zone_balancing_enabled = local.higher_environment_level } # Create the staging App Service @@ -26,7 +26,33 @@ resource "azurerm_linux_web_app" "api" { https_only = true - site_config {} + virtual_network_subnet_id = local.cdc_domain_environment ? azurerm_subnet.app.id : null + + site_config { + scm_use_main_ip_restriction = local.cdc_domain_environment ? true : null + + dynamic "ip_restriction" { + for_each = local.cdc_domain_environment ? [1] : [] + + content { + name = "deny_all_ipv4" + action = "Deny" + ip_address = "0.0.0.0/0" + priority = "200" + } + } + + dynamic "ip_restriction" { + for_each = local.cdc_domain_environment ? [1] : [] + + content { + name = "deny_all_ipv6" + action = "Deny" + ip_address = "::/0" + priority = "201" + } + } + } app_settings = { DOCKER_REGISTRY_SERVER_URL = "https://${azurerm_container_registry.registry.login_server}" diff --git a/operations/template/db.tf b/operations/template/db.tf index 6666a3f9d..8024cca65 100644 --- a/operations/template/db.tf +++ b/operations/template/db.tf @@ -11,6 +11,8 @@ resource "azurerm_postgresql_flexible_server" "database" { storage_mb = "32768" auto_grow_enabled = true backup_retention_days = "14" + delegated_subnet_id = local.cdc_domain_environment ? azurerm_subnet.database.id : null + private_dns_zone_id = local.cdc_domain_environment ? azurerm_private_dns_zone.dns_zone.id : null authentication { password_auth_enabled = "false" @@ -18,6 +20,8 @@ resource "azurerm_postgresql_flexible_server" "database" { tenant_id = data.azurerm_client_config.current.tenant_id } + depends_on = [azurerm_private_dns_zone_virtual_network_link.db_network_link] + lifecycle { ignore_changes = [ zone, @@ -45,6 +49,7 @@ resource "azurerm_postgresql_flexible_server_active_directory_administrator" "ad } resource "azurerm_postgresql_flexible_server_firewall_rule" "db_firewall_5" { + count = local.cdc_domain_environment ? 0 : 1 name = "AllowAzure" server_id = azurerm_postgresql_flexible_server.database.id start_ip_address = "0.0.0.0" diff --git a/operations/template/key.tf b/operations/template/key.tf index fe1dcdca3..85b02dc19 100644 --- a/operations/template/key.tf +++ b/operations/template/key.tf @@ -81,5 +81,3 @@ resource "azurerm_key_vault_secret" "trusted_intermediary_private_key" { } depends_on = [azurerm_key_vault_access_policy.allow_github_deployer] //wait for the permission that allows our deployer to write the secret } - - diff --git a/operations/template/main.tf b/operations/template/main.tf index 02060b435..4745b35af 100644 --- a/operations/template/main.tf +++ b/operations/template/main.tf @@ -7,6 +7,7 @@ locals { selected_rs_environment_prefix = lookup(local.environment_to_rs_environment_prefix_mapping, var.environment, "staging") rs_domain_prefix = "${local.selected_rs_environment_prefix}${length(local.selected_rs_environment_prefix) == 0 ? "" : "."}" higher_environment_level = var.environment == "stg" || var.environment == "prd" + cdc_domain_environment = var.environment == "dev" || var.environment == "stg" || var.environment == "prd" } data "azurerm_resource_group" "group" { diff --git a/operations/template/net.tf b/operations/template/net.tf new file mode 100644 index 000000000..c888d089f --- /dev/null +++ b/operations/template/net.tf @@ -0,0 +1,327 @@ +data "azurerm_virtual_network" "app" { + name = "csels-rsti-${var.environment}-moderate-app-vnet" + resource_group_name = data.azurerm_resource_group.group.name +} + +locals { + subnets_cidrs = cidrsubnets(data.azurerm_virtual_network.app.address_space[0], 2, 2, 2, 3, 3) +} + +resource "azurerm_subnet" "app" { + name = "app" + resource_group_name = data.azurerm_resource_group.group.name + virtual_network_name = data.azurerm_virtual_network.app.name + address_prefixes = [local.subnets_cidrs[0]] + + service_endpoints = [ + "Microsoft.AzureActiveDirectory", + "Microsoft.AzureCosmosDB", + "Microsoft.ContainerRegistry", + "Microsoft.EventHub", + "Microsoft.KeyVault", + "Microsoft.ServiceBus", + "Microsoft.Sql", + "Microsoft.Storage", + "Microsoft.Web", + ] + + delegation { + name = "delegation" + + service_delegation { + name = "Microsoft.Web/serverFarms" + actions = ["Microsoft.Network/virtualNetworks/subnets/join/action"] + } + } +} + +resource "azurerm_subnet" "database" { + name = "database" + resource_group_name = data.azurerm_resource_group.group.name + virtual_network_name = data.azurerm_virtual_network.app.name + address_prefixes = [local.subnets_cidrs[1]] + + service_endpoints = [ + "Microsoft.AzureActiveDirectory", + "Microsoft.AzureCosmosDB", + "Microsoft.ContainerRegistry", + "Microsoft.EventHub", + "Microsoft.KeyVault", + "Microsoft.ServiceBus", + "Microsoft.Sql", + "Microsoft.Storage", + "Microsoft.Web", + ] + + delegation { + name = "delegation" + + service_delegation { + name = "Microsoft.DBforPostgreSQL/flexibleServers" + actions = ["Microsoft.Network/virtualNetworks/subnets/join/action", "Microsoft.Network/virtualNetworks/subnets/prepareNetworkPolicies/action"] + } + } +} + +resource "azurerm_subnet" "vpn" { + name = "GatewaySubnet" + resource_group_name = data.azurerm_resource_group.group.name + virtual_network_name = data.azurerm_virtual_network.app.name + address_prefixes = [local.subnets_cidrs[2]] +} + +resource "azurerm_subnet" "resolver_inbound" { + name = "resolver-inbound" + resource_group_name = data.azurerm_resource_group.group.name + virtual_network_name = data.azurerm_virtual_network.app.name + address_prefixes = [local.subnets_cidrs[3]] + + delegation { + name = "delegation" + + service_delegation { + name = "Microsoft.Network/dnsResolvers" + actions = ["Microsoft.Network/virtualNetworks/subnets/join/action"] + } + } +} + +resource "azurerm_subnet" "resolver_outbound" { + name = "resolver-outbound" + resource_group_name = data.azurerm_resource_group.group.name + virtual_network_name = data.azurerm_virtual_network.app.name + address_prefixes = [local.subnets_cidrs[4]] + + delegation { + name = "delegation" + + service_delegation { + name = "Microsoft.Network/dnsResolvers" + actions = ["Microsoft.Network/virtualNetworks/subnets/join/action"] + } + } +} + +resource "azurerm_private_dns_zone" "dns_zone" { + name = "privateintermediary.postgres.database.azure.com" + resource_group_name = data.azurerm_resource_group.group.name +} + +resource "azurerm_private_dns_zone_virtual_network_link" "db_network_link" { + name = "db_network_link" + private_dns_zone_name = azurerm_private_dns_zone.dns_zone.name + virtual_network_id = data.azurerm_virtual_network.app.id + resource_group_name = data.azurerm_resource_group.group.name + depends_on = [azurerm_subnet.database] +} + +resource "azurerm_network_security_group" "db_security_group" { + name = "database-security-group" + location = data.azurerm_resource_group.group.location + resource_group_name = data.azurerm_resource_group.group.name +} + +resource "azurerm_route_table" "database" { + name = "database-route-table" + location = data.azurerm_resource_group.group.location + resource_group_name = data.azurerm_resource_group.group.name +} + +resource "azurerm_route" "entra_internet" { + name = "entra_internet" + resource_group_name = data.azurerm_resource_group.group.name + route_table_name = azurerm_route_table.database.name + address_prefix = "AzureActiveDirectory" + next_hop_type = "Internet" +} + +resource "azurerm_subnet_route_table_association" "database_database" { + subnet_id = azurerm_subnet.database.id + route_table_id = azurerm_route_table.database.id +} + +resource "azurerm_network_security_rule" "DB_Splunk_UF_omhsinf" { + name = "Splunk_UF_omhsinf" + priority = 103 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "9997-9998" + source_address_prefixes = ["10.65.8.211/32", "10.65.8.212/32", "10.65.7.212/32", "10.65.7.211/32", "10.65.8.210/32", "10.65.7.210/32"] + destination_address_prefix = "*" + resource_group_name = data.azurerm_resource_group.group.name + network_security_group_name = azurerm_network_security_group.db_security_group.name +} + +resource "azurerm_network_security_rule" "DB_Splunk_Indexer_Discovery_omhsinf" { + name = "Splunk_Indexer_Discovery_omhsinf" + priority = 104 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "8089" + source_address_prefix = "10.11.7.22/32" + destination_address_prefix = "*" + resource_group_name = data.azurerm_resource_group.group.name + network_security_group_name = azurerm_network_security_group.db_security_group.name +} + + +resource "azurerm_network_security_rule" "DB_Safe_Encase_Monitoring_omhsinf" { + name = "Safe_Encase_Monitoring_omhsinf" + priority = 105 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "34445" + source_address_prefix = "10.11.6.145/32" + destination_address_prefix = "*" + resource_group_name = data.azurerm_resource_group.group.name + network_security_group_name = azurerm_network_security_group.db_security_group.name +} + +resource "azurerm_network_security_rule" "DB_ForeScout_Manager_omhsinf" { + name = "ForeScout_Manager_omhsinf" + priority = 106 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_ranges = ["556", "443", "10003-10006"] + source_address_prefixes = ["10.64.8.184", "10.64.8.180/32"] + destination_address_prefix = "*" + resource_group_name = data.azurerm_resource_group.group.name + network_security_group_name = azurerm_network_security_group.db_security_group.name +} + +resource "azurerm_network_security_rule" "DB_BigFix_omhsinf" { + name = "BigFix_omhsinf" + priority = 107 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "52314" + source_address_prefix = "10.11.4.84/32" + destination_address_prefix = "*" + resource_group_name = data.azurerm_resource_group.group.name + network_security_group_name = azurerm_network_security_group.db_security_group.name +} + +resource "azurerm_network_security_rule" "DB_Allow_All_Out_omhsinf" { + name = "Allow_All_Out_omhsinf" + priority = 109 + direction = "Outbound" + access = "Allow" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "*" + destination_address_prefix = "*" + resource_group_name = data.azurerm_resource_group.group.name + network_security_group_name = azurerm_network_security_group.db_security_group.name +} + +resource "azurerm_subnet_network_security_group_association" "database_security_group" { + subnet_id = azurerm_subnet.database.id + network_security_group_id = azurerm_network_security_group.db_security_group.id +} + +resource "azurerm_network_security_group" "app_security_group" { + name = "app-security-group" + location = data.azurerm_resource_group.group.location + resource_group_name = data.azurerm_resource_group.group.name +} + +resource "azurerm_network_security_rule" "App_Splunk_UF_omhsinf" { + name = "Splunk_UF_omhsinf" + priority = 103 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "9997-9998" + source_address_prefixes = ["10.65.8.211/32", "10.65.8.212/32", "10.65.7.212/32", "10.65.7.211/32", "10.65.8.210/32", "10.65.7.210/32"] + destination_address_prefix = "*" + resource_group_name = data.azurerm_resource_group.group.name + network_security_group_name = azurerm_network_security_group.app_security_group.name +} + +resource "azurerm_network_security_rule" "App_Splunk_Indexer_Discovery_omhsinf" { + name = "Splunk_Indexer_Discovery_omhsinf" + priority = 104 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "8089" + source_address_prefix = "10.11.7.22/32" + destination_address_prefix = "*" + resource_group_name = data.azurerm_resource_group.group.name + network_security_group_name = azurerm_network_security_group.app_security_group.name +} + + +resource "azurerm_network_security_rule" "App_Safe_Encase_Monitoring_omhsinf" { + name = "Safe_Encase_Monitoring_omhsinf" + priority = 105 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "34445" + source_address_prefix = "10.11.6.145/32" + destination_address_prefix = "*" + resource_group_name = data.azurerm_resource_group.group.name + network_security_group_name = azurerm_network_security_group.app_security_group.name +} + +resource "azurerm_network_security_rule" "App_ForeScout_Manager_omhsinf" { + name = "ForeScout_Manager_omhsinf" + priority = 106 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_ranges = ["556", "443", "10003-10006"] + source_address_prefixes = ["10.64.8.184", "10.64.8.180/32"] + destination_address_prefix = "*" + resource_group_name = data.azurerm_resource_group.group.name + network_security_group_name = azurerm_network_security_group.app_security_group.name +} + +resource "azurerm_network_security_rule" "App_BigFix_omhsinf" { + name = "BigFix_omhsinf" + priority = 107 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "52314" + source_address_prefix = "10.11.4.84/32" + destination_address_prefix = "*" + resource_group_name = data.azurerm_resource_group.group.name + network_security_group_name = azurerm_network_security_group.app_security_group.name +} + +resource "azurerm_network_security_rule" "App_Allow_All_Out_omhsinf" { + name = "Allow_All_Out_omhsinf" + priority = 109 + direction = "Outbound" + access = "Allow" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "*" + destination_address_prefix = "*" + resource_group_name = data.azurerm_resource_group.group.name + network_security_group_name = azurerm_network_security_group.app_security_group.name +} + +resource "azurerm_subnet_network_security_group_association" "app_security_group" { + subnet_id = azurerm_subnet.app.id + network_security_group_id = azurerm_network_security_group.app_security_group.id +} diff --git a/operations/template/variables.tf b/operations/template/variables.tf index 566f1a881..c259c0fe1 100644 --- a/operations/template/variables.tf +++ b/operations/template/variables.tf @@ -7,3 +7,9 @@ variable "deployer_id" { type = string nullable = false } + +variable "vpn_root_certificate" { + type = string + nullable = true + default = null +} diff --git a/operations/template/vpn.tf b/operations/template/vpn.tf new file mode 100644 index 000000000..01ec6fec6 --- /dev/null +++ b/operations/template/vpn.tf @@ -0,0 +1,68 @@ +resource "azurerm_public_ip" "vpn" { + name = "vpn-public-ip" + location = data.azurerm_resource_group.group.location + resource_group_name = data.azurerm_resource_group.group.name + + allocation_method = "Dynamic" +} + +resource "azurerm_virtual_network_gateway" "vpn" { + name = "${var.environment}-vpn" + location = data.azurerm_resource_group.group.location + resource_group_name = data.azurerm_resource_group.group.name + + type = "Vpn" + vpn_type = "RouteBased" + + active_active = false + enable_bgp = false + sku = "VpnGw1" + + ip_configuration { + public_ip_address_id = azurerm_public_ip.vpn.id + private_ip_address_allocation = "Dynamic" + subnet_id = azurerm_subnet.vpn.id + } + + dynamic "vpn_client_configuration" { + for_each = var.vpn_root_certificate != null ? [1] : [] + content { + address_space = ["192.168.0.0/16"] + vpn_auth_types = ["Certificate"] + vpn_client_protocols = ["OpenVPN"] + + root_certificate { + name = "vpn-cert" + public_cert_data = var.vpn_root_certificate + } + } + } + + depends_on = [azurerm_subnet.app, azurerm_subnet.database, azurerm_subnet.resolver_inbound, azurerm_subnet.resolver_outbound, azurerm_subnet_network_security_group_association.app_security_group, azurerm_subnet_network_security_group_association.database_security_group, azurerm_subnet_route_table_association.database_database] # the VPN "locks" the subnets, so the VPN should wait until the subnet edits are done +} + +resource "azurerm_private_dns_resolver" "private_zone_resolver" { + name = "private-resolve-${var.environment}" + resource_group_name = data.azurerm_resource_group.group.name + location = data.azurerm_resource_group.group.location + virtual_network_id = data.azurerm_virtual_network.app.id +} + + +resource "azurerm_private_dns_resolver_inbound_endpoint" "resolver_inbound_endpoint" { + name = "endpoint-inbound-${var.environment}" + private_dns_resolver_id = azurerm_private_dns_resolver.private_zone_resolver.id + location = azurerm_private_dns_resolver.private_zone_resolver.location + + ip_configurations { + private_ip_allocation_method = "Dynamic" + subnet_id = azurerm_subnet.resolver_inbound.id + } +} + +resource "azurerm_private_dns_resolver_outbound_endpoint" "resolver_outbound_endpoint" { + name = "endpoint-outbound-${var.environment}" + private_dns_resolver_id = azurerm_private_dns_resolver.private_zone_resolver.id + location = azurerm_private_dns_resolver.private_zone_resolver.location + subnet_id = azurerm_subnet.resolver_outbound.id +} diff --git a/operations/vpn/dev.ovpn b/operations/vpn/dev.ovpn new file mode 100644 index 000000000..ff7a4d225 --- /dev/null +++ b/operations/vpn/dev.ovpn @@ -0,0 +1,48 @@ +client +remote azuregateway-d84e0077-7de0-48f3-b087-b3759344ba44-7322fc20c350.vpn.azure.com 443 +verify-x509-name d84e0077-7de0-48f3-b087-b3759344ba44.vpn.azure.com name +remote-cert-tls server + +dev tun +proto tcp +resolv-retry infinite +nobind + +auth SHA256 +cipher AES-256-GCM +persist-key +persist-tun + +tls-timeout 30 +tls-version-min 1.2 +key-direction 1 + +dhcp-option DNS 172.17.67.228 +dhcp-option DOMAIN azure.net +dhcp-option DOMAIN azure.com +dhcp-option DOMAIN azurewebsites.net +dhcp-option DOMAIN windows.net + +verb 3 + +# P2S CA root certificate +ca ca.crt + +# Pre Shared Key +tls-auth tls.key + +# P2S client certificate +# Please fill this field with a PEM formatted client certificate +# Alternatively, configure 'cert PATH_TO_CLIENT_CERT' to use input from a PEM certificate file. +cert user.crt + +# P2S client certificate private key +# Please fill this field with a PEM formatted private key of the client certificate. +# Alternatively, configure 'key PATH_TO_CLIENT_KEY' to use input from a PEM key file. +key user.key + +# DNS +script-security 2 +up /etc/openvpn/update-systemd-resolved +down /etc/openvpn/update-systemd-resolved +down-pre