From 05293f2a1b219d605dda9126b24bbd35419d98d6 Mon Sep 17 00:00:00 2001 From: eemrdog Date: Mon, 2 Dec 2024 12:06:53 +0100 Subject: [PATCH 1/4] dev-core --- .github/workflows/validation.yaml | 281 ++++++++++++++++++++++++++++++ 1 file changed, 281 insertions(+) create mode 100644 .github/workflows/validation.yaml diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml new file mode 100644 index 000000000..3039ecabc --- /dev/null +++ b/.github/workflows/validation.yaml @@ -0,0 +1,281 @@ +name: PR Validation + +on: + + pull_request: + branches: + - dev + - dev-core + - test + + workflow_dispatch: # Allows manual triggering of the workflow + inputs: + use_cases: + description: 'Enter the use cases to run (comma-separated)' + required: true + default: '["demo_ar_workflows_gitlab","demo_ar_workflows_ansible"]' + type: string + # custom_use_cases: + # description: 'Enter the custom use cases with GitHub URLs to run (comma-separated)' + # required: false + # default: '[""]' + # type: string + providers: + description: 'Select the providers to run (comma-separated)' + required: true + default: '["gcloud"]' + type: string + + destroy_resources: + description: 'Set to "true" to enable resource destruction' + required: true + default: 'false' + type: string + + + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + + preparation: + runs-on: self-hosted + env: + DEFAULT_USE_CASES: '[\"demo_all\"]' # '[\"demo_ar_workflows_gitlab\",\"demo_ar_workflows_ansible\"]' + DEFAULT_PROVIDERS: '[\"gcloud\"]' + outputs: + use_cases: ${{ steps.set_outputs.outputs.use_cases }} + providers: ${{ steps.set_outputs.outputs.providers }} + destroy_resources: ${{ steps.set_outputs.outputs.destroy_resources }} + steps: + - name: Parse Inputs + id: parse_inputs + run: | + echo ${{ github.event.inputs.use_cases }} + echo ${{ env.DEFAULT_USE_CASES }} + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "use_cases=$(echo '${{ github.event.inputs.use_cases }}' )" >> "$GITHUB_ENV" + echo "providers=$(echo '${{ github.event.inputs.providers }}' )" >> "$GITHUB_ENV" + echo "destroy_resources=$(echo '${{ github.event.inputs.destroy_resources }}' )" >> "$GITHUB_ENV" + else + echo "use_cases=${{ env.DEFAULT_USE_CASES }}" >> "$GITHUB_ENV" + echo "providers=${{ env.DEFAULT_PROVIDERS }}" >> "$GITHUB_ENV" + if [[ "${{ github.base_ref }}" == "dev" ]]; then + echo "destroy_resources=true" >> "$GITHUB_ENV" + elif [[ "${{ github.base_ref }}" == "test" ]]; then + echo "destroy_resources=false" >> "$GITHUB_ENV" + fi + fi + shell: bash + - name: Set Outputs + id: set_outputs + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "use_cases=$(echo '${{ github.event.inputs.use_cases }}' )" >> "$GITHUB_OUTPUT" + echo "providers=$(echo '${{ github.event.inputs.providers }}' )" >> "$GITHUB_OUTPUT" + echo "destroy_resources=$(echo '${{ github.event.inputs.destroy_resources }}' )" >> "$GITHUB_OUTPUT" + else + echo "use_cases=${{ env.DEFAULT_USE_CASES }}" >> "$GITHUB_OUTPUT" + echo "providers=${{ env.DEFAULT_PROVIDERS }}" >> "$GITHUB_OUTPUT" + if [[ "${{ github.base_ref }}" == "dev" ]]; then + echo "destroy_resources=true" >> "$GITHUB_OUTPUT" + elif [[ "${{ github.base_ref }}" == "test" ]]; then + echo "destroy_resources=false" >> "$GITHUB_OUTPUT" + fi + fi + shell: bash + + test: + needs: preparation + runs-on: self-hosted + + env: + DESTROY_RESOURCES: ${{ needs.preparation.outputs.destroy_resources }} # Set to "true" to enable resource destruction + TF_VAR_dt_tenant: ${{ vars.DT_TENANT }} + TF_VAR_dt_api_token: ${{ secrets.DT_API_TOKEN }} + TF_VAR_otel_export_enable: "true" + TF_VAR_name_prefix: "emre-ace-box" + TF_VAR_skip_domain_workspace_alignment: "true" + TF_VAR_extra_vars: | + { + "dt_environment_url_gen3": "${{ vars.DT_URL_GEN3 }}", + "dt_oauth_sso_endpoint": "${{ vars.DT_OAUTH_SSO_ENDPOINT }}", + "dt_oauth_client_id": "${{ secrets.DT_OAUTH_CLIENT_ID }}", + "dt_oauth_client_secret": "${{ secrets.DT_OAUTH_CLIENT_SECRET }}", + "dt_oauth_account_urn": "${{ vars.DT_OAUTH_ACCOUNT_URN }}", + } + + strategy: + matrix: + provider: ${{ fromJson(needs.preparation.outputs.providers) }} + use_case: ${{ fromJson(needs.preparation.outputs.use_cases) }} + include: + # - provider: aws + # working_directory: ./terraform/aws + # aws_region: "us-east-2" + # aws_instance_type: "t3.xlarge" + # disk_size: 60 + # acebox_user: "ubuntu" + # associate_eip: "false" + # ingress_protocol: "https" + # route53_zone_name: "gtm-innovation.com" + # route53_private_zone: "false" + # skip_domain_workspace_alignment: "true" + # backend_config: | + # bucket = "your-terraform-state-bucket" + # key = "path/to/terraform.tfstate" + # region = "us-east-2" + # encrypt = true + # dynamodb_table = "your-lock-table" + + # gcloud test with https and SSL certificates + - provider: gcloud + working_directory: ./terraform/gcloud + gcloud_project: "acl-gsi" #"myGCPProject" # GCP Project you want to use + gcloud_zone: "europe-west1-b" # zone + acebox_size: "n2-standard-16" + managed_zone_name: "ace-innovation-info" + backend_config_bucket: "ace-box-tf-state" + backend_config_prefix: "ace-box/ssl-enabled" + # gcloud test with http + # - provider: gcloud + # working_directory: ./terraform/gcloud + # gcloud_project: "acl-gsi" #"myGCPProject" # GCP Project you want to use + # gcloud_zone: "europe-west1-b" # zone + # acebox_size: "n2-standard-16" + # backend_config_bucket: "ace-box-tf-state" + # backend_config_prefix: "ace-box/no-ssl" + + steps: + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install Dependencies + run: | + sudo apt-get update + sudo apt-get install -y unzip nodejs + + - name: Set Custom Domain + id: set_custom_domain + run: echo "custom_domain=${{ matrix.use_case }}.ace-innovation.info" >> $GITHUB_OUTPUT + + # gcloud specific steps + - name: 'auth' + if: matrix.provider == 'gcloud' + uses: 'google-github-actions/auth@v2' + with: + credentials_json: '${{ secrets.GCP_CREDENTIALS_JSON }}' + + - name: 'Set up Cloud SDK' + if: matrix.provider == 'gcloud' + uses: 'google-github-actions/setup-gcloud@v2' + + - name: 'Use gcloud CLI' + if: matrix.provider == 'gcloud' + run: 'gcloud info' + + - name: Create Backend Config for gcloud + if: matrix.provider == 'gcloud' + run: | + echo 'terraform { + backend "gcs" { + bucket = "${{ matrix.backend_config_bucket }}" + prefix = "${{ matrix.backend_config_prefix }}/${{ matrix.use_case }}" + } + }' > ${{ matrix.working_directory }}/backend.tf + echo "backend.tf created" + cat ${{ matrix.working_directory }}/backend.tf + + + # aws specific steps + - name: Configure AWS Credentials + if: matrix.provider == 'aws' + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ matrix.aws_region }} + + - name: Create Backend Config for aws + if: matrix.provider == 'aws' + run: | + echo 'terraform { + backend "s3" { + bucket = "${{ matrix.backend_config_bucket }}" + key = "${{ matrix.backend_config_key }}" + region = "${{ matrix.backend_config_region }}" + encrypt = true + dynamodb_table = "${{ matrix.backend_config_dynamodb_table }}" + } + }' > ${{ matrix.working_directory }}/backend.tf + echo "backend.tf created" + cat ${{ matrix.working_directory }}/backend.tf + + - name: Set up Terraform + uses: hashicorp/setup-terraform@v1 + with: + terraform_version: 1.9 + + - name: Terraform Init + run: terraform init + working-directory: ${{ matrix.working_directory }} + + - name: Terraform Plan + run: terraform plan -out=tfplan -lock=false -replace="null_resource.provisioner_ace_enable" -var="gcloud_project=${{ matrix.gcloud_project }}" -var="gcloud_zone=${{ matrix.gcloud_zone }}" -var="acebox_size=${{ matrix.acebox_size }}" -var="managed_zone_name=${{ matrix.managed_zone_name }}" -var="use_case=${{ matrix.use_case }}" + working-directory: ${{ matrix.working_directory }} + + - name: Terraform Apply + run: | + retries=2 + count=0 + until terraform apply -auto-approve -lock=false tfplan; do + exit_code=$? + count=$((count + 1)) + if [ $count -ge $retries ]; then + echo "Terraform apply failed after $count attempts." + exit $exit_code + fi + echo "Retrying terraform apply... ($count/$retries)" + terraform plan -out=tfplan -lock=false -replace="null_resource.provisioner_ace_enable" -var="gcloud_project=${{ matrix.gcloud_project }}" -var="gcloud_zone=${{ matrix.gcloud_zone }}" -var="acebox_size=${{ matrix.acebox_size }}" -var="managed_zone_name=${{ matrix.managed_zone_name }}" -var="use_case=${{ matrix.use_case }}" + sleep 10 + done + working-directory: ${{ matrix.working_directory }} + + - name: Run Tests + run: | + # Add your test commands here + echo "Running tests..." + # Example: curl -f http://your-service-url/health || exit 1 + + - name: Conditional Terraform Destroy + if: always() && env.DESTROY_RESOURCES == true + run: terraform destroy -auto-approve -lock=false -var="gcloud_project=${{ matrix.gcloud_project }}" -var="gcloud_zone=${{ matrix.gcloud_zone }}" -var="acebox_size=${{ matrix.acebox_size }}" -var="managed_zone_name=${{ matrix.managed_zone_name }}" -var="use_case=${{ matrix.use_case }}" + working-directory: ${{ matrix.working_directory }} + + update-pr-status: + needs: test + runs-on: self-hosted + if: github.event_name == 'pull_request' + steps: + - name: Update PR Status + uses: actions/github-script@v4 + with: + script: | + const prNumber = context.payload.pull_request.number; + const repo = context.repo.repo; + const owner = context.repo.owner; + await github.issues.createComment({ + owner, + repo, + issue_number: prNumber, + body: 'All checks have passed. This PR is ready for review.' + }); + await github.pulls.update({ + owner, + repo, + pull_number: prNumber, + state: 'open' + }); \ No newline at end of file From 0f067e1a326ef9c95061710a8922d2a7d1caa75c Mon Sep 17 00:00:00 2001 From: eemrdog Date: Mon, 2 Dec 2024 12:08:53 +0100 Subject: [PATCH 2/4] docker runner --- .github/workflows/validation.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 3039ecabc..8a4d34ba6 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -41,7 +41,7 @@ concurrency: jobs: preparation: - runs-on: self-hosted + runs-on: docker-runner env: DEFAULT_USE_CASES: '[\"demo_all\"]' # '[\"demo_ar_workflows_gitlab\",\"demo_ar_workflows_ansible\"]' DEFAULT_PROVIDERS: '[\"gcloud\"]' @@ -89,7 +89,7 @@ jobs: test: needs: preparation - runs-on: self-hosted + runs-on: docker-runner env: DESTROY_RESOURCES: ${{ needs.preparation.outputs.destroy_resources }} # Set to "true" to enable resource destruction From 146a09142d0c6c47b2b216cb3f2b664bd0e31938 Mon Sep 17 00:00:00 2001 From: eemrdog Date: Mon, 2 Dec 2024 12:13:20 +0100 Subject: [PATCH 3/4] self hosted --- .github/workflows/validation.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 8a4d34ba6..3039ecabc 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -41,7 +41,7 @@ concurrency: jobs: preparation: - runs-on: docker-runner + runs-on: self-hosted env: DEFAULT_USE_CASES: '[\"demo_all\"]' # '[\"demo_ar_workflows_gitlab\",\"demo_ar_workflows_ansible\"]' DEFAULT_PROVIDERS: '[\"gcloud\"]' @@ -89,7 +89,7 @@ jobs: test: needs: preparation - runs-on: docker-runner + runs-on: self-hosted env: DESTROY_RESOURCES: ${{ needs.preparation.outputs.destroy_resources }} # Set to "true" to enable resource destruction From 7595bb8d7e9cfb9cf8ccf72cc3c251f38a93e213 Mon Sep 17 00:00:00 2001 From: eemrdog Date: Mon, 2 Dec 2024 12:52:57 +0100 Subject: [PATCH 4/4] branches --- .github/workflows/validation.yaml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 3039ecabc..6ca73e735 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -5,8 +5,8 @@ on: pull_request: branches: - dev - - dev-core - test + - demo-envs workflow_dispatch: # Allows manual triggering of the workflow inputs: @@ -15,11 +15,6 @@ on: required: true default: '["demo_ar_workflows_gitlab","demo_ar_workflows_ansible"]' type: string - # custom_use_cases: - # description: 'Enter the custom use cases with GitHub URLs to run (comma-separated)' - # required: false - # default: '[""]' - # type: string providers: description: 'Select the providers to run (comma-separated)' required: true @@ -29,7 +24,7 @@ on: destroy_resources: description: 'Set to "true" to enable resource destruction' required: true - default: 'false' + default: 'true' type: string @@ -64,7 +59,7 @@ jobs: echo "providers=${{ env.DEFAULT_PROVIDERS }}" >> "$GITHUB_ENV" if [[ "${{ github.base_ref }}" == "dev" ]]; then echo "destroy_resources=true" >> "$GITHUB_ENV" - elif [[ "${{ github.base_ref }}" == "test" ]]; then + elif [[ "${{ github.base_ref }}" == "test" || "${{ github.base_ref }}" == "demo-envs" ]]; then echo "destroy_resources=false" >> "$GITHUB_ENV" fi fi @@ -81,7 +76,7 @@ jobs: echo "providers=${{ env.DEFAULT_PROVIDERS }}" >> "$GITHUB_OUTPUT" if [[ "${{ github.base_ref }}" == "dev" ]]; then echo "destroy_resources=true" >> "$GITHUB_OUTPUT" - elif [[ "${{ github.base_ref }}" == "test" ]]; then + elif [[ "${{ github.base_ref }}" == "test" || "${{ github.base_ref }}" == "demo-envs" ]]; then echo "destroy_resources=false" >> "$GITHUB_OUTPUT" fi fi