From e10658c40f06848f7166c8f87517bc2e73e5a144 Mon Sep 17 00:00:00 2001 From: mojganii Date: Thu, 24 Oct 2024 15:53:04 +0200 Subject: [PATCH] Update CI script --- .../actions/build-ios-e2e-tests/action.yml | 43 +++++++---- .github/actions/run-ios-e2e-tests/action.yml | 42 ++++++++--- ...os-end-to-end-tests-settings-migration.yml | 50 +++++++------ .github/workflows/ios-end-to-end-tests.yml | 73 ++++++++++++------- 4 files changed, 132 insertions(+), 76 deletions(-) diff --git a/.github/actions/build-ios-e2e-tests/action.yml b/.github/actions/build-ios-e2e-tests/action.yml index 4d65cfbe3682..5c983b4bebad 100644 --- a/.github/actions/build-ios-e2e-tests/action.yml +++ b/.github/actions/build-ios-e2e-tests/action.yml @@ -1,36 +1,51 @@ -name: 'Build iOS end to end tests action' -description: 'Prepares and builds end to end tests on iOS device' +--- +name: Build iOS end-to-end tests action +description: Prepares and builds end-to-end tests on an iOS device inputs: ios_device_pin_code: - description: 'iOS Device Pin Code' + description: iOS Device Pin Code required: true test_device_identifier_uuid: - description: 'Test Device Identifier UUID' + description: Test Device Identifier UUID required: true has_time_account_number: - description: 'Has Time Account Number' + description: Has Time Account Number required: true no_time_account_number: - description: 'No Time Account Number' + description: No Time Account Number required: true test_device_udid: - description: 'Test Device UDID' + description: Test Device UDID required: true partner_api_token: - description: 'Partner API Token' + description: Partner API Token required: true test_name: - description: 'Test case/suite name. Will run all tests in the test plan if not provided.' + description: Test case/suite name. Will run all tests in the test plan if not provided. required: false outputs_path: - description: 'Path to store outputs. This should be unique for each job run in order to avoid concurrency issues.' + description: Path to store outputs. This should be unique for each job run in order to avoid concurrency issues. required: true + base_directory: + description: Base directory for the working paths. Defaults to the current directory ("."). + required: false + default: . runs: - using: 'composite' + using: composite steps: - - name: Configure Xcode project + - name: Set working-directory or default + id: set-working-directory run: | + if [ -z "${{ inputs.base_directory }}" ]; then + echo "base_directory=." >> $GITHUB_ENV + else + echo "base_directory=${{ inputs.base_directory }}" >> $GITHUB_ENV + fi + shell: bash + + - name: Configure Xcode project + run: > for file in *.xcconfig.template ; do cp $file ${file//.template/} ; done sed -i "" "/^HAS_TIME_ACCOUNT_NUMBER/d" UITests.xcconfig sed -i "" "/^NO_TIME_ACCOUNT_NUMBER/d" UITests.xcconfig @@ -56,7 +71,7 @@ runs: "/UNINSTALL_APP_IN_TEST_SUITE_TEAR_DOWN =/ s#= .*#= 0#" \ UITests.xcconfig shell: bash - working-directory: ios/Configurations + working-directory: ${{ env.base_directory }}/ios/Configurations env: IOS_DEVICE_PIN_CODE: ${{ inputs.ios_device_pin_code }} TEST_DEVICE_IDENTIFIER_UUID: ${{ inputs.test_device_identifier_uuid }} @@ -79,7 +94,7 @@ runs: -derivedDataPath derived-data \ clean build-for-testing 2>&1 shell: bash - working-directory: ios/ + working-directory: ${{ env.base_directory }}/ios env: TEST_DEVICE_UDID: ${{ inputs.test_device_udid }} TEST_NAME: ${{ inputs.test_name }} diff --git a/.github/actions/run-ios-e2e-tests/action.yml b/.github/actions/run-ios-e2e-tests/action.yml index 9e9b077c2699..ec5feb437e37 100644 --- a/.github/actions/run-ios-e2e-tests/action.yml +++ b/.github/actions/run-ios-e2e-tests/action.yml @@ -1,36 +1,52 @@ -name: 'Run iOS end to end tests action' -description: 'Runs end to end tests on iOS device' +--- +name: Run iOS end to end tests action +description: Runs end-to-end tests on iOS device inputs: test_name: - description: 'Test case/suite name. Will run all tests in the test plan if not provided.' + description: Test case/suite name. Will run all tests in the test plan if not provided. required: false test_device_udid: - description: 'Test Device UDID' + description: Test Device UDID required: true outputs_path: description: > - Path to where outputs are stored - both build outputs and outputs from running tests. - This should be unique for each job run in order to avoid concurrency issues. + Path to where outputs are stored - both build outputs and outputs + from running tests. This should be unique for each job run in order to + avoid concurrency issues. required: true + base_directory: + description: Base directory for the working paths. + required: false + default: . runs: - using: 'composite' + using: composite steps: - # Set up a unique output directory + - name: Set base directory or default + id: set-base-directory + run: | + if [ -z "${{ inputs.base_directory }}" ]; then + echo "base_directory=." >> $GITHUB_ENV + else + echo "base_directory=${{ inputs.base_directory }}" >> $GITHUB_ENV + fi + shell: bash + - name: Set up outputs directory run: | # Forcing the filesystem buffers to be flushed to ensure the # directory tree is updated sync + if [ -n "$TEST_NAME" ]; then # Strip slashes to avoid creating subdirectories test_name_sanitized=$(printf "$TEST_NAME" | sed 's/\//_/g') echo "Setting output directory tests-output-test-name-sanitized" echo "$test_name_sanitized" - test_output_directory="${{ env.OUTPUTS_PATH }}/tests-output-$test_name_sanitized" + test_output_directory="${{ inputs.outputs_path }}/tests-output-$test_name_sanitized" else echo "Setting output directory output" - test_output_directory="${{ env.OUTPUTS_PATH }}/tests-output" + test_output_directory="${{ inputs.outputs_path }}/tests-output" fi echo "TEST_OUTPUT_DIRECTORY=$test_output_directory" >> $GITHUB_ENV @@ -46,16 +62,18 @@ runs: env: TEST_DEVICE_UDID: ${{ inputs.test_device_udid }} - - name: Run end-to-end-tests + - name: Run end-to-end tests run: | # Forcing the filesystem buffers to be flushed to ensure the # directory tree is updated sync + if [ -n "$TEST_NAME" ]; then TEST_NAME_ARGUMENT=" -only-testing $TEST_NAME" else TEST_NAME_ARGUMENT="" fi + set -o pipefail && env NSUnbufferedIO=YES xcodebuild \ -project MullvadVPN.xcodeproj \ -scheme MullvadVPNUITests \ @@ -66,7 +84,7 @@ runs: test-without-building 2>&1 | xcbeautify --report junit \ --report-path ${{ env.TEST_OUTPUT_DIRECTORY }}/junit-test-report shell: bash - working-directory: ${{ inputs.outputs_path }}/mullvadvpn-app/ios + working-directory: ${{ env.base_directory }}/mullvadvpn-app/ios env: TEST_NAME: ${{ inputs.test_name }} TEST_DEVICE_UDID: ${{ inputs.test_device_udid }} diff --git a/.github/workflows/ios-end-to-end-tests-settings-migration.yml b/.github/workflows/ios-end-to-end-tests-settings-migration.yml index f291136ecf46..684bd2fd5501 100644 --- a/.github/workflows/ios-end-to-end-tests-settings-migration.yml +++ b/.github/workflows/ios-end-to-end-tests-settings-migration.yml @@ -3,41 +3,39 @@ name: iOS settings migration tests concurrency: group: ios-end-to-end-tests cancel-in-progress: false + permissions: contents: read issues: write pull-requests: write + on: - workflow_dispatch: + workflow_dispatch: {} schedule: - # At midnight every day. - # Notifications for scheduled workflows are sent to the user who last modified the cron - # syntax in the workflow file. If you update this you must have notifications for - # Github Actions enabled, so these don't go unnoticed. - # https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/notifications-for-workflow-runs - - cron: '0 0 * * *' + - cron: '0 0 * * *' # At midnight every day. + env: TEST_DEVICE_UDID: 00008130-0019181022F3803A - OLD_APP_COMMIT_HASH: 895b7d98825e678f5d7023d5ea3c9b7beee89280 + OLD_APP_COMMIT_HASH: f82b90126441e9f8afa8c820faa0e5b2c99ecc3e + jobs: - checkout_old_version: - name: Checkout Old Repository Version + checkout_version: + name: Checkout Current Repository Version runs-on: - self-hosted - macOS - ios-test steps: - - name: Checkout old repository version + - name: Checkout current repository uses: actions/checkout@v4 - with: - ref: ${{ env.OLD_APP_COMMIT_HASH }} + configure_rust: name: Configure Rust runs-on: - self-hosted - macOS - ios-test - needs: checkout_old_version + needs: checkout_version steps: - name: Configure Rust uses: actions-rs/toolchain@v1.0.6 @@ -45,46 +43,52 @@ jobs: toolchain: stable override: true target: aarch64-apple-ios + uninstall_app: - name: Uninstall Old App + name: Uninstall App runs-on: - self-hosted - macOS - ios-test - needs: checkout_old_version + needs: checkout_version steps: - name: Uninstall old app timeout-minutes: 5 - run: ios-deploy --id ${{ env.TEST_DEVICE_UDID }} --uninstall_only --bundle_id - net.mullvad.MullvadVPN + run: ios-deploy --id ${{ env.TEST_DEVICE_UDID }} --uninstall_only --bundle_id net.mullvad.MullvadVPN + change_dns_settings: name: Change DNS Settings on Old App Version needs: - - checkout_old_version + - checkout_version - configure_rust - uninstall_app - uses: mullvad/mullvadvpn-app/.github/workflows/ios-end-to-end-tests.yml@main + uses: ./.github/workflows/ios-end-to-end-tests.yml with: arg_tests_json_key: "MullvadVPNUITestsChangeDNSSettings" + commit_hash: ${{ env.OLD_APP_COMMIT_HASH }} secrets: inherit + verify_dns_settings: name: Verify DNS Settings Still Changed on Current App Version needs: change_dns_settings - uses: mullvad/mullvadvpn-app/.github/workflows/ios-end-to-end-tests.yml@main + uses: ./.github/workflows/ios-end-to-end-tests.yml with: arg_tests_json_key: "MullvadVPNUITestsVerifyDNSSettingsChanged" secrets: inherit + change_other_settings: name: Change All Other Settings on Old App Version needs: verify_dns_settings - uses: mullvad/mullvadvpn-app/.github/workflows/ios-end-to-end-tests.yml@main + uses: ./.github/workflows/ios-end-to-end-tests.yml with: arg_tests_json_key: "MullvadVPNUITestsChangeSettings" + commit_hash: ${{ env.OLD_APP_COMMIT_HASH }} secrets: inherit + verify_other_settings: name: Verify All Other Settings Still Changed on Current App Version needs: change_other_settings - uses: mullvad/mullvadvpn-app/.github/workflows/ios-end-to-end-tests.yml@main + uses: ./.github/workflows/ios-end-to-end-tests.yml with: arg_tests_json_key: "MullvadVPNUITestsVerifySettingsChanged" secrets: inherit diff --git a/.github/workflows/ios-end-to-end-tests.yml b/.github/workflows/ios-end-to-end-tests.yml index ea80d5d0b3fc..df91a16530e1 100644 --- a/.github/workflows/ios-end-to-end-tests.yml +++ b/.github/workflows/ios-end-to-end-tests.yml @@ -1,24 +1,29 @@ ---- name: iOS end-to-end tests + env: TEST_DEVICE_UDID: 00008130-0019181022F3803A + REPO_DIR: iOSWorkingDirectory + permissions: contents: read issues: write pull-requests: write + on: workflow_call: inputs: arg_tests_json_key: type: string required: false + commit_hash: + type: string + required: false workflow_dispatch: inputs: - # Optionally specify a test case or suite to run. - # Must be in the format MullvadVPNUITest// where test case name is optional. user_supplied_test_name: description: 'Only run test case/suite' required: false + jobs: set-up-outputs-directory: name: Set up outputs directory @@ -32,10 +37,8 @@ jobs: job_outputs_directory="$HOME/workflow-outputs/job-outputs-${{ github.run_id }}" echo "job_outputs_directory=$job_outputs_directory" >> "$GITHUB_OUTPUT" mkdir -p "$job_outputs_directory" - shell: bash - # Define the set of tests to run based on the event type and input define-test-suites-matrix: name: Define test suites matrix runs-on: [self-hosted, macOS, ios-test] @@ -45,40 +48,54 @@ jobs: id: test-runs-to-json run: | if [ -n "${{ inputs.arg_tests_json_key }}" ]; then - # JSON key supplied by another workflow calling this reusable workflow - echo "Using calling workflow supplied test suites JSON key: ${{ inputs.arg_tests_json_key }}" - # Forcing the filesystem buffers to be flushed to ensure the - # directory tree is updated - sync + echo "Using supplied test suites JSON key: ${{ inputs.arg_tests_json_key }}" test_suites_json=$(jq -r --compact-output '.tests."${{ inputs.arg_tests_json_key }}"' tests.json) - echo "test_suites_json=$test_suites_json" >> $GITHUB_ENV elif [ -n "${{ inputs.user_supplied_test_name }}" ]; then - # User specified test case/suite when manually triggering run echo "Using user supplied test name: ${{ inputs.user_supplied_test_name }}" - test_suites_json="['${{ inputs.user_supplied_test_name }}']" >> $GITHUB_ENV - echo "test_suites_json=$test_suites_json" >> $GITHUB_ENV + test_suites_json="['${{ inputs.user_supplied_test_name }}']" else - echo "Tests not specified, will fallback to running nightly(all) tests scope" + echo "Tests not specified, will fallback to running nightly (all) tests scope" test_suites_json=$(jq -r --compact-output '.tests.nightly' tests.json) - echo "test_suites_json=$test_suites_json" >> $GITHUB_ENV fi - + echo "test_suites_json=$test_suites_json" >> $GITHUB_ENV echo "Test suites/cases to run: $test_suites_json" - working-directory: ios/MullvadVPNUITests + working-directory: ${{ env.REPO_DIR }}/ios/MullvadVPNUITests outputs: test_suites_json: ${{ env.test_suites_json }} - # Build app and tests target build: - name: Build for end to end testing + name: Build for end-to-end testing runs-on: [self-hosted, macOS, ios-test] needs: set-up-outputs-directory timeout-minutes: 20 steps: + - name: Set commit hash or default to github.sha + id: set-commit-hash + run: | + if [ -z "${{ inputs.commit_hash }}" ]; then + echo "COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV + else + echo "COMMIT_HASH=${{ inputs.commit_hash }}" >> $GITHUB_ENV + fi + shell: bash + + - name: Debug commit hash + run: | + echo "Commit hash is: ${{ env.COMMIT_HASH }}" + shell: bash + - name: Checkout repository uses: actions/checkout@v4 with: clean: true + fetch-depth: 1 + ref: ${{ env.COMMIT_HASH }} + path: ${{ env.REPO_DIR }} + + - name: Show checkout directory + run: | + echo "Repository checked out to: ${{ github.workspace }}" + ls -R ${{ github.workspace }} # List files to verify contents - name: Configure Rust uses: actions-rs/toolchain@v1.0.6 @@ -87,7 +104,7 @@ jobs: override: true target: aarch64-apple-ios - - name: Build iOS end to end tests action + - name: Build iOS end-to-end tests action uses: ./.github/actions/build-ios-e2e-tests with: test_name: ${{ github.event.inputs.user_supplied_test_name }} @@ -98,6 +115,7 @@ jobs: test_device_udid: ${{ env.TEST_DEVICE_UDID }} partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }} outputs_path: ${{ needs.set-up-outputs-directory.outputs.job_outputs_directory }} + base_directory: ${{ env.REPO_DIR }} - name: Debug print job output directory run: | @@ -108,7 +126,7 @@ jobs: run: | mkdir -p "$JOB_OUTPUTS_DIRECTORY/derived-data" cp -R ios/derived-data "$JOB_OUTPUTS_DIRECTORY" - cp -R . $JOB_OUTPUTS_DIRECTORY/mullvadvpn-app + cp -R . "$JOB_OUTPUTS_DIRECTORY/mullvadvpn-app" shell: bash env: JOB_OUTPUTS_DIRECTORY: ${{ needs.set-up-outputs-directory.outputs.job_outputs_directory }} @@ -127,14 +145,15 @@ jobs: strategy: fail-fast: false matrix: - test_suite: ${{fromJson(needs.define-test-suites-matrix.outputs.test_suites_json)}} + test_suite: ${{ fromJson(needs.define-test-suites-matrix.outputs.test_suites_json) }} steps: - - name: Run iOS end to end tests action + - name: Run iOS end-to-end tests action uses: ./.github/actions/run-ios-e2e-tests with: test_name: "MullvadVPNUITests/${{ matrix.test_suite }}" test_device_udid: ${{ env.TEST_DEVICE_UDID }} outputs_path: ${{ needs.set-up-outputs-directory.outputs.job_outputs_directory }} + base_directory: ${{ env.REPO_DIR }} clean-up-outputs-directory: if: always() @@ -148,7 +167,7 @@ jobs: notify-on-failure: if: failure() && github.event_name == 'pull_request' - name: Notify team on failure(if PR related) + name: Notify team on failure (if PR related) runs-on: [self-hosted, macOS, ios-test] needs: test timeout-minutes: 5 @@ -156,7 +175,7 @@ jobs: - name: Comment PR on test failure uses: actions/github-script@v7 with: - github-token: ${{secrets.GITHUB_TOKEN}} + github-token: ${{ secrets.GITHUB_TOKEN }} script: | const issue_number = context.issue.number; const run_id = context.runId; @@ -165,5 +184,5 @@ jobs: owner: context.repo.owner, repo: context.repo.repo, issue_number: issue_number, - body: `🚨 End to end tests failed. Please check the [failed workflow run](${run_url}).` + body: `🚨 End-to-end tests failed. Please check the [failed workflow run](${run_url}).` });