From a87718418373c3a01a54dd4e2e0053295ca39df6 Mon Sep 17 00:00:00 2001 From: Petros Paraskevopoulos Date: Tue, 14 Jan 2025 13:20:18 +0200 Subject: [PATCH 01/10] CI: Update a8c-ci-toolkit to pr-sha commit (dependency cache) FYI: This change is done for testing purposes and until the below 'A8C CI Toolkit' #135 PR gets merged to 'trunk'. When that's done, the 'a8c-ci-toolkit' will be updated to '3.9.0' instead. A8C CI Toolkit PR: [Dependency Updates] [Dependency Cache] Dependency Cache on CI per Project [without GRADLE_RO_DEP_CACHE] #135 - https://github.com/Automattic/a8c-ci-toolkit-buildkite-plugin/pull/135 --- .buildkite/shared-pipeline-vars | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.buildkite/shared-pipeline-vars b/.buildkite/shared-pipeline-vars index 307dd62e628..d6c45f0bc27 100644 --- a/.buildkite/shared-pipeline-vars +++ b/.buildkite/shared-pipeline-vars @@ -3,6 +3,5 @@ # This file is `source`'d before calling `buildkite-agent pipeline upload`, and can be used # to set up some variables that will be interpolated in the `.yml` pipeline before uploading it. -export CI_TOOLKIT="automattic/a8c-ci-toolkit#3.7.1" +export CI_TOOLKIT="automattic/a8c-ci-toolkit#1dda7bd51d20fbb82d5435dc2cbee3b08f5b5153" export TEST_COLLECTOR="test-collector#v1.10.1" - From 684c6e6d8d1a8b3942705a10e4a43c3b2f7a484c Mon Sep 17 00:00:00 2001 From: Petros Paraskevopoulos Date: Tue, 14 Jan 2025 13:37:35 +0200 Subject: [PATCH 02/10] CI: Add a scheduled dependency cache job (save cache) FYI: This job will be then used by 'buildkite-ci' and configured as a 'buildkite_pipeline_schedule' with a weekly frequency. PS: The targeted 'pipeline' related jobs are: - Mobile App - Wear App - Lint - Unit Tests - Android tests --- .buildkite/commands/save-cache.sh | 32 +++++++++++++++++++++++ .buildkite/schedules/dependency-cache.yml | 24 +++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100755 .buildkite/commands/save-cache.sh create mode 100644 .buildkite/schedules/dependency-cache.yml diff --git a/.buildkite/commands/save-cache.sh b/.buildkite/commands/save-cache.sh new file mode 100755 index 00000000000..ccc5aa942cf --- /dev/null +++ b/.buildkite/commands/save-cache.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -euo pipefail + +echo "--- :rubygems: Setting up Gems" +install_gems + +echo "--- :closed_lock_with_key: Installing Secrets" +bundle exec fastlane run configure_apply + +echo "--- ๐Ÿ›  Download Mobile App Dependencies [Assemble Mobile App]" +./gradlew :WooCommerce:assembleJalapenoDebug +echo "" + +echo "--- ๐Ÿ›  Download Wear App Dependencies [Assemble Wear App]" +./gradlew :WooCommerce-Wear:assembleJalapenoDebug +echo "" + +echo "--- ๐Ÿงน Download Lint Dependencies [Lint Mobile App]" +./gradlew :WooCommerce:lintJalapenoDebug +echo "" + +echo "--- ๐Ÿงช Download Unit Test Dependencies [Assemble Unit Tests]" +./gradlew assembleJalapenoDebugUnitTest lib:cardreader:assembleDebugUnitTest +echo "" + +echo "--- ๐Ÿงช Download Android Test Dependencies [Assemble Android Tests]" +./gradlew assembleJalapenoDebugAndroidTest +echo "" + +echo "--- ๐Ÿ’พ Save Cache" +save_gradle_dependency_cache diff --git a/.buildkite/schedules/dependency-cache.yml b/.buildkite/schedules/dependency-cache.yml new file mode 100644 index 00000000000..ad18b3e4ef1 --- /dev/null +++ b/.buildkite/schedules/dependency-cache.yml @@ -0,0 +1,24 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +--- + +agents: + queue: "android" + +steps: + - label: "dependency cache" + command: | + echo "--- ๐Ÿ’พ Download and Cache Dependencies" + .buildkite/commands/save-cache.sh + plugins: [$CI_TOOLKIT] + +notify: + - slack: + channels: + - "#android-core-notifs" + message: "The dependency cache has been updated." + if: build.state == "passed" + - slack: + channels: + - "#android-core-notifs" + message: "Failure to update the dependency cache." + if: build.state == "failed" From 8e3e0aa11d416b961a43679fb6fe9f4da61a9ac4 Mon Sep 17 00:00:00 2001 From: Petros Paraskevopoulos Date: Tue, 14 Jan 2025 13:40:56 +0200 Subject: [PATCH 03/10] CI: Remove unnecessary echo linting from detekt pipeline job --- .buildkite/pipeline.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 360aad54c4b..059db869b7e 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -35,9 +35,7 @@ steps: queue: "linter" - label: "detekt" - command: | - echo "--- ๐Ÿงน Linting" - ./gradlew detektAll + command: ./gradlew detektAll plugins: [$CI_TOOLKIT] artifact_paths: - "**/build/reports/detekt/detekt.html" From fd52a287acb97117e034a39c13fdf0bb6d0fa366 Mon Sep 17 00:00:00 2001 From: Petros Paraskevopoulos Date: Tue, 14 Jan 2025 13:49:42 +0200 Subject: [PATCH 04/10] CI: Add restore cache and run on targeted pipeline related jobs only PS: The targeted 'pipeline' related jobs are: - Mobile App - Wear App - Lint - Unit Tests - Android tests PS: The 'detekt' job isn't targeted because it only takes about 20+ seconds of network request to download all the dependencies needed for this specific job, which is about the same time it take for the dependency cache to actually be restored (10+ seconds). Thus, this optimization is not really worth it for this specific job. For all those other jobs targeted, it take more than a minute of network requests to download all the dependencies needed, and as such worth the diff is worth it. --- .buildkite/commands/restore-cache.sh | 6 ++++++ .buildkite/pipeline.yml | 14 +++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) create mode 100755 .buildkite/commands/restore-cache.sh diff --git a/.buildkite/commands/restore-cache.sh b/.buildkite/commands/restore-cache.sh new file mode 100755 index 00000000000..480ae698790 --- /dev/null +++ b/.buildkite/commands/restore-cache.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -e + +echo "--- ๐Ÿ’พ Restore Cache" +restore_gradle_dependency_cache || true diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 059db869b7e..780d670a7eb 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -41,7 +41,9 @@ steps: - "**/build/reports/detekt/detekt.html" - label: "lint" - command: .buildkite/commands/lint.sh + command: | + .buildkite/commands/restore-cache.sh + .buildkite/commands/lint.sh plugins: [$CI_TOOLKIT] artifact_paths: - "**/build/reports/lint-results*.*" @@ -59,12 +61,14 @@ steps: - label: "๐Ÿ›  Prototype Build: Mobile App" command: | + .buildkite/commands/restore-cache.sh ".buildkite/commands/prototype-build.sh" "WooCommerce" if: build.pull_request.id != null plugins: [$CI_TOOLKIT] - label: "๐Ÿ›  Prototype Build: Wear App" command: | + .buildkite/commands/restore-cache.sh ".buildkite/commands/prototype-build.sh" "WooCommerce-Wear" if: build.pull_request.id != null plugins: [$CI_TOOLKIT] @@ -74,7 +78,9 @@ steps: steps: - label: "Unit tests" - command: .buildkite/commands/run-unit-tests.sh + command: | + .buildkite/commands/restore-cache.sh + .buildkite/commands/run-unit-tests.sh plugins: - $CI_TOOLKIT - $TEST_COLLECTOR : @@ -84,7 +90,9 @@ steps: - "**/build/test-results/merged-test-results.xml" - label: "Instrumented tests" - command: .buildkite/commands/run-instrumented-tests.sh + command: | + .buildkite/commands/restore-cache.sh + .buildkite/commands/run-instrumented-tests.sh plugins: - $CI_TOOLKIT - $TEST_COLLECTOR : From 3547e664e43bcb37d7e5c67190aa68bf7f492842 Mon Sep 17 00:00:00 2001 From: Petros Paraskevopoulos Date: Wed, 15 Jan 2025 10:33:35 +0200 Subject: [PATCH 05/10] CI: Update a8c-ci-toolkit to 3.9.0 Release Notes: https://github.com/Automattic/ a8c-ci-toolkit-buildkite-plugin/releases/tag/3.9.0 Related: 4475a6ad4bb6aa6be778c4b37374d87f92f717d2 --- .buildkite/shared-pipeline-vars | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/shared-pipeline-vars b/.buildkite/shared-pipeline-vars index d6c45f0bc27..4cfa585abd4 100644 --- a/.buildkite/shared-pipeline-vars +++ b/.buildkite/shared-pipeline-vars @@ -3,5 +3,5 @@ # This file is `source`'d before calling `buildkite-agent pipeline upload`, and can be used # to set up some variables that will be interpolated in the `.yml` pipeline before uploading it. -export CI_TOOLKIT="automattic/a8c-ci-toolkit#1dda7bd51d20fbb82d5435dc2cbee3b08f5b5153" +export CI_TOOLKIT="automattic/a8c-ci-toolkit#3.9.0" export TEST_COLLECTOR="test-collector#v1.10.1" From 49d771a9cf4f11e7efdf7dd0641e2b529bf39f07 Mon Sep 17 00:00:00 2001 From: Petros Paraskevopoulos Date: Wed, 15 Jan 2025 12:12:37 +0200 Subject: [PATCH 06/10] CI: Update a8c-ci-toolkit to pr-sha commit (fix dependency cache) FYI: This change is done for testing purposes and until the below 'A8C CI Toolkit' #138 PR gets merged to 'trunk'. When that's done, the 'a8c-ci-toolkit' will be updated to '3.9.1' instead. A8C CI Toolkit PR: [Dependency Cache] Fix Dependency Cache #138 - https://github.com/Automattic/a8c-ci-toolkit-buildkite-plugin/pull/138 --- .buildkite/shared-pipeline-vars | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/shared-pipeline-vars b/.buildkite/shared-pipeline-vars index 4cfa585abd4..c31a5c9c77c 100644 --- a/.buildkite/shared-pipeline-vars +++ b/.buildkite/shared-pipeline-vars @@ -3,5 +3,5 @@ # This file is `source`'d before calling `buildkite-agent pipeline upload`, and can be used # to set up some variables that will be interpolated in the `.yml` pipeline before uploading it. -export CI_TOOLKIT="automattic/a8c-ci-toolkit#3.9.0" +export CI_TOOLKIT="automattic/a8c-ci-toolkit#9a12d43b5df24ad661ee565cdff77513dd44216c" export TEST_COLLECTOR="test-collector#v1.10.1" From 6ac4eb4555927d660994fb8bb7e2fc3aae6faad6 Mon Sep 17 00:00:00 2001 From: Petros Paraskevopoulos Date: Wed, 15 Jan 2025 13:48:14 +0200 Subject: [PATCH 07/10] CI: Update a8c-ci-toolkit to 3.9.1 Release Notes: https://github.com/Automattic/ a8c-ci-toolkit-buildkite-plugin/releases/tag/3.9.1 Related: 49d771a9cf4f11e7efdf7dd0641e2b529bf39f07 --- .buildkite/shared-pipeline-vars | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/shared-pipeline-vars b/.buildkite/shared-pipeline-vars index c31a5c9c77c..969e154cb6c 100644 --- a/.buildkite/shared-pipeline-vars +++ b/.buildkite/shared-pipeline-vars @@ -3,5 +3,5 @@ # This file is `source`'d before calling `buildkite-agent pipeline upload`, and can be used # to set up some variables that will be interpolated in the `.yml` pipeline before uploading it. -export CI_TOOLKIT="automattic/a8c-ci-toolkit#9a12d43b5df24ad661ee565cdff77513dd44216c" +export CI_TOOLKIT="automattic/a8c-ci-toolkit#3.9.1" export TEST_COLLECTOR="test-collector#v1.10.1" From cbca6ac19fcc9320bccaac54d2a4dcd227a4fbb1 Mon Sep 17 00:00:00 2001 From: Petros Paraskevopoulos <9729923+ParaskP7@users.noreply.github.com> Date: Wed, 15 Jan 2025 16:59:46 +0200 Subject: [PATCH 08/10] CI: Update save cache to run assemble debug unit test on all lib modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Wojciech Ziฤ™ba --- .buildkite/commands/save-cache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/commands/save-cache.sh b/.buildkite/commands/save-cache.sh index ccc5aa942cf..e7598730434 100755 --- a/.buildkite/commands/save-cache.sh +++ b/.buildkite/commands/save-cache.sh @@ -21,7 +21,7 @@ echo "--- ๐Ÿงน Download Lint Dependencies [Lint Mobile App]" echo "" echo "--- ๐Ÿงช Download Unit Test Dependencies [Assemble Unit Tests]" -./gradlew assembleJalapenoDebugUnitTest lib:cardreader:assembleDebugUnitTest +./gradlew assembleJalapenoDebugUnitTest assembleDebugUnitTest echo "" echo "--- ๐Ÿงช Download Android Test Dependencies [Assemble Android Tests]" From 4798b6346c703359fd7b5e1aa929d537b379139e Mon Sep 17 00:00:00 2001 From: Petros Paraskevopoulos Date: Thu, 16 Jan 2025 11:06:47 +0200 Subject: [PATCH 09/10] CI: Move restore cache call to individual .sh script files PR Comment: https://github.com/woocommerce/woocommerce-android/pull/ 13303#discussion_r1916752265 --- .buildkite/commands/lint.sh | 2 ++ .buildkite/commands/prototype-build.sh | 2 ++ .buildkite/commands/run-instrumented-tests.sh | 2 ++ .buildkite/commands/run-unit-tests.sh | 2 ++ .buildkite/commands/save-cache.sh | 10 ++++++++++ .buildkite/pipeline.yml | 20 +++++-------------- 6 files changed, 23 insertions(+), 15 deletions(-) diff --git a/.buildkite/commands/lint.sh b/.buildkite/commands/lint.sh index 71e35516721..4b9590ec5cb 100755 --- a/.buildkite/commands/lint.sh +++ b/.buildkite/commands/lint.sh @@ -1,5 +1,7 @@ #!/bin/bash -u +.buildkite/commands/restore-cache.sh + echo "--- ๐Ÿงน Linting" ./gradlew :WooCommerce:lintJalapenoDebug app_lint_exit_code=$? diff --git a/.buildkite/commands/prototype-build.sh b/.buildkite/commands/prototype-build.sh index 28dfbcf1c15..5f91ec44302 100755 --- a/.buildkite/commands/prototype-build.sh +++ b/.buildkite/commands/prototype-build.sh @@ -1,5 +1,7 @@ #!/bin/bash -eu +.buildkite/commands/restore-cache.sh + APP_TO_BUILD="${1?You need to specify the app to build, WooCommerce or WooCommerce-Wear}" echo "--- :rubygems: Setting up Gems" diff --git a/.buildkite/commands/run-instrumented-tests.sh b/.buildkite/commands/run-instrumented-tests.sh index b570524de8d..66ebce299e1 100755 --- a/.buildkite/commands/run-instrumented-tests.sh +++ b/.buildkite/commands/run-instrumented-tests.sh @@ -1,5 +1,7 @@ #!/bin/bash -eu +.buildkite/commands/restore-cache.sh + echo "--- :rubygems: Setting up Gems" install_gems diff --git a/.buildkite/commands/run-unit-tests.sh b/.buildkite/commands/run-unit-tests.sh index 9d28c534e67..6d2fa7bc21c 100755 --- a/.buildkite/commands/run-unit-tests.sh +++ b/.buildkite/commands/run-unit-tests.sh @@ -1,5 +1,7 @@ #!/bin/bash -eu +.buildkite/commands/restore-cache.sh + echo "--- ๐Ÿงช Testing" set +e ./gradlew testJalapenoDebugUnitTest testDebugUnitTest diff --git a/.buildkite/commands/save-cache.sh b/.buildkite/commands/save-cache.sh index e7598730434..9a62673c420 100755 --- a/.buildkite/commands/save-cache.sh +++ b/.buildkite/commands/save-cache.sh @@ -8,22 +8,32 @@ install_gems echo "--- :closed_lock_with_key: Installing Secrets" bundle exec fastlane run configure_apply +# .buildkite/commands/prototype-build.sh -> build_and_upload_prototype_build +# -> prototype_flavor = 'Jalapeno' +# -> prototype_build_type = 'Debug' echo "--- ๐Ÿ›  Download Mobile App Dependencies [Assemble Mobile App]" ./gradlew :WooCommerce:assembleJalapenoDebug echo "" +# .buildkite/commands/prototype-build.sh -> build_and_upload_prototype_build +# -> prototype_flavor = 'Jalapeno' +# -> prototype_build_type = 'Debug' echo "--- ๐Ÿ›  Download Wear App Dependencies [Assemble Wear App]" ./gradlew :WooCommerce-Wear:assembleJalapenoDebug echo "" +# .buildkite/commands/lint.sh -> ./gradlew :WooCommerce:lintJalapenoDebug echo "--- ๐Ÿงน Download Lint Dependencies [Lint Mobile App]" ./gradlew :WooCommerce:lintJalapenoDebug echo "" +# .buildkite/commands/run-unit-tests.sh -> ./gradlew testJalapenoDebugUnitTest testDebugUnitTest echo "--- ๐Ÿงช Download Unit Test Dependencies [Assemble Unit Tests]" ./gradlew assembleJalapenoDebugUnitTest assembleDebugUnitTest echo "" +# .buildkite/commands/run-instrumented-tests.sh -> build_and_instrumented_test +# -> gradle(tasks: %w[assembleVanillaDebug assembleVanillaDebugAndroidTest]) echo "--- ๐Ÿงช Download Android Test Dependencies [Assemble Android Tests]" ./gradlew assembleJalapenoDebugAndroidTest echo "" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 780d670a7eb..a863d1a01ec 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -41,9 +41,7 @@ steps: - "**/build/reports/detekt/detekt.html" - label: "lint" - command: | - .buildkite/commands/restore-cache.sh - .buildkite/commands/lint.sh + command: .buildkite/commands/lint.sh plugins: [$CI_TOOLKIT] artifact_paths: - "**/build/reports/lint-results*.*" @@ -60,16 +58,12 @@ steps: steps: - label: "๐Ÿ›  Prototype Build: Mobile App" - command: | - .buildkite/commands/restore-cache.sh - ".buildkite/commands/prototype-build.sh" "WooCommerce" + command: ".buildkite/commands/prototype-build.sh" "WooCommerce" if: build.pull_request.id != null plugins: [$CI_TOOLKIT] - label: "๐Ÿ›  Prototype Build: Wear App" - command: | - .buildkite/commands/restore-cache.sh - ".buildkite/commands/prototype-build.sh" "WooCommerce-Wear" + command: ".buildkite/commands/prototype-build.sh" "WooCommerce-Wear" if: build.pull_request.id != null plugins: [$CI_TOOLKIT] @@ -78,9 +72,7 @@ steps: steps: - label: "Unit tests" - command: | - .buildkite/commands/restore-cache.sh - .buildkite/commands/run-unit-tests.sh + command: .buildkite/commands/run-unit-tests.sh plugins: - $CI_TOOLKIT - $TEST_COLLECTOR : @@ -90,9 +82,7 @@ steps: - "**/build/test-results/merged-test-results.xml" - label: "Instrumented tests" - command: | - .buildkite/commands/restore-cache.sh - .buildkite/commands/run-instrumented-tests.sh + command: .buildkite/commands/run-instrumented-tests.sh plugins: - $CI_TOOLKIT - $TEST_COLLECTOR : From ead005b6c92d0c5bb29b9ca03c67767a2afc09ae Mon Sep 17 00:00:00 2001 From: Petros Paraskevopoulos Date: Thu, 16 Jan 2025 11:23:40 +0200 Subject: [PATCH 10/10] CI: Fix wrongly defined command related double quotes on pipeline jobs When using 'single-cmd' and not using this double quotes structure it is is producing 'did not find expected key' type of errors. --- .buildkite/pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index a863d1a01ec..02cc3a2b6b0 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -58,12 +58,12 @@ steps: steps: - label: "๐Ÿ›  Prototype Build: Mobile App" - command: ".buildkite/commands/prototype-build.sh" "WooCommerce" + command: ".buildkite/commands/prototype-build.sh WooCommerce" if: build.pull_request.id != null plugins: [$CI_TOOLKIT] - label: "๐Ÿ›  Prototype Build: Wear App" - command: ".buildkite/commands/prototype-build.sh" "WooCommerce-Wear" + command: ".buildkite/commands/prototype-build.sh WooCommerce-Wear" if: build.pull_request.id != null plugins: [$CI_TOOLKIT]