Update CI documentation, renaming runs to "Canary" (apache#26151)

For quite some time we did not have the name of the main builds
However more and more the "main" builds are used to provide early
warnigns for some problems:

* 3rd-party dependencies breaking our builds
* our own dependencies breaking the constraints
* building ARM images
* building breeze images quickly
* running complete matrix of tests
* finding flaky tests

So effectively, those main builds are really "Canary" builds - when
those builds are failing, they give us a chance to react quickly,
without affecting the regular PR builds.

This PR clarifies the meaning and reasoning for those builds
and introduces "Canary" name for them.

During related documentation review, it also turned out that a number
of old environment variables are not used any more (after the breeze
changing to Python) and this PR also removes them from documentation
and removes the variables from all the scripts (including removal of
some unused scripts)

The new documentation also mentions somethign that we've learned
recently - that in case you use Breeze in non-airflow workflows in
GitHub Actions, you need to override the variables through command
line parameters rather than through environment variables, because
GitHub actions treats GITHUB_* variables as immutable for security.

.github/workflows/build-images.yml

@@ -26,8 +26,6 @@ permissions:
 env:
   MOUNT_SELECTED_LOCAL_SOURCES: "false"
   ANSWER: "yes"
-  CHECK_IMAGE_FOR_REBUILD: "true"
-  SKIP_CHECK_REMOTE_IMAGE: "true"
   DB_RESET: "true"
   VERBOSE: "true"
   GITHUB_REPOSITORY: ${{ github.repository }}

.github/workflows/ci.yml

@@ -31,8 +31,6 @@ permissions:
 env:
   MOUNT_SELECTED_LOCAL_SOURCES: "false"
   ANSWER: "yes"
-  CHECK_IMAGE_FOR_REBUILD: "true"
-  SKIP_CHECK_REMOTE_IMAGE: "true"
   DB_RESET: "true"
   VERBOSE: "true"
   GITHUB_REPOSITORY: ${{ github.repository }}
@@ -168,7 +166,7 @@ jobs:
       in-workflow-build: ${{ steps.source-run-info.outputs.in-workflow-build }}
       build-job-description: ${{ steps.source-run-info.outputs.build-job-description }}
       runs-on: ${{ steps.source-run-info.outputs.runs-on }}
-      merge-run: ${{ steps.source-run-info.outputs.merge-run }}
+      canary-run: ${{ steps.source-run-info.outputs.canary-run }}
       run-coverage: ${{ steps.source-run-info.outputs.run-coverage }}
     steps:
       - name: Cleanup repo
@@ -261,27 +259,27 @@ jobs:
     steps:
       - name: Cleanup repo
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
-        if: needs.build-info.outputs.merge-run == 'true'
+        if: needs.build-info.outputs.canary-run == 'true'
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         uses: actions/checkout@v3
         with:
           persist-credentials: false
-        if: needs.build-info.outputs.merge-run == 'true'
+        if: needs.build-info.outputs.canary-run == 'true'
       - name: "Setup python"
         uses: actions/setup-python@v4
         with:
           python-version: ${{ needs.build-info.outputs.default-python-version }}
           cache: 'pip'
           cache-dependency-path: ./dev/breeze/setup*
-        if: needs.build-info.outputs.merge-run == 'true'
+        if: needs.build-info.outputs.canary-run == 'true'
       - run: ./scripts/ci/install_breeze.sh
-        if: needs.build-info.outputs.merge-run == 'true'
+        if: needs.build-info.outputs.canary-run == 'true'
       - name: "Free space"
         run: breeze ci free-space
-        if: needs.build-info.outputs.merge-run == 'true'
+        if: needs.build-info.outputs.canary-run == 'true'
       - name: "Start ARM instance"
         run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
-        if: matrix.platform == 'linux/arm64' && needs.build-info.outputs.merge-run == 'true'
+        if: matrix.platform == 'linux/arm64' && needs.build-info.outputs.canary-run == 'true'
       - name: "Push CI cache ${{ matrix.platform }}"
         run: >
           breeze ci-image build
@@ -290,23 +288,23 @@ jobs:
           --run-in-parallel
           --force-build
           --platform ${{ matrix.platform }}
-        if: needs.build-info.outputs.merge-run == 'true'
+        if: needs.build-info.outputs.canary-run == 'true'
       - name: "Push CI latest image ${{ matrix.platform }}"
         run: >
           breeze ci-image build
           --tag-as-latest --push --run-in-parallel --platform ${{ matrix.platform }}
         # We only push "amd" image as it is really only needed for any kind of automated builds in CI
         # and currently there is not an easy way to make multi-platform image from two separate builds
-        if: matrix.platform == 'linux/amd64' && needs.build-info.outputs.merge-run == 'true'
+        if: matrix.platform == 'linux/amd64' && needs.build-info.outputs.canary-run == 'true'
       - name: "Stop ARM instance"
         run: ./scripts/ci/images/ci_stop_arm_instance.sh
-        if: always() && matrix.platform == 'linux/arm64' && needs.build-info.outputs.merge-run == 'true'
+        if: always() && matrix.platform == 'linux/arm64' && needs.build-info.outputs.canary-run == 'true'
       - name: "Clean docker cache for ${{ matrix.platform }}"
         run: docker system prune --all --force
-        if: matrix.platform == 'linux/amd64' && needs.build-info.outputs.merge-run == 'true'
+        if: matrix.platform == 'linux/amd64' && needs.build-info.outputs.canary-run == 'true'
       - name: "Fix ownership"
         run: breeze ci fix-ownership
-        if: always() && needs.build-info.outputs.merge-run == 'true'
+        if: always() && needs.build-info.outputs.canary-run == 'true'
 
   # Check that after earlier cache push, breeze command will build quickly
   chcek-that-image-builds-quicklly:
@@ -323,30 +321,30 @@ jobs:
     steps:
       - name: Cleanup repo
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
-        if: needs.build-info.outputs.merge-run == 'true'
+        if: needs.build-info.outputs.canary-run == 'true'
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         uses: actions/checkout@v3
         with:
           persist-credentials: false
-        if: needs.build-info.outputs.merge-run == 'true'
+        if: needs.build-info.outputs.canary-run == 'true'
       - name: "Setup python"
         uses: actions/setup-python@v4
         with:
           python-version: ${{ needs.build-info.outputs.default-python-version }}
           cache: 'pip'
           cache-dependency-path: ./dev/breeze/setup*
-        if: needs.build-info.outputs.merge-run == 'true'
+        if: needs.build-info.outputs.canary-run == 'true'
       - run: ./scripts/ci/install_breeze.sh
-        if: needs.build-info.outputs.merge-run == 'true'
+        if: needs.build-info.outputs.canary-run == 'true'
       - name: "Free space"
         run: breeze ci free-space
-        if: needs.build-info.outputs.merge-run == 'true'
+        if: needs.build-info.outputs.canary-run == 'true'
       - name: "Check that image builds quickly"
         run: breeze shell --max-time 120
-        if: matrix.platform == 'linux/amd64' && needs.build-info.outputs.merge-run == 'true'
+        if: matrix.platform == 'linux/amd64' && needs.build-info.outputs.canary-run == 'true'
       - name: "Fix ownership"
         run: breeze ci fix-ownership
-        if: always() && needs.build-info.outputs.merge-run == 'true'
+        if: always() && needs.build-info.outputs.canary-run == 'true'
 
   build-ci-images:
     permissions:
@@ -1636,20 +1634,20 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
       - name: "Set constraints branch name"
         id: constraints-branch
         run: ./scripts/ci/constraints/ci_branch_constraints.sh
-        if: needs.build-info.outputs.merge-run == 'true'
+        if: needs.build-info.outputs.canary-run == 'true'
       - name: Checkout ${{ steps.constraints-branch.outputs.branch }}
         uses: actions/checkout@v3
-        if: needs.build-info.outputs.merge-run == 'true'
+        if: needs.build-info.outputs.canary-run == 'true'
         with:
           path: "repo"
           ref: ${{ steps.constraints-branch.outputs.branch }}
           persist-credentials: false
       - name: "Commit changed constraint files for ${{needs.build-info.outputs.python-versions}}"
         run: ./scripts/ci/constraints/ci_commit_constraints.sh
-        if: needs.build-info.outputs.merge-run == 'true'
+        if: needs.build-info.outputs.canary-run == 'true'
       - name: "Push changes"
         uses: ./.github/actions/github-push-action
-        if: needs.build-info.outputs.merge-run == 'true'
+        if: needs.build-info.outputs.canary-run == 'true'
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           branch: ${{ steps.constraints-branch.outputs.branch }}
@@ -1672,7 +1670,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
       - build-info
       - constraints
       - docs
-    if: needs.build-info.outputs.merge-run == 'true'
+    if: needs.build-info.outputs.canary-run == 'true'
     strategy:
       fail-fast: false
       matrix:
@@ -1776,7 +1774,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
     if: >
       needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' &&
       needs.build-info.outputs.in-workflow-build == 'true' &&
-      needs.build-info.outputs.merge-run != 'true'
+      needs.build-info.outputs.canary-run != 'true'
     steps:
       - name: Cleanup repo
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"