Fix disk space issues on github runners (apache#46358)

* Triage disk space issues, DO NOT MERGE * Triage disk space issues, DO NOT MERGE * Triage disk space issues, DO NOT MERGE * Triage disk space issues, DO NOT MERGE * Triage disk space issues, DO NOT MERGE * Triage disk space issues, DO NOT MERGE * Triage disk space issues, DO NOT MERGE * Move docker storage to second drive in general * Cleanup debug and triaging stuff * Exception of docker volume for constraints building
boschglobal · Feb 3, 2025 · 8d8afdf · 8d8afdf
1 parent 3ff4f2d
commit 8d8afdf
Show file tree

Hide file tree

Showing 15 changed files with 78 additions and 0 deletions.
diff --git a/.github/actions/prepare_all_ci_images/action.yml b/.github/actions/prepare_all_ci_images/action.yml
@@ -22,6 +22,9 @@ inputs:
   python-versions-list-as-string:
     description: 'Stringified array of all Python versions to test - separated by spaces.'
     required: true
+  docker-volume-location:
+    description: File system location where to move docker space to
+    default: /mnt/var-lib-docker
   platform:
     description: 'Platform for the build - linux/amd64 or linux/arm64'
     required: true
@@ -30,6 +33,8 @@ runs:
   steps:
     - name: "Cleanup docker"
       run: ./scripts/ci/cleanup_docker.sh
+      env:
+        TARGET_DOCKER_VOLUME_LOCATION: ${{ inputs.docker-volume-location }}
       shell: bash
     # TODO: Currently we cannot loop through the list of python versions and have dynamic list of
     #       tasks. Instead we hardcode all possible python versions and they - but

diff --git a/.github/actions/prepare_breeze_and_image/action.yml b/.github/actions/prepare_breeze_and_image/action.yml
@@ -39,7 +39,10 @@ runs:
   using: "composite"
   steps:
     - name: "Cleanup docker"
+      # Move docker space to second partition to have more space
       run: ./scripts/ci/cleanup_docker.sh
+      env:
+        TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
       shell: bash
     - name: "Install Breeze"
       uses: ./.github/actions/breeze

diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml
@@ -142,7 +142,10 @@ jobs:
         with:
           persist-credentials: false
       - name: "Cleanup docker"
+        # Move docker space to second partition to have more space
         run: ./scripts/ci/cleanup_docker.sh
+        env:
+          TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
         with:

diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml
@@ -116,7 +116,10 @@ jobs:
           fetch-depth: 2
           persist-credentials: false
       - name: "Cleanup docker"
+        # Move docker space to second partition to have more space
         run: ./scripts/ci/cleanup_docker.sh
+        env:
+          TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
       - name: "Prepare breeze & PROD image: ${{ inputs.default-python-version }}"
         uses: ./.github/actions/prepare_breeze_and_image
         with:

diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml
@@ -77,7 +77,10 @@ jobs:
           fetch-depth: 0
           persist-credentials: false
       - name: "Cleanup docker"
+        # Move docker space to second partition to have more space
         run: ./scripts/ci/cleanup_docker.sh
+        env:
+          TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
         with:
@@ -98,7 +101,10 @@ jobs:
         with:
           persist-credentials: false
       - name: "Cleanup docker"
+        # Move docker space to second partition to have more space
         run: ./scripts/ci/cleanup_docker.sh
+        env:
+          TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
       - name: Setup pnpm
         uses: pnpm/action-setup@fe02b34f77f8bc703788d5817da081398fad5dd2  # v4.0.0
         with:
@@ -165,7 +171,10 @@ jobs:
         with:
           persist-credentials: false
       - name: "Cleanup docker"
+        # Move docker space to second partition to have more space
         run: ./scripts/ci/cleanup_docker.sh
+        env:
+          TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
       - name: "Setup node"
         uses: actions/setup-node@v4
         with:
@@ -233,7 +242,10 @@ jobs:
         with:
           persist-credentials: false
       - name: "Cleanup docker"
+        # Move docker space to second partition to have more space
         run: ./scripts/ci/cleanup_docker.sh
+        env:
+          TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
         with:
@@ -288,7 +300,10 @@ jobs:
         with:
           persist-credentials: false
       - name: "Cleanup docker"
+        # Move docker space to second partition to have more space
         run: ./scripts/ci/cleanup_docker.sh
+        env:
+          TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
         with:
@@ -367,7 +382,10 @@ jobs:
         with:
           persist-credentials: false
       - name: "Cleanup docker"
+        # Move docker space to second partition to have more space
         run: ./scripts/ci/cleanup_docker.sh
+        env:
+          TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
         with:

diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml
@@ -130,7 +130,10 @@ jobs:
         with:
           persist-credentials: false
       - name: "Cleanup docker"
+        # Move docker space to second partition to have more space
         run: ./scripts/ci/cleanup_docker.sh
+        env:
+          TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
         with:

diff --git a/.github/workflows/ci-image-checks.yml b/.github/workflows/ci-image-checks.yml
@@ -324,7 +324,10 @@ jobs:
         with:
           persist-credentials: false
       - name: "Cleanup docker"
+        # Move docker space to second partition to have more space
         run: ./scripts/ci/cleanup_docker.sh
+        env:
+          TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
       - name: "Download docs prepared as artifacts"
         uses: actions/download-artifact@v4
         with:
@@ -405,7 +408,10 @@ jobs:
           fetch-depth: 2
           persist-credentials: false
       - name: "Cleanup docker"
+        # Move docker space to second partition to have more space
         run: ./scripts/ci/cleanup_docker.sh
+        env:
+          TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         uses: actions/checkout@v4
         with:

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -147,7 +147,10 @@ jobs:
         with:
           persist-credentials: false
       - name: "Cleanup docker"
+        # Move docker space to second partition to have more space
         run: ./scripts/ci/cleanup_docker.sh
+        env:
+          TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
       - name: Fetch incoming commit ${{ github.sha }} with its parent
         uses: actions/checkout@v4
         with:

diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml
@@ -104,7 +104,10 @@ jobs:
           # Needed to perform push action
           persist-credentials: false
       - name: "Cleanup docker"
+        # Move docker space to second partition to have more space
         run: ./scripts/ci/cleanup_docker.sh
+        env:
+          TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
       - name: "Set constraints branch name"
         id: constraints-branch
         run: ./scripts/ci/constraints/ci_branch_constraints.sh >> ${GITHUB_OUTPUT}
@@ -192,7 +195,10 @@ jobs:
         with:
           persist-credentials: false
       - name: "Cleanup docker"
+        # Move docker space to second partition to have more space
         run: ./scripts/ci/cleanup_docker.sh
+        env:
+          TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
       - name: "Free up disk space"
         shell: bash
         run: ./scripts/tools/free_up_disk_space.sh

diff --git a/.github/workflows/generate-constraints.yml b/.github/workflows/generate-constraints.yml
@@ -78,6 +78,7 @@ jobs:
         with:
           platform: "linux/amd64"
           python-versions-list-as-string: ${{ inputs.python-versions-list-as-string }}
+          docker-volume-location: ""  # TODO(jscheffl): Understand why it fails here and fix it
       - name: "Verify all CI images ${{ inputs.python-versions-list-as-string }}"
         run: breeze ci-image verify --run-in-parallel
       - name: "Source constraints"

diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml
@@ -97,7 +97,10 @@ jobs:
         with:
           persist-credentials: false
       - name: "Cleanup docker"
+        # Move docker space to second partition to have more space
         run: ./scripts/ci/cleanup_docker.sh
+        env:
+          TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
         with:

diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml
@@ -137,7 +137,10 @@ jobs:
         with:
           persist-credentials: false
       - name: "Cleanup docker"
+        # Move docker space to second partition to have more space
         run: ./scripts/ci/cleanup_docker.sh
+        env:
+          TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
         if: inputs.upload-package-artifact == 'true'
       - name: "Cleanup dist and context file"
         shell: bash
@@ -222,7 +225,10 @@ jobs:
         with:
           persist-credentials: false
       - name: "Cleanup docker"
+        # Move docker space to second partition to have more space
         run: ./scripts/ci/cleanup_docker.sh
+        env:
+          TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
         with:

diff --git a/.github/workflows/push-image-cache.yml b/.github/workflows/push-image-cache.yml
@@ -126,7 +126,10 @@ jobs:
         with:
           persist-credentials: false
       - name: "Cleanup docker"
+        # Move docker space to second partition to have more space
         run: ./scripts/ci/cleanup_docker.sh
+        env:
+          TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
         with:
@@ -202,7 +205,10 @@ jobs:
         with:
           persist-credentials: false
       - name: "Cleanup docker"
+        # Move docker space to second partition to have more space
         run: ./scripts/ci/cleanup_docker.sh
+        env:
+          TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
         with:

diff --git a/.github/workflows/release_dockerhub_image.yml b/.github/workflows/release_dockerhub_image.yml
@@ -60,7 +60,10 @@ jobs:
         with:
           persist-credentials: false
       - name: "Cleanup docker"
+        # Move docker space to second partition to have more space
         run: ./scripts/ci/cleanup_docker.sh
+        env:
+          TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
         with:
@@ -99,7 +102,10 @@ jobs:
         with:
           persist-credentials: false
       - name: "Cleanup docker"
+        # Move docker space to second partition to have more space
         run: ./scripts/ci/cleanup_docker.sh
+        env:
+          TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
         with:

diff --git a/scripts/ci/cleanup_docker.sh b/scripts/ci/cleanup_docker.sh
@@ -19,6 +19,12 @@ function cleanup_docker {
     # This is faster than docker prune
     sudo systemctl stop docker
     sudo rm -rf /var/lib/docker
+    # If a path is provided in ENV, bind mount it to /var/lib/docker
+    if [ -n "${TARGET_DOCKER_VOLUME_LOCATION}" ]; then
+        echo "Mounting ${TARGET_DOCKER_VOLUME_LOCATION} to /var/lib/docker"
+        sudo mkdir -p "${TARGET_DOCKER_VOLUME_LOCATION}" /var/lib/docker
+        sudo mount --bind "${TARGET_DOCKER_VOLUME_LOCATION}" /var/lib/docker
+    fi
     sudo systemctl start docker
 }