Use zstd to compress yocto sstate for GHA caching

Change-type: patch Signed-off-by: Kyle Harding <[email protected]>
balena-os · Jun 17, 2024 · 8e421b1 · 8e421b1
1 parent bdd131f
commit 8e421b1
Showing 1 changed file with 104 additions and 37 deletions.
diff --git a/.github/workflows/yocto-build-deploy.yml b/.github/workflows/yocto-build-deploy.yml
@@ -41,7 +41,8 @@ on:
         default: >
           [
             "self-hosted",
-            "X64"
+            "X64",
+            "16cpu"
           ]
       device-repo:
         description: balenaOS device repository (owner/repo)
@@ -184,12 +185,10 @@ jobs:
       BALENARC_BALENA_URL: ${{ vars.BALENA_URL || 'balena-cloud.com' }}
       API_ENV: ${{ vars.BALENA_URL || 'balena-cloud.com' }}
 
-      # Yocto NFS sstate cache host
-      YOCTO_CACHE_HOST: ${{ vars.YOCTO_CACHE_HOST || 'nfs.product-os.io' }}
-      YOCTO_CACHE_DIR: ${{ github.workspace }}/shared/yocto-cache
-      # S3_CACHE_URL: s3://balena-yocto-cache/${{ inputs.machine }}
       BARYS_ARGUMENTS_VAR: ""
 
+      YOCTO_CACHE_DIR: ${{ github.workspace }}/shared/yocto-cache
+
       # URL for secure boot signing server API
       SIGN_API_URL: ${{ vars.SIGN_API_URL || 'https://sign.balena-cloud.com' }}
 
@@ -389,29 +388,6 @@ jobs:
             echo BARYS_ARGUMENTS_VAR="${BARYS_ARGUMENTS_VAR} -d" >> $GITHUB_ENV
           fi
 
-      # # https://docs.yoctoproject.org/dev/dev-manual/speeding-up-build.html#speeding-up-a-build
-      # # TODO: Delete  when using properly isolated self-hosted runner resources
-      # - name: Configure bitbake resource limits
-      #   env:
-      #     BB_NUMBER_THREADS: 4
-      #     BB_NUMBER_PARSE_THREADS: 4
-      #     PARALLEL_MAKE: -j4
-      #     PARALLEL_MAKEINST: -j4
-      #     BB_PRESSURE_MAX_CPU: 500
-      #     BB_PRESSURE_MAX_IO: 500
-      #     BB_PRESSURE_MAX_MEMORY: 500
-      #   run: |
-      #     nproc
-      #     free -h
-      #     BARYS_ARGUMENTS_VAR="${BARYS_ARGUMENTS_VAR} -a BB_NUMBER_THREADS=${BB_NUMBER_THREADS}"
-      #     BARYS_ARGUMENTS_VAR="${BARYS_ARGUMENTS_VAR} -a BB_NUMBER_PARSE_THREADS=${BB_NUMBER_PARSE_THREADS}"
-      #     BARYS_ARGUMENTS_VAR="${BARYS_ARGUMENTS_VAR} -a PARALLEL_MAKE=${PARALLEL_MAKE}"
-      #     BARYS_ARGUMENTS_VAR="${BARYS_ARGUMENTS_VAR} -a PARALLEL_MAKEINST=${PARALLEL_MAKEINST}"
-      #     BARYS_ARGUMENTS_VAR="${BARYS_ARGUMENTS_VAR} -a BB_PRESSURE_MAX_CPU=${BB_PRESSURE_MAX_CPU}"
-      #     BARYS_ARGUMENTS_VAR="${BARYS_ARGUMENTS_VAR} -a BB_PRESSURE_MAX_IO=${BB_PRESSURE_MAX_IO}"
-      #     BARYS_ARGUMENTS_VAR="${BARYS_ARGUMENTS_VAR} -a BB_PRESSURE_MAX_MEMORY=${BB_PRESSURE_MAX_MEMORY}"
-      #     echo "BARYS_ARGUMENTS_VAR=${BARYS_ARGUMENTS_VAR}" >> $GITHUB_ENV
-
       - name: Enable signed images
         if: inputs.sign-image == true
         env:
@@ -426,17 +402,66 @@ jobs:
           BARYS_ARGUMENTS_VAR="${BARYS_ARGUMENTS_VAR} --bitbake-args --no-setscene"
           echo "BARYS_ARGUMENTS_VAR=${BARYS_ARGUMENTS_VAR}" >> $GITHUB_ENV
 
-      # the directory is required even if we don't mount the NFS share
-      - name: Create shared cache mount point
+      - name: Install zstd
         run: |
-          sudo mkdir -p "${YOCTO_CACHE_DIR}/$(whoami)"
-          sudo chown -R $(id -u):$(id -g) "${YOCTO_CACHE_DIR}"
+          sudo apt-get update
+          sudo apt-get install -y zstd
 
-      - name: Mount shared NFS cache
-        if: env.YOCTO_CACHE_HOST != '' && contains(fromJSON(inputs.runs-on), 'self-hosted')
+      # https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28#list-github-actions-caches-for-a-repository
+      - name: List Action caches
+        continue-on-error: true
+        env:
+          # environment variables used by gh CLI
+          # https://cli.github.com/manual/gh_help_environment
+          GH_DEBUG: "true"
+          GH_PAGER: "cat"
+          GH_PROMPT_DISABLED: "true"
+          GH_REPO: "${{ github.repository }}"
+          GH_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+        run: |
+          gh api \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            /repos/${{ github.repository }}/actions/caches
+
+      # https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28#get-github-actions-cache-usage-for-a-repository
+      - name: Get Actions cache usage
+        continue-on-error: true
+        env:
+          # environment variables used by gh CLI
+          # https://cli.github.com/manual/gh_help_environment
+          GH_DEBUG: "true"
+          GH_PAGER: "cat"
+          GH_PROMPT_DISABLED: "true"
+          GH_REPO: "${{ github.repository }}"
+          GH_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
         run: |
-          sudo mount -t nfs "${YOCTO_CACHE_HOST}:/" "${YOCTO_CACHE_DIR}" -o fsc,nolock
-          ls -al "${YOCTO_CACHE_DIR}"/$(whoami)
+          gh api \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            /repos/${{ github.repository }}/actions/cache/usage
+
+      # https://github.com/actions/cache?tab=readme-ov-file
+      - name: Restore Yocto sstate cache
+        id: cache-yocto-sstate
+        uses: actions/[email protected]
+        with:
+          # Run the post step to save the cache even if another step before fails
+          save-always: true
+          path: yocto_sstate_cache.zst
+          key: yocto-sstate-${{ inputs.machine }}
+          restore-keys: |
+            yocto-sstate-${{ inputs.machine }}
+
+      - name: Decompress sstate cache
+        # if: steps.cache-yocto-sstate.outputs.cache-hit == 'true'
+        continue-on-error: true
+        run: |
+          mkdir -p "${YOCTO_CACHE_DIR}"
+          # zstd -d yocto_sstate_cache.zst -o "${YOCTO_CACHE_DIR}/"
+          zstd --decompress yocto_sstate_cache.tar.zst -o yocto_sstate_cache.tar
+          tar -xvf yocto_sstate_cache.tar -C "${YOCTO_CACHE_DIR}/"
+          du -sh yocto_sstate_cache.tar.zst yocto_sstate_cache.tar "${YOCTO_CACHE_DIR}/*"
 
       # All preperation complete before this step
       # Start building balenaOS
@@ -453,16 +478,58 @@ jobs:
           sudo sysctl -w vm.mmap_min_addr=65536
           sysctl vm.mmap_min_addr
 
+          mkdir -p "${YOCTO_CACHE_DIR}"
+
           ./balena-yocto-scripts/build/balena-build.sh \
             -d "${MACHINE}" \
             -t "${{ secrets.BALENA_API_KEY }}" \
-            -s "${YOCTO_CACHE_DIR}/$(whoami)" \
+            -s "${YOCTO_CACHE_DIR}" \
             -g "${BARYS_ARGUMENTS_VAR}"
 
           if grep -R "ERROR: " build/tmp/log/*; then
             exit 1
           fi
 
+      - name: Compress sstate cache
+        run: |
+          tar -cvf yocto_sstate_cache.tar -C "${YOCTO_CACHE_DIR}" .
+          zstd -19 yocto_sstate_cache.tar -o yocto_sstate_cache.tar.zst
+          du -sh yocto_sstate_cache.tar.zst yocto_sstate_cache.tar "${YOCTO_CACHE_DIR}/*"
+
+      # https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28#list-github-actions-caches-for-a-repository
+      - name: List Action caches
+        continue-on-error: true
+        env:
+          # environment variables used by gh CLI
+          # https://cli.github.com/manual/gh_help_environment
+          GH_DEBUG: "true"
+          GH_PAGER: "cat"
+          GH_PROMPT_DISABLED: "true"
+          GH_REPO: "${{ github.repository }}"
+          GH_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+        run: |
+          gh api \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            /repos/${{ github.repository }}/actions/caches
+
+      # https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28#get-github-actions-cache-usage-for-a-repository
+      - name: Get Actions cache usage
+        continue-on-error: true
+        env:
+          # environment variables used by gh CLI
+          # https://cli.github.com/manual/gh_help_environment
+          GH_DEBUG: "true"
+          GH_PAGER: "cat"
+          GH_PROMPT_DISABLED: "true"
+          GH_REPO: "${{ github.repository }}"
+          GH_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+        run: |
+          gh api \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            /repos/${{ github.repository }}/actions/cache/usage
+
       # TODO: pre-install on self-hosted-runners
       # Needed by the yocto job to zip artifacts - Don't remove
       - name: Install zip package