From 0ca27cf1fcc6d626931fe259a272a13210f4f758 Mon Sep 17 00:00:00 2001 From: Kyle Harding Date: Fri, 1 Nov 2024 13:03:25 -0400 Subject: [PATCH 1/5] Enable mirror tarballs bitbake feature Causes tarballs of the source control repositories (e.g. Git repositories), including metadata, to be placed in the DL_DIR directory Change-type: patch Signed-off-by: Kyle Harding --- .github/workflows/yocto-build-deploy.yml | 27 +++++------------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/.github/workflows/yocto-build-deploy.yml b/.github/workflows/yocto-build-deploy.yml index 796015c3c..5960d6a5a 100644 --- a/.github/workflows/yocto-build-deploy.yml +++ b/.github/workflows/yocto-build-deploy.yml @@ -425,28 +425,11 @@ jobs: # Move newly generated OS contract to location expected later on in the workflow cp "${CONTRACTS_OUTPUT_DIR}/${DEVICE_TYPE_SLUG}/balena-os/balena.yml" "${WORKSPACE}/balena.yml" - # # https://docs.yoctoproject.org/dev/dev-manual/speeding-up-build.html#speeding-up-a-build - # # TODO: Delete when using properly isolated self-hosted runner resources - # - name: Configure bitbake resource limits - # env: - # BB_NUMBER_THREADS: 4 - # BB_NUMBER_PARSE_THREADS: 4 - # PARALLEL_MAKE: -j4 - # PARALLEL_MAKEINST: -j4 - # BB_PRESSURE_MAX_CPU: 500 - # BB_PRESSURE_MAX_IO: 500 - # BB_PRESSURE_MAX_MEMORY: 500 - # run: | - # nproc - # free -h - # BARYS_ARGUMENTS_VAR="${BARYS_ARGUMENTS_VAR} -a BB_NUMBER_THREADS=${BB_NUMBER_THREADS}" - # BARYS_ARGUMENTS_VAR="${BARYS_ARGUMENTS_VAR} -a BB_NUMBER_PARSE_THREADS=${BB_NUMBER_PARSE_THREADS}" - # BARYS_ARGUMENTS_VAR="${BARYS_ARGUMENTS_VAR} -a PARALLEL_MAKE=${PARALLEL_MAKE}" - # BARYS_ARGUMENTS_VAR="${BARYS_ARGUMENTS_VAR} -a PARALLEL_MAKEINST=${PARALLEL_MAKEINST}" - # BARYS_ARGUMENTS_VAR="${BARYS_ARGUMENTS_VAR} -a BB_PRESSURE_MAX_CPU=${BB_PRESSURE_MAX_CPU}" - # BARYS_ARGUMENTS_VAR="${BARYS_ARGUMENTS_VAR} -a BB_PRESSURE_MAX_IO=${BB_PRESSURE_MAX_IO}" - # BARYS_ARGUMENTS_VAR="${BARYS_ARGUMENTS_VAR} -a BB_PRESSURE_MAX_MEMORY=${BB_PRESSURE_MAX_MEMORY}" - # echo "BARYS_ARGUMENTS_VAR=${BARYS_ARGUMENTS_VAR}" >>"${GITHUB_ENV}" + # Causes tarballs of the source control repositories (e.g. Git repositories), including metadata, to be placed in the DL_DIR directory. + # https://docs.yoctoproject.org/4.0.5/ref-manual/variables.html?highlight=compress#term-BB_GENERATE_MIRROR_TARBALLS + - name: Enable mirror tarballs + run: | + BARYS_ARGUMENTS_VAR="${BARYS_ARGUMENTS_VAR} -a BB_GENERATE_MIRROR_TARBALLS=1" >> "${GITHUB_ENV}" - name: Enable signed images if: inputs.sign-image == true From 57f91ba647a753098fa058ba973578b366798379 Mon Sep 17 00:00:00 2001 From: Kyle Harding Date: Fri, 1 Nov 2024 15:56:01 -0400 Subject: [PATCH 2/5] Use legacy yocto NFS cache as a source mirror This change makes the legacy NFS cache mount optional, and will not fail the build if unset or unreachable. Instead, if the NFS is available it will be used as a read-only mirror for shared downloads, but will not be updated with new files. Change-type: minor Signed-off-by: Kyle Harding --- .github/workflows/yocto-build-deploy.yml | 63 ++++++++++++++++++------ 1 file changed, 49 insertions(+), 14 deletions(-) diff --git a/.github/workflows/yocto-build-deploy.yml b/.github/workflows/yocto-build-deploy.yml index 5960d6a5a..540dd694e 100644 --- a/.github/workflows/yocto-build-deploy.yml +++ b/.github/workflows/yocto-build-deploy.yml @@ -186,11 +186,10 @@ jobs: automation_dir: "${{ github.workspace }}/balena-yocto-scripts/automation" BALENARC_BALENA_URL: ${{ vars.BALENA_HOST || inputs.deploy-environment || 'balena-cloud.com' }} API_ENV: ${{ vars.BALENA_HOST || inputs.deploy-environment || 'balena-cloud.com' }} - - # Yocto NFS sstate cache host - YOCTO_CACHE_HOST: ${{ vars.YOCTO_CACHE_HOST || 'nfs.product-os.io' }} - YOCTO_CACHE_DIR: ${{ github.workspace }}/shared/yocto-cache BARYS_ARGUMENTS_VAR: "" + # https://docs.yoctoproject.org/3.1.21/overview-manual/overview-manual-concepts.html#user-configuration + # Create an autobuilder configuration file that is loaded before local.conf + AUTO_CONF_FILE: "${{ github.workspace }}/build/conf/auto.conf" outputs: os_version: ${{ steps.balena-lib.outputs.os_version }} @@ -446,17 +445,48 @@ jobs: BARYS_ARGUMENTS_VAR="${BARYS_ARGUMENTS_VAR} --bitbake-args --no-setscene" echo "BARYS_ARGUMENTS_VAR=${BARYS_ARGUMENTS_VAR}" >>"${GITHUB_ENV}" - # the directory is required even if we don't mount the NFS share - - name: Create shared cache mount point - run: | - sudo mkdir -p "${YOCTO_CACHE_DIR}/$(whoami)" - sudo chown -R "$(id -u):$(id -g)" "${YOCTO_CACHE_DIR}" - - name: Mount shared NFS cache - if: env.YOCTO_CACHE_HOST != '' && contains(fromJSON(inputs.build-runs-on), 'self-hosted') + if: vars.YOCTO_CACHE_HOST && contains(fromJSON(inputs.build-runs-on), 'self-hosted') + continue-on-error: true + id: jenkins-nfs + env: + YOCTO_CACHE_HOST: ${{ vars.YOCTO_CACHE_HOST }} + MOUNTPOINT: ${{ github.workspace}}/nfs/yocto run: | - sudo mount -t nfs "${YOCTO_CACHE_HOST}:/" "${YOCTO_CACHE_DIR}" -o fsc,nolock - ls -al "${YOCTO_CACHE_DIR}/$(whoami)" + sudo mkdir -p "${MOUNTPOINT}" + sudo chown -R "$(id -u):$(id -g)" "${MOUNTPOINT}" + sudo mount -t nfs "${YOCTO_CACHE_HOST}:/" "${MOUNTPOINT}" -o fsc,nolock + + # https://wiki.yoctoproject.org/wiki/Enable_sstate_cache + # https://docs.yoctoproject.org/4.0.10/ref-manual/variables.html#term-MIRRORS + # https://docs.yoctoproject.org/4.0.10/ref-manual/variables.html#term-PREMIRRORS + # https://docs.yoctoproject.org/4.0.10/ref-manual/variables.html#term-SSTATE_MIRRORS + # https://docs.yoctoproject.org/4.0.10/overview-manual/concepts.html#source-mirror-s + # https://docs.yoctoproject.org/4.0.10/ref-manual/classes.html?highlight=source_mirror#own-mirrors-bbclass + # https://github.com/openembedded/openembedded/blob/master/classes/own-mirrors.bbclass + # https://github.com/openembedded/openembedded/blob/master/classes/mirrors.bbclass + - name: Add NFS shared-downloads to PREMIRRORS + if: steps.jenkins-nfs.outcome == 'success' + env: + # Relative to the build container working dir, not the workspace + SOURCE_MIRROR_URL: file:///work/nfs/yocto/runner/shared-downloads/ + SSTATE_MIRROR_URL: file:///work/nfs/yocto/runner/${{ inputs.machine }}/sstate/PATH + run: | + mkdir -p "$(dirname "${AUTO_CONF_FILE}")" + cat <> "${AUTO_CONF_FILE}" + + PREMIRRORS:prepend = "\\ + cvs://.*/.* ${SOURCE_MIRROR_URL} \\ + svn://.*/.* ${SOURCE_MIRROR_URL} \\ + git://.*/.* ${SOURCE_MIRROR_URL} \\ + hg://.*/.* ${SOURCE_MIRROR_URL} \\ + bzr://.*/.* ${SOURCE_MIRROR_URL} \\ + https?$://.*/.* ${SOURCE_MIRROR_URL} \\ + ftp://.*/.* ${SOURCE_MIRROR_URL} \\ + " + + EOF + cat "${AUTO_CONF_FILE}" # All preperation complete before this step # Start building balenaOS @@ -466,6 +496,7 @@ jobs: id: build env: HELPER_IMAGE_REPO: ghcr.io/balena-os/balena-yocto-scripts + SHARED_BUILD_DIR: ${{ github.workspace }}/shared run: | # When building for non-x86 device types, meson, after building binaries must try to run them via qemu if possible , maybe as some sanity check or test? # Therefore qemu must be used - and our runner mmap_min_addr is set to 4096 (default, set here: https://github.com/product-os/github-runner-kernel/blob/ef5a66951599dc64bf2920d896c36c6d9eda8df6/config/5.10/microvm-kernel-x86_64-5.10.config#L858 @@ -475,10 +506,14 @@ jobs: sudo sysctl -w vm.mmap_min_addr=65536 sysctl vm.mmap_min_addr + mkdir -p "${SHARED_BUILD_DIR}" + + cat "${AUTO_CONF_FILE}" + ./balena-yocto-scripts/build/balena-build.sh \ -d "${MACHINE}" \ -t "${{ secrets.BALENA_API_DEPLOY_KEY }}" \ - -s "${YOCTO_CACHE_DIR}/$(whoami)" \ + -s "${SHARED_BUILD_DIR}" \ -g "${BARYS_ARGUMENTS_VAR}" if grep -R "ERROR: " build/tmp/log/*; then From 25aa1b959f251a35733465cfffae14fc403ea21f Mon Sep 17 00:00:00 2001 From: Kyle Harding Date: Mon, 4 Nov 2024 09:27:31 -0500 Subject: [PATCH 3/5] Sync the shared sstate to github actions cache This allows the per-device sstate to be ephemeral and best effort, with automatic size and expiry limits enforced by GitHub and scoped to the repository and branch. Change-type: patch Signed-off-by: Kyle Harding --- .github/workflows/yocto-build-deploy.yml | 36 ++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/.github/workflows/yocto-build-deploy.yml b/.github/workflows/yocto-build-deploy.yml index 540dd694e..5915e0ed8 100644 --- a/.github/workflows/yocto-build-deploy.yml +++ b/.github/workflows/yocto-build-deploy.yml @@ -488,6 +488,23 @@ jobs: EOF cat "${AUTO_CONF_FILE}" + # https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows + # https://github.com/actions/cache/blob/main/README.md#creating-a-cache-key + # https://github.com/actions/cache + # https://github.com/actions/cache/blob/main/restore/README.md + # Caches are scoped to the current branch context, with fallback to the default branch context. + # GitHub will remove any cache entries that have not been accessed in over 7 days. + # There is no limit on the number of caches you can store, but the total size of all caches in a repository is limited to 10 GB. + # Once a repository has reached its maximum cache storage, the cache eviction policy will create space by deleting the oldest caches in the repository. + - name: Restore sstate cache + id: cache-restore + uses: actions/cache/restore@v4.0.2 + with: + path: ${{ github.workspace }}/shared/${{ inputs.machine }}/sstate + key: ${{ inputs.machine }}-sstate-${{ github.sha }} + restore-keys: | + ${{ inputs.machine }}-sstate- + # All preperation complete before this step # Start building balenaOS # We use the BALENA_API_DEPLOY_KEY secret to preload the supervisor image @@ -520,6 +537,25 @@ jobs: exit 1 fi + # https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows + # https://github.com/actions/cache/blob/main/README.md#creating-a-cache-key + # https://github.com/actions/cache + # https://github.com/actions/cache/blob/main/save/README.md + # Caches are scoped to the current branch context, with fallback to the default branch context. + # GitHub will remove any cache entries that have not been accessed in over 7 days. + # There is no limit on the number of caches you can store, but the total size of all caches in a repository is limited to 10 GB. + # Once a repository has reached its maximum cache storage, the cache eviction policy will create space by deleting the oldest caches in the repository. + - name: Save sstate cache + uses: actions/cache/save@v4.0.2 + # Do not save cache for pull_request_target events + # as they run in the context of the main branch and would be vulnerable to cache poisoning + # https://0xn3va.gitbook.io/cheat-sheets/ci-cd/github/actions#cache-poisoning + # https://adnanthekhan.com/2024/05/06/the-monsters-in-your-build-cache-github-actions-cache-poisoning/ + if: github.event_name != 'pull_request_target' + with: + path: ${{ github.workspace }}/shared/${{ inputs.machine }}/sstate + key: ${{ steps.cache-restore.outputs.cache-primary-key }} + # TODO: pre-install on self-hosted-runners # Needed by the yocto job to zip artifacts - Don't remove - name: Install zip package From 4a8f644ba7afeb8770a0a381ce8cd96e4e29f93e Mon Sep 17 00:00:00 2001 From: Kyle Harding Date: Mon, 4 Nov 2024 09:56:59 -0500 Subject: [PATCH 4/5] Sync shared downloads to S3 to use as a sources mirror Change-type: minor Signed-off-by: Kyle Harding --- .github/workflows/yocto-build-deploy.yml | 75 ++++++++++++++++++------ 1 file changed, 57 insertions(+), 18 deletions(-) diff --git a/.github/workflows/yocto-build-deploy.yml b/.github/workflows/yocto-build-deploy.yml index 5915e0ed8..4c675b724 100644 --- a/.github/workflows/yocto-build-deploy.yml +++ b/.github/workflows/yocto-build-deploy.yml @@ -217,7 +217,7 @@ jobs: timeout-minutes: 90 uses: product-os/review-commit-action@cddebf4cec8e40ea8f698b6dcce8cd70e38b7320 # v0.1.7 with: - poll-interval: '10' + poll-interval: "10" allow-authors: false # this must be done before putting files in the workspace @@ -456,7 +456,7 @@ jobs: sudo mkdir -p "${MOUNTPOINT}" sudo chown -R "$(id -u):$(id -g)" "${MOUNTPOINT}" sudo mount -t nfs "${YOCTO_CACHE_HOST}:/" "${MOUNTPOINT}" -o fsc,nolock - + # https://wiki.yoctoproject.org/wiki/Enable_sstate_cache # https://docs.yoctoproject.org/4.0.10/ref-manual/variables.html#term-MIRRORS # https://docs.yoctoproject.org/4.0.10/ref-manual/variables.html#term-PREMIRRORS @@ -488,6 +488,23 @@ jobs: EOF cat "${AUTO_CONF_FILE}" + # https://docs.yoctoproject.org/4.0.10/ref-manual/classes.html?highlight=source_mirror#own-mirrors-bbclass + # https://github.com/openembedded/openembedded/blob/master/classes/own-mirrors.bbclass + # The own-mirrors class makes it easier to set up your own PREMIRRORS from which to first fetch source before + # attempting to fetch it from the upstream specified in SRC_URI within each recipe. + - name: Add S3 shared-downloads to PREMIRRORS + env: + SOURCE_MIRROR_URL: https://${{ vars.AWS_S3_BUCKET || vars.S3_BUCKET }}.s3.${{ vars.AWS_REGION || 'us-east-1' }}.amazonaws.com/shared-downloads/ + run: | + mkdir -p "$(dirname "${AUTO_CONF_FILE}")" + cat <> "${AUTO_CONF_FILE}" + + INHERIT += "own-mirrors" + SOURCE_MIRROR_URL = "${SOURCE_MIRROR_URL}" + + EOF + cat "${AUTO_CONF_FILE}" + # https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows # https://github.com/actions/cache/blob/main/README.md#creating-a-cache-key # https://github.com/actions/cache @@ -556,6 +573,44 @@ jobs: path: ${{ github.workspace }}/shared/${{ inputs.machine }}/sstate key: ${{ steps.cache-restore.outputs.cache-primary-key }} + # https://github.com/unfor19/install-aws-cli-action + - name: Setup awscli + uses: unfor19/install-aws-cli-action@e8b481e524a99f37fbd39fdc1dcb3341ab091367 # v1 + + # https://github.com/aws-actions/configure-aws-credentials + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 + with: + role-to-assume: ${{ vars.AWS_IAM_ROLE }} + role-session-name: github-${{ github.job }}-${{ github.run_id }}-${{ github.run_attempt }} + aws-region: ${{ vars.AWS_REGION || 'us-east-1' }} + # https://github.com/orgs/community/discussions/26636#discussioncomment-3252664 + mask-aws-account-id: false + + # Sync shared downloads to S3 to use as a sources mirror in case original sources are not available. + # Exlude all directories and temp files as we only want the content and the .done files. + # https://awscli.amazonaws.com/v2/documentation/api/latest/reference/s3/sync.html + - name: Sync shared downloads to S3 + # Do not publish shared downloads for pull_request_target events to prevent cache poisoning + # Do not publish shared downloads for private device-types as the mirror is public-read + if: github.event_name != 'pull_request_target' && steps.balena-lib.outputs.is_private == 'false' + # Ignore errors for now, as we may have upload conflicts with other jobs + continue-on-error: true + env: + SHARED_DOWNLOADS_DIR: ${{ github.workspace }}/shared/shared-downloads + S3_ACL: public-read + S3_SSE: AES256 + # FIXME: This should be a public bucket that does not differ between production and staging deploys + S3_URL: "s3://${{ vars.AWS_S3_BUCKET || vars.S3_BUCKET }}/shared-downloads" + S3_REGION: ${{ vars.AWS_REGION || 'us-east-1' }} + # Create a symlink to the from the relative container path to the workspace in order to resolve symlinks + # created in the build container runtime. + run: | + sudo ln -sf "${{ github.workspace }}" /work + ls -al "${SHARED_DOWNLOADS_DIR}/" + aws s3 sync --sse="${S3_SSE}" --acl="${S3_ACL}" "${SHARED_DOWNLOADS_DIR}/" "${S3_URL}/" \ + --exclude "*/*" --exclude "*.tmp" --size-only --follow-symlinks --no-progress + # TODO: pre-install on self-hosted-runners # Needed by the yocto job to zip artifacts - Don't remove - name: Install zip package @@ -675,22 +730,6 @@ jobs: if: steps.should-deploy.outputs.deploy && steps.esr-check.outputs.is-esr run: echo "string=esr-images" >>"${GITHUB_OUTPUT}" - # https://github.com/unfor19/install-aws-cli-action - - name: Setup awscli - if: steps.should-deploy.outputs.deploy - uses: unfor19/install-aws-cli-action@e8b481e524a99f37fbd39fdc1dcb3341ab091367 # v1 - - # # https://github.com/aws-actions/configure-aws-credentials - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 - if: steps.should-deploy.outputs.deploy - with: - role-to-assume: ${{ vars.AWS_IAM_ROLE }} - role-session-name: github-${{ github.job }}-${{ github.run_id }}-${{ github.run_attempt }} - aws-region: ${{ vars.AWS_REGION || 'us-east-1' }} - # https://github.com/orgs/community/discussions/26636#discussioncomment-3252664 - mask-aws-account-id: false - # "If no keys are provided, but an IAM role is associated with the EC2 instance, it will be used transparently". # https://awscli.amazonaws.com/v2/documentation/api/latest/reference/s3/rm.html # https://awscli.amazonaws.com/v2/documentation/api/latest/reference/s3/cp.html From 826c5d9662085dbf6b95bb7702d64d6b9a44059c Mon Sep 17 00:00:00 2001 From: Kyle Harding Date: Tue, 5 Nov 2024 10:22:06 -0500 Subject: [PATCH 5/5] Add improved success check to build step Signed-off-by: Kyle Harding --- .github/workflows/yocto-build-deploy.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/yocto-build-deploy.yml b/.github/workflows/yocto-build-deploy.yml index 4c675b724..696870d99 100644 --- a/.github/workflows/yocto-build-deploy.yml +++ b/.github/workflows/yocto-build-deploy.yml @@ -548,12 +548,16 @@ jobs: -d "${MACHINE}" \ -t "${{ secrets.BALENA_API_DEPLOY_KEY }}" \ -s "${SHARED_BUILD_DIR}" \ - -g "${BARYS_ARGUMENTS_VAR}" + -g "${BARYS_ARGUMENTS_VAR}" | tee balena-build.log if grep -R "ERROR: " build/tmp/log/*; then exit 1 fi + if ! grep -q "Build for ${{ inputs.machine }} suceeded" balena-build.log; then + exit 1 + fi + # https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows # https://github.com/actions/cache/blob/main/README.md#creating-a-cache-key # https://github.com/actions/cache