Sync shared downloads to S3 for download mirroring

Change-type: minor
Signed-off-by: Kyle Harding <[email protected]>

.github/workflows/yocto-build-deploy.yml

@@ -494,6 +494,25 @@ jobs:
           EOF
           cat "${AUTO_CONF_FILE}"
 
+      # https://docs.yoctoproject.org/4.0.10/ref-manual/classes.html?highlight=source_mirror#own-mirrors-bbclass
+      # https://github.com/openembedded/openembedded/blob/master/classes/own-mirrors.bbclass
+      # The own-mirrors class makes it easier to set up your own PREMIRRORS from which to first fetch source before
+      # attempting to fetch it from the upstream specified in SRC_URI within each recipe.
+      - name: Add S3 shared-downloads to PREMIRRORS
+        env:
+          SOURCE_MIRROR_URL: https://${{ vars.AWS_S3_BUCKET || vars.S3_BUCKET }}.s3.${{ vars.AWS_REGION || 'us-east-1' }}.amazonaws.com/shared-downloads/
+        run: |
+          mkdir -p "$(dirname "${AUTO_CONF_FILE}")"
+          cat <<EOF >> "${AUTO_CONF_FILE}"
+
+          INHERIT += "own-mirrors"
+          SOURCE_MIRROR_URL = "${SOURCE_MIRROR_URL}"
+
+          # BB_FETCH_PREMIRRORONLY = "1"
+
+          EOF
+          cat "${AUTO_CONF_FILE}"
+
       # https://github.com/actions/cache
       - name: Sync sstate cache
         uses: actions/[email protected]
@@ -537,6 +556,37 @@ jobs:
             exit 1
           fi
 
+      # https://github.com/unfor19/install-aws-cli-action
+      - name: Setup awscli
+        uses: unfor19/install-aws-cli-action@e8b481e524a99f37fbd39fdc1dcb3341ab091367 # v1
+
+      # https://github.com/aws-actions/configure-aws-credentials
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
+        with:
+          role-to-assume: ${{ vars.AWS_IAM_ROLE }}
+          role-session-name: github-${{ github.job }}-${{ github.run_id }}-${{ github.run_attempt }}
+          aws-region: ${{ vars.AWS_REGION || 'us-east-1' }}
+          # https://github.com/orgs/community/discussions/26636#discussioncomment-3252664
+          mask-aws-account-id: false
+
+      # If the device-type is public sync the shared downloads to S3 for use as a yocto mirror.
+      # Exlude all directories as we only want the top level files.
+      # https://awscli.amazonaws.com/v2/documentation/api/latest/reference/s3/sync.html
+      - name: Sync shared downloads to S3
+        if: steps.balena-lib.outputs.is_private == 'false'
+        # Ignore errors for now, as we may have upload conflicts with other jobs
+        continue-on-error: true
+        env:
+          SHARED_DOWNLOADS_DIR: ${{ github.workspace }}/shared/shared-downloads
+          S3_ACL: public-read
+          S3_SSE: AES256
+          # FIXME: This should be a public bucket that does not differ between production and staging deploys
+          S3_URL: "s3://${{ vars.AWS_S3_BUCKET || vars.S3_BUCKET }}/shared-downloads"
+          S3_REGION: ${{ vars.AWS_REGION || 'us-east-1' }}
+        run:
+          aws s3 sync --no-progress --sse="${S3_SSE}" --acl="${S3_ACL}" "${SHARED_DOWNLOADS_DIR}/" "${S3_URL}/" --exclude "*/*"
+
       # TODO: pre-install on self-hosted-runners
       # Needed by the yocto job to zip artifacts - Don't remove
       - name: Install zip package
@@ -656,22 +706,6 @@ jobs:
         if: steps.should-deploy.outputs.deploy && steps.esr-check.outputs.is-esr
         run: echo "string=esr-images" >>"${GITHUB_OUTPUT}"
 
-      # https://github.com/unfor19/install-aws-cli-action
-      - name: Setup awscli
-        if: steps.should-deploy.outputs.deploy
-        uses: unfor19/install-aws-cli-action@e8b481e524a99f37fbd39fdc1dcb3341ab091367 # v1
-
-      # # https://github.com/aws-actions/configure-aws-credentials
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
-        if: steps.should-deploy.outputs.deploy
-        with:
-          role-to-assume: ${{ vars.AWS_IAM_ROLE }}
-          role-session-name: github-${{ github.job }}-${{ github.run_id }}-${{ github.run_attempt }}
-          aws-region: ${{ vars.AWS_REGION || 'us-east-1' }}
-          # https://github.com/orgs/community/discussions/26636#discussioncomment-3252664
-          mask-aws-account-id: false
-
       # "If no keys are provided, but an IAM role is associated with the EC2 instance, it will be used transparently".
       # https://awscli.amazonaws.com/v2/documentation/api/latest/reference/s3/rm.html
       # https://awscli.amazonaws.com/v2/documentation/api/latest/reference/s3/cp.html