Add support for "external pins" in "diff-pr.sh"

This uses `crane` to download some of the JSON files of the remote images to give us at least *some* amount of diffing between updates to external pins.
vakarisbk · Jan 13, 2023 · cdc2c8a · cdc2c8a
1 parent aad0efa
commit cdc2c8a
Show file tree

Hide file tree

Showing 3 changed files with 77 additions and 23 deletions.
diff --git a/.github/workflows/munge-pr.yml b/.github/workflows/munge-pr.yml
@@ -31,6 +31,15 @@ jobs:
         run: |
           (set +x; echo "::stop-commands::$(echo -n ${{ github.token }} | sha256sum | head -c 64)")
           git fetch --quiet https://github.com/docker-library/official-images.git master
+          externalPins="$(git diff --no-renames --name-only FETCH_HEAD...HEAD -- '.external-pins/*/**')"
+          externalPinTags="$(
+            if [ -n "$externalPins" ]; then
+              # doing backflips to run "tag.sh" from master instead of from the PR
+              git show FETCH_HEAD:.external-pins/tag.sh > ~/master-external-pins-tag.sh
+              chmod +x ~/master-external-pins-tag.sh
+              ~/master-external-pins-tag.sh $externalPins
+            fi
+          )"
           images="$(git diff --no-renames --name-only FETCH_HEAD...HEAD -- library/)"
           if [ -n "$images" ]; then
             new="$(git diff --no-renames --name-only --diff-filter=A FETCH_HEAD...HEAD -- $images)"
@@ -39,40 +48,49 @@ jobs:
             new=
             deleted=
           fi
-          export images new deleted
+          export images new deleted externalPins externalPinTags
           images="$(jq -cn '
             (env.images | rtrimstr("\n") | split("\n")) as $images
             | (env.new | rtrimstr("\n") | split("\n")) as $new
             | (env.deleted | rtrimstr("\n") | split("\n")) as $deleted
+            | (env.externalPins | rtrimstr("\n") | split("\n")) as $externalPins
+            | (env.externalPinTags | rtrimstr("\n") | split("\n")) as $externalPinTags
             | {
               images: $images,
               count: ($images | length),
               new: $new,
               deleted: $deleted,
+              externalPins: $externalPins,
+              externalPinTags: $externalPinTags,
+              externalPinsCount: ($externalPins | length),
             }
+            | .imagesAndExternalPinsCount = (.count + .externalPinsCount) # man, I *really* do not love GitHub Actions expressions...
           ')"
           jq . <<<"$images"
           set +x
           echo "::$(echo -n ${{ github.token }} | sha256sum | head -c 64)::"
-          echo "::set-output name=images::$images"
+          echo "images=$images" >> "$GITHUB_OUTPUT"
     outputs:
       images: '${{ steps.gather.outputs.images }}'
 
   apply-labels:
     name: Apply Labels
     runs-on: ubuntu-latest
     needs: gather
-    if: fromJSON(needs.gather.outputs.images).count > 0
+    if: fromJSON(needs.gather.outputs.images).imagesAndExternalPinsCount > 0
     steps:
       - name: Apply Labels
-        uses: actions/github-script@v5
+        uses: actions/github-script@v6
         env:
           IMAGES: ${{ needs.gather.outputs.images }}
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
             const data = JSON.parse(process.env.IMAGES);
-            var labels = data.images;
+            var labels = [
+              ...data.images, // "library/debian", ...
+              ...new Set(data.externalPinTags.map(x => 'external/' + x.replace(/:.+$/, ''))), // "external/mcr.microsoft.com/windows/servercore", ...
+            ];
             if (data.new.length > 0) {
               labels.push('new-image');
             }
@@ -88,7 +106,7 @@ jobs:
     name: Diff Comment
     runs-on: ubuntu-latest
     needs: gather
-    if: fromJSON(needs.gather.outputs.images).count > 0
+    if: fromJSON(needs.gather.outputs.images).imagesAndExternalPinsCount > 0
     steps:
       - uses: actions/checkout@v3
         with:
@@ -120,7 +138,7 @@ jobs:
         run: |
           docker run --rm --read-only --tmpfs /tmp oisupport/bashbrew:diff-pr ./diff-pr.sh "$GITHUB_PR_NUMBER" | tee "$GITHUB_WORKSPACE/oi-pr.diff"
       - name: Comment
-        uses: actions/github-script@v5
+        uses: actions/github-script@v6
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |

diff --git a/Dockerfile b/Dockerfile
@@ -13,11 +13,18 @@ RUN set -eux; \
 		gawk \
 # tar -tf in diff-pr.sh
 		bzip2 \
+# jq for diff-pr.sh
+		jq \
 	; \
 	rm -rf /var/lib/apt/lists/*
 
 ENV DIR /usr/src/official-images
 ENV BASHBREW_LIBRARY $DIR/library
 
+# crane for diff-pr.sh
+# https://gcr.io/go-containerregistry/crane:latest
+# https://explore.ggcr.dev/?image=gcr.io/go-containerregistry/crane:latest
+COPY --from=gcr.io/go-containerregistry/crane@sha256:d0e5cc313e7388a573bb4cfb980a935bb740c5787df7d90f7066b8e8146455ed /ko-app/crane /usr/local/bin/
+
 WORKDIR $DIR
 COPY . $DIR
diff --git a/diff-pr.sh b/diff-pr.sh
@@ -79,10 +79,15 @@ else
 		"$diffDir" HEAD:refs/heads/pull
 fi
 
+externalPins=
 if [ "$#" -eq 0 ]; then
+	externalPins="$(git -C oi/.external-pins diff --no-renames --name-only HEAD...pull -- '*/**')"
+
 	images="$(git -C oi/library diff --no-renames --name-only HEAD...pull -- .)"
-	[ -n "$images" ] || exit 0
-	images="$(xargs -n1 basename <<<"$images")"
+	if [ -z "$images" ] && [ -z "$externalPins" ]; then
+		exit 0
+	fi
+	images="$(xargs -rn1 basename <<<"$images")"
 	set -- $images
 fi
 
@@ -261,21 +266,45 @@ templateLastTags='
 '
 
 _metadata-files() {
-	bashbrew list "$@" 2>>temp/_bashbrew.err | sort -uV > temp/_bashbrew-list || :
-
-	"$diffDir/_bashbrew-cat-sorted.sh" "$@" 2>>temp/_bashbrew.err > temp/_bashbrew-cat || :
-
-	bashbrew list --uniq "$@" \
-		| sort -V \
-		| xargs -r bashbrew list --uniq --build-order 2>>temp/_bashbrew.err \
-		| xargs -r bashbrew cat --format "$templateLastTags" 2>>temp/_bashbrew.err \
-		> temp/_bashbrew-list-build-order || :
+	if [ "$#" -gt 0 ]; then
+		bashbrew list "$@" 2>>temp/_bashbrew.err | sort -uV > temp/_bashbrew-list || :
+
+		"$diffDir/_bashbrew-cat-sorted.sh" "$@" 2>>temp/_bashbrew.err > temp/_bashbrew-cat || :
+
+		bashbrew list --uniq "$@" \
+			| sort -V \
+			| xargs -r bashbrew list --uniq --build-order 2>>temp/_bashbrew.err \
+			| xargs -r bashbrew cat --format "$templateLastTags" 2>>temp/_bashbrew.err \
+			> temp/_bashbrew-list-build-order || :
+
+		script="$(bashbrew cat --format "$template" "$@")"
+		mkdir tar
+		( eval "$script" | tar -xiC tar )
+		copy-tar tar temp
+		rm -rf tar
+	fi
 
-	script="$(bashbrew cat --format "$template" "$@")"
-	mkdir tar
-	( eval "$script" | tar -xiC tar )
-	copy-tar tar temp
-	rm -rf tar
+	if [ -n "$externalPins" ] && command -v crane &> /dev/null && command -v jq &> /dev/null; then
+		local file
+		for file in $externalPins; do
+			[ -e "oi/$file" ] || continue
+			local pin digest dir
+			pin="$("$diffDir/.external-pins/tag.sh" "$file")"
+			digest="$(< "oi/$file")"
+			dir="temp/$file"
+			mkdir -p "$dir"
+			bashbrew remote arches --json "$pin@$digest" | jq -S . > "$dir/bashbrew.json"
+			local manifests manifest
+			manifests="$(jq -r '[ .arches[][].digest | @sh ] | join(" ")' "$dir/bashbrew.json")"
+			eval "manifests=( $manifests )"
+			for manifest in "${manifests[@]}"; do
+				crane manifest "$pin@$manifest" | jq -S . > "$dir/manifest-${manifest//:/_}.json"
+				local config
+				config="$(jq -r '.config.digest' "$dir/manifest-${manifest//:/_}.json")"
+				crane blob "$pin@$config" | jq -S . > "$dir/manifest-${manifest//:/_}-config.json"
+			done
+		done
+	fi
 }
 
 mkdir temp