diff --git a/.github/workflows/.bashbrew/action.yml b/.github/workflows/.bashbrew/action.yml new file mode 100644 index 0000000000000..09950a24d6eb1 --- /dev/null +++ b/.github/workflows/.bashbrew/action.yml @@ -0,0 +1,19 @@ +# https://github.com/docker-library/official-images/pull/13556#issuecomment-1319181339 🙈 +name: 'Shared Bashbrew Action' +description: 'Install the same version of Bashbrew consistently in all other GitHub Actions' +inputs: + build: + default: 'host' # or 'docker' or 'none' +runs: + using: 'composite' + steps: + + # these two version numbers are intentionally as close together as I could possibly get them because no matter what I tried, GitHub will not allow me to DRY them (can't have any useful variables in `uses:` and can't even have YAML references to steal it in `env:` or something) + - shell: 'bash -Eeuo pipefail -x {0}' + run: echo BASHBREW_VERSION=v0.1.5 >> "$GITHUB_ENV" + - uses: docker-library/bashbrew@v0.1.5 + if: inputs.build == 'host' + + - run: docker build --pull --tag oisupport/bashbrew:base "https://github.com/docker-library/bashbrew.git#$BASHBREW_VERSION" + shell: 'bash -Eeuo pipefail -x {0}' + if: inputs.build == 'docker' diff --git a/.github/workflows/generate.sh b/.github/workflows/generate.sh index 789fa092b59d9..2c9a203088c2e 100755 --- a/.github/workflows/generate.sh +++ b/.github/workflows/generate.sh @@ -5,14 +5,8 @@ set -Eeuo pipefail # NOTE: this is *not* a good example for integrating these tests into your own repository! # If you want that, check out https://github.com/docker-library/golang/blob/3f2c52653043f067156ce4f41182c2a758c4c857/.github/workflows/ci.yml instead. # - -bashbrewDir="$1"; shift -dir="$(dirname "$BASH_SOURCE")" # ./.github/workflows -oiDir="$(dirname "$dir")" # ./.github -oiDir="$(dirname "$oiDir")" # ./ -bashbrewVersion="$(< "$oiDir/bashbrew-version")" -export bashbrewVersion +[ -d "$BASHBREW_SCRIPTS/github-actions" ] if [ "$#" -eq 0 ]; then git fetch --quiet https://github.com/docker-library/official-images.git master @@ -23,18 +17,17 @@ fi strategy='{}' for repo; do - newStrategy="$(GITHUB_REPOSITORY="$repo" GENERATE_STACKBREW_LIBRARY='cat "library/$GITHUB_REPOSITORY"' "$bashbrewDir/scripts/github-actions/generate.sh")" + newStrategy="$(GITHUB_REPOSITORY="$repo" GENERATE_STACKBREW_LIBRARY='cat "library/$GITHUB_REPOSITORY"' "$BASHBREW_SCRIPTS/github-actions/generate.sh")" newStrategy="$(jq -c --arg repo "$repo" '.matrix.include = [ .matrix.include[] | ([ .meta.entries[].tags[0] ]) as $tags | .name = ($tags | join(", ")) # replace "build" steps with something that uses "bashbrew" instead of "docker build" - # https://github.com/docker-library/bashbrew/blob/a40a54d4d81b9fd2e39b4d7ba3fe203e8b022a67/scripts/github-actions/generate.sh#L74-L93 - | .runs.prepare += "\ngit clone --depth 1 https://github.com/docker-library/bashbrew.git -b " + ("v" + env.bashbrewVersion | @sh) + " ~/bashbrew\n~/bashbrew/bashbrew.sh --version" + # https://github.com/docker-library/bashbrew/blob/20b5a50a4eafee1e92fadca5f9cbbce6b16d80b1/scripts/github-actions/generate.sh#L79-L105 | .runs.build = ( (if .os | startswith("windows-") then "export BASHBREW_ARCH=windows-amd64 BASHBREW_CONSTRAINTS=" + ([ .meta.entries[].constraints[] ] | join(", ") | @sh) + "\n" else "" end) + "export BASHBREW_LIBRARY=\"$PWD/library\"\n" - + ([ $tags[] | "~/bashbrew/bashbrew.sh build " + @sh ] | join("\n")) + + ([ $tags[] | "bashbrew build " + @sh ] | join("\n")) ) # use our local clone of official-images for running tests (so test changes can be tested too, if they live in the PR with the image change) # https://github.com/docker-library/bashbrew/blob/a40a54d4d81b9fd2e39b4d7ba3fe203e8b022a67/scripts/github-actions/generate.sh#L95 diff --git a/.github/workflows/munge-pr.yml b/.github/workflows/munge-pr.yml index a8e449b39ee35..b59409c0f3a85 100644 --- a/.github/workflows/munge-pr.yml +++ b/.github/workflows/munge-pr.yml @@ -21,7 +21,7 @@ jobs: name: Gather Metadata runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: # ideally this would be "github.event.pull_request.merge_commit_sha" but according to https://docs.github.com/en/free-pro-team@latest/rest/reference/pulls#get-a-pull-request if "mergeable" is null (meaning there's a background job in-progress to check mergeability), that value is undefined... ref: ${{ github.event.pull_request.head.sha }} @@ -90,16 +90,17 @@ jobs: needs: gather if: fromJSON(needs.gather.outputs.images).count > 0 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: # again, this would ideally be "github.event.pull_request.merge_commit_sha" but we might not have that yet when this runs, so we compromise by checkout out the latest code from the target branch (so we get the latest "diff-pr.sh" script to run) ref: ${{ github.event.pull_request.base.ref }} fetch-depth: 0 + - uses: ./.github/workflows/.bashbrew + with: + build: 'docker' - name: Prepare Environment run: | - # this mimics "test-pr.sh", but without running repo-local scripts (to avoid CVE-2020-15228 via the scripts being updated to write nasty things to $GITHUB_ENV) - bashbrewVersion="$(< bashbrew-version)" - docker build --pull --tag oisupport/bashbrew:base "https://github.com/docker-library/bashbrew.git#v$bashbrewVersion" + # this avoids running repo-local scripts (to avoid CVE-2020-15228 via the scripts being updated to write nasty things to $GITHUB_ENV) docker build --tag oisupport/bashbrew:diff-pr . - name: Gather Maintainers env: diff --git a/.github/workflows/test-pr.yml b/.github/workflows/test-pr.yml index dc6c7b25a316a..dbba8b9dd526c 100644 --- a/.github/workflows/test-pr.yml +++ b/.github/workflows/test-pr.yml @@ -25,17 +25,12 @@ jobs: name: Naughty runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 + - uses: ./.github/workflows/.bashbrew - name: Check for Common Issues - run: | - bashbrewVersion="$(< bashbrew-version)" - git clone --depth 1 https://github.com/docker-library/bashbrew.git -b "v$bashbrewVersion" ~/bashbrew - ~/bashbrew/bashbrew.sh --version > /dev/null - export PATH="$HOME/bashbrew/bin:$PATH" - bashbrew --version - .github/workflows/naughty.sh + run: .github/workflows/naughty.sh generate-jobs: name: Generate Jobs @@ -44,19 +39,18 @@ jobs: strategy: ${{ steps.generate-jobs.outputs.strategy }} length: ${{ steps.generate-jobs.outputs.length }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 + - uses: ./.github/workflows/.bashbrew - id: generate-jobs name: Generate Jobs run: | - bashbrewVersion="$(< bashbrew-version)" - git clone --depth 1 https://github.com/docker-library/bashbrew.git -b "v$bashbrewVersion" ~/bashbrew - strategy="$(.github/workflows/generate.sh ~/bashbrew)" + strategy="$(.github/workflows/generate.sh)" + echo "strategy=$strategy" >> "$GITHUB_OUTPUT" jq . <<<"$strategy" # sanity check / debugging aid - echo "::set-output name=strategy::$strategy" length="$(jq <<<"$strategy" -r '.matrix.include | length')" - echo "::set-output name=length::$length" + echo "length=$length" >> "$GITHUB_OUTPUT" test: needs: generate-jobs @@ -73,7 +67,8 @@ jobs: echo 'MSYS=winsymlinks:nativestrict' >> "$GITHUB_ENV" # https://github.com/docker-library/bashbrew/blob/a40a54d4d81b9fd2e39b4d7ba3fe203e8b022a67/scripts/github-actions/generate.sh#L146-L149 if: runner.os == 'Windows' - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - uses: ./.github/workflows/.bashbrew - name: Prepare Environment run: ${{ matrix.runs.prepare }} - name: Pull Dependencies diff --git a/Dockerfile b/Dockerfile index dfc2afd5e60c0..44f0ad5fc9e29 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# FYI, this base image is built via test-pr.sh (from https://github.com/docker-library/bashbrew/tree/master/Dockerfile) +# FYI, this base image is built via ".github/workflows/.bashbrew/action.yml" (from https://github.com/docker-library/bashbrew/tree/master/Dockerfile) FROM oisupport/bashbrew:base RUN set -eux; \ diff --git a/bashbrew-version b/bashbrew-version deleted file mode 100644 index 845639eef26c0..0000000000000 --- a/bashbrew-version +++ /dev/null @@ -1 +0,0 @@ -0.1.4 diff --git a/pr-urls.sh b/pr-urls.sh new file mode 100755 index 0000000000000..7d050d32fe5bd --- /dev/null +++ b/pr-urls.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# +# usage: +# $ ./pr-urls.sh PR-NUMBER +# $ ./pr-urls.sh PR-NUMBER IMAGE1 IMAGE2:TAG1 IMAGE3:TAG2 +# +# $ ./pr-urls.sh 12072 +# $ ./pr-urls.sh 12072 hello-world:linux +# $ ./pr-urls.sh 12072 | xargs -rt bashbrew build +# $ ./pr-urls.sh 12072 | xargs -rt bashbrew list --uniq +# $ ./pr-urls.sh 12072 | xargs -rt bashbrew list --uniq | xargs -rt ./test/run.sh +# +# (rough replacement for the old "test-pr.sh" script and its associated complexity) +# + +pr="$1" +shift + +patch="$(wget -qO- "https://github.com/docker-library/official-images/pull/$pr.patch")" + +commit="$(grep <<<"$patch" -oE '^From [0-9a-f]+ ' | tail -1 | cut -d' ' -f2)" + +if [ "$#" -eq 0 ]; then + files="$(grep <<<"$patch" -oE '^[+]{3} b/library/.+' | cut -d/ -f3 | sort -u)" + set -- $files +fi + +for file; do + echo "https://github.com/docker-library/official-images/raw/$commit/library/$file" +done diff --git a/test-pr.sh b/test-pr.sh deleted file mode 100755 index b1909ce1c0666..0000000000000 --- a/test-pr.sh +++ /dev/null @@ -1,217 +0,0 @@ -#!/bin/bash -set -eo pipefail - -# TODO something clever with this pattern to get the exact list of _tags_ which have changed, not just repos: -#format='{{ range .Entries }}{{ join " " (join ":" $.RepoName (.Tags | first)) .GitRepo .GitFetch .GitCommit .Directory }}{{ "\n" }}{{ end }}' -#comm -13 \ -# <(bashbrew cat -f "$format" https://github.com/docker-library/official-images/raw/master/library/docker | sort) \ -# <(bashbrew cat -f "$format" https://raw.githubusercontent.com/infosiftr/stackbrew/d92ffa4b5f8a558c22c5d0a7e0f33bff8fae990b/library/docker | sort) \ -# | cut -d' ' -f1 - -# make sure we can GTFO -trap 'echo >&2 Ctrl+C captured, exiting; exit 1' SIGINT - -# start with an error if Docker isn't working... -docker version > /dev/null - -dir="$(dirname "$(readlink -f "$BASH_SOURCE")")" - -usage() { - cat <<-EOUSAGE - usage: $0 [PR number] [repo[:tag]] - ie: $0 1024 - $0 9001 debian php django - $0 0 hylang # special case that runs against local directory - - This script builds and tests the specified pull request to official-images and - provides ouput in markdown for commenting on the pull request. - EOUSAGE -} - -pull="$1" -shift || { usage >&2 && exit 1; } - -if [ -z "$BASHBREW_SECOND_STAGE" ]; then - dockerRepo='oisupport/bashbrew' - dockerBase="$dockerRepo:base" - dockerImage="$dockerRepo:test-pr" - - bashbrewVersion="$(< "$dir/bashbrew-version")" - docker build -t "$dockerBase" --pull "https://github.com/docker-library/bashbrew.git#v$bashbrewVersion" > /dev/null - docker build -t "$dockerImage" "$dir" > /dev/null - - args=( --init ) - - if [ "$pull" = '0' ]; then - args+=( --name "bashbrew-test-local-$RANDOM" ) - else - args+=( --name "bashbrew-test-pr-$pull" ) - fi - - args+=( - -v /var/run/docker.sock:/var/run/docker.sock - --group-add 0 - - -v /etc/passwd:/etc/passwd:ro - -v /etc/group:/etc/group:ro - ) - if getent group docker &> /dev/null; then - args+=( --group-add "$(getent group docker | cut -d: -f3)" ) - fi - - # if we don't have DOCKER_HOST set, let's bind-mount cache for speed! - if [ -z "$DOCKER_HOST" ]; then - export BASHBREW_CACHE="${BASHBREW_CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/bashbrew}" # resolve path to current "host-side" cache directory - mkdir -p "$BASHBREW_CACHE" # ensure it's created by our user, not root - export BASHBREW_CACHE="$(cd "$BASHBREW_CACHE" && pwd -P)" # readlink -f - args+=( - -v "$BASHBREW_CACHE":/bashbrew-cache - -e BASHBREW_CACHE=/bashbrew-cache - # make sure our user in the container can read it - --group-add "$(stat -c '%g' "$BASHBREW_CACHE")" - ) - else - dockerGid="$( - docker run -i --rm "${args[@]}" "$dockerImage" sh -e <<-'EOSH' - exec 2>/dev/null - stat -c '%g' /var/run/docker.sock \ - || getent group docker | cut -d: -f3 - EOSH - )" || true - if [ "$dockerGid" ]; then - args+=( --group-add "$dockerGid" ) - fi - fi - - args+=( - --user "$(id -u)":"$(id -g)" - $(id -G | xargs -n1 echo --group-add) - - -e BASHBREW_SECOND_STAGE=1 - ) - - for e in "${!BASHBREW_@}"; do - case "$e" in - BASHBREW_SECOND_STAGE|BASHBREW_CACHE|BASHBREW_LIBRARY) ;; - *) - args+=( -e "$e" ) - ;; - esac - done - - cmd=( ./test-pr.sh "$pull" "$@" ) - - if [ -t 0 ] && [ -t 1 ]; then - # only add "-t" if we have a TTY - args+=( -t ) - fi - - exec docker run -i --rm "${args[@]}" "$dockerImage" "${cmd[@]}" -fi - -if [ -d .git ]; then - echo >&2 'error: something has gone horribly wrong; .git already exists' - echo >&2 ' why do you have BASHBREW_SECOND_STAGE set?' - exit 1 -fi - -if [ "$pull" = '0' ]; then - commit='FAKE' -else - dir="$(mktemp -d)" - trap "rm -rf '$dir'" EXIT - cd "$dir" - - # TODO we only have "git version 2.4.1" which doesn't support "clone -q" :( - git init -q . - git remote add origin https://github.com/docker-library/official-images.git - git fetch -q origin - git reset -q --hard origin/master - git config user.name 'nobody' - git config user.email 'nobody@nowhere.noplace' - git fetch -q origin "pull/$pull/head:pr-$pull" - git merge -q --no-edit "pr-$pull" > /dev/null - - commit="$(git log -1 --format=format:%h "pr-$pull")" -fi - -export BASHBREW_LIBRARY="$PWD/library" - -if [ "$#" -eq 0 ]; then - IFS=$'\n' - files=( $(git diff --name-only origin/master...HEAD -- library | xargs -n1 basename) ) - unset IFS - - # TODO narrow this down into groups of the exact tags for each image that changed >:) -else - files=( "$@" ) -fi - -if [ ${#files[@]} -eq 0 ]; then - echo >&2 'no files in library/ changed in PR #'"$pull" - exit 0 -fi - -join() { - sep="$1" - arg1="$2" - shift 2 - echo -n "$arg1" - [ $# -gt 0 ] && printf "${sep}%s" "$@" -} - -IFS=$'\n' -files=( $(bashbrew list --repos --uniq --build-order "${files[@]}") ) -unset IFS - -echo 'Build test of' '#'"$pull"';' "$commit"';' '`'"${BASHBREW_ARCH:-amd64}"'`' '(`'"$(join '`, `' "${files[@]}")"'`):' -declare -A failedBuild=() failedTests=() -for img in "${files[@]}"; do - IFS=$'\n' - uniqImgs=( $(bashbrew list --uniq --build-order "$img") ) - uniqImgs=( $(bashbrew cat --format '{{ if .TagEntry.HasArchitecture arch }}{{ $.RepoName }}:{{ .TagEntry.Tags | first }}{{ end }}' "${uniqImgs[@]}") ) # filter to just the set supported by the current BASHBREW_ARCH - unset IFS - - echo - echo '```console' - for uniqImg in "${uniqImgs[@]}"; do - imgRepo="${uniqImg%%:*}" - echo - echo '$ bashbrew build' "$uniqImg" - if bashbrew build --pull=missing "$uniqImg"; then - echo - echo '$ test/run.sh' "$uniqImg" - if ! ./test/run.sh "$uniqImg"; then - failedTests[$imgRepo]+=" $uniqImg" - fi - else - failedBuild[$imgRepo]+=" $uniqImg" - fi - echo - done - echo '```' -done -echo -if [ "${#failedBuild[@]}" -gt 0 ]; then - echo 'The following images failed to build:' - echo - for repo in "${!failedBuild[@]}"; do - echo '- `'"$repo"'`:' - for img in ${failedBuild[$repo]}; do - echo ' - `'"$img"'`' - done - done - echo -fi -if [ "${#failedTests[@]}" -gt 0 ]; then - echo - echo 'The following images failed at least one test:' - echo - for repo in "${!failedTests[@]}"; do - echo '- `'"$repo"'`:' - for img in ${failedTests[$repo]}; do - echo ' - `'"$img"'`' - done - done - echo -fi