diff --git a/.github/workflows/runtimes-digest-updater-upstream.yaml b/.github/workflows/runtimes-digest-updater-upstream.yaml index 4883b5f4e..cee906c9c 100644 --- a/.github/workflows/runtimes-digest-updater-upstream.yaml +++ b/.github/workflows/runtimes-digest-updater-upstream.yaml @@ -1,19 +1,25 @@ --- -# The aim of this GitHub workflow is to update the runtimes across `/jupyter/datascience/ubi*-python-*/runtime-images/*.json` paths. -name: Update runtime images +# The aim of this GitHub workflow is to update the runtimes ImageStreams +name: Update runtime ImageStreams SHA digests on: # yamllint disable-line rule:truthy workflow_dispatch: inputs: branch: required: true - description: "Provide the name of the branch you want to update ex main, vYYYYx etc: " - # Put the scheduler on comment until automate the full release procedure - # schedule: - # - cron: "0 0 * * 5" #Scheduled every Friday + description: "Which branch do you want to update?" + tag_version: + required: true + description: "Provide tag version: main or YYYYx" + user-hash: + required: false + description: "Optional: Specify a Git hash (it should exist in the provided branch's history)" + env: - DIGEST_UPDATER_BRANCH: digest-updater-${{ github.run_id }} + TMP_BRANCH: tmp-branch-${{ github.run_id }} BRANCH_NAME: ${{ github.event.inputs.branch || 'main' }} - RELEASE_VERSION_N: 2024b + TAG_VERSION: ${{ github.event.inputs.tag_version || 'main' }} + USER_HASH: ${{ github.event.inputs.user-hash }} + jobs: initialize: runs-on: ubuntu-latest @@ -24,20 +30,18 @@ jobs: shell: bash run: | sudo apt-get -y update - sudo apt-get -y install skopeo + sudo apt-get -y install skopeo yq jq - # Checkout the branch - name: Checkout branch uses: actions/checkout@v4 with: ref: ${{ env.BRANCH_NAME }} - # Create a new branch - name: Create a new branch run: | - echo ${{ env.DIGEST_UPDATER_BRANCH }} - git checkout -b ${{ env.DIGEST_UPDATER_BRANCH }} - git push --set-upstream origin ${{ env.DIGEST_UPDATER_BRANCH }} + echo ${{ env.TMP_BRANCH }} + git checkout -b ${{ env.TMP_BRANCH }} + git push --set-upstream origin ${{ env.TMP_BRANCH }} update-runtimes: needs: [initialize] @@ -50,59 +54,29 @@ jobs: git config --global user.email "github-actions[bot]@users.noreply.github.com" git config --global user.name "GitHub Actions" - # Get latest build commit from the https://github.com/opendatahub-io/notebooks/${release_branch} using this as identifier for the latest tag name - - name: Retrive latest commit hash from the release branch - id: hash-n - shell: bash - run: | - PAYLOAD=$(curl --silent -H 'Accept: application/vnd.github.v4.raw' https://api.github.com/repos/opendatahub-io/notebooks/commits?sha=$RELEASE_VERSION_N&per_page=1) - echo "HASH_N=$(echo $PAYLOAD | jq -r '.[0].sha[0:7]')" >> ${GITHUB_OUTPUT} - - # Checkout the release branch to apply the updates - name: Checkout release branch uses: actions/checkout@v4 with: - ref: ${{ env.DIGEST_UPDATER_BRANCH }} + ref: ${{ env.TMP_BRANCH }} + fetch-depth: 0 - - name: Update Runtimes + - name: Invoke script to handle the update + shell: bash run: | - echo "Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N }}" - - find . -name runtime-images -type d -exec find {} -type f -print \; | grep python-3.11 | while read -r path; do - echo "Processing the '${path}' file." + chmod +x "${GITHUB_WORKSPACE}/ci/runtimes-digest-updater.sh" + bash ${GITHUB_WORKSPACE}/ci/runtimes-digest-updater.sh ${{ env.TAG_VERSION }} ${{ env.USER_HASH }} - img=$(jq -r '.metadata.image_name' "${path}") - name=$(echo "$path" | sed 's#.*runtime-images/\(.*\)-py.*#\1#') - py_version=$(echo "$path" | grep -o 'python-[0-9]\.[0-9]*') - # Handling specific cases - if [[ $name == tensorflow* ]]; then - name="cuda-$name" - elif [[ $name == ubi* ]]; then - name="minimal-$name" - fi - registry=$(echo "$img" | cut -d '@' -f1) - regex="^runtime-$name-$py_version-${{ env.RELEASE_VERSION_N }}-\d+-${{ steps.hash-n.outputs.HASH_N }}\$" - latest_tag=$(skopeo inspect --retry-times 3 "docker://$img" | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]') - echo "CHECKING: ${latest_tag}" - if [[ -z "${latest_tag}" ]]; then - echo "No matching tag found" - exit 1 - fi - digest=$(skopeo inspect --retry-times 3 "docker://$registry:$latest_tag" | jq .Digest | tr -d '"') - output="${registry}@${digest}" - echo "NEW: ${output}" - jq --arg output "$output" '.metadata.image_name = $output' "$path" > "$path.tmp" && mv "$path.tmp" "$path" - done - - if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then - git fetch origin "${{ env.DIGEST_UPDATER_BRANCH }}" && \ - git pull origin "${{ env.DIGEST_UPDATER_BRANCH }}" && \ - git add jupyter/datascience/* && \ - git commit -m "Update file via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && \ - git push origin "${{ env.DIGEST_UPDATER_BRANCH }}" - else - echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N }}" - fi + - name: Commit the changes + run: | + if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then + git fetch origin "${{ env.TMP_BRANCH }}" && \ + git pull origin "${{ env.TMP_BRANCH }}" && \ + git add "manifests/base/runtime-*" && \ + git commit -m "Update file via ${{ env.TMP_BRANCH }} GitHub action" && \ + git push origin "${{ env.TMP_BRANCH }}" + else + echo "There were no changes detected in the images for the ${{ env.BRANCH_NAME }}" + fi open-pull-request: needs: [update-runtimes] @@ -113,19 +87,30 @@ jobs: - name: Checkout repo uses: actions/checkout@v4 + - name: Read skipped files (if any) + id: skipped-files + run: | + if [[ -f "${GITHUB_WORKSPACE}/skipped_files.txt" && -s "${GITHUB_WORKSPACE}/skipped_files.txt" ]]; then + echo "### Skipped Files" >> pr_body.txt + echo "Some files were skipped during the update process due to missing tags in the registry:" >> pr_body.txt + cat ${GITHUB_WORKSPACE}/skipped_files.txt >> pr_body.txt + echo "" >> pr_body.txt + fi + + - name: Generate PR Body + run: | + echo ":rocket: This is an automated Pull Request." > full_pr_body.txt + echo "Created by \`/.github/workflows/runtimes-digest-updater-upstream.yaml\`" >> full_pr_body.txt + echo "" >> full_pr_body.txt + if [[ -f "pr_body.txt" ]]; then cat pr_body.txt >> full_pr_body.txt; fi + echo ":exclamation: **IMPORTANT NOTE**: Remember to delete the \`${{ env.TMP_BRANCH }}\` branch after merging the changes" >> full_pr_body.txt + - name: pull-request uses: repo-sync/pull-request@v2 with: - source_branch: ${{ env.DIGEST_UPDATER_BRANCH }} + source_branch: ${{ env.TMP_BRANCH }} destination_branch: ${{ env.BRANCH_NAME }} github_token: ${{ secrets.GITHUB_TOKEN }} pr_label: "automated pr" - pr_title: "[Digest Updater Action] Update Runtimes Images" - pr_body: | - :rocket: This is an automated Pull Request. - Created by `/.github/workflows/runtimes-digest-updater-upstream.yaml` - - This PR updates the following files: - - All the runtime images across `/jupyter/datascience/ubi*-python-*/runtime-images/*.json` paths - - :exclamation: **IMPORTANT NOTE**: Remember to delete the ` ${{ env.DIGEST_UPDATER_BRANCH }}` branch after merging the changes + pr_title: "[Digest Updater Action] Update Runtime ImageStreams" + pr_body_path: full_pr_body.txt diff --git a/ci/runtimes-digest-updater.sh b/ci/runtimes-digest-updater.sh new file mode 100755 index 000000000..718899369 --- /dev/null +++ b/ci/runtimes-digest-updater.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +TAG_VERSION=$1 +USER_HASH=$2 + +REPO_OWNER="atheo89" +REPO_NAME="notebooks" +GITHUB_API_URL="https://api.github.com/repos/$REPO_OWNER/$REPO_NAME" + +if [[ -n "$USER_HASH" ]]; then + HASH=$USER_HASH + echo "Using user-provided HASH: $HASH" +else + PAYLOAD=$(curl --silent -H 'Accept: application/vnd.github.v4.raw' "$GITHUB_API_URL/commits?sha=$TAG_VERSION&per_page=1") + HASH=$(echo "$PAYLOAD" | jq -r '.[0].sha' | cut -c1-7) + echo "Extracted HASH: $HASH" +fi + +REPO_ROOT=$(git rev-parse --show-toplevel) + +SKIPPED_FILE_LIST="skipped_files.txt" +touch "$SKIPPED_FILE_LIST" # Ensure file exists before use +echo "" > "$SKIPPED_FILE_LIST" # Reset file at the beginning + +MANIFEST_DIR="$REPO_ROOT/manifests/base" +# Find matching files +files=$(find "$MANIFEST_DIR" -type f -name "runtime-*.yaml") +for file in $files; do + echo "PROCESSING: $file" + + # Extract values + img=$(yq e '.spec.tags[].annotations."opendatahub.io/runtime-image-metadata" | fromjson | .[].metadata.image_name' "$file" 2>/dev/null) + name=$(yq e '.spec.tags[].name' "$file" 2>/dev/null) + ubi=$(yq e '.metadata.annotations."opendatahub.io/runtime-image-name"' "$file" 2>/dev/null | grep -oE 'UBI[0-9]+' | tr '[:upper:]' '[:lower:]') + py_version=$(yq e '.metadata.annotations."opendatahub.io/runtime-image-name"' "$file" 2>/dev/null | grep -oE 'Python [0-9]+\.[0-9]+' | sed 's/ /-/g' | tr '[:upper:]' '[:lower:]') + registry=$(echo "$img" | cut -d '@' -f1) + + # Handling specific cases + [[ $name == tensorflow* ]] && name="cuda-$name" + + if [[ $TAG_VERSION == main ]]; then + # This should match with the runtime-image tag name as is on quay.io registry + regex="^runtime-$name-$ubi-$py_version-[0-9]{8}-$HASH$" + else + # This should match with the runtime-image tag name as is on quay.io registry + regex="^runtime-$name-$ubi-$py_version-$TAG_VERSION-[0-9]{8}-$HASH$" + fi + + latest_tag=$(skopeo inspect --retry-times 3 "docker://$img" | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]') + echo "CHECKING: ${latest_tag}" + + if [[ -z "$latest_tag" || "$latest_tag" == "null" ]]; then + echo "No matching tag found on registry for $file. Skipping." + echo "- $file (No matching tag found)" >> "$SKIPPED_FILE_LIST" + continue + fi + + # Extract the digest sha from the latest tag + digest=$(skopeo inspect --retry-times 3 "docker://$registry:$latest_tag" | jq .Digest | tr -d '"') + output="${registry}@${digest}" + echo "NEW: ${output}" + + # Updates the ImageStream with the new SHAs + yq e -i '(.spec.tags[] | .from.name) = "'"$output"'"' "$file" + sed -i "s|\(\"image_name\": \"\)[^\"]*|\1${output}|" "$file" + +done + +echo "Final skipped file list contents:" +cat "$SKIPPED_FILE_LIST" \ No newline at end of file