From ed7a2043e0b0e71b17233eb1de4f019f33aef25c Mon Sep 17 00:00:00 2001 From: atheo89 Date: Tue, 25 Feb 2025 17:09:43 +0100 Subject: [PATCH] Update Runtime updater GHA --- .../runtimes-digest-updater-upstream.yaml | 84 +++++++------------ ci/runtimes-updater.sh | 62 ++++++++++++++ 2 files changed, 91 insertions(+), 55 deletions(-) create mode 100755 ci/runtimes-updater.sh diff --git a/.github/workflows/runtimes-digest-updater-upstream.yaml b/.github/workflows/runtimes-digest-updater-upstream.yaml index 4883b5f4e..c1a4c4c6c 100644 --- a/.github/workflows/runtimes-digest-updater-upstream.yaml +++ b/.github/workflows/runtimes-digest-updater-upstream.yaml @@ -1,18 +1,23 @@ --- -# The aim of this GitHub workflow is to update the runtimes across `/jupyter/datascience/ubi*-python-*/runtime-images/*.json` paths. -name: Update runtime images +# The aim of this GitHub workflow is to update the runtimes ImageStreams +name: Update runtime ImageStreams SHA digests on: # yamllint disable-line rule:truthy workflow_dispatch: inputs: branch: required: true - description: "Provide the name of the branch you want to update ex main, vYYYYx etc: " - # Put the scheduler on comment until automate the full release procedure - # schedule: - # - cron: "0 0 * * 5" #Scheduled every Friday + description: "Provide branch name: " + tag: + required: true + description: "Provide tag: main or YYYYx: " + user-hash: + required: false + description: "Provide git-hash: " + env: - DIGEST_UPDATER_BRANCH: digest-updater-${{ github.run_id }} + DIGEST_UPDATER_BRANCH: runtimes-updater-${{ github.run_id }} BRANCH_NAME: ${{ github.event.inputs.branch || 'main' }} + TAG: ${{ github.event.inputs.tag || 'main' }} RELEASE_VERSION_N: 2024b jobs: initialize: @@ -24,7 +29,7 @@ jobs: shell: bash run: | sudo apt-get -y update - sudo apt-get -y install skopeo + sudo apt-get -y install skopeo yq jq # Checkout the branch - name: Checkout branch @@ -50,59 +55,29 @@ jobs: git config --global user.email "github-actions[bot]@users.noreply.github.com" git config --global user.name "GitHub Actions" - # Get latest build commit from the https://github.com/opendatahub-io/notebooks/${release_branch} using this as identifier for the latest tag name - - name: Retrive latest commit hash from the release branch - id: hash-n - shell: bash - run: | - PAYLOAD=$(curl --silent -H 'Accept: application/vnd.github.v4.raw' https://api.github.com/repos/opendatahub-io/notebooks/commits?sha=$RELEASE_VERSION_N&per_page=1) - echo "HASH_N=$(echo $PAYLOAD | jq -r '.[0].sha[0:7]')" >> ${GITHUB_OUTPUT} - - # Checkout the release branch to apply the updates - name: Checkout release branch uses: actions/checkout@v4 with: ref: ${{ env.DIGEST_UPDATER_BRANCH }} + fetch-depth: 0 - - name: Update Runtimes + - name: Invoke script to handle the runtimes update + shell: bash run: | - echo "Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N }}" - - find . -name runtime-images -type d -exec find {} -type f -print \; | grep python-3.11 | while read -r path; do - echo "Processing the '${path}' file." + chmod +x "${GITHUB_WORKSPACE}/ci/runtimes-updater.sh" + bash ${GITHUB_WORKSPACE}/ci/runtimes-updater.sh ${{ env.TAG }} ${{ inputs.user_hash }} - img=$(jq -r '.metadata.image_name' "${path}") - name=$(echo "$path" | sed 's#.*runtime-images/\(.*\)-py.*#\1#') - py_version=$(echo "$path" | grep -o 'python-[0-9]\.[0-9]*') - # Handling specific cases - if [[ $name == tensorflow* ]]; then - name="cuda-$name" - elif [[ $name == ubi* ]]; then - name="minimal-$name" - fi - registry=$(echo "$img" | cut -d '@' -f1) - regex="^runtime-$name-$py_version-${{ env.RELEASE_VERSION_N }}-\d+-${{ steps.hash-n.outputs.HASH_N }}\$" - latest_tag=$(skopeo inspect --retry-times 3 "docker://$img" | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]') - echo "CHECKING: ${latest_tag}" - if [[ -z "${latest_tag}" ]]; then - echo "No matching tag found" - exit 1 - fi - digest=$(skopeo inspect --retry-times 3 "docker://$registry:$latest_tag" | jq .Digest | tr -d '"') - output="${registry}@${digest}" - echo "NEW: ${output}" - jq --arg output "$output" '.metadata.image_name = $output' "$path" > "$path.tmp" && mv "$path.tmp" "$path" - done - - if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then + - name: Commit the changes + run: | + if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then git fetch origin "${{ env.DIGEST_UPDATER_BRANCH }}" && \ - git pull origin "${{ env.DIGEST_UPDATER_BRANCH }}" && \ - git add jupyter/datascience/* && \ - git commit -m "Update file via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && \ - git push origin "${{ env.DIGEST_UPDATER_BRANCH }}" - else - echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N }}" - fi + git pull origin "${{ env.DIGEST_UPDATER_BRANCH }}" && \ + git add . && \ + git commit -m "Update file via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && \ + git push origin "${{ env.DIGEST_UPDATER_BRANCH }}" + else + echo "There were no changes detected in the images for the ${{ env.BRANCH_NAME }}" + fi open-pull-request: needs: [update-runtimes] @@ -125,7 +100,6 @@ jobs: :rocket: This is an automated Pull Request. Created by `/.github/workflows/runtimes-digest-updater-upstream.yaml` - This PR updates the following files: - - All the runtime images across `/jupyter/datascience/ubi*-python-*/runtime-images/*.json` paths + This PR updates all the runtime ImageStream files. :exclamation: **IMPORTANT NOTE**: Remember to delete the ` ${{ env.DIGEST_UPDATER_BRANCH }}` branch after merging the changes diff --git a/ci/runtimes-updater.sh b/ci/runtimes-updater.sh new file mode 100755 index 000000000..e48808911 --- /dev/null +++ b/ci/runtimes-updater.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +TAG=$1 +USER_HASH=$2 + +REPO_OWNER="opendatahub-io" +REPO_NAME="notebooks" +GITHUB_API_URL="https://api.github.com/repos/$REPO_OWNER/$REPO_NAME" + +if [[ -n "$USER_HASH" ]]; then + HASH=$USER_HASH + echo "Using user-provided HASH: $HASH" +else + PAYLOAD=$(curl --silent -H 'Accept: application/vnd.github.v4.raw' "$GITHUB_API_URL/commits?sha=$TAG&per_page=1") + HASH=$(echo "$PAYLOAD" | jq -r '.[0].sha' | cut -c1-7) + echo "Extracted HASH: $HASH" +fi + +REPO_ROOT=$(git rev-parse --show-toplevel) +MANIFEST_DIR="$REPO_ROOT/manifests/base" +# Find matching files +files=$(find "$MANIFEST_DIR" -type f -name "runtime-*.yaml") +for file in $files; do + echo "Processing: $file" + + # Extract values + img=$(yq e '.spec.tags[].annotations."opendatahub.io/runtime-image-metadata" | fromjson | .[].metadata.image_name' "$file" 2>/dev/null) + name=$(yq e '.spec.tags[].name' "$file" 2>/dev/null) + ubi=$(yq e '.metadata.annotations."opendatahub.io/runtime-image-name"' "$file" 2>/dev/null | grep -oE 'UBI[0-9]+' | tr '[:upper:]' '[:lower:]') + py_version=$(yq e '.metadata.annotations."opendatahub.io/runtime-image-name"' "$file" 2>/dev/null | grep -oE 'Python [0-9]+\.[0-9]+' | sed 's/ /-/g' | tr '[:upper:]' '[:lower:]') + registry=$(echo "$img" | cut -d '@' -f1) + + # Handling specific cases + [[ $name == tensorflow* ]] && name="cuda-$name" + + if [[ $TAG == main ]]; then + # This should match with the runtime-image tag name as is on quay.io registry + regex="^runtime-$name-$ubi-$py_version-[0-9]{8}-$HASH$" + else + # This should match with the runtime-image tag name as is on quay.io registry + regex="^runtime-$name-$ubi-$py_version-$TAG-[0-9]{8}-$HASH$" + fi + echo "regex: ${regex}" + + latest_tag=$(skopeo inspect --retry-times 3 "docker://$img" | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]') + echo "CHECKING: ${latest_tag}" + + if [[ -z "$latest_tag" || "$latest_tag" == "null" ]]; then + echo "No matching tag found on registry for $file. Skipping." + continue + fi + + # Extract the digest sha from the latest tag + digest=$(skopeo inspect --retry-times 3 "docker://$registry:$latest_tag" | jq .Digest | tr -d '"') + output="${registry}@${digest}" + echo "NEW: ${output}" + + # Updates the ImageStream with the new SHAs + yq e -i '(.spec.tags[] | .from.name) = "'"$output"'"' "$file" + sed -i "s|\(\"image_name\": \"\)[^\"]*|\1${output}|" "$file" + +done \ No newline at end of file