Skip to content

Commit

Permalink
Update Runtime updater GHA
Browse files Browse the repository at this point in the history
  • Loading branch information
atheo89 committed Feb 26, 2025
1 parent 76ca8b2 commit 51419ff
Show file tree
Hide file tree
Showing 2 changed files with 91 additions and 55 deletions.
84 changes: 29 additions & 55 deletions .github/workflows/runtimes-digest-updater-upstream.yaml
Original file line number Diff line number Diff line change
@@ -1,18 +1,23 @@
---
# The aim of this GitHub workflow is to update the runtimes across `/jupyter/datascience/ubi*-python-*/runtime-images/*.json` paths.
name: Update runtime images
# The aim of this GitHub workflow is to update the runtimes ImageStreams
name: Update runtime ImageStreams SHA digests
on: # yamllint disable-line rule:truthy
workflow_dispatch:
inputs:
branch:
required: true
description: "Provide the name of the branch you want to update ex main, vYYYYx etc: "
# Put the scheduler on comment until automate the full release procedure
# schedule:
# - cron: "0 0 * * 5" #Scheduled every Friday
description: "Provide branch name: "
tag:
required: true
description: "Provide tag: main or YYYYx: "
user-hash:
required: false
description: "Provide git-hash: "

env:
DIGEST_UPDATER_BRANCH: digest-updater-${{ github.run_id }}
DIGEST_UPDATER_BRANCH: runtimes-updater-${{ github.run_id }}
BRANCH_NAME: ${{ github.event.inputs.branch || 'main' }}
TAG: ${{ github.event.inputs.tag || 'main' }}
RELEASE_VERSION_N: 2024b
jobs:
initialize:
Expand All @@ -24,7 +29,7 @@ jobs:
shell: bash
run: |
sudo apt-get -y update
sudo apt-get -y install skopeo
sudo apt-get -y install skopeo yq jq
# Checkout the branch
- name: Checkout branch
Expand All @@ -50,59 +55,29 @@ jobs:
git config --global user.email "github-actions[bot]@users.noreply.github.com"
git config --global user.name "GitHub Actions"
# Get latest build commit from the https://github.com/opendatahub-io/notebooks/${release_branch} using this as identifier for the latest tag name
- name: Retrive latest commit hash from the release branch
id: hash-n
shell: bash
run: |
PAYLOAD=$(curl --silent -H 'Accept: application/vnd.github.v4.raw' https://api.github.com/repos/opendatahub-io/notebooks/commits?sha=$RELEASE_VERSION_N&per_page=1)
echo "HASH_N=$(echo $PAYLOAD | jq -r '.[0].sha[0:7]')" >> ${GITHUB_OUTPUT}
# Checkout the release branch to apply the updates
- name: Checkout release branch
uses: actions/checkout@v4
with:
ref: ${{ env.DIGEST_UPDATER_BRANCH }}
fetch-depth: 0

- name: Update Runtimes
- name: Invoke script to handle the runtimes update
shell: bash
run: |
echo "Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N }}"
find . -name runtime-images -type d -exec find {} -type f -print \; | grep python-3.11 | while read -r path; do
echo "Processing the '${path}' file."
chmod +x "${GITHUB_WORKSPACE}/ci/runtimes-updater.sh"
bash ${GITHUB_WORKSPACE}/ci/runtimes-updater.sh ${{ env.TAG }} ${{ inputs.user_hash }}
img=$(jq -r '.metadata.image_name' "${path}")
name=$(echo "$path" | sed 's#.*runtime-images/\(.*\)-py.*#\1#')
py_version=$(echo "$path" | grep -o 'python-[0-9]\.[0-9]*')
# Handling specific cases
if [[ $name == tensorflow* ]]; then
name="cuda-$name"
elif [[ $name == ubi* ]]; then
name="minimal-$name"
fi
registry=$(echo "$img" | cut -d '@' -f1)
regex="^runtime-$name-$py_version-${{ env.RELEASE_VERSION_N }}-\d+-${{ steps.hash-n.outputs.HASH_N }}\$"
latest_tag=$(skopeo inspect --retry-times 3 "docker://$img" | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
echo "CHECKING: ${latest_tag}"
if [[ -z "${latest_tag}" ]]; then
echo "No matching tag found"
exit 1
fi
digest=$(skopeo inspect --retry-times 3 "docker://$registry:$latest_tag" | jq .Digest | tr -d '"')
output="${registry}@${digest}"
echo "NEW: ${output}"
jq --arg output "$output" '.metadata.image_name = $output' "$path" > "$path.tmp" && mv "$path.tmp" "$path"
done
if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
- name: Commit the changes
run: |
if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
git fetch origin "${{ env.DIGEST_UPDATER_BRANCH }}" && \
git pull origin "${{ env.DIGEST_UPDATER_BRANCH }}" && \
git add jupyter/datascience/* && \
git commit -m "Update file via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && \
git push origin "${{ env.DIGEST_UPDATER_BRANCH }}"
else
echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N }}"
fi
git pull origin "${{ env.DIGEST_UPDATER_BRANCH }}" && \
git add . && \
git commit -m "Update file via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && \
git push origin "${{ env.DIGEST_UPDATER_BRANCH }}"
else
echo "There were no changes detected in the images for the ${{ env.BRANCH_NAME }}"
fi
open-pull-request:
needs: [update-runtimes]
Expand All @@ -125,7 +100,6 @@ jobs:
:rocket: This is an automated Pull Request.
Created by `/.github/workflows/runtimes-digest-updater-upstream.yaml`
This PR updates the following files:
- All the runtime images across `/jupyter/datascience/ubi*-python-*/runtime-images/*.json` paths
This PR updates all the runtime ImageStream files.
:exclamation: **IMPORTANT NOTE**: Remember to delete the ` ${{ env.DIGEST_UPDATER_BRANCH }}` branch after merging the changes
62 changes: 62 additions & 0 deletions ci/runtimes-updater.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
#!/bin/bash

TAG=$1
USER_HASH=$2

REPO_OWNER="atheo89"
REPO_NAME="notebooks"
GITHUB_API_URL="https://api.github.com/repos/$REPO_OWNER/$REPO_NAME"

if [[ -n "$USER_HASH" ]]; then
HASH=$USER_HASH
echo "Using user-provided HASH: $HASH"
else
PAYLOAD=$(curl --silent -H 'Accept: application/vnd.github.v4.raw' "$GITHUB_API_URL/commits?sha=$TAG&per_page=1")
HASH=$(echo "$PAYLOAD" | jq -r '.[0].sha' | cut -c1-7)
echo "Extracted HASH: $HASH"
fi

REPO_ROOT=$(git rev-parse --show-toplevel)
MANIFEST_DIR="$REPO_ROOT/manifests/base"
# Find matching files
files=$(find "$MANIFEST_DIR" -type f -name "runtime-*.yaml")
for file in $files; do
echo "Processing: $file"

# Extract values
img=$(yq e '.spec.tags[].annotations."opendatahub.io/runtime-image-metadata" | fromjson | .[].metadata.image_name' "$file" 2>/dev/null)
name=$(yq e '.spec.tags[].name' "$file" 2>/dev/null)
ubi=$(yq e '.metadata.annotations."opendatahub.io/runtime-image-name"' "$file" 2>/dev/null | grep -oE 'UBI[0-9]+' | tr '[:upper:]' '[:lower:]')
py_version=$(yq e '.metadata.annotations."opendatahub.io/runtime-image-name"' "$file" 2>/dev/null | grep -oE 'Python [0-9]+\.[0-9]+' | sed 's/ /-/g' | tr '[:upper:]' '[:lower:]')
registry=$(echo "$img" | cut -d '@' -f1)

# Handling specific cases
[[ $name == tensorflow* ]] && name="cuda-$name"

if [[ $TAG == main ]]; then
# This should match with the runtime-image tag name as is on quay.io registry
regex="^runtime-$name-$ubi-$py_version-[0-9]{8}-$HASH$"
else
# This should match with the runtime-image tag name as is on quay.io registry
regex="^runtime-$name-$ubi-$py_version-$TAG-[0-9]{8}-$HASH$"
fi
echo "regex: ${regex}"

latest_tag=$(skopeo inspect --retry-times 3 "docker://$img" | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
echo "CHECKING: ${latest_tag}"

if [[ -z "$latest_tag" || "$latest_tag" == "null" ]]; then
echo "No matching tag found on registry for $file. Skipping."
continue
fi

# Extract the digest sha from the latest tag
digest=$(skopeo inspect --retry-times 3 "docker://$registry:$latest_tag" | jq .Digest | tr -d '"')
output="${registry}@${digest}"
echo "NEW: ${output}"

# Updates the ImageStream with the new SHAs
yq e -i '(.spec.tags[] | .from.name) = "'"$output"'"' "$file"
sed -i "s|\(\"image_name\": \"\)[^\"]*|\1${output}|" "$file"

done

0 comments on commit 51419ff

Please sign in to comment.