Update Runtime updater GHA

.github/workflows/runtimes-digest-updater-upstream.yaml

@@ -1,19 +1,25 @@
 ---
-# The aim of this GitHub workflow is to update the runtimes across `/jupyter/datascience/ubi*-python-*/runtime-images/*.json` paths.
-name: Update runtime images
+# The aim of this GitHub workflow is to update the runtimes ImageStreams
+name: Update runtime ImageStreams SHA digests
 on:  # yamllint disable-line rule:truthy
   workflow_dispatch:
     inputs:
       branch:
         required: true
-        description: "Provide the name of the branch you want to update ex main, vYYYYx etc: "
-        # Put the scheduler on comment until automate the full release procedure
-        # schedule:
-        #   - cron: "0 0 * * 5"  #Scheduled every Friday
+        description: "Which branch do you want to update?"
+      tag_version:
+        required: true
+        description: "Provide tag version: main or YYYYx"
+      user-hash:
+        required: false
+        description: "Optional: Specify a Git hash (it should exist in the provided branch's history)"
+
 env:
-  DIGEST_UPDATER_BRANCH: digest-updater-${{ github.run_id }}
+  TMP_BRANCH: tmp-branch-${{ github.run_id }}
   BRANCH_NAME: ${{ github.event.inputs.branch || 'main' }}
-  RELEASE_VERSION_N: 2024b
+  TAG_VERSION: ${{ github.event.inputs.tag_version || 'main' }}
+  USER_HASH: ${{ github.event.inputs.user-hash }}
+
 jobs:
   initialize:
     runs-on: ubuntu-latest
@@ -24,20 +30,18 @@ jobs:
         shell: bash
         run: |
           sudo apt-get -y update
-          sudo apt-get -y install skopeo
+          sudo apt-get -y install skopeo yq jq
 
-      # Checkout the branch
       - name: Checkout branch
         uses: actions/checkout@v4
         with:
           ref: ${{ env.BRANCH_NAME }}
 
-      # Create a new branch
       - name: Create a new branch
         run: |
-         echo ${{ env.DIGEST_UPDATER_BRANCH }}
-         git checkout -b ${{ env.DIGEST_UPDATER_BRANCH }}
-         git push --set-upstream origin ${{ env.DIGEST_UPDATER_BRANCH }}
+         echo ${{ env.TMP_BRANCH }}
+         git checkout -b ${{ env.TMP_BRANCH }}
+         git push --set-upstream origin ${{ env.TMP_BRANCH }}
 
   update-runtimes:
     needs: [initialize]
@@ -50,59 +54,29 @@ jobs:
           git config --global user.email "github-actions[bot]@users.noreply.github.com"
           git config --global user.name "GitHub Actions"
 
-      # Get latest build commit from the https://github.com/opendatahub-io/notebooks/${release_branch} using this as identifier for the latest tag name
-      - name: Retrive latest commit hash from the release branch
-        id: hash-n
-        shell: bash
-        run: |
-          PAYLOAD=$(curl --silent -H 'Accept: application/vnd.github.v4.raw' https://api.github.com/repos/opendatahub-io/notebooks/commits?sha=$RELEASE_VERSION_N&per_page=1)
-          echo "HASH_N=$(echo $PAYLOAD | jq -r '.[0].sha[0:7]')" >> ${GITHUB_OUTPUT}
-
-      # Checkout the release branch to apply the updates
       - name: Checkout release branch
         uses: actions/checkout@v4
         with:
-          ref: ${{ env.DIGEST_UPDATER_BRANCH }}
+          ref: ${{ env.TMP_BRANCH }}
+          fetch-depth: 0
 
-      - name: Update Runtimes
+      - name: Invoke script to handle the update
+        shell: bash
         run: |
-            echo "Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N }}"
-
-            find . -name runtime-images -type d -exec find {} -type f -print \; | grep python-3.11 | while read -r path; do
-              echo "Processing the '${path}' file."
+          chmod +x "${GITHUB_WORKSPACE}/ci/runtimes-digest-updater.sh"
+          bash ${GITHUB_WORKSPACE}/ci/runtimes-digest-updater.sh ${{ env.TAG_VERSION }} ${{ env.USER_HASH }}
 
-              img=$(jq -r '.metadata.image_name' "${path}")
-              name=$(echo "$path" | sed 's#.*runtime-images/\(.*\)-py.*#\1#')
-              py_version=$(echo "$path" | grep -o 'python-[0-9]\.[0-9]*')
-              # Handling specific cases
-              if [[ $name == tensorflow* ]]; then
-                name="cuda-$name"
-              elif [[ $name == ubi* ]]; then
-                name="minimal-$name"
-              fi
-              registry=$(echo "$img" | cut -d '@' -f1)
-              regex="^runtime-$name-$py_version-${{ env.RELEASE_VERSION_N }}-\d+-${{ steps.hash-n.outputs.HASH_N }}\$"
-              latest_tag=$(skopeo inspect --retry-times 3 "docker://$img" | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
-              echo "CHECKING: ${latest_tag}"
-              if [[ -z "${latest_tag}" ]]; then
-                echo "No matching tag found"
-                exit 1
-              fi
-              digest=$(skopeo inspect --retry-times 3 "docker://$registry:$latest_tag" | jq .Digest | tr -d '"')
-              output="${registry}@${digest}"
-              echo "NEW: ${output}"
-              jq --arg output "$output" '.metadata.image_name = $output' "$path" > "$path.tmp" && mv "$path.tmp" "$path"
-            done
-
-            if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
-              git fetch origin "${{ env.DIGEST_UPDATER_BRANCH }}" && \
-                git pull origin "${{ env.DIGEST_UPDATER_BRANCH }}" && \
-                git add jupyter/datascience/* && \
-                git commit -m "Update file via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && \
-                git push origin "${{ env.DIGEST_UPDATER_BRANCH }}"
-            else
-              echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N }}"
-            fi
+      - name: Commit the changes
+        run: |
+          if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
+              git fetch origin "${{ env.TMP_BRANCH }}" && \
+              git pull origin "${{ env.TMP_BRANCH }}" && \
+              git add "manifests/base/runtime-*" && \
+              git commit -m "Update file via ${{ env.TMP_BRANCH }} GitHub action" && \
+              git push origin "${{ env.TMP_BRANCH }}"
+          else
+              echo "There were no changes detected in the images for the ${{ env.BRANCH_NAME }}"
+          fi
 
   open-pull-request:
     needs: [update-runtimes]
@@ -113,19 +87,30 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@v4
 
+      - name: Read skipped files (if any)
+        id: skipped-files
+        run: |
+          if [[ -f "${GITHUB_WORKSPACE}/skipped_files.txt" && -s "${GITHUB_WORKSPACE}/skipped_files.txt" ]]; then
+            echo "### Skipped Files" >> pr_body.txt
+            echo "Some files were skipped during the update process due to missing tags in the registry:" >> pr_body.txt
+            cat ${GITHUB_WORKSPACE}/skipped_files.txt >> pr_body.txt
+            echo "" >> pr_body.txt
+          fi
+
+      - name: Generate PR Body
+        run: |
+          echo ":rocket: This is an automated Pull Request." > full_pr_body.txt
+          echo "Created by \`/.github/workflows/runtimes-digest-updater-upstream.yaml\`" >> full_pr_body.txt
+          echo "" >> full_pr_body.txt
+          if [[ -f "pr_body.txt" ]]; then cat pr_body.txt >> full_pr_body.txt; fi
+          echo ":exclamation: **IMPORTANT NOTE**: Remember to delete the \`${{ env.TMP_BRANCH }}\` branch after merging the changes" >> full_pr_body.txt
+
       - name: pull-request
         uses: repo-sync/pull-request@v2
         with:
-          source_branch: ${{ env.DIGEST_UPDATER_BRANCH }}
+          source_branch: ${{ env.TMP_BRANCH }}
           destination_branch: ${{ env.BRANCH_NAME }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
           pr_label: "automated pr"
-          pr_title: "[Digest Updater Action] Update Runtimes Images"
-          pr_body: |
-            :rocket: This is an automated Pull Request.
-            Created by `/.github/workflows/runtimes-digest-updater-upstream.yaml`
-
-            This PR updates the following files:
-            - All the runtime images across `/jupyter/datascience/ubi*-python-*/runtime-images/*.json` paths
-
-            :exclamation: **IMPORTANT NOTE**: Remember to delete the ` ${{ env.DIGEST_UPDATER_BRANCH }}` branch after merging the changes
+          pr_title: "[Digest Updater Action] Update Runtime ImageStreams"
+          pr_body_path: full_pr_body.txt

ci/runtimes-digest-updater.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+TAG_VERSION=$1
+USER_HASH=$2
+
+REPO_OWNER="atheo89"
+REPO_NAME="notebooks"
+GITHUB_API_URL="https://api.github.com/repos/$REPO_OWNER/$REPO_NAME"
+
+if [[ -n "$USER_HASH" ]]; then
+  HASH=$USER_HASH
+  echo "Using user-provided HASH: $HASH"
+else
+  PAYLOAD=$(curl --silent -H 'Accept: application/vnd.github.v4.raw' "$GITHUB_API_URL/commits?sha=$TAG_VERSION&per_page=1")
+  HASH=$(echo "$PAYLOAD" | jq -r '.[0].sha' | cut -c1-7)
+  echo "Extracted HASH: $HASH"
+fi
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+SKIPPED_FILE_LIST="skipped_files.txt"
+touch "$SKIPPED_FILE_LIST"  # Ensure file exists before use
+echo "" > "$SKIPPED_FILE_LIST"  # Reset file at the beginning
+
+MANIFEST_DIR="$REPO_ROOT/manifests/base"
+# Find matching files
+files=$(find "$MANIFEST_DIR" -type f -name "runtime-*.yaml")
+for file in $files; do
+  echo "PROCESSING: $file"
+
+  # Extract values
+  img=$(yq e '.spec.tags[].annotations."opendatahub.io/runtime-image-metadata" | fromjson | .[].metadata.image_name' "$file" 2>/dev/null)
+  name=$(yq e '.spec.tags[].name' "$file" 2>/dev/null)
+  ubi=$(yq e '.metadata.annotations."opendatahub.io/runtime-image-name"' "$file" 2>/dev/null | grep -oE 'UBI[0-9]+' | tr '[:upper:]' '[:lower:]')
+  py_version=$(yq e '.metadata.annotations."opendatahub.io/runtime-image-name"' "$file" 2>/dev/null | grep -oE 'Python [0-9]+\.[0-9]+' | sed 's/ /-/g' | tr '[:upper:]' '[:lower:]')
+  registry=$(echo "$img" | cut -d '@' -f1)
+
+  # Handling specific cases
+  [[ $name == tensorflow* ]] && name="cuda-$name"
+
+  if [[ $TAG_VERSION == main ]]; then
+    # This should match with the runtime-image tag name as is on quay.io registry
+    regex="^runtime-$name-$ubi-$py_version-[0-9]{8}-$HASH$"
+  else
+    # This should match with the runtime-image tag name as is on quay.io registry
+    regex="^runtime-$name-$ubi-$py_version-$TAG_VERSION-[0-9]{8}-$HASH$"
+  fi
+
+  latest_tag=$(skopeo inspect --retry-times 3 "docker://$img" | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
+  echo "CHECKING: ${latest_tag}"
+
+  if [[ -z "$latest_tag" || "$latest_tag" == "null" ]]; then
+    echo "No matching tag found on registry for $file. Skipping."
+    echo "- $file (No matching tag found)" >> "$SKIPPED_FILE_LIST"
+    continue
+  fi
+
+  # Extract the digest sha from the latest tag
+  digest=$(skopeo inspect --retry-times 3 "docker://$registry:$latest_tag" | jq .Digest | tr -d '"')
+  output="${registry}@${digest}"
+  echo "NEW: ${output}"
+
+  # Updates the ImageStream with the new SHAs
+  yq e -i '(.spec.tags[] | .from.name) = "'"$output"'"' "$file"
+  sed -i "s|\(\"image_name\": \"\)[^\"]*|\1${output}|" "$file"
+
+done
+
+echo "Final skipped file list contents:"
+cat "$SKIPPED_FILE_LIST"