CSUB-1345: Migrate away from Azure in favor of Linode

.github/dependabot.yml

@@ -10,3 +10,14 @@ updates:
     labels: ["A2-insubstantial", "B0-silent", "C1-low", "E2-dependencies"]
     schedule:
       interval: daily
+
+  # Python dependencies for Linode
+  - package-ecosystem: "pip"
+    directories:
+      - "/.github"
+    groups:
+      linode-dependencies:
+        dependency-type: "production"
+        applies-to: version-updates
+    schedule:
+      interval: "monthly"

.github/install-docker-engine-from-upstream.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+if [ ! -f "/etc/apt/sources.list.d/docker.list" ]; then
+    sudo apt-get remove -y docker docker-engine docker.io containerd runc || echo
+    sudo apt-get update
+    sudo apt-get install -y ca-certificates curl gnupg lsb-release
+    sudo mkdir -p /etc/apt/keyrings
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+    echo \
+        "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
+        $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+    sudo apt-get update
+    sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+fi

.github/linode-cloud-init.template

@@ -0,0 +1,14 @@
+#cloud-config
+
+users:
+  - default
+  - name: ubuntu
+    shell: /bin/bash
+    lock_passwd: true
+    groups:
+      - sudo
+    sudo:
+      - ALL=(ALL) NOPASSWD:ALL
+    # WARNING: always keep this line at the end
+    # b/c runner provisioning appends the actual keys
+    ssh_authorized_keys:

.github/provision-github-runner.sh

@@ -3,13 +3,21 @@
 mkdir actions-runner
 pushd actions-runner || exit 1
 
-curl -L https://github.com/actions/runner/releases/download/v2.302.1/actions-runner-linux-x64-2.302.1.tar.gz > runner.tar.gz
+curl -L https://github.com/actions/runner/releases/download/v2.317.0/actions-runner-linux-x64-2.317.0.tar.gz > runner.tar.gz
 
 tar xzf ./runner.tar.gz
 sudo ./bin/installdependencies.sh
-sudo apt install -y jq
+# for 3rd party dependencies and building the code
+sudo apt install curl gnupg -y
+curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
+echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
+sudo apt-get update
+sudo apt install -y build-essential \
+    cmake clang libclang-dev llvm curl gcc git-lfs jq \
+    libssl-dev pipx pkg-config protobuf-compiler \
+    unzip yarn
 
-OWNER_REPO_SLUG="gluwa/substrate"
+OWNER_REPO_SLUG="${LC_OWNER_REPO_SLUG}"
 REPOSITORY_URL="https://github.com/$OWNER_REPO_SLUG"
 EPHEMERAL=${LC_RUNNER_EPHEMERAL:-true}
 
@@ -20,7 +28,19 @@ REGISTRATION_TOKEN=$(curl --silent -X POST \
     -H "X-GitHub-Api-Version: 2022-11-28" \
     "https://api.github.com/repos/$OWNER_REPO_SLUG/actions/runners/registration-token" | jq -r '.token')
 
+if [ "$REGISTRATION_TOKEN" == "null" ]; then
+    echo "ERROR: REGISTRATION_TOKEN is null"
+    exit 1
+fi
+
+if [ -z "$REGISTRATION_TOKEN" ]; then
+    echo "ERROR: REGISTRATION_TOKEN is empty"
+    exit 2
+fi
+
 # Important: ephemeral runners are removed after a single job is executed on them
 # which is inline with the VM lifecycle
-./config.sh --unattended --ephemeral "$EPHEMERAL" --url "$REPOSITORY_URL" --token "$REGISTRATION_TOKEN" --labels "$LC_RUNNER_VM_NAME"
+./config.sh --unattended --ephemeral "$EPHEMERAL" --url "$REPOSITORY_URL" --token "$REGISTRATION_TOKEN" \
+    --name "$LC_RUNNER_VM_NAME" \
+    --labels "$LC_RUNNER_VM_NAME,workflow-$LC_WORKFLOW_ID,proxy-$LC_PROXY_ENABLED,secret-$LC_PROXY_SECRET_VARIANT,type-$LC_PROXY_TYPE"
 nohup ./run.sh >/dev/null 2>&1 </dev/null &

.github/provision-linode-vm.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+set -x
+
+# Install linode-cli
+python3 --version
+pip install -r .github/requirements.txt
+linode-cli --version
+
+# Authorize hosted-runner
+mkdir -p ~/.ssh/
+ssh-keygen -q -t rsa -N '' -f ~/.ssh/id_rsa
+cat ~/.ssh/id_rsa.pub >> .github/authorized_keys
+
+
+# Provision VM
+echo "INFO: From ENVs: RUNNER_VM_NAME=$LC_RUNNER_VM_NAME"
+
+# inject authorized keys into cloud-init for the `ubuntu@` user
+while read -r LINE; do
+  echo "      - $LINE" >> .github/linode-cloud-init.template
+done < .github/authorized_keys
+
+# retry until we get a VM
+IP_ADDRESS=""
+COUNTER=0
+while [ -z "$IP_ADDRESS" ]; do
+    # if all jobs retry rate-limited queries at the same time they still hit the limit
+    # and subsequently fail. Max number of retries is hard-coded to 3 in linodecli
+    # use up to 60 sec random delay to avoid everything being scheduled at once!
+    sleep $((RANDOM % 60))
+
+    # WARNING: we do not specify --authorized_keys for root b/c
+    # linode-cli expects each key as a separate argument and iteratively constructing
+    # the argument list hits issues with quoting the jey values b/c of white-space.
+    # All SSH logins should be via the `ubuntu@` user. For more info see:
+    # https://www.linode.com/community/questions/21290/how-to-pass-multiple-ssh-public-keys-with-linode-cli-linodes-create
+    linode-cli linodes create --json \
+        --image 'linode/ubuntu22.04' --region "$LINODE_REGION" \
+        --type "$LINODE_VM_SIZE" --label "$LC_RUNNER_VM_NAME" \
+        --root_pass "$(uuidgen)" --backups_enabled false --booted true --private_ip false \
+        --metadata.user_data "$(base64 --wrap 0 < .github/linode-cloud-init.template)" > "retry_$COUNTER.json"
+
+    IP_ADDRESS=$(jq -r '.[0].ipv4[0]' < "retry_$COUNTER.json")
+
+    (( COUNTER=COUNTER+1 ))
+done
+
+# provision the GitHub Runner binary on the VM
+# passing additional ENV values
+SSH_USER_AT_HOSTNAME="ubuntu@$IP_ADDRESS"
+echo "INFO: $SSH_USER_AT_HOSTNAME"
+
+until ssh -i ~/.ssh/id_rsa \
+  -o StrictHostKeyChecking=no "$SSH_USER_AT_HOSTNAME" < .github/install-docker-engine-from-upstream.sh; do
+  echo "DEBUG: retrying ssh connection ..."
+  sleep 30
+done
+
+until ssh -i ~/.ssh/id_rsa \
+  -o SendEnv=LC_GITHUB_REPO_ADMIN_TOKEN,LC_RUNNER_VM_NAME,LC_WORKFLOW_ID,LC_PROXY_ENABLED,LC_PROXY_SECRET_VARIANT,LC_PROXY_TYPE \
+  -o StrictHostKeyChecking=no "$SSH_USER_AT_HOSTNAME" < .github/provision-github-runner.sh; do
+  echo "DEBUG: retrying ssh connection ..."
+  sleep 30
+done

.github/remove-linode-vm.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -x
+
+# Install linode-cli
+python3 --version
+pipx install linode-cli
+export PATH="$PATH:~/.local/bin"
+linode-cli --version
+
+# if a specific VM was not specified then remove all zombies
+if [ -z "$LC_RUNNER_VM_NAME" ]; then
+    THRESHOLD=$(date --utc "+%Y-%m-%dT%H:%M:%S" -d "5 hours ago")
+
+    for VM_ID in $(linode-cli linodes list --json | jq ".[] | select(.created <= \"$THRESHOLD\")" | jq -r '.id'); do
+        echo "INFO: going to remove expired VM $VM_ID"
+        linode-cli linodes delete "$VM_ID"
+    done
+else
+    VM_ID=$(linode-cli linodes list --json --label "$LC_RUNNER_VM_NAME" | jq -r '.[0].id')
+    linode-cli linodes delete "$VM_ID"
+fi

.github/requirements.txt

@@ -0,0 +1 @@
+linode-cli==5.54.0