bootstrap: improve validation and update tasks (onedr0p#1623)

.github/tests/config-talos.yaml

@@ -1,8 +1,8 @@
 ---
 skip_tests: true
 
-boostrap_talos:
-  schematic_id: "376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba"
+bootstrap_cluster_name: fake
+bootstrap_schematic_id: "376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba"
 bootstrap_node_network: 10.10.10.0/24
 bootstrap_node_default_gateway: 10.10.10.1
 bootstrap_node_inventory:

.taskfiles/bootstrap/Taskfile.yaml

@@ -29,12 +29,10 @@ tasks:
       TALHELPER_SECRET_EXISTS:
         sh: test -f {{.TALHELPER_SECRET_FILE}} && echo true || echo false
     preconditions:
-      - msg: Missing talhelper config file
-        sh: test -f {{.TALHELPER_CONFIG_FILE}}
-      - msg: Missing Sops config file
-        sh: test -f {{.SOPS_CONFIG_FILE}}
-      - msg: Missing Sops Age key file
-        sh: test -f {{.SOPS_AGE_KEY_FILE}}
+      - test -f {{.SOPS_CONFIG_FILE}}
+      - test -f {{.SOPS_AGE_KEY_FILE}}
+      - test -f {{.TALHELPER_CONFIG_FILE}}
+      - which talhelper sops
 
   flux:
     desc: Bootstrap Flux into the Talos cluster
@@ -52,10 +50,9 @@ tasks:
       SOPS_SECRET_EXISTS:
         sh: kubectl --namespace flux-system get secret sops-age &>/dev/null && echo true || echo false
     preconditions:
-      - msg: Missing kubeconfig
-        sh: test -f {{.KUBECONFIG}}
-      - msg: Missing Sops Age key file
-        sh: test -f {{.SOPS_AGE_KEY_FILE}}
+      - test -f {{.KUBECONFIG}}
+      - test -f {{.SOPS_AGE_KEY_FILE}}
+      - which flux kubectl sops
 
   apps:
     internal: true
@@ -65,29 +62,26 @@ tasks:
       - helmfile --file {{.KUBERNETES_DIR}}/bootstrap/helmfile.yaml apply --skip-diff-on-install --suppress-diff
       - until kubectl wait --for=condition=Ready nodes --all --timeout=600s; do sleep 10; done
     preconditions:
-      - msg: Missing kubeconfig
-        sh: test -f {{.KUBECONFIG}}
-      - msg: Missing helmfile
-        sh: test -f {{.KUBERNETES_DIR}}/bootstrap/helmfile.yaml
+      - test -f {{.KUBECONFIG}}
+      - test -f {{.KUBERNETES_DIR}}/bootstrap/helmfile.yaml
+      - which helmfile kubectl
 
   age-keygen:
     desc: Bootstrap the Sops Age key
     cmd: age-keygen --output {{.SOPS_AGE_KEY_FILE}}
     status:
       - test -f {{.SOPS_AGE_KEY_FILE}}
+    preconditions:
+      - which age-keygen
 
   template:
     internal: true
     cmd: '{{.VIRTUAL_ENV}}/bin/makejinja'
     preconditions:
-      - msg: Missing virtual environment
-        sh: test -d {{.VIRTUAL_ENV}}
-      - msg: Missing Makejinja config file
-        sh: test -f {{.MAKEJINJA_CONFIG_FILE}}
-      - msg: Missing Makejinja plugin file
-        sh: test -f {{.BOOTSTRAP_DIR}}/scripts/plugin.py
-      - msg: Missing bootstrap config file
-        sh: test -f {{.BOOTSTRAP_CONFIG_FILE}}
+      - test -d {{.VIRTUAL_ENV}}
+      - test -f {{.BOOTSTRAP_CONFIG_FILE}}
+      - test -f {{.BOOTSTRAP_DIR}}/scripts/plugin.py
+      - test -f {{.MAKEJINJA_CONFIG_FILE}}
 
   secrets:
     internal: true
@@ -101,10 +95,9 @@ tasks:
       SECRET_FILES:
         sh: find "{{.KUBERNETES_DIR}}" -type f -name "*.sops.*"
     preconditions:
-      - msg: Missing Sops config file
-        sh: test -f {{.SOPS_CONFIG_FILE}}
-      - msg: Missing Sops Age key file
-        sh: test -f {{.SOPS_AGE_KEY_FILE}}
+      - test -f {{.SOPS_AGE_KEY_FILE}}
+      - test -f {{.SOPS_CONFIG_FILE}}
+      - which sops
 
   clean:
     desc: Clean files and directories no longer needed after cluster bootstrap
@@ -125,7 +118,5 @@ tasks:
     vars:
       TS: '{{now | unixEpoch}}'
     preconditions:
-      - msg: Missing bootstrap directory
-        sh: test -d {{.BOOTSTRAP_DIR}}
-      - msg: Missing Renovate config file
-        sh: test -f {{.ROOT_DIR}}/.github/renovate.json5
+      - test -d {{.BOOTSTRAP_DIR}}
+      - test -f {{.ROOT_DIR}}/.github/renovate.json5

.taskfiles/kubernetes/Taskfile.yaml

@@ -8,39 +8,35 @@ vars:
 tasks:
 
   apply-ks:
-    desc: Apply a Flux Kustomization resource for a cluster
-    summary: |-
-      PATH: Path to the Flux Kustomization resource from the apps base dir (required, e.g. network/echo-server)
-      NS: Namespace the Flux Kustomization exists in (default: flux-system)
+    desc: Apply a Flux KS for a cluster [PATH=required]
     cmd: >
-      flux build --namespace {{.NS}} ks {{base .PATH}}
+      flux build --namespace flux-system ks {{base .PATH}}
       --kustomization-file {{.KUBERNETES_DIR}}/apps/{{.PATH}}/ks.yaml
-      --path {{.KUBERNETES_DIR}}/apps/{{.PATH}}
-      {{- if contains "not found" .KS }}--dry-run \{{ end }}
-      | yq 'with(select(.apiVersion == "kustomize.toolkit.fluxcd.io/v1" and .kind == "Kustomization"); .metadata.namespace = "{{.NS}}")' -
-      | kubectl apply --server-side --field-manager=kustomize-controller -f -
+      --path {{.KUBERNETES_DIR}}/apps/{{.PATH}}{{- if contains "not found" .KS }}--dry-run \{{ end }}
+      | yq 'with(select(.apiVersion == "kustomize.toolkit.fluxcd.io/v1" and .kind == "Kustomization"); .metadata.namespace = "flux-system")' -
+      | kubectl apply --server-side --field-manager=kustomize-controller --filename -
     requires:
       vars: [PATH]
     vars:
-      NS: '{{.NS | default "flux-system"}}'
       KS:
-        sh: flux --namespace {{.NS}} get kustomizations {{base .PATH}} 2>&1
+        sh: flux --namespace flux-system get kustomizations {{base .PATH}} 2>&1
     preconditions:
       - test -f {{.KUBERNETES_DIR}}/apps/{{.PATH}}/ks.yaml
+      - which flux kubectl yq
 
   reconcile:
     desc: Force update Flux to pull in changes from your Git repository
-    cmd: flux reconcile --namespace flux-system kustomization cluster --with-source
+    cmd: flux --namespace flux-system reconcile kustomization cluster --with-source
     preconditions:
-      - msg: Missing kubeconfig
-        sh: test -f {{.KUBECONFIG}}
+      - test -f {{.KUBECONFIG}}
+      - which flux
 
   kubeconform:
     desc: Validate Kubernetes manifests with kubeconform
     cmd: bash {{.KUBECONFORM_SCRIPT}} {{.KUBERNETES_DIR}}
     preconditions:
-      - msg: Missing kubeconform script
-        sh: test -f {{.KUBECONFORM_SCRIPT}}
+      - test -f {{.KUBECONFORM_SCRIPT}}
+      - which kubeconform
 
   resources:
     desc: Gather common resources in your cluster, useful when asking for support
@@ -59,5 +55,5 @@ tasks:
         ingresses
         pods
     preconditions:
-      - msg: Missing kubeconfig
-        sh: test -f {{.KUBECONFIG}}
+      - test -f {{.KUBECONFIG}}
+      - which kubectl

.taskfiles/talos/Taskfile.yaml

@@ -17,41 +17,36 @@ tasks:
     desc: Generate Talos configuration
     cmd: talhelper genconfig --config-file {{.TALHELPER_CONFIG_FILE}} --secret-file {{.TALHELPER_SECRET_FILE}} --out-dir {{.TALHELPER_CLUSTER_DIR}}
     preconditions:
-      - msg: Missing talhelper config file
-        sh: test -f {{.TALHELPER_CONFIG_FILE}}
-      - msg: Missing Sops config file
-        sh: test -f {{.SOPS_CONFIG_FILE}}
-      - msg: Missing Sops Age key file
-        sh: test -f {{.SOPS_AGE_KEY_FILE}}
+      - test -f {{.TALHELPER_CONFIG_FILE}}
+      - test -f {{.SOPS_CONFIG_FILE}}
+      - test -f {{.SOPS_AGE_KEY_FILE}}
+      - which talhelper
 
-  apply-config:
-    desc: Apply Talos configuration to a node
+  apply-node:
+    desc: Apply Talos config to a node [HOSTNAME=required]
     cmds:
-      - talosctl --nodes {{.HOSTNAME}} apply-config --mode=staged --file {{.TALHELPER_CLUSTER_DIR}}/{{.CLUSTER_NAME}}-{{.HOSTNAME}}.yaml
-      - talosctl --nodes {{.HOSTNAME}} reboot
+      - talosctl --nodes {{.HOSTNAME}} apply-config --mode={{.MODE}}}} --file {{.TALHELPER_CLUSTER_DIR}}/{{.CLUSTER_NAME}}-{{.HOSTNAME}}.yaml
       - talosctl --nodes {{.HOSTNAME}} health --wait-timeout=10m --server=false
     vars:
       CLUSTER_NAME:
         sh: yq '.clusterName' {{.TALHELPER_CONFIG_FILE}}
+      MODE: '{{.MODE | default "auto"}}'
     requires:
       vars: [HOSTNAME]
     preconditions:
-      - msg: Missing talosconfig
-        sh: test -f {{.TALOSCONFIG}}
-      - msg: Unable to retrieve Talos config
-        sh: talosctl config info &>/dev/null
-      - msg: Node not found
-        sh: talosctl --nodes {{.HOSTNAME}} get machineconfig &>/dev/null
-      - msg: Talos config for node not found
-        sh: test -f {{.TALHELPER_CLUSTER_DIR}}/{{.CLUSTER_NAME}}-{{.HOSTNAME}}.yaml
+      - talosctl --nodes {{.HOSTNAME}} get machineconfig &>/dev/null
+      - talosctl config info &>/dev/null
+      - test -f {{.TALHELPER_CLUSTER_DIR}}/{{.CLUSTER_NAME}}-{{.HOSTNAME}}.yaml
+      - test -f {{.TALOSCONFIG}}
+      - which talosctl yq
 
   upgrade-node:
-    desc: Upgrade Talos on a single node
+    desc: Upgrade Talos on a single node [HOSTNAME=required]
     cmds:
-      - task: '{{if ne .ROLLOUT true}}down{{else}}noop{{end}}'
+      - task: down
       - talosctl --nodes {{.HOSTNAME}} upgrade --image="factory.talos.dev/installer/{{.TALOS_SCHEMATIC_ID}}:{{.TALOS_VERSION}}" --timeout=10m
       - talosctl --nodes {{.HOSTNAME}} health --wait-timeout=10m --server=false
-      - task: '{{if ne .ROLLOUT true}}up{{else}}noop{{end}}'
+      - task: up
     vars:
       TALOS_SCHEMATIC_ID:
         sh: kubectl get node {{.HOSTNAME}} --output=jsonpath='{.metadata.annotations.extensions\.talos\.dev/schematic}'
@@ -60,28 +55,11 @@ tasks:
     requires:
       vars: [HOSTNAME]
     preconditions:
-      - msg: Missing talosconfig
-        sh: test -f {{.TALOSCONFIG}}
-      - msg: Unable to retrieve Talos config
-        sh: talosctl config info &>/dev/null
-      - msg: Node not found
-        sh: talosctl --nodes {{.HOSTNAME}} get machineconfig &>/dev/null
-      - msg: Upstream Talos version not found
-        sh: curl -fsSL -o /dev/null --fail https://github.com/siderolabs/talos/releases/tag/{{.TALOS_VERSION}}
-
-  upgrade-cluster:
-    desc: Upgrade Talos on the whole cluster
-    cmds:
-      - task: down
-      - for: { var: HOSTNAMES }
-        task: upgrade-node
-        vars:
-          HOSTNAME: '{{.ITEM}}'
-          ROLLOUT: true
-      - task: up
-    vars:
-      HOSTNAMES:
-        sh: kubectl get nodes --output=jsonpath='{.items[*].metadata.name}'
+      - curl -fsSL -o /dev/null --fail https://github.com/siderolabs/talos/releases/tag/{{.TALOS_VERSION}}
+      - talosctl --nodes {{.HOSTNAME}} get machineconfig &>/dev/null
+      - talosctl config info &>/dev/null
+      - test -f {{.TALOSCONFIG}}
+      - which kubectl talosctl yq
 
   upgrade-k8s:
     desc: Upgrade Kubernetes
@@ -92,31 +70,28 @@ tasks:
       KUBERNETES_VERSION:
         sh: yq '.kubernetesVersion' {{.TALHELPER_CONFIG_FILE}}
     preconditions:
-      - msg: Missing talosconfig
-        sh: test -f {{.TALOSCONFIG}}
-      - msg: Unable to retrieve Talos config
-        sh: talosctl config info &>/dev/null
-      - msg: Node not found
-        sh: talosctl --nodes {{.KUBERNETES_CONTROLLER}} get machineconfig &>/dev/null
-      - msg: Upstream Kubernetes version not found
-        sh: curl -fsSL -o /dev/null --fail https://github.com/siderolabs/kubelet/releases/tag/{{.KUBERNETES_VERSION}}
+      - curl -fsSL -o /dev/null --fail https://github.com/siderolabs/kubelet/releases/tag/{{.KUBERNETES_VERSION}}
+      - talosctl --nodes {{.KUBERNETES_CONTROLLER}} get machineconfig &>/dev/null
+      - talosctl config info &>/dev/null
+      - test -f {{.TALOSCONFIG}}
+      - which talosctl yq
 
   reset:
     desc: Resets nodes back to maintenance mode
     dir: "{{.KUBERNETES_DIR}}/bootstrap/talos"
     prompt: This will destroy your cluster and reset the nodes back to maintenance mode... continue?
     cmd: talhelper gencommand reset --config-file {{.TALHELPER_CONFIG_FILE}} --out-dir {{.TALHELPER_CLUSTER_DIR}} --extra-flags="--reboot {{- if eq .CLI_FORCE false }} --system-labels-to-wipe STATE --system-labels-to-wipe EPHEMERAL{{ end }} --graceful=false --wait=false" | bash
+  preconditions:
+    - which talhelper
 
   down:
     internal: true
     cmd: flux --namespace flux-system suspend kustomization --all
+    preconditions:
+      - which flux
 
   up:
     internal: true
     cmd: flux --namespace flux-system resume kustomization --all
-
-  # Ref: https://github.com/go-task/task/issues/608
-  noop:
-    internal: true
-    silent: true
-    cmd: noop() { :; }
+    preconditions:
+      - which flux

.taskfiles/workstation/Taskfile.yaml

@@ -6,42 +6,38 @@ tasks:
 
   arch:
     desc: Set up Arch Linux tools
-    cmd: '{{.PKGMGR}} -Syu --needed --noconfirm --noprogressbar $(cat {{.ROOT_DIR}}/.taskfiles/workstation/Archfile | xargs)'
+    cmd: >
+      {{.PKGMGR}} -Syu --needed --noconfirm --noprogressbar
+      $(cat {{.ROOT_DIR}}/.taskfiles/workstation/Archfile | xargs)
     vars:
       PKGMGR:
-        sh: command -v paru || command -v yay
+        sh: which paru || which yay
     preconditions:
-      - msg: Missing paru or yay
-        sh: command -v paru &>/dev/null || command -v yay &>/dev/null
-      - msg: Missing Archfile
-        sh: test -f {{.ROOT_DIR}}/.taskfiles/workstation/Archfile
+      - test -f {{.ROOT_DIR}}/.taskfiles/workstation/Archfile
+      - which paru || which yay
 
   brew:
     desc: Set up Homebrew tools
-    cmds:
-      - brew bundle --file {{.ROOT_DIR}}/.taskfiles/workstation/Brewfile
+    cmd: brew bundle --file {{.ROOT_DIR}}/.taskfiles/workstation/Brewfile
     sources:
       - '{{.ROOT_DIR}}/.taskfiles/workstation/Brewfile'
     generates:
       - '{{.ROOT_DIR}}/.taskfiles/workstation/Brewfile.lock.json'
     preconditions:
-      - msg: Missing brew
-        sh: command -v brew &>/dev/null
-      - msg: Missing Brewfile
-        sh: test -f {{.ROOT_DIR}}/.taskfiles/workstation/Brewfile
+      - test -f {{.ROOT_DIR}}/.taskfiles/workstation/Brewfile
+      - which brew
 
   direnv:
-    desc: Run direnv hooks
+    desc: Set up direnv hooks
     cmd: direnv allow .
     status:
       - '[[ $(direnv status --json | jq ".state.foundRC.allowed") == 0 ]]'
       - '[[ $(direnv status --json | jq ".state.loadedRC.allowed") == 0 ]]'
     preconditions:
-      - msg: Missing direnv
-        sh: command -v direnv &>/dev/null
+      - which direnv
 
   generic-linux:
-    desc: Setup CLI tools into the projects .bin directory
+    desc: Set up CLI tools into the projects .bin directory
     dir: '{{.ROOT_DIR}}/.bin'
     platforms: ['linux/amd64', 'linux/arm64']
     cmds:
@@ -85,7 +81,5 @@ tasks:
     generates:
       - '{{.VIRTUAL_ENV}}/pyvenv.cfg'
     preconditions:
-      - msg: Missing python3
-        sh: command -v python3 &>/dev/null
-      - msg: Missing Pip requirements file
-        sh: test -f {{.ROOT_DIR}}/requirements.txt
+      - test -f {{.ROOT_DIR}}/requirements.txt
+      - which python3

README.md

@@ -298,20 +298,14 @@ task talos:reset # --force
 # (Re)generate the Talos config
 task talos:generate-config
 # Apply the config to the node
-task talos:apply-config HOSTNAME=? MODE=?
-# e.g. task talos:apply-config HOSTNAME=k8s-0 MODE=reboot
+task talos:apply-node HOSTNAME=? MODE=?
+# e.g. task talos:apply-config HOSTNAME=k8s-0 MODE=auto
 ```
 
 #### ⬆️ Updating Talos and Kubernetes versions
 
 📍 _Ensure the `talosVersion` and `kubernetesVersion` in `talhelper.yaml` are up-to-date with the version you wish to upgrade to._
 
-```sh
-# Upgrade the whole cluster to a newer Talos version
-task talos:upgrade-cluster
-# e.g. task talos:upgrade-cluster
-```
-
 ```sh
 # Upgrade node to a newer Talos version
 task talos:upgrade-node HOSTNAME=?