feat: add raspi cluster

kashalls · Oct 12, 2024 · 670d24b · 670d24b
1 parent e963c60
commit 670d24b
Show file tree

Hide file tree

Showing 45 changed files with 1,657 additions and 319 deletions.
diff --git a/.envrc b/.envrc
@@ -1,3 +1,3 @@
 #shellcheck disable=SC2148,SC2155
-export KUBECONFIG="$(expand_path ./kubernetes/main/kubeconfig):$(expand_path ./kubernetes/storage/kubeconfig)"
+export KUBECONFIG="$(expand_path ./kubernetes/main/kubeconfig):$(expand_path ./kubernetes/raspi/kubeconfig)"
 export SOPS_AGE_KEY_FILE="$(expand_path ./age.key)"
diff --git a/.taskfiles/Talos/Taskfile.yaml b/.taskfiles/Talos/Taskfile.yaml
@@ -16,18 +16,17 @@ tasks:
   apply-config:
     desc: Apply Talos configuration to a node
     cmd: |
-      sops --decrypt {{.TALOS_DIR}}/{{.cluster}}/{{.type}}.secret.sops.yaml | \
+      sops --decrypt {{.TALOS_DIR}}/{{.cluster}}/{{.hostname}}.secret.sops.yaml | \
           $GOPATH/bin/envsubst | \
-              talosctl --context {{.cluster}} apply-config --nodes {{.node}} --file /dev/stdin --mode={{.mode}} --insecure={{.insecure}}
+              talosctl --context {{.cluster}} apply-config --nodes {{if .endpoint}}{{.endpoint}}{{else}}{{.hostname}}{{end}} --file /dev/stdin --mode={{.mode}} {{if .insecure}}--insecure{{end}} {{if .endpoint}}--endpoints={{.endpoint}}{{end}}
     env: *vars
     vars:
-      type: '{{.type | default "worker"}}'
       mode: '{{.mode | default "no-reboot"}}'
       insecure: '{{.insecure | default "false" }}'
     requires:
-      vars: ["cluster", "node", "type"]
+      vars: ["cluster", "hostname"]
     preconditions:
-      - test -f {{.TALOS_DIR}}/{{.cluster}}/{{.type}}.secret.sops.yaml
+      - test -f {{.TALOS_DIR}}/{{.cluster}}/{{.hostname}}.secret.sops.yaml
       #- talosctl --context {{.cluster}} --nodes {{.hostname}} get machineconfig >/dev/null 2>&1
 
   upgrade:

diff --git a/kubernetes/raspi/apps/kube-system/cilium/app/helm-values.yaml b/kubernetes/raspi/apps/kube-system/cilium/app/helm-values.yaml
@@ -0,0 +1,63 @@
+---
+autoDirectNodeRoutes: true
+bandwidthManager:
+  enabled: true
+  bbr: true
+bpf:
+  masquerade: true
+  tproxy: true
+cgroup:
+  automount:
+    enabled: false
+  hostRoot: /sys/fs/cgroup
+cluster:
+  id: 2
+  name: raspi
+devices: enp+
+enableRuntimeDeviceDetection: true
+endpointRoutes:
+  enabled: true
+hubble:
+  enabled: false
+envoy:
+  enabled: false
+ipam:
+  mode: kubernetes
+ipv4:
+  enabled: true
+ipv4NativeRoutingCIDR: 172.32.0.0/16
+ipv6:
+  enabled: false
+k8sServiceHost: 127.0.0.1
+k8sServicePort: 7445
+kubeProxyReplacement: true
+kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
+l2announcements:
+  enabled: true
+loadBalancer:
+  algorithm: maglev
+  mode: dsr
+localRedirectPolicy: true
+operator:
+  replicas: 1
+  rollOutPods: true
+rollOutCiliumPods: true
+routingMode: native
+securityContext:
+  capabilities:
+    ciliumAgent:
+      - CHOWN
+      - KILL
+      - NET_ADMIN
+      - NET_RAW
+      - IPC_LOCK
+      - SYS_ADMIN
+      - SYS_RESOURCE
+      - DAC_OVERRIDE
+      - FOWNER
+      - SETGID
+      - SETUID
+    cleanCiliumState:
+      - NET_ADMIN
+      - SYS_ADMIN
+      - SYS_RESOURCE
diff --git a/kubernetes/raspi/apps/kube-system/cilium/app/helmrelease.yaml b/kubernetes/raspi/apps/kube-system/cilium/app/helmrelease.yaml
@@ -0,0 +1,73 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.ok8.sh/helm.toolkit.fluxcd.io/helmrelease_v2.json
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: cilium
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: cilium
+      version: 1.16.2
+      sourceRef:
+        kind: HelmRepository
+        name: cilium
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      strategy: rollback
+      retries: 3
+  valuesFrom:
+    - kind: ConfigMap
+      name: cilium-helm-values
+  values:
+    hubble:
+      enabled: true
+      metrics:
+        enabled:
+          - dns:query
+          - drop
+          - tcp
+          - flow
+          - port-distribution
+          - icmp
+          - http
+        serviceMonitor:
+          enabled: true
+        dashboards:
+          enabled: true
+      relay:
+        enabled: true
+        rollOutPods: true
+        prometheus:
+          serviceMonitor:
+            enabled: true
+      ui:
+        enabled: true
+        rollOutPods: true
+        ingress:
+          enabled: true
+          className: internal
+          annotations:
+            external-dns.alpha.kubernetes.io/target: "internal.ok8.sh"
+          hosts: ["hubble.ok8.sh"]
+    operator:
+      replicas: 1
+      prometheus:
+        enabled: true
+        serviceMonitor:
+          enabled: true
+      dashboards:
+        enabled: true
+    prometheus:
+      enabled: true
+      serviceMonitor:
+        enabled: true
+        trustCRDsExist: true
+    dashboards:
+      enabled: true
diff --git a/kubernetes/raspi/apps/kube-system/cilium/app/kustomization.yaml b/kubernetes/raspi/apps/kube-system/cilium/app/kustomization.yaml
@@ -0,0 +1,12 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./helmrelease.yaml
+configMapGenerator:
+  - name: cilium-helm-values
+    files:
+      - values.yaml=./helm-values.yaml
+configurations:
+  - kustomizeconfig.yaml
diff --git a/kubernetes/raspi/apps/kube-system/cilium/app/kustomizeconfig.yaml b/kubernetes/raspi/apps/kube-system/cilium/app/kustomizeconfig.yaml
@@ -0,0 +1,7 @@
+---
+nameReference:
+  - kind: ConfigMap
+    version: v1
+    fieldSpecs:
+      - path: spec/valuesFrom/name
+        kind: HelmRelease
diff --git a/kubernetes/raspi/apps/kube-system/cilium/config/kustomization.yaml b/kubernetes/raspi/apps/kube-system/cilium/config/kustomization.yaml
@@ -0,0 +1,6 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./l2.yaml
diff --git a/kubernetes/raspi/apps/kube-system/cilium/config/l2.yaml b/kubernetes/raspi/apps/kube-system/cilium/config/l2.yaml
@@ -0,0 +1,35 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.ok8.sh/cilium.io/ciliuml2announcementpolicy_v2alpha1.json
+apiVersion: cilium.io/v2alpha1
+kind: CiliumL2AnnouncementPolicy
+metadata:
+  name: l2-policy
+spec:
+  loadBalancerIPs: true
+  interfaces: ["^enp.*"]
+  nodeSelector:
+    matchLabels:
+      kubernetes.io/os: linux
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.ok8.sh/cilium.io/ciliumloadbalancerippool_v2alpha1.json
+apiVersion: cilium.io/v2alpha1
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: l2-pool
+spec:
+  allowFirstLastIPs: "Yes"
+  disabled: true
+  blocks:
+    - start: 10.69.0.100
+      stop: 10.69.0.149
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.ok8.sh/cilium.io/ciliumloadbalancerippool_v2alpha1.json
+apiVersion: cilium.io/v2alpha1
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: teng-pool
+spec:
+  allowFirstLastIPs: "Yes"
+  blocks:
+    - start: 172.16.0.10
+      stop: 172.16.0.60
diff --git a/kubernetes/raspi/apps/kube-system/cilium/ks.yaml b/kubernetes/raspi/apps/kube-system/cilium/ks.yaml
@@ -0,0 +1,44 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.ok8.sh/kustomize.toolkit.fluxcd.io/kustomization_v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app cilium
+  namespace: flux-system
+spec:
+  targetNamespace: kube-system
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  path: ./kubernetes/main/apps/kube-system/cilium/app
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-kubernetes
+  wait: true
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.ok8.sh/kustomize.toolkit.fluxcd.io/kustomization_v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app cilium-config
+  namespace: flux-system
+spec:
+  targetNamespace: kube-system
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  dependsOn:
+    - name: cilium
+  path: ./kubernetes/main/apps/kube-system/cilium/config
+  prune: false # never should be deleted
+  sourceRef:
+    kind: GitRepository
+    name: home-kubernetes
+  wait: false
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
diff --git a/kubernetes/raspi/apps/kube-system/coredns/app/helm-values.yaml b/kubernetes/raspi/apps/kube-system/coredns/app/helm-values.yaml
@@ -0,0 +1,57 @@
+---
+fullnameOverride: coredns
+replicaCount: 1
+k8sAppLabelOverride: kube-dns
+serviceAccount:
+  create: true
+service:
+  name: kube-dns
+  clusterIP: 172.31.0.10
+servers:
+  - zones:
+      - zone: .
+        scheme: dns://
+        use_tcp: true
+    port: 53
+    plugins:
+      - name: log
+      - name: errors
+      - name: health
+        configBlock: |-
+          lameduck 5s
+      - name: ready
+      - name: kubernetes
+        parameters: cluster.local in-addr.arpa ip6.arpa
+        configBlock: |-
+          pods insecure
+          fallthrough in-addr.arpa ip6.arpa
+          ttl 30
+      - name: prometheus
+        parameters: 0.0.0.0:9153
+      - name: forward
+        parameters: . /etc/resolv.conf
+      - name: cache
+        parameters: 30
+      - name: loop
+      - name: reload
+      - name: loadbalance
+affinity:
+  nodeAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: node-role.kubernetes.io/control-plane
+              operator: Exists
+tolerations:
+  - key: CriticalAddonsOnly
+    operator: Exists
+  - key: node-role.kubernetes.io/control-plane
+    operator: Exists
+    effect: NoSchedule
+topologySpreadConstraints:
+  - maxSkew: 1
+    topologyKey: kubernetes.io/hostname
+    whenUnsatisfiable: DoNotSchedule
+    labelSelector:
+      matchLabels:
+        app.kubernetes.io/instance: coredns
diff --git a/kubernetes/raspi/apps/kube-system/coredns/app/helmrelease.yaml b/kubernetes/raspi/apps/kube-system/coredns/app/helmrelease.yaml
@@ -0,0 +1,27 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.ok8.sh/helm.toolkit.fluxcd.io/helmrelease_v2.json
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: coredns
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: coredns
+      version: 1.35.1
+      sourceRef:
+        kind: HelmRepository
+        name: coredns
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      strategy: rollback
+      retries: 3
+  valuesFrom:
+    - kind: ConfigMap
+      name: coredns-helm-values
diff --git a/kubernetes/raspi/apps/kube-system/coredns/app/kustomization.yaml b/kubernetes/raspi/apps/kube-system/coredns/app/kustomization.yaml
@@ -0,0 +1,12 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./helmrelease.yaml
+configMapGenerator:
+  - name: coredns-helm-values
+    files:
+      - values.yaml=./helm-values.yaml
+configurations:
+  - kustomizeconfig.yaml
diff --git a/kubernetes/raspi/apps/kube-system/coredns/app/kustomizeconfig.yaml b/kubernetes/raspi/apps/kube-system/coredns/app/kustomizeconfig.yaml
@@ -0,0 +1,7 @@
+---
+nameReference:
+  - kind: ConfigMap
+    version: v1
+    fieldSpecs:
+      - path: spec/valuesFrom/name
+        kind: HelmRelease