Add secrets propagation from management cluster to othres using Clust…

…erProfile

Makefile

@@ -36,7 +36,7 @@ dev:
 
 lint-chart-%:
 	$(HELM) dependency update $(TEMPLATES_DIR)/$*
-	$(HELM) lint --strict $(TEMPLATES_DIR)/$*
+	$(HELM) lint --strict $(TEMPLATES_DIR)/$* --set global.lint=true
 
 package-chart-%: lint-chart-%
 	$(HELM) package --destination $(CHARTS_PACKAGE_DIR) $(TEMPLATES_DIR)/$*
@@ -105,8 +105,8 @@ dev-storage-deploy: dev ## Deploy kof-storage helm chart to the K8s cluster spec
 dev-ms-deploy-aws: dev ## Deploy Mothership helm chart to the K8s cluster specified in ~/.kube/config for a remote storage cluster
 	cp -f $(TEMPLATES_DIR)/kof-mothership/values.yaml dev/mothership-values.yaml
 	@$(YQ) eval -i '.kcm.installTemplates = true' dev/mothership-values.yaml
-	@$(YQ) eval -i '.grafana.logSources = [{"name": "$(USER)-storage", "url": "https://vmauth.$(STORAGE_DOMAIN)/vls", "type": "victoriametrics-logs-datasource", "auth": {"credentials_secret_name": "grafana-admin-credentials"}}]' dev/mothership-values.yaml
-	@$(YQ) eval -i '.promxy.config.serverGroups = [{"clusterName": "$(USER)-storage", "targets": ["vmauth.$(STORAGE_DOMAIN):443"], "auth": {"credentials_secret_name": "grafana-admin-credentials"}}]' dev/mothership-values.yaml
+	@$(YQ) eval -i '.grafana.logSources = [{"name": "$(USER)-aws-storage", "url": "https://vmauth.$(STORAGE_DOMAIN)/vls", "type": "victoriametrics-logs-datasource", "auth": {"credentials_secret_name": "storage-vmuser-credentials", "username_key": "username", "password_key": "password"}}]' dev/mothership-values.yaml
+	@$(YQ) eval -i '.promxy.config.serverGroups = [{"clusterName": "$(USER)-aws-storage", "targets": ["vmauth.$(STORAGE_DOMAIN):443"], "auth": {"credentials_secret_name": "storage-vmuser-credentials", "create_secret": true, "username_key": "username", "password_key": "password"}}]' dev/mothership-values.yaml
 
 	@$(YQ) eval -i '.kcm.kof.charts.collectors.version = "$(COLLECTORS_VERSION)"' dev/mothership-values.yaml
 	@$(YQ) eval -i '.kcm.kof.charts.storage.version = "$(STORAGE_VERSION)"' dev/mothership-values.yaml
@@ -117,31 +117,31 @@ dev-ms-deploy-aws: dev ## Deploy Mothership helm chart to the K8s cluster specif
 	else \
 		$(YQ) eval -i '.kcm.kof.repo.url = "$(REGISTRY_REPO)"' dev/mothership-values.yaml; \
 	fi; \
-	$(HELM) upgrade -i kof ./charts/kof-mothership -n kof --create-namespace -f dev/mothership-values.yaml
+	$(HELM) upgrade -i kof-mothership ./charts/kof-mothership -n kof --create-namespace -f dev/mothership-values.yaml
 
 .PHONY: dev-storage-deploy-aws
 dev-storage-deploy-aws: dev ## Deploy Regional Managed cluster using KCM
 	cp -f demo/cluster/aws-storage.yaml dev/aws-storage.yaml
 	@$(YQ) eval -i '.metadata.name = "$(USER)-aws-storage"' dev/aws-storage.yaml
-	@$(YQ) '.spec.services[] | select(.name == "kof-storage") | .values' dev/aws-storage.yaml > dev/kof-storage-values.yaml
+	@$(YQ) '.spec.serviceSpec.services[] | select(.name == "kof-storage") | .values' dev/aws-storage.yaml > dev/kof-storage-values.yaml
 	@$(YQ) eval -i '.["cert-manager"].email = "$(USER_EMAIL)"' dev/kof-storage-values.yaml
 	@$(YQ) eval -i '.victoriametrics.vmauth.ingress.host = "vmauth.$(STORAGE_DOMAIN)"' dev/kof-storage-values.yaml
 	@$(YQ) eval -i '.grafana.ingress.host = "grafana.$(STORAGE_DOMAIN)"' dev/kof-storage-values.yaml
 	@$(YQ) eval -i '.["external-dns"].enabled = true' dev/kof-storage-values.yaml
-	@$(YQ) eval -i '(.spec.services[] | select(.name == "kof-storage")).values |= load_str("dev/kof-storage-values.yaml")' dev/aws-storage.yaml
+	@$(YQ) eval -i '(.spec.serviceSpec.services[] | select(.name == "kof-storage")).values |= load_str("dev/kof-storage-values.yaml")' dev/aws-storage.yaml
 	kubectl apply -f dev/aws-storage.yaml
 
 .PHONY: dev-managed-deploy-aws
 dev-managed-deploy-aws: dev ## Deploy Regional Managed cluster using KCM
 	cp -f demo/cluster/aws-managed.yaml dev/aws-managed.yaml
 	@$(YQ) eval -i '.metadata.name = "$(MANAGED_CLUSTER_NAME)"' dev/aws-managed.yaml
-	@$(YQ) '.spec.services[] | select(.name == "kof-collectors") | .values' dev/aws-managed.yaml > dev/kof-managed-values.yaml
+	@$(YQ) '.spec.serviceSpec.services[] | select(.name == "kof-collectors") | .values' dev/aws-managed.yaml > dev/kof-managed-values.yaml
 	@$(YQ) eval -i '.global.clusterName = "$(MANAGED_CLUSTER_NAME)"' dev/kof-managed-values.yaml
 	@$(YQ) eval -i '.opencost.opencost.exporter.defaultClusterId = "$(MANAGED_CLUSTER_NAME)"' dev/kof-managed-values.yaml
 	@$(YQ) eval -i '.opencost.opencost.prometheus.external.url = "https://vmauth.$(STORAGE_DOMAIN)/vm/select/0/prometheus"' dev/kof-managed-values.yaml
 	@$(YQ) eval -i '.kof.logs.endpoint = "https://vmauth.$(STORAGE_DOMAIN)/vls/insert/opentelemetry/v1/logs"' dev/kof-managed-values.yaml
 	@$(YQ) eval -i '.kof.metrics.endpoint = "https://vmauth.$(STORAGE_DOMAIN)/vm/insert/0/prometheus/api/v1/write"' dev/kof-managed-values.yaml
-	@$(YQ) eval -i '(.spec.services[] | select(.name == "kof-collectors")).values |= load_str("dev/kof-managed-values.yaml")' dev/aws-managed.yaml
+	@$(YQ) eval -i '(.spec.serviceSpec.services[] | select(.name == "kof-collectors")).values |= load_str("dev/kof-managed-values.yaml")' dev/aws-managed.yaml
 	kubectl apply -f dev/aws-managed.yaml
 
 ## Tool Binaries

README.md

@@ -12,19 +12,14 @@ This repo contains 4 charts to deploy an observability stack using [k0rdent](htt
 ### Demo deployment
 In `demo/demo-mothership-values.yaml` set your target ingress names that you are going to use for your storage clusters, but they can always be changed after the fact
 
-Create secrets for grafana admin user and storage clusters datasources endpoint access. By default the secret below be reused everywhere, but it is customizable.
-
-```yaml
----
-kind: Secret
-apiVersion: v1
-metadata:
-  name: grafana-admin-credentials
-  namespace: kof
-stringData:
-  GF_SECURITY_ADMIN_USER: username # Grafana username
-  GF_SECURITY_ADMIN_PASSWORD: password # Grafana password
-type: Opaque
+By default the secrets defined in the `values.yaml` are created automatically and propagated to managed clusters using Sveltos cluster profile.
+
+You can retrieve grafana password and username using the following command
+
+```bash
+kubectl get secret grafana-admin-credentials -o jsonpath="{.data.GF_SECURITY_ADMIN_USER}" -n kof | base64 -d; echo
+
+kubectl get secret grafana-admin-credentials -o jsonpath="{.data.GF_SECURITY_ADMIN_PASSWORD}" -n kof | base64 -d; echo
 ```
 
 ```bash

charts/kof-collectors/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 name: kof-collectors
 description: A Helm chart that deploys OpenTelemetryCollector resources
-version: 0.0.1
+version: 0.0.2
 appVersion: "1.0"
 dependencies:
   - name: prometheus-node-exporter

charts/kof-collectors/templates/_helpers.tpl

@@ -2,7 +2,7 @@
 {{- define "basic_auth_extensions" -}}
 {{- range tuple "metrics" "logs" }}
 {{- $secret := (lookup "v1" "Secret" $.Release.Namespace (index $.Values "kof" . "credentials_secret_name")) }}
-{{- if $secret }}
+{{- if not $.Values.global.lint }}
 basicauth/{{ . }}:
   client_auth:
     username: {{ index $secret.data (index $.Values "kof" . "username_key") | b64dec | quote }}

charts/kof-collectors/values.yaml

@@ -4,14 +4,14 @@ global:
 kof:
   logs:
     endpoint: http://victoria-logs-single-server:9428/insert/opentelemetry/v1/logs
-    credentials_secret_name: grafana-admin-credentials
-    username_key: GF_SECURITY_ADMIN_USER
-    password_key: GF_SECURITY_ADMIN_PASSWORD
+    credentials_secret_name: storage-vmuser-credentials
+    username_key: username
+    password_key: password
   metrics:
     endpoint: http://vminsert-cluster:8480/insert/0/prometheus/api/v1/write
-    credentials_secret_name: grafana-admin-credentials
-    username_key: GF_SECURITY_ADMIN_USER
-    password_key: GF_SECURITY_ADMIN_PASSWORD
+    credentials_secret_name: storage-vmuser-credentials
+    username_key: username
+    password_key: password
   instrumentation:
     enabled: true
 prometheus-node-exporter:
@@ -41,9 +41,9 @@ opencost:
   enabled: true
   opencost:
     prometheus:
-      existingSecretName: grafana-admin-credentials
-      username_key: GF_SECURITY_ADMIN_USER
-      password_key: GF_SECURITY_ADMIN_PASSWORD
+      existingSecretName: storage-vmuser-credentials
+      username_key: username
+      password_key: password
       external:
         enabled: true
         url: "https://vmauth.storage0.example.net/vm/select/0/prometheus"

charts/kof-mothership/templates/grafana/logs-datasource.yaml

@@ -1,5 +1,6 @@
 {{- if .Values.grafana.enabled }}
 {{- range .Values.grafana.logSources }}
+---
 apiVersion: grafana.integreatly.org/v1beta1
 kind: GrafanaDatasource
 metadata:
@@ -12,12 +13,12 @@ spec:
     - targetPath: "basicAuthUser"
       valueFrom:
         secretKeyRef:
-          key: {{ .auth.username_key | default "GF_SECURITY_ADMIN_USER" }}
+          key: {{ .auth.username_key }}
           name: {{ .auth.credentials_secret_name }}
     - targetPath: "secureJsonData.basicAuthPassword"
       valueFrom:
         secretKeyRef:
-          key: {{ .auth.password_key | default "GF_SECURITY_ADMIN_PASSWORD" }}
+          key: {{ .auth.password_key }}
           name: {{ .auth.credentials_secret_name }}
   datasource:
     access: proxy
@@ -30,6 +31,5 @@ spec:
     matchLabels:
       dashboards: grafana
   resyncPeriod: 5m
----
 {{- end }}
 {{- end }}

charts/kof-mothership/templates/grafana/secret.yaml

@@ -0,0 +1,15 @@
+{{- if .Values.grafana.security.create_secret }}
+{{- $secret := (lookup "v1" "Secret" .Release.Namespace .Values.grafana.security.credentials_secret_name) }}
+{{- if not $secret }}
+---
+kind: Secret
+apiVersion: v1
+metadata:
+  name: {{ .Values.grafana.security.credentials_secret_name }}
+  namespace: {{ .Release.Namespace }}
+stringData:
+  GF_SECURITY_ADMIN_USER: {{ randAlpha  (.Values.global.random_username_length | int) | quote }} # Grafana username
+  GF_SECURITY_ADMIN_PASSWORD: {{ randAlpha (.Values.global.random_password_length | int) | quote }} # Grafana password
+type: Opaque
+{{- end }}
+{{- end }}

charts/kof-mothership/templates/hmc/kof-svctmpl.yaml → charts/kof-mothership/templates/kcm/kof/svctmpl.yaml

@@ -5,7 +5,7 @@ apiVersion: hmc.mirantis.com/v1alpha1
 kind: ServiceTemplate
 metadata:
   name: kof-{{ $name }}
-  namespace: hmc-system
+  namespace: {{ $.Values.kcm.namespace }}
 spec:
   helm:
     chartRef:

charts/kof-mothership/templates/kcm/sveltos/copy-template.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: copy-resource-template
+  namespace: {{ $.Release.Namespace }}
+  annotations:
+    projectsveltos.io/template: "true"  # add annotation to indicate Sveltos content is a template
+data:
+  secret.yaml: |
+    {{` {{ copy "Resource" }} `}}

charts/kof-mothership/templates/promxy/secret.yaml

@@ -1,4 +1,29 @@
+{{- /* Tracking secrets with the same name to avoid conflict during creation */}}
+{{ $all_secrets := dict }}
+{{- range .Values.promxy.config.serverGroups }}
+{{- if .auth.create_secret }}
+{{- /* Checking that the secret isn't created yet to avoid credentials regeneration */}}
+{{- $secret := (lookup "v1" "Secret" $.Release.Namespace .auth.credentials_secret_name) }}
+{{- if $secret }}
+{{- $_ := set $all_secrets .auth.credentials_secret_name (dict "username" (index $secret.data .auth.username_key | b64dec) "password" (index $secret.data .auth.password_key | b64dec))}}
+{{- end }}
+{{- if and (not $secret) (not (hasKey $all_secrets .auth.credentials_secret_name)) }}
+{{- $_ := set $all_secrets .auth.credentials_secret_name (dict "username" (randAlpha ($.Values.global.random_username_length | int)) "password" (randAlpha ($.Values.global.random_password_length | int)) )}}
+---
+kind: Secret
+apiVersion: v1
+metadata:
+  name: {{ .auth.credentials_secret_name }}
+  namespace: {{ $.Release.Namespace }}
+stringData:
+  {{ .auth.username_key}}: {{ index $all_secrets .auth.credentials_secret_name "username" | quote }}
+  {{ .auth.password_key}}: {{ index $all_secrets .auth.credentials_secret_name "password" | quote }}
+type: Opaque
+{{- end }}
+{{- end }}
+{{- end }}
 {{- if eq .Values.promxy.secret "" }}
+---
 apiVersion: v1
 kind: Secret
 metadata:
@@ -32,10 +57,15 @@ stringData:
             tls_config:
               insecure_skip_verify: true
             basic_auth:
+{{- if not .auth.create_secret }}
 {{- $secret := (lookup "v1" "Secret" $.Release.Namespace .auth.credentials_secret_name) }}
-{{- if $secret }}
-              username: {{ index $secret.data (.auth.username_key | default "GF_SECURITY_ADMIN_USER") | b64dec | quote }}
-              password: {{ index $secret.data (.auth.password_key | default "GF_SECURITY_ADMIN_PASSWORD") | b64dec | quote }}
+{{- if not $.Values.global.lint }}
+              username: {{ index $secret.data .auth.username_key | b64dec | quote }}
+              password: {{ index $secret.data .auth.password_key | b64dec | quote }}
+{{- end }}
+{{- else }}
+              username: {{ index $all_secrets .auth.credentials_secret_name "username" | quote }}
+              password: {{ index $all_secrets .auth.credentials_secret_name "password" | quote }}
 {{- end }}
           labels:
             promxyCluster: {{ .clusterName | quote }}

charts/kof-mothership/templates/sveltos/copy-secrets-cluster-profile.yaml

@@ -0,0 +1,24 @@
+{{- if .Values.kcm.kof.secrets_copy.secrets }}
+{{- /* Distributing secrets to clusters automatically*/}}
+---
+apiVersion: config.projectsveltos.io/v1beta1
+kind: ClusterProfile
+metadata:
+  name: kof-secrets
+spec:
+  clusterSelector:
+    matchLabels: {{ .Values.kcm.kof.secrets_copy.matchLabels | toYaml | nindent 6 }}
+  templateResourceRefs:
+{{- range .Values.kcm.kof.secrets_copy.secrets }}
+    - resource:
+        apiVersion: v1
+        kind: Secret
+        name: {{ . }}
+        namespace: {{ $.Release.Namespace }}
+      identifier: Resource
+{{- end }}
+  policyRefs:
+    - kind: ConfigMap
+      name: copy-resource-template
+      namespace: {{ $.Release.Namespace }}
+{{- end }}

charts/kof-mothership/values.yaml

@@ -2,6 +2,8 @@ global:
   clusterLabel: clusterName
   storageClass: standard
   clusterName: mothership
+  random_username_length: 8
+  random_password_length: 12
 kcm:
   installTemplates: false
   kof:
@@ -14,9 +16,14 @@ kcm:
       operators:
         version: 0.0.1
       collectors:
-        version: 0.0.1
+        version: 0.0.2
       storage:
-        version: 0.1.2
+        version: 0.1.4
+    secrets_copy:
+      matchLabels:
+        k0rdent.mirantis.com/kof-storage-secrets: "true"
+      secrets:
+        - storage-vmuser-credentials
 victoriametrics:
   enabled: true
   vmcluster:
@@ -38,14 +45,19 @@ grafana:
       url: https://vmauth.storage0.example.net/vls
       type: victoriametrics-logs-datasource
       auth:
-        credentials_secret_name: grafana-admin-credentials
+        credentials_secret_name: storage-vmuser-credentials
+        username_key: username
+        password_key: password
     - name: storage1-logs
       url: https://vmauth.storage1.example.net/vls
       type: victoriametrics-logs-datasource
       auth:
-        credentials_secret_name: grafana-admin-credentials
+        credentials_secret_name: storage-vmuser-credentials
+        username_key: username
+        password_key: password
   security:
     credentials_secret_name: grafana-admin-credentials
+    create_secret: true
 victoria-metrics-operator:
   enabled: true
   crds:
@@ -103,9 +115,15 @@ promxy:
         targets:
           - vmauth.storage0.example.net:443
         auth:
-          credentials_secret_name: grafana-admin-credentials
+          credentials_secret_name: storage-vmuser-credentials
+          username_key: username
+          password_key: password
+          create_secret: true
       - clusterName: storage1
         targets:
           - vmauth.storage1.example.net:443
         auth:
-          credentials_secret_name: grafana-admin-credentials
+          credentials_secret_name: storage-vmuser-credentials
+          username_key: username
+          password_key: password
+          create_secret: true

charts/kof-storage/Chart.lock

@@ -4,12 +4,12 @@ dependencies:
   version: v5.15.1
 - name: victoria-metrics-operator
   repository: https://victoriametrics.github.io/helm-charts/
-  version: 0.40.3
+  version: 0.40.4
 - name: victoria-logs-single
   repository: https://victoriametrics.github.io/helm-charts/
   version: 0.8.12
 - name: external-dns
   repository: https://kubernetes-sigs.github.io/external-dns/
   version: 1.15.0
-digest: sha256:765fa797ecae93345bb5eff857a8ab89803f9cfb0048e560625820a1d4567df0
-generated: "2025-01-06T15:31:00.919698+02:00"
+digest: sha256:e4ba96403921fa721337ab6cb4a005fe205407c01badf77a91b7f9549dfb5737
+generated: "2025-01-13T15:11:53.857929+02:00"

charts/kof-storage/templates/grafana/secret.yaml

@@ -0,0 +1,15 @@
+{{- if .Values.grafana.security.create_secret }}
+{{- $secret := (lookup "v1" "Secret" .Release.Namespace .Values.grafana.security.credentials_secret_name) }}
+{{- if not $secret }}
+---
+kind: Secret
+apiVersion: v1
+metadata:
+  name: {{ .Values.grafana.security.credentials_secret_name }}
+  namespace: {{ .Release.Namespace }}
+stringData:
+  GF_SECURITY_ADMIN_USER: {{ randAlpha  (.Values.global.random_username_length | int) | quote }} # Grafana username
+  GF_SECURITY_ADMIN_PASSWORD: {{ randAlpha (.Values.global.random_password_length | int) | quote }} # Grafana password
+type: Opaque
+{{- end }}
+{{- end }}

charts/kof-storage/templates/victoria/vmuser.yaml

@@ -22,12 +22,12 @@ spec:
     - /vm/insert/.*
     static:
       url: http://vminsert-cluster.{{ .Release.Namespace }}.svc:8480
-{{- $secret := (lookup "v1" "Secret" .Release.Namespace .Values.victoriametrics.vmauth.credentials.password_secret_name) }}
-{{- if $secret }}
+{{- $secret := (lookup "v1" "Secret" .Release.Namespace .Values.victoriametrics.vmauth.credentials.credentials_secret_name) }}
+{{- if not $.Values.global.lint }}
   username: {{ index $secret.data .Values.victoriametrics.vmauth.credentials.username_key | b64dec | quote }}
 {{- end }}
   passwordRef:
     key: {{ .Values.victoriametrics.vmauth.credentials.password_key }}
-    name: {{ .Values.victoriametrics.vmauth.credentials.password_secret_name }}
+    name: {{ .Values.victoriametrics.vmauth.credentials.credentials_secret_name }}
 {{- end }}
 {{- end }}