Merge pull request #28 from gmlexx/main

Rework secrets creation, usage and propagation

Makefile

@@ -15,7 +15,7 @@ $(CHARTS_PACKAGE_DIR): | $(LOCALBIN)
 	rm -rf $(CHARTS_PACKAGE_DIR)
 	mkdir -p $(CHARTS_PACKAGE_DIR)
 
-REGISTRY_NAME ?= hmc-local-registry
+REGISTRY_NAME ?= kcm-local-registry
 REGISTRY_PORT ?= 5001
 REGISTRY_REPO ?= oci://127.0.0.1:$(REGISTRY_PORT)/charts
 REGISTRY_IS_OCI = $(shell echo $(REGISTRY_REPO) | grep -q oci && echo true || echo false)
@@ -36,7 +36,7 @@ dev:
 
 lint-chart-%:
 	$(HELM) dependency update $(TEMPLATES_DIR)/$*
-	$(HELM) lint --strict $(TEMPLATES_DIR)/$*
+	$(HELM) lint --strict $(TEMPLATES_DIR)/$* --set global.lint=true
 
 package-chart-%: lint-chart-%
 	$(HELM) package --destination $(CHARTS_PACKAGE_DIR) $(TEMPLATES_DIR)/$*
@@ -72,7 +72,7 @@ helm-push: helm-package
 				echo "REGISTRY_USERNAME and REGISTRY_PASSWORD must be populated to push the chart to an HTTPS repository"; \
 				exit 1; \
 			else \
-				$(HELM) repo add hmc $(REGISTRY_REPO); \
+				$(HELM) repo add kcm $(REGISTRY_REPO); \
 				echo "Pushing $$chart to $(REGISTRY_REPO)"; \
 				$(HELM) cm-push "$$chart" $(REGISTRY_REPO) --username $$REGISTRY_USERNAME --password $$REGISTRY_PASSWORD; \
 			fi; \
@@ -105,8 +105,8 @@ dev-storage-deploy: dev ## Deploy kof-storage helm chart to the K8s cluster spec
 dev-ms-deploy-aws: dev ## Deploy Mothership helm chart to the K8s cluster specified in ~/.kube/config for a remote storage cluster
 	cp -f $(TEMPLATES_DIR)/kof-mothership/values.yaml dev/mothership-values.yaml
 	@$(YQ) eval -i '.kcm.installTemplates = true' dev/mothership-values.yaml
-	@$(YQ) eval -i '.grafana.logSources = [{"name": "$(USER)-storage", "url": "https://vmauth.$(STORAGE_DOMAIN)/vls", "type": "victoriametrics-logs-datasource", "auth": {"credentials_secret_name": "grafana-admin-credentials"}}]' dev/mothership-values.yaml
-	@$(YQ) eval -i '.promxy.config.serverGroups = [{"clusterName": "$(USER)-storage", "targets": ["vmauth.$(STORAGE_DOMAIN):443"], "auth": {"credentials_secret_name": "grafana-admin-credentials"}}]' dev/mothership-values.yaml
+	@$(YQ) eval -i '.grafana.logSources = [{"name": "$(USER)-aws-storage", "url": "https://vmauth.$(STORAGE_DOMAIN)/vls", "type": "victoriametrics-logs-datasource", "auth": {"credentials_secret_name": "storage-vmuser-credentials", "username_key": "username", "password_key": "password"}}]' dev/mothership-values.yaml
+	@$(YQ) eval -i '.promxy.config.serverGroups = [{"clusterName": "$(USER)-aws-storage", "targets": ["vmauth.$(STORAGE_DOMAIN):443"], "auth": {"credentials_secret_name": "storage-vmuser-credentials", "create_secret": true, "username_key": "username", "password_key": "password"}}]' dev/mothership-values.yaml
 
 	@$(YQ) eval -i '.kcm.kof.charts.collectors.version = "$(COLLECTORS_VERSION)"' dev/mothership-values.yaml
 	@$(YQ) eval -i '.kcm.kof.charts.storage.version = "$(STORAGE_VERSION)"' dev/mothership-values.yaml
@@ -117,31 +117,31 @@ dev-ms-deploy-aws: dev ## Deploy Mothership helm chart to the K8s cluster specif
 	else \
 		$(YQ) eval -i '.kcm.kof.repo.url = "$(REGISTRY_REPO)"' dev/mothership-values.yaml; \
 	fi; \
-	$(HELM) upgrade -i kof ./charts/kof-mothership -n kof --create-namespace -f dev/mothership-values.yaml
+	$(HELM) upgrade -i kof-mothership ./charts/kof-mothership -n kof --create-namespace -f dev/mothership-values.yaml
 
 .PHONY: dev-storage-deploy-aws
 dev-storage-deploy-aws: dev ## Deploy Regional Managed cluster using KCM
 	cp -f demo/cluster/aws-storage.yaml dev/aws-storage.yaml
 	@$(YQ) eval -i '.metadata.name = "$(USER)-aws-storage"' dev/aws-storage.yaml
-	@$(YQ) '.spec.services[] | select(.name == "kof-storage") | .values' dev/aws-storage.yaml > dev/kof-storage-values.yaml
+	@$(YQ) '.spec.serviceSpec.services[] | select(.name == "kof-storage") | .values' dev/aws-storage.yaml > dev/kof-storage-values.yaml
 	@$(YQ) eval -i '.["cert-manager"].email = "$(USER_EMAIL)"' dev/kof-storage-values.yaml
 	@$(YQ) eval -i '.victoriametrics.vmauth.ingress.host = "vmauth.$(STORAGE_DOMAIN)"' dev/kof-storage-values.yaml
 	@$(YQ) eval -i '.grafana.ingress.host = "grafana.$(STORAGE_DOMAIN)"' dev/kof-storage-values.yaml
 	@$(YQ) eval -i '.["external-dns"].enabled = true' dev/kof-storage-values.yaml
-	@$(YQ) eval -i '(.spec.services[] | select(.name == "kof-storage")).values |= load_str("dev/kof-storage-values.yaml")' dev/aws-storage.yaml
+	@$(YQ) eval -i '(.spec.serviceSpec.services[] | select(.name == "kof-storage")).values |= load_str("dev/kof-storage-values.yaml")' dev/aws-storage.yaml
 	kubectl apply -f dev/aws-storage.yaml
 
 .PHONY: dev-managed-deploy-aws
 dev-managed-deploy-aws: dev ## Deploy Regional Managed cluster using KCM
 	cp -f demo/cluster/aws-managed.yaml dev/aws-managed.yaml
 	@$(YQ) eval -i '.metadata.name = "$(MANAGED_CLUSTER_NAME)"' dev/aws-managed.yaml
-	@$(YQ) '.spec.services[] | select(.name == "kof-collectors") | .values' dev/aws-managed.yaml > dev/kof-managed-values.yaml
+	@$(YQ) '.spec.serviceSpec.services[] | select(.name == "kof-collectors") | .values' dev/aws-managed.yaml > dev/kof-managed-values.yaml
 	@$(YQ) eval -i '.global.clusterName = "$(MANAGED_CLUSTER_NAME)"' dev/kof-managed-values.yaml
 	@$(YQ) eval -i '.opencost.opencost.exporter.defaultClusterId = "$(MANAGED_CLUSTER_NAME)"' dev/kof-managed-values.yaml
 	@$(YQ) eval -i '.opencost.opencost.prometheus.external.url = "https://vmauth.$(STORAGE_DOMAIN)/vm/select/0/prometheus"' dev/kof-managed-values.yaml
 	@$(YQ) eval -i '.kof.logs.endpoint = "https://vmauth.$(STORAGE_DOMAIN)/vls/insert/opentelemetry/v1/logs"' dev/kof-managed-values.yaml
 	@$(YQ) eval -i '.kof.metrics.endpoint = "https://vmauth.$(STORAGE_DOMAIN)/vm/insert/0/prometheus/api/v1/write"' dev/kof-managed-values.yaml
-	@$(YQ) eval -i '(.spec.services[] | select(.name == "kof-collectors")).values |= load_str("dev/kof-managed-values.yaml")' dev/aws-managed.yaml
+	@$(YQ) eval -i '(.spec.serviceSpec.services[] | select(.name == "kof-collectors")).values |= load_str("dev/kof-managed-values.yaml")' dev/aws-managed.yaml
 	kubectl apply -f dev/aws-managed.yaml
 
 ## Tool Binaries

README.md

@@ -12,19 +12,14 @@ This repo contains 4 charts to deploy an observability stack using [k0rdent](htt
 ### Demo deployment
 In `demo/demo-mothership-values.yaml` set your target ingress names that you are going to use for your storage clusters, but they can always be changed after the fact
 
-Create secrets for grafana admin user and storage clusters datasources endpoint access. By default the secret below be reused everywhere, but it is customizable.
-
-```yaml
----
-kind: Secret
-apiVersion: v1
-metadata:
-  name: grafana-admin-credentials
-  namespace: kof
-stringData:
-  GF_SECURITY_ADMIN_USER: username # Grafana username
-  GF_SECURITY_ADMIN_PASSWORD: password # Grafana password
-type: Opaque
+By default the secrets defined in the `values.yaml` are created automatically and propagated to managed clusters using Sveltos cluster profile.
+
+You can retrieve grafana password and username using the following command
+
+```bash
+kubectl get secret grafana-admin-credentials -o jsonpath="{.data.GF_SECURITY_ADMIN_USER}" -n kof | base64 -d; echo
+
+kubectl get secret grafana-admin-credentials -o jsonpath="{.data.GF_SECURITY_ADMIN_PASSWORD}" -n kof | base64 -d; echo
 ```
 
 ```bash
@@ -52,13 +47,13 @@ To deploy storage `clusterdeployment` configure desired ingress names for vmauth
 ```bash
 kubectl apply -f demo/cluster/aws-storage.yaml
 # you can check helm chart deployment status using ClusterSummary object:
-kubectl get clustersummaries.config.projectsveltos.io -n hmc-system
+kubectl get clustersummaries.config.projectsveltos.io -n kcm-system
 ```
 Once the storage clusterdeploymet is ready - retrieve its kubeconfig and get loadbalancer IP/DNS name for your ingress-nginx service.
 
 ```bash
-kubectl get secret -n hmc-system aws-storage-kubeconfig -o jsonpath={.data.value} | base64 -d  > /tmp/hmc-aws-storage-kubeconfig.yaml
-export KUBECONFIG=/tmp/hmc-aws-storage-kubeconfig.yaml
+kubectl get secret -n kcm-system aws-storage-kubeconfig -o jsonpath={.data.value} | base64 -d  > /tmp/kcm-aws-storage-kubeconfig.yaml
+export KUBECONFIG=/tmp/kcm-aws-storage-kubeconfig.yaml
 kubectl get svc -n ingress-nginx ingress-nginx-controller
 ```
 
@@ -83,7 +78,7 @@ To deploy operators and collectors to a `clusterdeployment` configure ingress na
 ```
 kubectl apply -f demo/cluster/aws-managed.yaml
 # you can check helm chart deployment status using ClusterSummary object:
-kubectl get clustersummaries.config.projectsveltos.io -n hmc-system
+kubectl get clustersummaries.config.projectsveltos.io -n kcm-system
 ```
 
 Once your managed clusters are up, create secrets for storage cluster authentication, it should start pushing metrics and logs to your storage one, through ingress domain you've configured.

charts/kof-collectors/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 name: kof-collectors
 description: A Helm chart that deploys OpenTelemetryCollector resources
-version: 0.0.1
+version: 0.0.2
 appVersion: "1.0"
 dependencies:
   - name: prometheus-node-exporter

charts/kof-collectors/templates/_helpers.tpl

@@ -2,7 +2,7 @@
 {{- define "basic_auth_extensions" -}}
 {{- range tuple "metrics" "logs" }}
 {{- $secret := (lookup "v1" "Secret" $.Release.Namespace (index $.Values "kof" . "credentials_secret_name")) }}
-{{- if $secret }}
+{{- if not $.Values.global.lint }}
 basicauth/{{ . }}:
   client_auth:
     username: {{ index $secret.data (index $.Values "kof" . "username_key") | b64dec | quote }}

charts/kof-collectors/values.yaml

@@ -4,14 +4,14 @@ global:
 kof:
   logs:
     endpoint: http://victoria-logs-single-server:9428/insert/opentelemetry/v1/logs
-    credentials_secret_name: grafana-admin-credentials
-    username_key: GF_SECURITY_ADMIN_USER
-    password_key: GF_SECURITY_ADMIN_PASSWORD
+    credentials_secret_name: storage-vmuser-credentials
+    username_key: username
+    password_key: password
   metrics:
     endpoint: http://vminsert-cluster:8480/insert/0/prometheus/api/v1/write
-    credentials_secret_name: grafana-admin-credentials
-    username_key: GF_SECURITY_ADMIN_USER
-    password_key: GF_SECURITY_ADMIN_PASSWORD
+    credentials_secret_name: storage-vmuser-credentials
+    username_key: username
+    password_key: password
   instrumentation:
     enabled: true
 prometheus-node-exporter:
@@ -41,9 +41,9 @@ opencost:
   enabled: true
   opencost:
     prometheus:
-      existingSecretName: grafana-admin-credentials
-      username_key: GF_SECURITY_ADMIN_USER
-      password_key: GF_SECURITY_ADMIN_PASSWORD
+      existingSecretName: storage-vmuser-credentials
+      username_key: username
+      password_key: password
       external:
         enabled: true
         url: "https://vmauth.storage0.example.net/vm/select/0/prometheus"

charts/kof-mothership/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 name: kof-mothership
 description: A Helm chart that deploys Grafana, Promxy, and VictoriaMetrics.
-version: 0.1.6
+version: 0.1.8
 appVersion: "1.0"
 dependencies:
   - name: grafana-operator

charts/kof-mothership/templates/grafana/logs-datasource.yaml

@@ -1,5 +1,6 @@
 {{- if .Values.grafana.enabled }}
 {{- range .Values.grafana.logSources }}
+---
 apiVersion: grafana.integreatly.org/v1beta1
 kind: GrafanaDatasource
 metadata:
@@ -12,12 +13,12 @@ spec:
     - targetPath: "basicAuthUser"
       valueFrom:
         secretKeyRef:
-          key: {{ .auth.username_key | default "GF_SECURITY_ADMIN_USER" }}
+          key: {{ .auth.username_key }}
           name: {{ .auth.credentials_secret_name }}
     - targetPath: "secureJsonData.basicAuthPassword"
       valueFrom:
         secretKeyRef:
-          key: {{ .auth.password_key | default "GF_SECURITY_ADMIN_PASSWORD" }}
+          key: {{ .auth.password_key }}
           name: {{ .auth.credentials_secret_name }}
   datasource:
     access: proxy
@@ -30,6 +31,5 @@ spec:
     matchLabels:
       dashboards: grafana
   resyncPeriod: 5m
----
 {{- end }}
 {{- end }}

charts/kof-mothership/templates/grafana/secret.yaml

@@ -0,0 +1,15 @@
+{{- if .Values.grafana.security.create_secret }}
+{{- $secret := (lookup "v1" "Secret" .Release.Namespace .Values.grafana.security.credentials_secret_name) }}
+{{- if not $secret }}
+---
+kind: Secret
+apiVersion: v1
+metadata:
+  name: {{ .Values.grafana.security.credentials_secret_name }}
+  namespace: {{ .Release.Namespace }}
+stringData:
+  GF_SECURITY_ADMIN_USER: {{ randAlpha  (.Values.global.random_username_length | int) | quote }} # Grafana username
+  GF_SECURITY_ADMIN_PASSWORD: {{ randAlpha (.Values.global.random_password_length | int) | quote }} # Grafana password
+type: Opaque
+{{- end }}
+{{- end }}

charts/kof-mothership/templates/hmc/cert-manager/flux-helm.yaml → charts/kof-mothership/templates/kcm/cert-manager/flux-helm.yaml

@@ -4,19 +4,19 @@ apiVersion: source.toolkit.fluxcd.io/v1
 kind: HelmRepository
 metadata:
   name: jetstack
-  namespace: hmc-system
+  namespace: {{ .Values.kcm.namespace }}
   labels:
-    hmc.mirantis.com/managed: "true"
+    k0rdent.mirantis.com/managed: "true"
 spec:
   url: https://charts.jetstack.io
 ---
 apiVersion: source.toolkit.fluxcd.io/v1
 kind: HelmChart
 metadata:
   name: cert-manager
-  namespace: hmc-system
+  namespace: {{ .Values.kcm.namespace }}
   labels:
-    hmc.mirantis.com/managed: "true"
+    k0rdent.mirantis.com/managed: "true"
 spec:
   chart: cert-manager
   interval: 10m0s
@@ -25,4 +25,4 @@ spec:
     kind: HelmRepository
     name: jetstack
   version: 1.16.1
-{{- end }}
+{{- end }}

charts/kof-mothership/templates/hmc/cert-manager/svctmpl.yaml → charts/kof-mothership/templates/kcm/cert-manager/svctmpl.yaml

@@ -1,15 +1,15 @@
 {{- if .Values.kcm.installTemplates }}
-apiVersion: hmc.mirantis.com/v1alpha1
+apiVersion: k0rdent.mirantis.com/v1alpha1
 kind: ServiceTemplate
 metadata:
   name: cert-manager
-  namespace: hmc-system
+  namespace: {{ .Values.kcm.namespace }}
 spec:
   helm:
     chartRef:
       apiVersion: source.toolkit.fluxcd.io/v1
       kind: HelmChart
       name: cert-manager
-      namespace: hmc-system
+      namespace: {{ .Values.kcm.namespace }}
   providers: []
 {{- end }}

charts/kof-mothership/templates/hmc/kof-flux-helm.yaml → charts/kof-mothership/templates/kcm/kof/flux-helm.yaml

@@ -5,9 +5,9 @@ apiVersion: source.toolkit.fluxcd.io/v1
 kind: HelmChart
 metadata:
   name: kof-{{ $name }}
-  namespace: hmc-system
+  namespace: {{ $.Values.kcm.namespace }}
   labels:
-    hmc.mirantis.com/managed: "true"
+    k0rdent.mirantis.com/managed: "true"
 spec:
   chart: kof-{{ $name }}
   interval: 10m0s

charts/kof-mothership/templates/hmc/kof-repo.yaml → charts/kof-mothership/templates/kcm/kof/flux-repo.yaml

@@ -4,9 +4,9 @@ apiVersion: source.toolkit.fluxcd.io/v1
 kind: HelmRepository
 metadata:
   name: {{ .Values.kcm.kof.repo.name }}
-  namespace: hmc-system
+  namespace: {{ .Values.kcm.namespace }}
   labels:
-    hmc.mirantis.com/managed: "true"
+    k0rdent.mirantis.com/managed: "true"
 spec:
   url: {{ .Values.kcm.kof.repo.url }}
   insecure: {{ .Values.kcm.kof.repo.insecure }}

charts/kof-mothership/templates/hmc/kof-svctmpl.yaml → charts/kof-mothership/templates/kcm/kof/svctmpl.yaml

@@ -1,18 +1,18 @@
 {{- if .Values.kcm.installTemplates }}
 {{- range $name, $values := .Values.kcm.kof.charts }}
 ---
-apiVersion: hmc.mirantis.com/v1alpha1
+apiVersion: k0rdent.mirantis.com/v1alpha1
 kind: ServiceTemplate
 metadata:
   name: kof-{{ $name }}
-  namespace: hmc-system
+  namespace: {{ $.Values.kcm.namespace }}
 spec:
   helm:
     chartRef:
       apiVersion: source.toolkit.fluxcd.io/v1
       kind: HelmChart
       name: kof-{{ $name }}
-      namespace: hmc-system
+      namespace: {{ $.Values.kcm.namespace }}
   providers: []
 {{- end }}
 {{- end }}

charts/kof-mothership/templates/kcm/sveltos/copy-template.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: copy-resource-template
+  namespace: {{ $.Release.Namespace }}
+  annotations:
+    projectsveltos.io/template: "true"  # add annotation to indicate Sveltos content is a template
+data:
+  secret.yaml: |
+    {{` {{ copy "Resource" }} `}}

charts/kof-mothership/templates/promxy/secret.yaml

@@ -1,4 +1,29 @@
+{{- /* Tracking secrets with the same name to avoid conflict during creation */}}
+{{ $all_secrets := dict }}
+{{- range .Values.promxy.config.serverGroups }}
+{{- if .auth.create_secret }}
+{{- /* Checking that the secret isn't created yet to avoid credentials regeneration */}}
+{{- $secret := (lookup "v1" "Secret" $.Release.Namespace .auth.credentials_secret_name) }}
+{{- if $secret }}
+{{- $_ := set $all_secrets .auth.credentials_secret_name (dict "username" (index $secret.data .auth.username_key | b64dec) "password" (index $secret.data .auth.password_key | b64dec))}}
+{{- end }}
+{{- if and (not $secret) (not (hasKey $all_secrets .auth.credentials_secret_name)) }}
+{{- $_ := set $all_secrets .auth.credentials_secret_name (dict "username" (randAlpha ($.Values.global.random_username_length | int)) "password" (randAlpha ($.Values.global.random_password_length | int)) )}}
+---
+kind: Secret
+apiVersion: v1
+metadata:
+  name: {{ .auth.credentials_secret_name }}
+  namespace: {{ $.Release.Namespace }}
+stringData:
+  {{ .auth.username_key}}: {{ index $all_secrets .auth.credentials_secret_name "username" | quote }}
+  {{ .auth.password_key}}: {{ index $all_secrets .auth.credentials_secret_name "password" | quote }}
+type: Opaque
+{{- end }}
+{{- end }}
+{{- end }}
 {{- if eq .Values.promxy.secret "" }}
+---
 apiVersion: v1
 kind: Secret
 metadata:
@@ -32,10 +57,15 @@ stringData:
             tls_config:
               insecure_skip_verify: true
             basic_auth:
+{{- if not .auth.create_secret }}
 {{- $secret := (lookup "v1" "Secret" $.Release.Namespace .auth.credentials_secret_name) }}
-{{- if $secret }}
-              username: {{ index $secret.data (.auth.username_key | default "GF_SECURITY_ADMIN_USER") | b64dec | quote }}
-              password: {{ index $secret.data (.auth.password_key | default "GF_SECURITY_ADMIN_PASSWORD") | b64dec | quote }}
+{{- if not $.Values.global.lint }}
+              username: {{ index $secret.data .auth.username_key | b64dec | quote }}
+              password: {{ index $secret.data .auth.password_key | b64dec | quote }}
+{{- end }}
+{{- else }}
+              username: {{ index $all_secrets .auth.credentials_secret_name "username" | quote }}
+              password: {{ index $all_secrets .auth.credentials_secret_name "password" | quote }}
 {{- end }}
           labels:
             promxyCluster: {{ .clusterName | quote }}