From 0c26dc3978fa81ea1ed7a54d3fbaf3d3e6304f10 Mon Sep 17 00:00:00 2001 From: Tarun Gupta Akirala Date: Thu, 23 Jan 2025 14:42:13 -0800 Subject: [PATCH 1/7] feat: add harbor cosi transformer Signed-off-by: Tarun Gupta Akirala --- stable/cosi-bucket-kit/Chart.yaml | 5 +- .../templates/job-readiness.yaml | 135 ++++++++++++++++++ stable/cosi-bucket-kit/values.yaml | 8 ++ 3 files changed, 146 insertions(+), 2 deletions(-) create mode 100644 stable/cosi-bucket-kit/templates/job-readiness.yaml diff --git a/stable/cosi-bucket-kit/Chart.yaml b/stable/cosi-bucket-kit/Chart.yaml index c7cbd2865..e95e3bdba 100644 --- a/stable/cosi-bucket-kit/Chart.yaml +++ b/stable/cosi-bucket-kit/Chart.yaml @@ -8,7 +8,8 @@ keywords: - bucket - storage - ceph -version: 0.0.1-alpha.1 -appVersion: 0.0.1-alpha.1 +version: 0.0.1-alpha.2 +appVersion: 0.0.1-alpha.2 maintainers: - name: takirala + - name: mhrabovcin diff --git a/stable/cosi-bucket-kit/templates/job-readiness.yaml b/stable/cosi-bucket-kit/templates/job-readiness.yaml new file mode 100644 index 000000000..7b579059d --- /dev/null +++ b/stable/cosi-bucket-kit/templates/job-readiness.yaml @@ -0,0 +1,135 @@ +{{- if .Values.cosiBucketKit.enabled }} # COSI Bucket chart is enabled +{{- if gt (len .Values.cosiBucketKit.bucketAccesses) 0 }} # At least one BucketAccess is created +{{- if or .Values.cosiBucketKit.transformations.kubecost.enabled .Values.cosiBucketKit.transformations.harbor }} # At least one transformation is enabled +# Add rbac for the Job Readiness Check +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-readiness + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ .Release.Name }}-readiness + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +rules: + - apiGroups: [ "" ] + resources: [ "configmaps" ] + verbs: [ "get", "list", "create", "patch"] # A new configmap is created for harbor + - apiGroups: [ "" ] + resources: [ "secrets" ] + verbs: [ "get", "list", "patch" ] # Existing secrets are updated for kubecost and harbor. +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Release.Name }}-readiness + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cosi-job-readiness-role +subjects: +- kind: ServiceAccount + name: cosi-bucket-kit + namespace: {{ .Release.Namespace }} +--- +# This job has a container for each transformation enabled. There is a shared container for bucket readiness check. +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-readiness + namespace: {{ .Release.Namespace }} +spec: + template: + metadata: + name: {{ .Release.Name }}-readiness + spec: + serviceAccountName: {{ .Release.Name }}-readiness + restartPolicy: OnFailure + {{- if .Values.cosiBucketKit.transformations.priorityClassName }} + priorityClassName: {{ .Values.cosiBucketKit.transformations.priorityClassName }} + {{- end }} + containers: + - name: wait-for-cosi-secret + image: {{ .Values.cosiBucketKit.transformations.kubectlImage }} + command: + - bash + - -c + - | + set -o nounset + set -o pipefail + + echo() { + command echo $(date) "$@" + } + + {{- $ns := .Release.Namespace -}} + {{- range .Values.cosiBucketKit.bucketAccesses }} + # For each bucketAccess, check if the secret exists. + while ! kubectl get secret -n {{ $ns }} {{ .credentialsSecretName }}; do + echo "Secret {{ .credentialsSecretName }} not found in namespace {{ $ns }}. Waiting for it to be created." + sleep 5 + done + # TODO(takirala): Wait for .data.BucketInfo field to be populated with a valid cosi secret value. + {{- end }} + {{- if .Values.cosiBucketKit.transformations.kubecost.enabled }} + # This job will exit with one of the following outcomes: + # - If namespace is not kommander namespace then exit successfully (targets attached clusters). + # - If kubecostClusterMode is not set to primary then exit successfully (targets attached clusters that have been expanded). + # - In kommander namespace and when running in primary mode, wait until the federated-store secret is found. Could be a user created secret or be created by cosi-bucket-kit helmrelease. + - name: transform-kubecost-cosi-secret + image: {{ .Values.cosiBucketKit.transformations.kubectlImage }} + command: + - bash + - -c + - | + set -o nounset + set -o pipefail + + echo() { + command echo $(date) "$@" + } + {{- end }} + {{- if .Values.cosiBucketKit.transformations.harbor.enabled }} + - name: transform-harbor-cosi-secret + image: {{ .Values.cosiBucketKit.transformations.kubectlImage }} + command: + - bash + - -c + - | + set -o nounset + set -o pipefail + + echo() { + command echo $(date) "$@" + } + + {{- $cmName := .Values.cosiBucketKit.transformations.harbor.cmName }} + {{- $ns := .Release.Namespace -}} + # TODO(takirala): Fail fast if there is more than one bucketAccess with harbor is found. + {{- range .Values.cosiBucketKit.bucketAccesses }} + # Update the cosi secret with harbor specific keys. + kubectl create secret generic {{ .credentialsSecretName }} -n {{ $ns }} \ + --from-literal=REGISTRY_STORAGE_S3_ACCESSKEY=$(kubectl get secret {{ .credentialsSecretName }} -n {{ $ns }} -o jsonpath="{.data.BucketInfo}" | base64 --decode | jq -r '.spec.secretS3.accessKeyID') \ + --from-literal=REGISTRY_STORAGE_S3_SECRETKEY=$(kubectl get secret {{ .credentialsSecretName }} -n {{ $ns }} -o jsonpath="{.data.BucketInfo}" | base64 --decode | jq -r '.spec.secretS3.accessSecretKey') \ + --dry-run=client -o yaml | kubectl apply -f - + # Create a configmap with the name of the secret from above. + kubectl create configmap {{ $cmName }} -n {{ $ns }} \ + --from-literal="values.yaml=$(printf -- '---\npersistence:\n imageChartStorage:\n s3:\n existingSecret: {{ .credentialsSecretName }}')" \ + --dry-run=client -o yaml | kubectl apply -f - + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/stable/cosi-bucket-kit/values.yaml b/stable/cosi-bucket-kit/values.yaml index 006ac11b8..2099bf75e 100644 --- a/stable/cosi-bucket-kit/values.yaml +++ b/stable/cosi-bucket-kit/values.yaml @@ -53,3 +53,11 @@ cosiBucketKit: # capabilities: # bucket: "*" # user: "*" + transformations: + priorityClassName: dkp-high-priority + kubectlImage: bitnami/kubectl:1.31.4 + kubecost: + enabled: false + harbor: + enabled: false + cmName: harbor-cosi-overrides From 0ecc39e2721fe15b6b1be5baf3e94e859283eedd Mon Sep 17 00:00:00 2001 From: Tarun Gupta Akirala Date: Thu, 23 Jan 2025 15:45:26 -0800 Subject: [PATCH 2/7] fix: typo in service account name Signed-off-by: Tarun Gupta Akirala --- stable/cosi-bucket-kit/templates/job-readiness.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/stable/cosi-bucket-kit/templates/job-readiness.yaml b/stable/cosi-bucket-kit/templates/job-readiness.yaml index 7b579059d..bc534b78c 100644 --- a/stable/cosi-bucket-kit/templates/job-readiness.yaml +++ b/stable/cosi-bucket-kit/templates/job-readiness.yaml @@ -38,10 +38,10 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: cosi-job-readiness-role + name: {{ .Release.Name }}-readiness subjects: - kind: ServiceAccount - name: cosi-bucket-kit + name: {{ .Release.Name }}-readiness namespace: {{ .Release.Namespace }} --- # This job has a container for each transformation enabled. There is a shared container for bucket readiness check. @@ -50,6 +50,9 @@ kind: Job metadata: name: {{ .Release.Name }}-readiness namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded spec: template: metadata: @@ -114,10 +117,10 @@ spec: echo() { command echo $(date) "$@" } + # TODO(takirala): Fail fast if there is more than one bucketAccess with harbor is found. {{- $cmName := .Values.cosiBucketKit.transformations.harbor.cmName }} {{- $ns := .Release.Namespace -}} - # TODO(takirala): Fail fast if there is more than one bucketAccess with harbor is found. {{- range .Values.cosiBucketKit.bucketAccesses }} # Update the cosi secret with harbor specific keys. kubectl create secret generic {{ .credentialsSecretName }} -n {{ $ns }} \ From 351652e4817c9e89e9610adc9af4543cdb669141 Mon Sep 17 00:00:00 2001 From: Tarun Gupta Akirala Date: Sat, 25 Jan 2025 15:12:24 -0800 Subject: [PATCH 3/7] chore: add kubecost transformation Signed-off-by: Tarun Gupta Akirala --- .../templates/job-readiness.yaml | 66 ++++++++++++++++--- 1 file changed, 58 insertions(+), 8 deletions(-) diff --git a/stable/cosi-bucket-kit/templates/job-readiness.yaml b/stable/cosi-bucket-kit/templates/job-readiness.yaml index bc534b78c..175013134 100644 --- a/stable/cosi-bucket-kit/templates/job-readiness.yaml +++ b/stable/cosi-bucket-kit/templates/job-readiness.yaml @@ -1,5 +1,4 @@ {{- if .Values.cosiBucketKit.enabled }} # COSI Bucket chart is enabled -{{- if gt (len .Values.cosiBucketKit.bucketAccesses) 0 }} # At least one BucketAccess is created {{- if or .Values.cosiBucketKit.transformations.kubecost.enabled .Values.cosiBucketKit.transformations.harbor }} # At least one transformation is enabled # Add rbac for the Job Readiness Check apiVersion: v1 @@ -84,13 +83,8 @@ spec: echo "Secret {{ .credentialsSecretName }} not found in namespace {{ $ns }}. Waiting for it to be created." sleep 5 done - # TODO(takirala): Wait for .data.BucketInfo field to be populated with a valid cosi secret value. {{- end }} {{- if .Values.cosiBucketKit.transformations.kubecost.enabled }} - # This job will exit with one of the following outcomes: - # - If namespace is not kommander namespace then exit successfully (targets attached clusters). - # - If kubecostClusterMode is not set to primary then exit successfully (targets attached clusters that have been expanded). - # - In kommander namespace and when running in primary mode, wait until the federated-store secret is found. Could be a user created secret or be created by cosi-bucket-kit helmrelease. - name: transform-kubecost-cosi-secret image: {{ .Values.cosiBucketKit.transformations.kubectlImage }} command: @@ -103,6 +97,61 @@ spec: echo() { command echo $(date) "$@" } + + while true; do # Wait until BucketInfo is found in the secret. + bucketInfo=$(kubectl get secret -n {{ .Release.Namespace }} federated-store -o jsonpath='{.data.BucketInfo}' | base64 -d) + federatedStoreYAML=$(kubectl get secret -n {{ .Release.Namespace }} federated-store -o jsonpath='{.data.federated-store\.yaml}' | base64 -d) + if [ -n "$bucketInfo" ] || [ -n "$federatedStoreYAML" ]; then + break + fi + echo "BucketInfo and federated-store.yaml not found in federated-store secret. Waiting for it to be populated." + sleep 5 + done + + # If bucketInfo is empty and federatedStoreYAML is not empty, then the secret is already updated (probably manually by the user). + if [ -z "$bucketInfo" ] && [ -n "$federatedStoreYAML" ]; then + echo "BucketInfo is empty and federated-store.yaml is not empty. Using the federated-store.yaml as-is." + kubectl label secret federated-store -n {{ .Release.Namespace }} app.kubernetes.io/kommander-kubecost-federated-store=true --overwrite + kubectl annotate secret federated-store -n {{ .Release.Namespace }} app.kubernetes.io/kommander-kubecost-federated-store-unprocessed=true --overwrite + exit 0 + fi + + # Update the cosi secret with kubecost specific format. + tmpfile=$(mktemp /tmp/federated-store.XXXXXX) + echo "Fetched bucketInfo from federated-store secret. Processing it..." + yq eval ' + { + "type": "S3", + "config": { + "bucket": .spec.bucketName, + "endpoint": .spec.secretS3.endpoint | sub(":\\d+$", "") | sub("^http://", "") | sub("^https://", ""), # Remove port and protocol (if any). + "region": .spec.secretS3.region, + "access_key": .spec.secretS3.accessKeyID, + "secret_key": .spec.secretS3.accessSecretKey, + "insecure": .spec.secretS3.endpoint | test("^http://"), # Use insecure if endpoint is http (e.g.: cluster internal endpoint). + "signature_version2": false, # Use signature version 4. + "put_user_metadata": { + "X-Amz-Acl": "bucket-owner-full-control" + }, + "http_config": { + "idle_conn_timeout": "90s", + "response_header_timeout": "2m", + "insecure_skip_verify": false + }, + "trace": { + "enable": false # Enable to debug errors (if any) + }, + "part_size": 134217728 + } + }' <<< "$bucketInfo" > "$tmpfile" + echo "Transformed bucketInfo to federated-store.yaml. Updating federated-store secret..." + + kubectl create secret generic federated-store -n {{ .Release.Namespace }} --from-file=federated-store.yaml="$tmpfile" --dry-run=client -o yaml | kubectl apply -f - + kubectl label secret federated-store -n {{ .Release.Namespace }} app.kubernetes.io/kommander-kubecost-federated-store=true --overwrite + kubectl annotate secret federated-store -n {{ .Release.Namespace }} app.kubernetes.io/kommander-kubecost-federated-store-processed=true --overwrite + + kubectl create configmap kubecost-object-store-config -n {{ .Release.Namespace }} --save-config --from-literal=objectStoreStatus=ready --dry-run=client -o yaml | kubectl apply -f - + rm "$tmpfile" {{- end }} {{- if .Values.cosiBucketKit.transformations.harbor.enabled }} - name: transform-harbor-cosi-secret @@ -117,7 +166,9 @@ spec: echo() { command echo $(date) "$@" } - # TODO(takirala): Fail fast if there is more than one bucketAccess with harbor is found. + {{- if gt (len .Values.cosiBucketKit.bucketAccesses) 1 }} + {{- fail "Error: .Values.cosiBucketKit.bucketAccesses array size must not exceed 1 if harbor transformation is enabled." }} + {{- end }} {{- $cmName := .Values.cosiBucketKit.transformations.harbor.cmName }} {{- $ns := .Release.Namespace -}} @@ -135,4 +186,3 @@ spec: {{- end }} {{- end }} {{- end }} -{{- end }} From 35eeb92b90589a25f6fbee596d3119d3e1efa614 Mon Sep 17 00:00:00 2001 From: Tarun Gupta Akirala Date: Mon, 27 Jan 2025 10:06:43 -0800 Subject: [PATCH 4/7] chore: apply suggestions from code review Co-authored-by: Martin Hrabovcin --- .../cosi-bucket-kit/templates/job-readiness.yaml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/stable/cosi-bucket-kit/templates/job-readiness.yaml b/stable/cosi-bucket-kit/templates/job-readiness.yaml index 175013134..f54505415 100644 --- a/stable/cosi-bucket-kit/templates/job-readiness.yaml +++ b/stable/cosi-bucket-kit/templates/job-readiness.yaml @@ -177,10 +177,22 @@ spec: kubectl create secret generic {{ .credentialsSecretName }} -n {{ $ns }} \ --from-literal=REGISTRY_STORAGE_S3_ACCESSKEY=$(kubectl get secret {{ .credentialsSecretName }} -n {{ $ns }} -o jsonpath="{.data.BucketInfo}" | base64 --decode | jq -r '.spec.secretS3.accessKeyID') \ --from-literal=REGISTRY_STORAGE_S3_SECRETKEY=$(kubectl get secret {{ .credentialsSecretName }} -n {{ $ns }} -o jsonpath="{.data.BucketInfo}" | base64 --decode | jq -r '.spec.secretS3.accessSecretKey') \ + --from-literal=REGISTRY_STORAGE_S3_REGION=none \ + --from-literal=REGISTRY_STORAGE_S3_REGIONENDPOINT=$(kubectl get secret {{ .credentialsSecretName }} -n {{ $ns }} -o jsonpath="{.data.BucketInfo}" | base64 --decode | jq -r '.spec.secretS3.nedpoint') \ + --from-literal=REGISTRY_STORAGE_S3_BUCKET=$(kubectl get secret {{ .credentialsSecretName }} -n {{ $ns }} -o jsonpath="{.data.BucketInfo}" | base64 --decode | jq -r '.spec.bucketName') \ + --from-literal=REGISTRY_STORAGE_S3_SECURE=false \ + --from-literal=REGISTRY_STORAGE_REDIRECT_DISABLE=true \ --dry-run=client -o yaml | kubectl apply -f - # Create a configmap with the name of the secret from above. kubectl create configmap {{ $cmName }} -n {{ $ns }} \ - --from-literal="values.yaml=$(printf -- '---\npersistence:\n imageChartStorage:\n s3:\n existingSecret: {{ .credentialsSecretName }}')" \ + --from-file=values.yaml=<(cat < Date: Mon, 27 Jan 2025 10:43:55 -0800 Subject: [PATCH 5/7] chore: parametrize harbor configmap namespace Signed-off-by: Tarun Gupta Akirala --- stable/cosi-bucket-kit/templates/job-readiness.yaml | 3 ++- stable/cosi-bucket-kit/values.yaml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/stable/cosi-bucket-kit/templates/job-readiness.yaml b/stable/cosi-bucket-kit/templates/job-readiness.yaml index f54505415..68d3baadf 100644 --- a/stable/cosi-bucket-kit/templates/job-readiness.yaml +++ b/stable/cosi-bucket-kit/templates/job-readiness.yaml @@ -171,6 +171,7 @@ spec: {{- end }} {{- $cmName := .Values.cosiBucketKit.transformations.harbor.cmName }} + {{- $cmNamespace := .Values.cosiBucketKit.transformations.harbor.cmNamespace }} {{- $ns := .Release.Namespace -}} {{- range .Values.cosiBucketKit.bucketAccesses }} # Update the cosi secret with harbor specific keys. @@ -184,7 +185,7 @@ spec: --from-literal=REGISTRY_STORAGE_REDIRECT_DISABLE=true \ --dry-run=client -o yaml | kubectl apply -f - # Create a configmap with the name of the secret from above. - kubectl create configmap {{ $cmName }} -n {{ $ns }} \ + kubectl create configmap {{ $cmName }} -n {{ $cmNamespace }} \ --from-file=values.yaml=<(cat < Date: Mon, 27 Jan 2025 10:52:20 -0800 Subject: [PATCH 6/7] fix: make linter happy Signed-off-by: Tarun Gupta Akirala --- stable/cosi-bucket-kit/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stable/cosi-bucket-kit/values.yaml b/stable/cosi-bucket-kit/values.yaml index 404705dc6..7dc5bd1e8 100644 --- a/stable/cosi-bucket-kit/values.yaml +++ b/stable/cosi-bucket-kit/values.yaml @@ -61,4 +61,4 @@ cosiBucketKit: harbor: enabled: false cmName: harbor-cosi-overrides - cmNamespace: # This should point to the namespace where the harbor flux helm release is created (not the .targetNamespace of the flux helm release). + cmNamespace: # This should point to the namespace where the harbor flux helm release is created (not the .targetNamespace of the flux helm release). From e5a9353e964ebbfed11f89f1db1c0a21d9d74cc8 Mon Sep 17 00:00:00 2001 From: Tarun Gupta Akirala Date: Mon, 27 Jan 2025 12:12:02 -0800 Subject: [PATCH 7/7] fix: typo Co-authored-by: Martin Hrabovcin --- stable/cosi-bucket-kit/templates/job-readiness.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stable/cosi-bucket-kit/templates/job-readiness.yaml b/stable/cosi-bucket-kit/templates/job-readiness.yaml index 68d3baadf..7d2270a79 100644 --- a/stable/cosi-bucket-kit/templates/job-readiness.yaml +++ b/stable/cosi-bucket-kit/templates/job-readiness.yaml @@ -179,7 +179,7 @@ spec: --from-literal=REGISTRY_STORAGE_S3_ACCESSKEY=$(kubectl get secret {{ .credentialsSecretName }} -n {{ $ns }} -o jsonpath="{.data.BucketInfo}" | base64 --decode | jq -r '.spec.secretS3.accessKeyID') \ --from-literal=REGISTRY_STORAGE_S3_SECRETKEY=$(kubectl get secret {{ .credentialsSecretName }} -n {{ $ns }} -o jsonpath="{.data.BucketInfo}" | base64 --decode | jq -r '.spec.secretS3.accessSecretKey') \ --from-literal=REGISTRY_STORAGE_S3_REGION=none \ - --from-literal=REGISTRY_STORAGE_S3_REGIONENDPOINT=$(kubectl get secret {{ .credentialsSecretName }} -n {{ $ns }} -o jsonpath="{.data.BucketInfo}" | base64 --decode | jq -r '.spec.secretS3.nedpoint') \ + --from-literal=REGISTRY_STORAGE_S3_REGIONENDPOINT=$(kubectl get secret {{ .credentialsSecretName }} -n {{ $ns }} -o jsonpath="{.data.BucketInfo}" | base64 --decode | jq -r '.spec.secretS3.endpoint') \ --from-literal=REGISTRY_STORAGE_S3_BUCKET=$(kubectl get secret {{ .credentialsSecretName }} -n {{ $ns }} -o jsonpath="{.data.BucketInfo}" | base64 --decode | jq -r '.spec.bucketName') \ --from-literal=REGISTRY_STORAGE_S3_SECURE=false \ --from-literal=REGISTRY_STORAGE_REDIRECT_DISABLE=true \