feat: add cosi transformer for harbor and kubecost

stable/cosi-bucket-kit/Chart.yaml

@@ -8,7 +8,8 @@ keywords:
   - bucket
   - storage
   - ceph
-version: 0.0.1-alpha.1
-appVersion: 0.0.1-alpha.1
+version: 0.0.1-alpha.2
+appVersion: 0.0.1-alpha.2
 maintainers:
   - name: takirala
+  - name: mhrabovcin

stable/cosi-bucket-kit/templates/job-readiness.yaml

@@ -0,0 +1,200 @@
+{{- if .Values.cosiBucketKit.enabled }} # COSI Bucket chart is enabled
+{{- if or .Values.cosiBucketKit.transformations.kubecost.enabled .Values.cosiBucketKit.transformations.harbor }} # At least one transformation is enabled
+# Add rbac for the Job Readiness Check
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Release.Name }}-readiness
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ .Release.Name }}-readiness
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+rules:
+  - apiGroups: [ "" ]
+    resources: [ "configmaps" ]
+    verbs: [ "get", "list", "create", "patch"] # A new configmap is created for harbor
+  - apiGroups: [ "" ]
+    resources: [ "secrets" ]
+    verbs: [ "get", "list", "patch" ] # Existing secrets are updated for kubecost and harbor.
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ .Release.Name }}-readiness
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ .Release.Name }}-readiness
+subjects:
+- kind: ServiceAccount
+  name: {{ .Release.Name }}-readiness
+  namespace: {{ .Release.Namespace }}
+---
+# This job has a container for each transformation enabled. There is a shared container for bucket readiness check.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ .Release.Name }}-readiness
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+spec:
+  template:
+    metadata:
+      name: {{ .Release.Name }}-readiness
+    spec:
+      serviceAccountName: {{ .Release.Name }}-readiness
+      restartPolicy: OnFailure
+      {{- if .Values.cosiBucketKit.transformations.priorityClassName }}
+      priorityClassName: {{ .Values.cosiBucketKit.transformations.priorityClassName }}
+      {{- end }}
+      containers:
+        - name: wait-for-cosi-secret
+          image: {{ .Values.cosiBucketKit.transformations.kubectlImage }}
+          command:
+            - bash
+            - -c
+            - |
+              set -o nounset
+              set -o pipefail
+
+              echo() {
+                  command echo $(date) "$@"
+              }
+
+              {{- $ns := .Release.Namespace -}}
+              {{- range .Values.cosiBucketKit.bucketAccesses }}
+              # For each bucketAccess, check if the secret exists.
+              while ! kubectl get secret -n {{ $ns }} {{ .credentialsSecretName }}; do
+                  echo "Secret {{ .credentialsSecretName }} not found in namespace {{ $ns }}. Waiting for it to be created."
+                  sleep 5
+              done
+              {{- end }}
+        {{- if .Values.cosiBucketKit.transformations.kubecost.enabled }}
+        - name: transform-kubecost-cosi-secret
+          image: {{ .Values.cosiBucketKit.transformations.kubectlImage }}
+          command:
+            - bash
+            - -c
+            - |
+              set -o nounset
+              set -o pipefail
+
+              echo() {
+                command echo $(date) "$@"
+              }
+
+              while true; do # Wait until BucketInfo is found in the secret.
+                  bucketInfo=$(kubectl get secret -n {{ .Release.Namespace }} federated-store -o jsonpath='{.data.BucketInfo}' | base64 -d)
+                  federatedStoreYAML=$(kubectl get secret -n {{ .Release.Namespace }} federated-store -o jsonpath='{.data.federated-store\.yaml}' | base64 -d)
+                  if [ -n "$bucketInfo" ] || [ -n "$federatedStoreYAML" ]; then
+                      break
+                  fi
+                  echo "BucketInfo and federated-store.yaml not found in federated-store secret. Waiting for it to be populated."
+                  sleep 5
+              done
+
+              # If bucketInfo is empty and federatedStoreYAML is not empty, then the secret is already updated (probably manually by the user).
+              if [ -z "$bucketInfo" ] && [ -n "$federatedStoreYAML" ]; then
+                  echo "BucketInfo is empty and federated-store.yaml is not empty. Using the federated-store.yaml as-is."
+                  kubectl label secret federated-store -n {{ .Release.Namespace }} app.kubernetes.io/kommander-kubecost-federated-store=true --overwrite
+                  kubectl annotate secret federated-store -n {{ .Release.Namespace }} app.kubernetes.io/kommander-kubecost-federated-store-unprocessed=true --overwrite
+                  exit 0
+              fi
+
+              # Update the cosi secret with kubecost specific format.
+              tmpfile=$(mktemp /tmp/federated-store.XXXXXX)
+              echo "Fetched bucketInfo from federated-store secret. Processing it..."
+              yq eval '
+              {
+                "type": "S3",
+                "config": {
+                  "bucket": .spec.bucketName,
+                  "endpoint": .spec.secretS3.endpoint | sub(":\\d+$", "") | sub("^http://", "") | sub("^https://", ""), # Remove port and protocol (if any).
+                  "region": .spec.secretS3.region,
+                  "access_key": .spec.secretS3.accessKeyID,
+                  "secret_key": .spec.secretS3.accessSecretKey,
+                  "insecure": .spec.secretS3.endpoint | test("^http://"), # Use insecure if endpoint is http (e.g.: cluster internal endpoint).
+                  "signature_version2": false, # Use signature version 4.
+                  "put_user_metadata": {
+                    "X-Amz-Acl": "bucket-owner-full-control"
+                  },
+                  "http_config": {
+                    "idle_conn_timeout": "90s",
+                    "response_header_timeout": "2m",
+                    "insecure_skip_verify": false
+                  },
+                  "trace": {
+                    "enable": false # Enable to debug errors (if any)
+                  },
+                  "part_size": 134217728
+                }
+              }' <<< "$bucketInfo" > "$tmpfile"
+              echo "Transformed bucketInfo to federated-store.yaml. Updating federated-store secret..."
+
+              kubectl create secret generic federated-store -n {{ .Release.Namespace }} --from-file=federated-store.yaml="$tmpfile" --dry-run=client -o yaml | kubectl apply -f -
+              kubectl label secret federated-store -n {{ .Release.Namespace }} app.kubernetes.io/kommander-kubecost-federated-store=true --overwrite
+              kubectl annotate secret federated-store -n {{ .Release.Namespace }} app.kubernetes.io/kommander-kubecost-federated-store-processed=true --overwrite
+
+              kubectl create configmap kubecost-object-store-config -n {{ .Release.Namespace }} --save-config --from-literal=objectStoreStatus=ready --dry-run=client -o yaml | kubectl apply -f -
+              rm "$tmpfile"
+        {{- end }}
+        {{- if .Values.cosiBucketKit.transformations.harbor.enabled }}
+        - name: transform-harbor-cosi-secret
+          image: {{ .Values.cosiBucketKit.transformations.kubectlImage }}
+          command:
+            - bash
+            - -c
+            - |
+              set -o nounset
+              set -o pipefail
+
+              echo() {
+                  command echo $(date) "$@"
+              }
+              {{- if gt (len .Values.cosiBucketKit.bucketAccesses) 1 }}
+              {{- fail "Error: .Values.cosiBucketKit.bucketAccesses array size must not exceed 1 if harbor transformation is enabled." }}
+              {{- end }}
+
+              {{- $cmName := .Values.cosiBucketKit.transformations.harbor.cmName }}
+              {{- $ns := .Release.Namespace -}}
+              {{- range .Values.cosiBucketKit.bucketAccesses }}
+              # Update the cosi secret with harbor specific keys.
+              kubectl create secret generic {{ .credentialsSecretName }} -n {{ $ns }} \
+                --from-literal=REGISTRY_STORAGE_S3_ACCESSKEY=$(kubectl get secret {{ .credentialsSecretName }} -n {{ $ns }} -o jsonpath="{.data.BucketInfo}" | base64 --decode | jq -r '.spec.secretS3.accessKeyID') \
+                --from-literal=REGISTRY_STORAGE_S3_SECRETKEY=$(kubectl get secret {{ .credentialsSecretName }} -n {{ $ns }} -o jsonpath="{.data.BucketInfo}" | base64 --decode | jq -r '.spec.secretS3.accessSecretKey') \
+                --from-literal=REGISTRY_STORAGE_S3_REGION=none \
+                --from-literal=REGISTRY_STORAGE_S3_REGIONENDPOINT=$(kubectl get secret {{ .credentialsSecretName }} -n {{ $ns }} -o jsonpath="{.data.BucketInfo}" | base64 --decode | jq -r '.spec.secretS3.nedpoint') \
+                --from-literal=REGISTRY_STORAGE_S3_BUCKET=$(kubectl get secret {{ .credentialsSecretName }} -n {{ $ns }} -o jsonpath="{.data.BucketInfo}" | base64 --decode | jq -r '.spec.bucketName') \
+                --from-literal=REGISTRY_STORAGE_S3_SECURE=false \
+                --from-literal=REGISTRY_STORAGE_REDIRECT_DISABLE=true \
+                --dry-run=client -o yaml | kubectl apply -f -
+              # Create a configmap with the name of the secret from above.
+              kubectl create configmap {{ $cmName }} -n {{ $ns }} \
+                --from-file=values.yaml=<(cat <<EOF
+                persistence:
+                  imageChartStorage:
+                    type: s3
+                    s3:
+                      existingSecret: {{ .credentialsSecretName }}
+                EOF
+                ) \
+                --dry-run=client -o yaml | kubectl apply -f -
+              {{- end }}
+        {{- end }}
+{{- end }}
+{{- end }}

stable/cosi-bucket-kit/values.yaml

@@ -53,3 +53,11 @@ cosiBucketKit:
 #         capabilities:
 #           bucket: "*"
 #           user: "*"
+  transformations:
+    priorityClassName: dkp-high-priority
+    kubectlImage: bitnami/kubectl:1.31.4
+    kubecost:
+      enabled: false
+    harbor:
+      enabled: false
+      cmName: harbor-cosi-overrides