Add Helm hook to upgrade CRDs

Signed-off-by: Yi Chen <[email protected]>

.dockerignore

@@ -2,7 +2,6 @@
 .idea/
 .vscode/
 bin/
-charts/
 docs/
 config/
 examples/

Dockerfile

@@ -55,6 +55,8 @@ USER ${SPARK_UID}:${SPARK_GID}
 
 COPY --from=builder /workspace/bin/spark-operator /usr/bin/spark-operator
 
+COPY --from=builder /workspace/charts/spark-operator-chart/crds /etc/spark-operator/crds
+
 COPY entrypoint.sh /usr/bin/
 
 ENTRYPOINT ["/usr/bin/entrypoint.sh"]

charts/spark-operator-chart/README.md

@@ -83,6 +83,8 @@ See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall) for command docum
 | image.tag | string | If not set, the chart appVersion will be used. | Image tag. |
 | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy. |
 | image.pullSecrets | list | `[]` | Image pull secrets for private image registry. |
+| hook.upgradeCrd | bool | `true` | Specifies whether to update CRDs with a Helm hook job. |
+| hook.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"privileged":false,"readOnlyRootFilesystem":true,"runAsNonRoot":true}` | Security context for hook containers. |
 | controller.replicas | int | `1` | Number of replicas of controller. |
 | controller.workers | int | `10` | Reconcile concurrency, higher values might increase memory usage. |
 | controller.logLevel | string | `"info"` | Configure the verbosity of logging, can be one of `debug`, `info`, `error`. |

charts/spark-operator-chart/templates/hook/_helpers.tpl

@@ -0,0 +1,65 @@
+{{/*
+Copyright 2024 The Kubeflow authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+Create the name of Helm hook
+*/}}
+{{- define "spark-operator.hook.name" -}}
+{{- include "spark-operator.fullname" . }}-hook
+{{- end -}}
+
+{{/*
+Common labels for the Helm hook
+*/}}
+{{- define "spark-operator.hook.labels" -}}
+{{ include "spark-operator.labels" . }}
+app.kubernetes.io/component: hook
+{{- end -}}
+
+{{/*
+Selector labels for the Helm hook
+*/}}
+{{- define "spark-operator.hook.selectorLabels" -}}
+{{ include "spark-operator.hook.labels" . }}
+{{- end -}}
+
+{{/*
+Create the name of the service account to be used by the Helm hooks.
+*/}}
+{{- define "spark-operator.hook.serviceAccountName" -}}
+{{ include "spark-operator.hook.name" . }}
+{{- end -}}
+
+{{/*
+Create the name of the cluster role to be used by the Helm hooks.
+*/}}
+{{- define "spark-operator.hook.clusterRoleName" -}}
+{{ include "spark-operator.hook.name" . }}
+{{- end }}
+
+{{/*
+Create the name of the cluster role binding to be used by the Helm hooks.
+*/}}
+{{- define "spark-operator.hook.clusterRoleBindingName" -}}
+{{ include "spark-operator.hook.clusterRoleName" . }}
+{{- end }}
+
+{{/*
+Create the name of the Helm hook job.
+*/}}
+{{- define "spark-operator.hook.jobName" -}}
+{{ include "spark-operator.hook.name" . }}-job
+{{- end }}

charts/spark-operator-chart/templates/hook/job.yaml

@@ -0,0 +1,49 @@
+{{/*
+Copyright 2024 The Kubeflow authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.hook.upgradeCrd }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "spark-operator.hook.jobName" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "spark-operator.hook.labels" . | nindent 4 }}
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+    helm.sh/hook-weight: "3"
+spec:
+  template:
+    spec:
+      containers:
+      - name: spark-operator-hook
+        image: {{ include "spark-operator.image" . }}
+        {{- with .Values.image.pullPolicy }}
+        imagePullPolicy: {{ . }}
+        {{- end }}
+        args:
+        - hook
+        - start
+        - --upgrade-crds
+        - --crds-path
+        - /etc/spark-operator/crds
+      {{- with .Values.image.pullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "spark-operator.hook.serviceAccountName" . }}
+{{- end }}

charts/spark-operator-chart/templates/hook/rbac.yaml

@@ -0,0 +1,61 @@
+{{/*
+Copyright 2024 The Kubeflow authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.hook.upgradeCrd }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "spark-operator.hook.clusterRoleName" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "spark-operator.hook.labels" . | nindent 4 }}
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed
+    helm.sh/hook-weight: "2"
+rules:
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  resourceNames:
+  - sparkapplications.sparkoperator.k8s.io
+  - scheduledsparkapplications.sparkoperator.k8s.io
+  verbs:
+  - get
+  - update
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "spark-operator.hook.clusterRoleBindingName" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "spark-operator.hook.labels" . | nindent 4 }}
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed
+    helm.sh/hook-weight: "2"
+subjects:
+- kind: ServiceAccount
+  name: {{ include "spark-operator.hook.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "spark-operator.hook.clusterRoleName" . }}
+{{- end }}

charts/spark-operator-chart/templates/hook/serviceaccount.yaml

@@ -0,0 +1,29 @@
+{{/*
+Copyright 2024 The Kubeflow authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.hook.upgradeCrd }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "spark-operator.hook.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "spark-operator.hook.labels" . | nindent 4 }}
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed
+    helm.sh/hook-weight: "1"
+{{- end }}

charts/spark-operator-chart/tests/hook/job_test.yaml

@@ -0,0 +1,40 @@
+#
+# Copyright 2024 The Kubeflow authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+suite: Test hook job
+
+templates:
+  - hook/job.yaml
+
+release:
+  name: spark-operator
+  namespace: spark-operator
+
+tests:
+  - it: Should not create hook job if `hook.upgradeCrd` is false
+    set:
+      hook:
+        upgradeCrd: false
+    asserts:
+      - hasDocuments:
+          count: 0
+
+  - it: Should create hook job by default
+    asserts:
+      - containsDocument:
+          apiVersion: batch/v1
+          kind: Job
+          name: spark-operator-hook-job

charts/spark-operator-chart/tests/hook/rbac_test.yaml

@@ -0,0 +1,75 @@
+#
+# Copyright 2024 The Kubeflow authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+suite: Test hook rbac
+
+templates:
+  - hook/rbac.yaml
+
+release:
+  name: spark-operator
+  namespace: spark-operator
+
+tests:
+  - it: Should not create hook RBAC resources if `hook.upgradeCrd` is false
+    set:
+      hook:
+        upgradeCrd: false
+    asserts:
+      - hasDocuments:
+          count: 0
+
+  - it: Should create hook ClusterRole by default
+    documentIndex: 0
+    asserts:
+      - containsDocument:
+          apiVersion: rbac.authorization.k8s.io/v1
+          kind: ClusterRole
+          name: spark-operator-hook
+      - contains:
+          path: rules
+          content:
+            apiGroups:
+              - apiextensions.k8s.io
+            resources:
+              - customresourcedefinitions
+            resourceNames:
+              - sparkapplications.sparkoperator.k8s.io
+              - scheduledsparkapplications.sparkoperator.k8s.io
+            verbs:
+              - get
+              - update
+
+  - it: Should create hook ClusterRoleBinding by default
+    documentIndex: 1
+    asserts:
+      - containsDocument:
+          apiVersion: rbac.authorization.k8s.io/v1
+          kind: ClusterRoleBinding
+          name: spark-operator-hook
+      - contains:
+          path: subjects
+          content:
+            kind: ServiceAccount
+            name: spark-operator-hook
+            namespace: spark-operator
+          count: 1
+      - equal:
+          path: roleRef
+          value:
+            apiGroup: rbac.authorization.k8s.io
+            kind: ClusterRole
+            name: spark-operator-hook

charts/spark-operator-chart/tests/hook/serviceaccount_test.yaml

@@ -0,0 +1,40 @@
+#
+# Copyright 2024 The Kubeflow authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+suite: Test hook service account
+
+templates:
+  - hook/serviceaccount.yaml
+
+release:
+  name: spark-operator
+  namespace: spark-operator
+
+tests:
+  - it: Should not create hook service account if `hook.upgradeCrd` is false
+    set:
+      hook:
+        upgradeCrd: false
+    asserts:
+      - hasDocuments:
+          count: 0
+
+  - it: Should create hook service account by default
+    asserts:
+      - containsDocument:
+          apiVersion: v1
+          kind: ServiceAccount
+          name: spark-operator-hook