feat(deployments) split deployment to webhooks and controllers (#808)

charts/manager/templates/deployment-controller.yaml

@@ -0,0 +1,92 @@
+{{/*
+SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors
+SPDX-License-Identifier: Apache-2.0
+*/}}
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "manager.fullname" . }}-controller-manager
+  namespace: greenhouse
+  labels:
+    app: greenhouse
+  {{- include "manager.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.controllerManager.replicas }}
+  selector:
+    matchLabels:
+      app: greenhouse
+    {{- include "manager.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        app: greenhouse
+      {{- include "manager.selectorLabels" . | nindent 8 }}
+      annotations:
+        kubectl.kubernetes.io/default-container: manager
+        prometheus.io/scrape: "true"
+    spec:
+      containers:
+        - command:
+            - /greenhouse
+          args:
+            - --dns-domain={{ required ".Values.global.dnsDomain missing" .Values.global.dnsDomain }}
+        {{- if gt (len .Values.controllerManager.args) 0 }}
+        {{- include "manager.params" . | indent 8 }}
+        {{- end }}
+          env:
+            - name: CONTROLLERS_ONLY
+              value: "true"
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+            - name: GOMEMLIMIT
+              valueFrom:
+                resourceFieldRef:
+                  containerName: manager
+                  divisor: "0"
+                  resource: limits.memory
+            - name: GOMAXPROCS
+              valueFrom:
+                resourceFieldRef:
+                  containerName: manager
+                  divisor: "0"
+                  resource: limits.cpu
+          image: {{ .Values.controllerManager.image.repository }}:{{ .Values.controllerManager.image.tag | default .Chart.AppVersion }}
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: 8081
+            initialDelaySeconds: 15
+            periodSeconds: 20
+          name: manager
+          ports:
+            - containerPort: 9443
+              name: webhook-server
+              protocol: TCP
+            - containerPort: 8080
+              name: metrics
+            - containerPort: 8081
+              name: probes
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: 8081
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          securityContext: {{- toYaml .Values.controllerManager.containerSecurityContext | nindent 10 }}
+          volumeMounts:
+            - mountPath: /tmp/k8s-webhook-server/serving-certs
+              name: cert
+              readOnly: true
+      securityContext:
+        runAsNonRoot: true
+      serviceAccountName: {{ include "manager.fullname" . }}-controller-manager
+      terminationGracePeriodSeconds: 10
+      volumes:
+        - name: cert
+          secret:
+            defaultMode: 420
+            secretName: {{ include "manager.fullname" . }}-webhook-server-cert

charts/manager/templates/deployment.yaml → charts/manager/templates/deployment-webhook.yaml

@@ -6,7 +6,7 @@ SPDX-License-Identifier: Apache-2.0
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: {{ include "manager.fullname" . }}-controller-manager
+  name: {{ include "manager.fullname" . }}-webhook-manager
   namespace: greenhouse
   labels:
     app: greenhouse
@@ -35,6 +35,8 @@ spec:
         {{- include "manager.params" . | indent 8 }}
         {{- end }}
         env:
+        - name: WEBHOOK_ONLY
+          value: "true"
         - name: POD_NAMESPACE
           valueFrom:
             fieldRef:

e2e/shared/e2e.go

@@ -39,7 +39,7 @@ const (
 	RemoteKubeConfigPathEnv    = "GREENHOUSE_REMOTE_KUBECONFIG"
 	remoteIntKubeConfigPathEnv = "GREENHOUSE_REMOTE_INT_KUBECONFIG"
 	ControllerLogsPathEnv      = "CONTROLLER_LOGS_PATH"
-	managerDeploymentName      = "greenhouse-controller-manager"
+	managerDeploymentName      = "greenhouse-webhook-manager"
 	managerDeploymentNamespace = "greenhouse"
 	remoteExecutionEnv         = "EXECUTION_ENV"
 	realCluster                = "GARDENER"

pkg/internal/local/commands/webhook.go

@@ -41,13 +41,7 @@ func processWebhook(cmd *cobra.Command, _ []string) error {
 	ctx := cmd.Context()
 	hookCfg := &setup.Webhook{
 		DockerFile: dockerFile,
-		Envs: []setup.WebhookEnv{
-			{
-				Name:  "WEBHOOK_ONLY",
-				Value: "true",
-			},
-		},
-		DevMode: devMode,
+		DevMode:    devMode,
 	}
 	manifest := &setup.Manifest{
 		ReleaseName:  releaseName,

pkg/internal/local/setup/manifest.go

@@ -92,11 +92,16 @@ func webhookManifestSetup(ctx context.Context, m *Manifest) Step {
 			noWbManifests := excludeResources(filtered, []string{"MutatingWebhookConfiguration", "ValidatingWebhookConfiguration"})
 			return m.applyManifests(noWbManifests, namespace, env.cluster.kubeConfigPath)
 		}
-		webHookManifests, err := m.setupWebhookManifest(resources, clusterName)
+		webHookManifests, err := m.setupWebhookManifest(resources, clusterName, WebhookDeploymentNameSuffix)
 		if err != nil {
 			return err
 		}
 		filtered = append(filtered, webHookManifests...)
+		controllerManifests, err := m.setupWebhookManifest(resources, clusterName, ControllersDeploymentNameSuffix)
+		if err != nil {
+			return err
+		}
+		filtered = append(filtered, controllerManifests...)
 		err = m.applyManifests(filtered, namespace, env.cluster.kubeConfigPath)
 		if err != nil {
 			return err
@@ -107,7 +112,11 @@ func webhookManifestSetup(ctx context.Context, m *Manifest) Step {
 				return err
 			}
 		}
-		return m.waitUntilDeploymentReady(ctx, clusterName, namespace)
+		if err := m.waitUntilDeploymentReady(ctx, clusterName, namespace, WebhookDeploymentNameSuffix); err != nil {
+			return err
+		}
+
+		return m.waitUntilDeploymentReady(ctx, clusterName, namespace, ControllersDeploymentNameSuffix)
 	}
 }
 

pkg/internal/local/setup/webhook.go

@@ -44,7 +44,8 @@ const (
 	MangerIMG                          = "greenhouse/manager:local"
 	MangerContainer                    = "manager"
 	DeploymentKind                     = "Deployment"
-	DeploymentNameSuffix               = "-controller-manager"
+	WebhookDeploymentNameSuffix        = "-webhook-manager"
+	ControllersDeploymentNameSuffix    = "-controller-manager"
 	JobKind                            = "Job"
 	JobNameSuffix                      = "-kube-webhook-certgen"
 	MutatingWebhookConfigurationKind   = "MutatingWebhookConfiguration"
@@ -57,10 +58,10 @@ const (
 // modifies cert job (charts/manager/templates/kube-webhook-certgen.yaml) to include host.docker.internal
 // if devMode is enabled, modifies mutating and validating webhook configurations to use host.docker.internal URL and removes service from clientConfig
 // extracts the webhook certs from the secret and writes them to tmp/k8s-webhook-server/serving-certs directory
-func (m *Manifest) setupWebhookManifest(resources []map[string]interface{}, clusterName string) ([]map[string]interface{}, error) {
+func (m *Manifest) setupWebhookManifest(resources []map[string]interface{}, clusterName string, resourceSuffix string) ([]map[string]interface{}, error) {
 	webhookManifests := make([]map[string]interface{}, 0)
 	releaseName := m.ReleaseName
-	managerDeployment, err := extractResourceByNameKind(resources, releaseName+DeploymentNameSuffix, DeploymentKind)
+	managerDeployment, err := extractResourceByNameKind(resources, releaseName+resourceSuffix, DeploymentKind)
 	if err != nil {
 		return nil, err
 	}
@@ -323,8 +324,8 @@ func extractResourceByKind(resources []map[string]interface{}, kind string) map[
 	return nil
 }
 
-func (m *Manifest) waitUntilDeploymentReady(ctx context.Context, clusterName, namespace string) error {
-	deploymentName := m.ReleaseName + DeploymentNameSuffix
+func (m *Manifest) waitUntilDeploymentReady(ctx context.Context, clusterName, namespace string, nameSuffix string) error {
+	deploymentName := m.ReleaseName + nameSuffix
 	cl, err := getKubeClient(clusterName)
 	if err != nil {
 		return err
@@ -341,7 +342,7 @@ func (m *Manifest) waitUntilDeploymentReady(ctx context.Context, clusterName, na
 			return err
 		}
 		if deployment.Status.Conditions == nil {
-			return errors.New("deployment is not yet ready")
+			return errors.New(fmt.Sprintf("deployment %s is not yet ready", deploymentName))
 		}
 		available := false
 		for _, condition := range deployment.Status.Conditions {
@@ -351,7 +352,7 @@ func (m *Manifest) waitUntilDeploymentReady(ctx context.Context, clusterName, na
 			}
 		}
 		if !available {
-			return errors.New("deployment is not yet ready")
+			return errors.New(fmt.Sprintf("deployment %s is not yet ready", deploymentName))
 		}
 		return nil
 	}, b)