From 5f00e943be261278d68d6b12dcf6d2808d7ee18d Mon Sep 17 00:00:00 2001 From: grzegorz-ciezkowski Date: Thu, 16 Jan 2025 16:50:05 +0100 Subject: [PATCH] feat(deployments) split deployment to webhooks and controllers (#808) --- .../templates/deployment-controller.yaml | 92 +++++++++++++++++++ ...eployment.yaml => deployment-webhook.yaml} | 4 +- e2e/shared/e2e.go | 2 +- pkg/internal/local/commands/webhook.go | 8 +- pkg/internal/local/setup/manifest.go | 13 ++- pkg/internal/local/setup/webhook.go | 15 +-- 6 files changed, 116 insertions(+), 18 deletions(-) create mode 100644 charts/manager/templates/deployment-controller.yaml rename charts/manager/templates/{deployment.yaml => deployment-webhook.yaml} (96%) diff --git a/charts/manager/templates/deployment-controller.yaml b/charts/manager/templates/deployment-controller.yaml new file mode 100644 index 000000000..e92f288b4 --- /dev/null +++ b/charts/manager/templates/deployment-controller.yaml @@ -0,0 +1,92 @@ +{{/* +SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors +SPDX-License-Identifier: Apache-2.0 +*/}} + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "manager.fullname" . }}-controller-manager + namespace: greenhouse + labels: + app: greenhouse + {{- include "manager.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.controllerManager.replicas }} + selector: + matchLabels: + app: greenhouse + {{- include "manager.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app: greenhouse + {{- include "manager.selectorLabels" . | nindent 8 }} + annotations: + kubectl.kubernetes.io/default-container: manager + prometheus.io/scrape: "true" + spec: + containers: + - command: + - /greenhouse + args: + - --dns-domain={{ required ".Values.global.dnsDomain missing" .Values.global.dnsDomain }} + {{- if gt (len .Values.controllerManager.args) 0 }} + {{- include "manager.params" . | indent 8 }} + {{- end }} + env: + - name: CONTROLLERS_ONLY + value: "true" + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: GOMEMLIMIT + valueFrom: + resourceFieldRef: + containerName: manager + divisor: "0" + resource: limits.memory + - name: GOMAXPROCS + valueFrom: + resourceFieldRef: + containerName: manager + divisor: "0" + resource: limits.cpu + image: {{ .Values.controllerManager.image.repository }}:{{ .Values.controllerManager.image.tag | default .Chart.AppVersion }} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + - containerPort: 8080 + name: metrics + - containerPort: 8081 + name: probes + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + securityContext: {{- toYaml .Values.controllerManager.containerSecurityContext | nindent 10 }} + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + securityContext: + runAsNonRoot: true + serviceAccountName: {{ include "manager.fullname" . }}-controller-manager + terminationGracePeriodSeconds: 10 + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: {{ include "manager.fullname" . }}-webhook-server-cert diff --git a/charts/manager/templates/deployment.yaml b/charts/manager/templates/deployment-webhook.yaml similarity index 96% rename from charts/manager/templates/deployment.yaml rename to charts/manager/templates/deployment-webhook.yaml index cec2d1274..b9c1b8580 100644 --- a/charts/manager/templates/deployment.yaml +++ b/charts/manager/templates/deployment-webhook.yaml @@ -6,7 +6,7 @@ SPDX-License-Identifier: Apache-2.0 apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "manager.fullname" . }}-controller-manager + name: {{ include "manager.fullname" . }}-webhook-manager namespace: greenhouse labels: app: greenhouse @@ -35,6 +35,8 @@ spec: {{- include "manager.params" . | indent 8 }} {{- end }} env: + - name: WEBHOOK_ONLY + value: "true" - name: POD_NAMESPACE valueFrom: fieldRef: diff --git a/e2e/shared/e2e.go b/e2e/shared/e2e.go index ebe7452a9..9adb912a0 100644 --- a/e2e/shared/e2e.go +++ b/e2e/shared/e2e.go @@ -39,7 +39,7 @@ const ( RemoteKubeConfigPathEnv = "GREENHOUSE_REMOTE_KUBECONFIG" remoteIntKubeConfigPathEnv = "GREENHOUSE_REMOTE_INT_KUBECONFIG" ControllerLogsPathEnv = "CONTROLLER_LOGS_PATH" - managerDeploymentName = "greenhouse-controller-manager" + managerDeploymentName = "greenhouse-webhook-manager" managerDeploymentNamespace = "greenhouse" remoteExecutionEnv = "EXECUTION_ENV" realCluster = "GARDENER" diff --git a/pkg/internal/local/commands/webhook.go b/pkg/internal/local/commands/webhook.go index bc823fc8e..974bd7b27 100644 --- a/pkg/internal/local/commands/webhook.go +++ b/pkg/internal/local/commands/webhook.go @@ -41,13 +41,7 @@ func processWebhook(cmd *cobra.Command, _ []string) error { ctx := cmd.Context() hookCfg := &setup.Webhook{ DockerFile: dockerFile, - Envs: []setup.WebhookEnv{ - { - Name: "WEBHOOK_ONLY", - Value: "true", - }, - }, - DevMode: devMode, + DevMode: devMode, } manifest := &setup.Manifest{ ReleaseName: releaseName, diff --git a/pkg/internal/local/setup/manifest.go b/pkg/internal/local/setup/manifest.go index ab871bee7..17fa95533 100644 --- a/pkg/internal/local/setup/manifest.go +++ b/pkg/internal/local/setup/manifest.go @@ -92,11 +92,16 @@ func webhookManifestSetup(ctx context.Context, m *Manifest) Step { noWbManifests := excludeResources(filtered, []string{"MutatingWebhookConfiguration", "ValidatingWebhookConfiguration"}) return m.applyManifests(noWbManifests, namespace, env.cluster.kubeConfigPath) } - webHookManifests, err := m.setupWebhookManifest(resources, clusterName) + webHookManifests, err := m.setupWebhookManifest(resources, clusterName, WebhookDeploymentNameSuffix) if err != nil { return err } filtered = append(filtered, webHookManifests...) + controllerManifests, err := m.setupWebhookManifest(resources, clusterName, ControllersDeploymentNameSuffix) + if err != nil { + return err + } + filtered = append(filtered, controllerManifests...) err = m.applyManifests(filtered, namespace, env.cluster.kubeConfigPath) if err != nil { return err @@ -107,7 +112,11 @@ func webhookManifestSetup(ctx context.Context, m *Manifest) Step { return err } } - return m.waitUntilDeploymentReady(ctx, clusterName, namespace) + if err := m.waitUntilDeploymentReady(ctx, clusterName, namespace, WebhookDeploymentNameSuffix); err != nil { + return err + } + + return m.waitUntilDeploymentReady(ctx, clusterName, namespace, ControllersDeploymentNameSuffix) } } diff --git a/pkg/internal/local/setup/webhook.go b/pkg/internal/local/setup/webhook.go index 65ab595b7..30929940a 100644 --- a/pkg/internal/local/setup/webhook.go +++ b/pkg/internal/local/setup/webhook.go @@ -44,7 +44,8 @@ const ( MangerIMG = "greenhouse/manager:local" MangerContainer = "manager" DeploymentKind = "Deployment" - DeploymentNameSuffix = "-controller-manager" + WebhookDeploymentNameSuffix = "-webhook-manager" + ControllersDeploymentNameSuffix = "-controller-manager" JobKind = "Job" JobNameSuffix = "-kube-webhook-certgen" MutatingWebhookConfigurationKind = "MutatingWebhookConfiguration" @@ -57,10 +58,10 @@ const ( // modifies cert job (charts/manager/templates/kube-webhook-certgen.yaml) to include host.docker.internal // if devMode is enabled, modifies mutating and validating webhook configurations to use host.docker.internal URL and removes service from clientConfig // extracts the webhook certs from the secret and writes them to tmp/k8s-webhook-server/serving-certs directory -func (m *Manifest) setupWebhookManifest(resources []map[string]interface{}, clusterName string) ([]map[string]interface{}, error) { +func (m *Manifest) setupWebhookManifest(resources []map[string]interface{}, clusterName string, resourceSuffix string) ([]map[string]interface{}, error) { webhookManifests := make([]map[string]interface{}, 0) releaseName := m.ReleaseName - managerDeployment, err := extractResourceByNameKind(resources, releaseName+DeploymentNameSuffix, DeploymentKind) + managerDeployment, err := extractResourceByNameKind(resources, releaseName+resourceSuffix, DeploymentKind) if err != nil { return nil, err } @@ -323,8 +324,8 @@ func extractResourceByKind(resources []map[string]interface{}, kind string) map[ return nil } -func (m *Manifest) waitUntilDeploymentReady(ctx context.Context, clusterName, namespace string) error { - deploymentName := m.ReleaseName + DeploymentNameSuffix +func (m *Manifest) waitUntilDeploymentReady(ctx context.Context, clusterName, namespace string, nameSuffix string) error { + deploymentName := m.ReleaseName + nameSuffix cl, err := getKubeClient(clusterName) if err != nil { return err @@ -341,7 +342,7 @@ func (m *Manifest) waitUntilDeploymentReady(ctx context.Context, clusterName, na return err } if deployment.Status.Conditions == nil { - return errors.New("deployment is not yet ready") + return errors.New(fmt.Sprintf("deployment %s is not yet ready", deploymentName)) } available := false for _, condition := range deployment.Status.Conditions { @@ -351,7 +352,7 @@ func (m *Manifest) waitUntilDeploymentReady(ctx context.Context, clusterName, na } } if !available { - return errors.New("deployment is not yet ready") + return errors.New(fmt.Sprintf("deployment %s is not yet ready", deploymentName)) } return nil }, b)