From dc2c277bb1bd41dd24e8ff96696b88a0465b998c Mon Sep 17 00:00:00 2001 From: Rohith Jayawardene Date: Wed, 16 Oct 2024 14:15:17 +0100 Subject: [PATCH] [FEATURE] - Helm Webhooks Adding the ability for the helm charts to manage the deployment of the validating and mutating webhooks --- Makefile | 5 +- .../templates/_helpers.tpl | 23 ++ .../terranetes-controller/templates/ca.yaml | 18 +- .../templates/deployment.yaml | 1 + .../templates/webhooks.yaml | 233 ++++++++++++++++++ charts/terranetes-controller/values.yaml | 16 +- cmd/controller/main.go | 15 +- pkg/server/server.go | 4 +- pkg/server/types.go | 3 + pkg/server/webhooks.go | 37 ++- pkg/utils/jobs/jobs_test.go | 5 +- test/e2e/integration/setup.bats | 4 + 12 files changed, 332 insertions(+), 32 deletions(-) create mode 100644 charts/terranetes-controller/templates/webhooks.yaml diff --git a/Makefile b/Makefile index 90094dd7e..4fc3f4592 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ ROOT_DIR=${PWD} UNAME := $(shell uname) LFLAGS ?= -X github.com/appvia/terranetes-controller/pkg/version.Version=${VERSION} -X github.com/appvia/terranetes-controller/pkg/version.GitCommit=${GIT_SHA} VERSION ?= latest -DOCKER_BUILD_PLATFORM?=linux/amd64 +DOCKER_BUILD_PLATFORM?=linux/amd64,linux/arm64 # IMPORTANT NOTE: On CircleCI RELEASE_TAG will be set to the string '' if no tag is in use, so # use the local RELEASE variable being 'true' to switch on release build logic. @@ -80,7 +80,8 @@ controller-gen: crd \ output:crd:dir=charts/terranetes-controller/crds \ webhook \ - output:webhook:dir=deploy/webhooks + output:webhook:dir=deploy/webhooks + @cp deploy/webhooks/manifests.yaml charts/terranetes-controller/templates/webhooks.yaml @./hack/patch-crd-gen.sh @./hack/gofmt.sh pkg/apis/*/*/zz_generated.deepcopy.go diff --git a/charts/terranetes-controller/templates/_helpers.tpl b/charts/terranetes-controller/templates/_helpers.tpl index 7dd401e78..07276d8c4 100644 --- a/charts/terranetes-controller/templates/_helpers.tpl +++ b/charts/terranetes-controller/templates/_helpers.tpl @@ -61,3 +61,26 @@ Create the name of the service account to use {{- default "default" .Values.serviceAccount.name }} {{- end }} {{- end }} + +{{/* +Generate a signed certificate using `genSignedCert` or include the logic for generating a cert. +Replace this with the actual logic if using an external tool or script. +*/}} +{{- define "webhook.generateCerts" -}} +{{- $secret := lookup "v1" "Secret" .Release.Namespace .Values.controller.webhooks.caSecret }} +{{- if $secret }} + {{- $_ := set .Values.controller.webhooks "caBundle" (index $secret.data "ca.pem") -}} + {{- $_ := set .Values.controller.webhooks "cert" (index $secret.data "tls.pem") -}} + {{- $_ := set .Values.controller.webhooks "key" (index $secret.data "tls-key.pem") -}} +{{- else }} + {{- if not .Values.controller.webhooks.caBundle }} + {{ $ca := genCA "terranetes-controller" 7300 }} + {{ $dn := printf "controller.%s.svc.cluster.local" .Release.Namespace }} + {{ $sn := printf "controller.%s.svc" .Release.Namespace }} + {{ $server := genSignedCert "" (list "127.0.0.1") (list "localhost" "controller" $sn $dn) 3650 $ca }} + {{- $_ := set .Values.controller.webhooks "caBundle" ($ca.Cert | b64enc) -}} + {{- $_ := set .Values.controller.webhooks "cert" ($server.Cert | b64enc) -}} + {{- $_ := set .Values.controller.webhooks "key" ($server.Key | b64enc) -}} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/terranetes-controller/templates/ca.yaml b/charts/terranetes-controller/templates/ca.yaml index d443d428f..02f9e18f3 100644 --- a/charts/terranetes-controller/templates/ca.yaml +++ b/charts/terranetes-controller/templates/ca.yaml @@ -1,5 +1,5 @@ {{- if .Values.controller.webhooks.ca }} -{{- $secret := lookup "v1" "Secret" .Release.Namespace .Values.controller.webhooks.caSecret }} +{{- include "webhook.generateCerts" . }} --- apiVersion: v1 kind: Secret @@ -9,17 +9,7 @@ metadata: {{- include "terranetes-controller.labels" . | nindent 4 }} type: Opaque data: -{{- if $secret }} - ca.pem: {{ index $secret.data "ca.pem" }} - tls.pem: {{ index $secret.data "tls.pem" }} - tls-key.pem: {{ index $secret.data "tls-key.pem" }} -{{- else }} -{{ $ca := genCA "terranetes-controller" 7300 }} -{{ $dn := printf "controller.%s.svc.cluster.local" .Release.Namespace }} -{{ $sn := printf "controller.%s.svc" .Release.Namespace }} -{{ $server := genSignedCert "" (list "127.0.0.1") (list "localhost" "controller" $sn $dn) 3650 $ca }} - ca.pem: {{ $ca.Cert | b64enc }} - tls.pem: {{ $server.Cert | b64enc }} - tls-key.pem: {{ $server.Key | b64enc }} -{{- end }} + ca.pem: {{ .Values.controller.webhooks.caBundle }} + tls.pem: {{ .Values.controller.webhooks.cert }} + tls-key.pem: {{ .Values.controller.webhooks.key }} {{- end }} diff --git a/charts/terranetes-controller/templates/deployment.yaml b/charts/terranetes-controller/templates/deployment.yaml index 8fce5b94f..eff01bbcf 100644 --- a/charts/terranetes-controller/templates/deployment.yaml +++ b/charts/terranetes-controller/templates/deployment.yaml @@ -69,6 +69,7 @@ spec: - --enable-terraform-versions={{ .Values.controller.enableTerraformVersions }} - --enable-watchers={{ .Values.controller.enableWatchers }} - --enable-webhook-prefix={{ .Values.controller.webhooks.prefix }} + - --enable-webhooks-registration={{ .Values.controller.enableControllerWebhookRegistration }} - --enable-webhooks={{ .Values.controller.webhooks.enabled }} - --executor-image={{ .Values.controller.images.executor }} {{- range .Values.controller.executorSecrets }} diff --git a/charts/terranetes-controller/templates/webhooks.yaml b/charts/terranetes-controller/templates/webhooks.yaml new file mode 100644 index 000000000..e6599d964 --- /dev/null +++ b/charts/terranetes-controller/templates/webhooks.yaml @@ -0,0 +1,233 @@ +{{- if .Values.controller.enableHelmWebhookRegistration }} +{{- include "webhook.generateCerts" . }} +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: terranetes-controller + labels: + {{- include "terranetes-controller.labels" . | nindent 4 }} +webhooks: + {{- if .Values.controller.enableNamespaceProtection }} + - admissionReviewVersions: + - v1 + clientConfig: + caBundle: {{ .Values.controller.webhooks.caBundle }} + service: + name: controller + namespace: {{ .Release.Namespace }} + path: /validate/terraform.appvia.io/namespaces + failurePolicy: Fail + name: cloudresources.terraform.appvia.io + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - DELETE + resources: + - namespaces + sideEffects: None + {{- end }} + - admissionReviewVersions: + - v1 + clientConfig: + caBundle: {{ .Values.controller.webhooks.caBundle }} + service: + name: controller + namespace: {{ .Release.Namespace }} + path: /validate/terraform.appvia.io/cloudresources + failurePolicy: Fail + name: cloudresources.terraform.appvia.io + rules: + - apiGroups: + - terraform.appvia.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - cloudresources + sideEffects: None + - admissionReviewVersions: + - v1 + clientConfig: + caBundle: {{ .Values.controller.webhooks.caBundle }} + service: + name: controller + namespace: {{ .Release.Namespace }} + path: /validate/terraform.appvia.io/configurations + failurePolicy: Fail + name: configurations.terraform.appvia.io + rules: + - apiGroups: + - terraform.appvia.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - configurations + sideEffects: None + - admissionReviewVersions: + - v1 + clientConfig: + caBundle: {{ .Values.controller.webhooks.caBundle }} + service: + name: controller + namespace: {{ .Release.Namespace }} + path: /validate/terraform.appvia.io/contexts + failurePolicy: Fail + name: contexts.terraform.appvia.io + rules: + - apiGroups: + - terraform.appvia.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - DELETE + - UPDATE + resources: + - contexts + sideEffects: None + - admissionReviewVersions: + - v1 + clientConfig: + caBundle: {{ .Values.controller.webhooks.caBundle }} + service: + name: controller + namespace: {{ .Release.Namespace }} + path: /validate/terraform.appvia.io/policies + failurePolicy: Fail + name: policies.terraform.appvia.io + rules: + - apiGroups: + - terraform.appvia.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - DELETE + - UPDATE + resources: + - policies + sideEffects: None + - admissionReviewVersions: + - v1 + clientConfig: + caBundle: {{ .Values.controller.webhooks.caBundle }} + service: + name: controller + namespace: {{ .Release.Namespace }} + path: /validate/terraform.appvia.io/providers + failurePolicy: Fail + name: providers.terraform.appvia.io + rules: + - apiGroups: + - terraform.appvia.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - providers + sideEffects: None + - admissionReviewVersions: + - v1 + clientConfig: + caBundle: {{ .Values.controller.webhooks.caBundle }} + service: + name: controller + namespace: {{ .Release.Namespace }} + path: /validate/terraform.appvia.io/revisions + failurePolicy: Fail + name: revisions.terraform.appvia.io + rules: + - apiGroups: + - terraform.appvia.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - DELETE + - UPDATE + resources: + - revisions + sideEffects: None +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: terranetes-controller + labels: + {{- include "terranetes-controller.labels" . | nindent 4 }} +webhooks: + - admissionReviewVersions: + - v1 + clientConfig: + caBundle: {{ .Values.controller.webhooks.caBundle }} + service: + name: controller + namespace: {{ .Release.Namespace }} + path: /mutate/terraform.appvia.io/cloudresources + failurePolicy: Fail + name: cloudresources.terraform.appvia.io + rules: + - apiGroups: + - terraform.appvia.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - cloudresources + sideEffects: None + - admissionReviewVersions: + - v1 + clientConfig: + caBundle: {{ .Values.controller.webhooks.caBundle }} + service: + name: controller + namespace: {{ .Release.Namespace }} + path: /mutate/terraform.appvia.io/configurations + failurePolicy: Fail + name: configurations.terraform.appvia.io + rules: + - apiGroups: + - terraform.appvia.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - configurations + sideEffects: None + - admissionReviewVersions: + - v1 + clientConfig: + caBundle: {{ .Values.controller.webhooks.caBundle }} + service: + name: controller + namespace: {{ .Release.Namespace }} + path: /mutate/terraform.appvia.io/revisions + failurePolicy: Fail + name: revisions.terraform.appvia.io + rules: + - apiGroups: + - terraform.appvia.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - revisions + sideEffects: None +{{- end }} diff --git a/charts/terranetes-controller/values.yaml b/charts/terranetes-controller/values.yaml index 934ffd530..dd3e807f2 100644 --- a/charts/terranetes-controller/values.yaml +++ b/charts/terranetes-controller/values.yaml @@ -71,11 +71,16 @@ controller: # indicate we create the watcher jobs in user namespace, these allow users # to view the terraform output enableWatchers: true + ## Indicates we should forgo the controller registering it's own webhooks and allowing + ## helm to manage the webhooks for us + enableHelmWebhookRegistration: false + ## indicates if the controller should register the validation and mutation webhooks + ## for the Configuration, Revision and CloudResource resources + enableControllerWebhookRegistration: true # indicates if namespace deletion should be blocked if the namespace contains one # or more Configuration resources, forcing the user to delete correctly enableNamespaceProtection: false - # indicates if the controller should deny updates to Revisions which are currently - # in use + # indicates if the controller should deny updates to Revisions which are currently in use enableRevisionUpdateProtection: true # enableTerraformVersions indicates configurations are permitted to override # the terraform version in their spec. @@ -109,6 +114,13 @@ controller: tlsDir: /certs # name of the file containing the tls private key tlsKey: tls-key.pem + # the base64 encoded certificate authority for the webhook + caBundle: "" + # the base64 encoded certificate for the webhook service + cert: "" + # the base64 encoded private key for the webhook service + key: "" + # extraArgs is used for passing additional command line arguments to the # controller. extraArgs: diff --git a/cmd/controller/main.go b/cmd/controller/main.go index b29c5e6f9..21a88cd4c 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -56,15 +56,15 @@ func main() { flags := cmd.Flags() flags.Bool("verbose", false, "Enable verbose logging") - flags.IntVar(&config.BackoffLimit, "backoff-limit", 1, "The number of times we are willing to allow a terraform job to error before marking as a failure") flags.BoolVar(&config.EnableContextInjection, "enable-context-injection", false, "Indicates the controller should inject Configuration context into the terraform variables") flags.BoolVar(&config.EnableNamespaceProtection, "enable-namespace-protection", false, "Indicates the controller should protect the controller namespace from being deleted") flags.BoolVar(&config.EnableRevisionUpdateProtection, "enable-revision-update-protection", false, "Indicates we should protect the revisions in use from being updated") flags.BoolVar(&config.EnableTerraformVersions, "enable-terraform-versions", true, "Indicates the terraform version can be overridden by configurations") flags.BoolVar(&config.EnableWatchers, "enable-watchers", true, "Indicates we create watcher jobs in the configuration namespaces") + flags.BoolVar(&config.EnableWebhookPrefix, "enable-webhook-prefix", false, "Indicates the controller should prefix webhook configuration names with the controller name") flags.BoolVar(&config.EnableWebhooks, "enable-webhooks", true, "Indicates we should register the webhooks") + flags.BoolVar(&config.EnableWebhooksRegistration, "enable-webhooks-registration", true, "Indicates the controller should register the webhooks for the controller") flags.BoolVar(&config.RegisterCRDs, "register-crds", true, "Indicates the controller to register its own CRDs") - flags.BoolVar(&config.EnableWebhookPrefix, "enable-webhook-prefix", false, "Indicates the controller should prefix webhook configuration names with the controller name") flags.DurationVar(&config.DriftControllerInterval, "drift-controller-interval", 5*time.Minute, "Is the check interval for the controller to search for configurations which should be checked for drift") flags.DurationVar(&config.DriftInterval, "drift-interval", 3*time.Hour, "The minimum duration the controller will wait before triggering a drift check") flags.DurationVar(&config.ResyncPeriod, "resync-period", 5*time.Hour, "The resync period for the controller") @@ -72,19 +72,20 @@ func main() { flags.Float64Var(&config.ConfigurationThreshold, "configurations-threshold", 0, "The maximum percentage of configurations that can be run at any one time") flags.Float64Var(&config.DriftThreshold, "drift-threshold", 0.10, "The maximum percentage of configurations that can be run drift detection at any one time") flags.IntVar(&config.APIServerPort, "apiserver-port", 10080, "The port the apiserver should be listening on") + flags.IntVar(&config.BackoffLimit, "backoff-limit", 1, "The number of times we are willing to allow a terraform job to error before marking as a failure") flags.IntVar(&config.MetricsPort, "metrics-port", 9090, "The port the metric endpoint binds to") flags.IntVar(&config.WebhookPort, "webhooks-port", 10081, "The port the webhook endpoint binds to") flags.StringSliceVar(&config.ExecutorSecrets, "executor-secret", []string{}, "Name of a secret in controller namespace which should be added to the job") - flags.StringVar(&config.ExecutorMemoryRequest, "executor-memory-request", "32Mi", "The default memory request for the executor container") - flags.StringVar(&config.ExecutorMemoryLimit, "executor-memory-limit", "", "The default memory limit for the executor container (default is no limit)") - flags.StringVar(&config.ExecutorCPURequest, "executor-cpu-request", "5m", "The default CPU request for the executor container") - flags.StringVar(&config.ExecutorCPULimit, "executor-cpu-limit", "", "The default CPU limit for the executor container (default is no limit)") + flags.StringSliceVar(&config.JobLabels, "job-label", []string{}, "A collection of key=values to add to all jobs") flags.StringVar(&config.BackendTemplate, "backend-template", "", "Name of secret in the controller namespace containing a template for the terraform state") + flags.StringVar(&config.ExecutorCPULimit, "executor-cpu-limit", "", "The default CPU limit for the executor container (default is no limit)") + flags.StringVar(&config.ExecutorCPURequest, "executor-cpu-request", "5m", "The default CPU request for the executor container") flags.StringVar(&config.ExecutorImage, "executor-image", fmt.Sprintf("ghcr.io/appvia/terranetes-executor:%s", version.Version), "The image to use for the executor") + flags.StringVar(&config.ExecutorMemoryLimit, "executor-memory-limit", "", "The default memory limit for the executor container (default is no limit)") + flags.StringVar(&config.ExecutorMemoryRequest, "executor-memory-request", "32Mi", "The default memory request for the executor container") flags.StringVar(&config.InfracostsImage, "infracost-image", "infracosts/infracost:latest", "The image to use for the infracosts") flags.StringVar(&config.InfracostsSecretName, "cost-secret", "", "Name of the secret on the controller namespace containing your infracost token") flags.StringVar(&config.JobTemplate, "job-template", "", "Name of configmap in the controller namespace containing a template for the job") - flags.StringSliceVar(&config.JobLabels, "job-label", []string{}, "A collection of key=values to add to all jobs") flags.StringVar(&config.Namespace, "namespace", os.Getenv("KUBE_NAMESPACE"), "The namespace the controller is running in and where jobs will run") flags.StringVar(&config.PolicyImage, "policy-image", "bridgecrew/checkov:latest", "The image to use for the policy") flags.StringVar(&config.PreloadImage, "preload-image", fmt.Sprintf("ghcr.io/appvia/terranetes-executor:%s", version.Version), "The image to use for the preload") diff --git a/pkg/server/server.go b/pkg/server/server.go index 15ad418da..c40ec2103 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -275,7 +275,9 @@ func New(cfg *rest.Config, config Config) (*Server, error) { // Start is called to begin the service func (s *Server) Start(ctx context.Context) error { if s.config.EnableWebhooks { - if err := s.registerWebhooks(ctx); err != nil { + log.Info("registering the terranetes controller webhooks") + + if err := s.manageWebhooks(ctx, s.config.EnableWebhooksRegistration); err != nil { return fmt.Errorf("failed to register the webhooks, error: %w", err) } } diff --git a/pkg/server/types.go b/pkg/server/types.go index d1f971162..97e5cd0c5 100644 --- a/pkg/server/types.go +++ b/pkg/server/types.go @@ -51,6 +51,9 @@ type Config struct { EnableNamespaceProtection bool // EnableWebhooks enables the webhooks registration EnableWebhooks bool + // EnableWebhooksRegistration indicates the controller should register the webhooks + // for the controller + EnableWebhooksRegistration bool // EnableWatchers enables the creation of watcher jobs EnableWatchers bool // EnableTerraformVersions indicates if configurations can override the default terraform version diff --git a/pkg/server/webhooks.go b/pkg/server/webhooks.go index fabc1d209..afeae18dc 100644 --- a/pkg/server/webhooks.go +++ b/pkg/server/webhooks.go @@ -23,6 +23,7 @@ import ( "fmt" "os" + log "github.com/sirupsen/logrus" admissionv1 "k8s.io/api/admissionregistration/v1" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -34,12 +35,13 @@ import ( "github.com/appvia/terranetes-controller/pkg/version" ) -// registerWebhooks is responsible for registering the webhooks -func (s *Server) registerWebhooks(ctx context.Context) error { +// manageWebhooks is responsible for registering or unregistering the webhooks +func (s *Server) manageWebhooks(ctx context.Context, managed bool) error { cc, err := client.New(s.cfg, client.Options{Scheme: schema.GetScheme()}) if err != nil { return err } + log.WithField("managed", managed).Info("attempting to manage the controller webhooks") // @step: read the certificate authority ca, err := os.ReadFile(s.config.TLSAuthority) @@ -86,8 +88,20 @@ func (s *Server) registerWebhooks(ctx context.Context) error { return fmt.Errorf("expected a validating or mutating webhook, got %T", o) } - if err := kubernetes.CreateOrForceUpdate(ctx, cc, o); err != nil { - return fmt.Errorf("failed to create / update the webhook, %w", err) + switch managed { + case true: + if err := kubernetes.CreateOrForceUpdate(ctx, cc, o); err != nil { + return fmt.Errorf("failed to create / update the webhook, %w", err) + } + + default: + log.WithFields(log.Fields{ + "webhook": o.GetName(), + }).Info("deleting any previous webhooks") + + if err := kubernetes.DeleteIfExists(ctx, cc, o); err != nil { + return fmt.Errorf("failed to delete any previous webhook, %w", err) + } } } @@ -125,6 +139,21 @@ func (s *Server) registerWebhooks(ctx context.Context) error { }, } + // @step: if we are not managing the webhooks we can delete them completely + if !managed { + log.WithFields(log.Fields{ + "webhook": wh.GetName(), + }).Info("deleting any previous namespace webhooks") + + if err := kubernetes.DeleteIfExists(ctx, cc, wh); err != nil { + return fmt.Errorf("failed to delete the namespace webhook, %w", err) + } + + return nil + } + + // @step: we manage the webhooks, we either need to create, update or delete + // the namespace webhook based on the controller configuration switch s.config.EnableNamespaceProtection { case true: if err := kubernetes.CreateOrForceUpdate(ctx, cc, wh); err != nil { diff --git a/pkg/utils/jobs/jobs_test.go b/pkg/utils/jobs/jobs_test.go index 27324d07a..8ddfd66b5 100644 --- a/pkg/utils/jobs/jobs_test.go +++ b/pkg/utils/jobs/jobs_test.go @@ -5,11 +5,12 @@ import ( batchv1 "k8s.io/api/batch/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/appvia/terranetes-controller/pkg/apis/terraform/v1alpha1" "github.com/appvia/terranetes-controller/pkg/assets" "github.com/appvia/terranetes-controller/pkg/utils/jobs" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestNewTerraformPlan(t *testing.T) { diff --git a/test/e2e/integration/setup.bats b/test/e2e/integration/setup.bats index 97f2fec65..95b2a10a3 100644 --- a/test/e2e/integration/setup.bats +++ b/test/e2e/integration/setup.bats @@ -39,6 +39,8 @@ teardown() { cat << EOF > ${BATS_TMPDIR}/my_values.yaml replicaCount: 2 controller: + enableControllerWebhookRegistration: false + enableHelmWebhookRegistration: true enableNamespaceProtection: true images: controller: "ghcr.io/appvia/terranetes-controller:${VERSION}" @@ -52,6 +54,8 @@ EOF cat << EOF > ${BATS_TMPDIR}/my_values.yaml controller: + enableControllerWebhookRegistration: false + enableHelmWebhookRegistration: true enableNamespaceProtection: true costs: secret: infracost-api