diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index 596f0bf..fc71ed4 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -24,3 +24,4 @@ assignees: '' * Kubernetes version (execute `kubectl version`): * kctrl version (execute `kctrl version`): * kapp-controller version (execute `kubectl get deployment -n kapp-controller kapp-controller -o yaml` and the annotation is `kbld.k14s.io/images`): +* package version (execute `kctrl package installed get -i kyverno -n `): diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f178585..8d1717d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,7 +14,7 @@ jobs: name: Integration Tests strategy: matrix: - k8s_version: [v1.25, v1.26, v1.27] + k8s_version: [v1.26, v1.27, v1.28] permissions: contents: read uses: kadras-io/github-reusable-workflows/.github/workflows/carvel-package-test-integration.yml@main diff --git a/Makefile b/Makefile index e5b70cc..1c426c2 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -K8S_VERSION=v1.27 +K8S_VERSION=v1.28 # Build package configuration build: package diff --git a/README.md b/README.md index 336e601..501a475 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ A Carvel package for [Kyverno](https://kyverno.io), a policy engine designed for ### Prerequisites -* Kubernetes 1.25+ +* Kubernetes 1.26+ * Carvel [`kctrl`](https://carvel.dev/kapp-controller/docs/latest/install/#installing-kapp-controller-cli-kctrl) CLI. * Carvel [kapp-controller](https://carvel.dev/kapp-controller) deployed in your Kubernetes cluster. You can install it with Carvel [`kapp`](https://carvel.dev/kapp/docs/latest/install) (recommended choice) or `kubectl`. @@ -69,6 +69,82 @@ Verify the installed packages and their status: Documentation, tutorials and examples for this package are available in the [docs](docs) folder. For documentation specific to Kyverno, check out [kyverno.io](https://kyverno.io). +## 🎯  Configuration + +The Kyverno package can be customized via a `values.yml` file. + + ```yaml + tracing: + enabled: true + endpoint: opentelemetrycollector.kyverno.svc.cluster.local + port: 4317 + ``` + +Reference the `values.yml` file from the `kctrl` command when installing or upgrading the package. + + ```shell + kctrl package install -i kyverno \ + -p kyverno.packages.kadras.io \ + -v ${VERSION} \ + -n kadras-packages \ + --values-file values.yml + ``` + +### Values + +The Kyverno package has the following configurable properties. + +
Configurable properties + +| Config | Default | Description | +|-------|-------------------|-------------| +| `optional_components.background_controller` | `true` | Whether to deploy the Background Controller, responsible for processing of generate and mutate-existing rules. | +| `optional_components.cleanup_controller` | `true` | Whether to deploy the Cleanup Controller, responsible for processing `CleanupPolicy` resources. | +| `optional_components.reports_controller` | `true` | Whether to deploy the Reports Controller, responsible for handling `PolicyReport` resources. | +| `ca_cert_data` | `""` | PEM-encoded certificate data to trust TLS connections with a custom CA. | + +Settings for the controllers. + +| Config | Default | Description | +|-------|-------------------|-------------| +| `admission_controller.replicas` | `1` | The number of replicas for the Admission Controller. In order to enable high availability, it should be at least 3 (2 is not permitted). | +| `background_controller.replicas` | `1` | The number of replicas for the Background Controller. In order to enable high availability, it should be greater than 1. | +| `cleanup_controller.replicas` | `1` | The number of replicas for the Cleanup Controller. In order to enable high availability, it should be greater than 1. | +| `reports_controller.replicas` | `1` | The number of replicas for the Reports Controller. In order to enable high availability, it should be greater than 1. | + +Settings for logging. + +| Config | Default | Description | +|-------|-------------------|-------------| +| `logging.level` | `2` | Number of the log level verbosity (from `1` to `6`). | +| `logging.encoding` | `text` | Log encoding format. Options: `text`, `json`. | + +Settings for metrics. + +| Config | Default | Description | +|-------|-------------------|-------------| +| `metrics.type` | `prometheus` | Whether to use OpenTelemetry (`grpc`) or Prometheus (`prometheus`) for exporting metrics. | +| `metrics.collector` | `""` | The endpoint where the OpenTelemetry-based collector receives telemetry data. For example, `opentelemetrycollector.kyverno.svc.cluster.local:4317`. | + +Settings for tracing. + +| Config | Default | Description | +|-------|-------------------|-------------| +| `tracing.enabled` | `false` | Whether to configure Kyverno to export OpenTelemetry traces to a distributed tracing backend. | +| `tracing.endpoint` | `""` | The endpoint where the distributed tracing backend accepts OpenTelemetry traces. For example, `opentelemetrycollector.kyverno.svc.cluster.local`. | +| `tracing.port` | `4317` | The port exposed by the distributed tracing backend to accept OpenTelemetry traces. | +| `tracing.ca_cert_secret` | `""` | The Secret containing the certificate which is used by the Opentelemetry Tracing Client. If empty string is set, an insecure connection will be used. | + +Settings for the corporate proxy. + +| Config | Default | Description | +|-------|-------------------|-------------| +| `proxy.https_proxy` | `""` | The HTTPS proxy to use for network traffic. | +| `proxy.http_proxy` | `""` | The HTTP proxy to use for network traffic. | +| `proxy.no_proxy` | `""` | A comma-separated list of hostnames, IP addresses, or IP ranges in CIDR format that should not use the proxy. | + +
+ ## 🛡️  Security The security process for reporting vulnerabilities is described in [SECURITY.md](SECURITY.md). diff --git a/docs/corporate-proxy.md b/docs/corporate-proxy.md new file mode 100644 index 0000000..7ff5764 --- /dev/null +++ b/docs/corporate-proxy.md @@ -0,0 +1,12 @@ +# Using a Corporate Proxy + +When running Kyverno behind a corporate proxy, you can configure the controllers to proxy communications with external services. + +```yaml +proxy: + http_proxy: "proxy.kadras.io" + https_proxy: "proxy.kadras.io" + no_proxy: ".cluster.local., .cluster.local, .svc" +``` + +For more information, check the Kyverno documentation for configuring a [proxy](https://kyverno.io/docs/installation/customization/#proxy). diff --git a/docs/custom-ca.md b/docs/custom-ca.md new file mode 100644 index 0000000..a7b46a4 --- /dev/null +++ b/docs/custom-ca.md @@ -0,0 +1,26 @@ +# Configuring a Self-Signed Certificate for a Private Registry + +If you need to interact with a private registry using a self-signed certificate, you can pass it as a PEM-encoded bundle when installing/upgrading the package. + +```yaml +ca_cert_data: | + -----BEGIN CERTIFICATE----- + MIICvjCCAaYCCQDhcJuwMw6yZzANBgkqhkiG9w0BAQsFADAhMQswCQYDVQQGEwJE + SzESMBAGA1UEAwwJa2FkcmFzLmlvMB4XDTIzMDEzMTIxMDQ0M1oXDTI4MDEzMTIx + MDQ0M1owITELMAkGA1UEBhMCREsxEjAQBgNVBAMMCWthZHJhcy5pbzCCASIwDQYJ + KoZIhvcNAQEBBQADggEPADCCAQoCggEBAK+rBzsOM95TDd1Ve7dTDJGHhP4snO8Y + 95rHl6LTdxe4x6uDQ7riqpV6uqCSaH0vQJZPhkdH/vgQRKtuNU1JrUNW/gY0t8pO + ITkh8PBctzM8R+28IPQ80qA/vyGk4aaN/TZUMcYAtswk54Izy2M7ZnMvNEOiNSYs + lHlKsj3oyrbkcWQrEcooPzsFoJZsMFnhJQjJ2MM+meSR2+x/edtS1+aw4/HUX9zw + jkbqWoPMzBGjLzHqcb9V/GLg/x4P1BLAMaiRFF1mmxOMbNP2KmUdjiNBnDo0KZRb + xm+898FF2yBWLLVs8ZMYpPGhmN7LSoNmLIueBrNrjau7K+8WePam8O0CAwEAATAN + BgkqhkiG9w0BAQsFAAOCAQEAmq21ZJqoXXfs1U3HDk20+ay4HH9m76B1Vw5q5D9j + t3sfjyl/RhvIObGoIGnrt59H+gfJ9aQFqm+2LeZHDCzDubHa+63Z7KQIoRO3uHGX + XnEhiAckIaxllBhJeO/UJmhr833hKPnS4e2xHgI83oAyplec4UtoicJMmUGULZvS + fZ81unl1Ia6j0MVQrGYG93T80DyiPyaGPnoLHQQpnbO3IXgQL+ZmtNUBP0wk7IiR + 71vOWfDFcY4Od3863+diyyL7uL7Nlfhl7bmbvmRjZ2HJadTi9pSlxLPDDJ6ATPIA + 83rObyM7bWgv+bpQlqZrNAZlLWb3ICBHFumx4CGh/g6pqg== + -----END CERTIFICATE----- +``` + +For more information, check the Kyverno documentation for [self-signed certificates](https://kyverno.io/docs/writing-policies/verify-images/sigstore/#trust). diff --git a/docs/high-availability.md b/docs/high-availability.md new file mode 100644 index 0000000..458054d --- /dev/null +++ b/docs/high-availability.md @@ -0,0 +1,18 @@ +# Configuring High Availability + +High availability for the Kyverno controllers can be achieved when more than 1 replica is configured. Admission Controller and Cleanup Controller use a combination of stateless replication and leader election strategy based on the active/passive model. Background Controller and Reports Controller rely fully on a leader election strategy based on the active/passive model. + +When more than 1 replica is configured (more than 2 for the Admission Controller), a `PodDisruptionBudget` is automatically created to prevent downtime during node unavailability. + +```yaml +admission_controller: + replicas: 3 +background_controller: + replicas: 2 +cleanup_controller: + replicas: 2 +reports_controller: + replicas: 2 +``` + +For more information, check the Kyverno documentation for [high availability](https://kyverno.io/docs/high-availability/). diff --git a/docs/observability.md b/docs/observability.md new file mode 100644 index 0000000..efe9896 --- /dev/null +++ b/docs/observability.md @@ -0,0 +1,46 @@ +# Configuring Observability + +Monitor and observe the operation of Kyverno using logs, metrics, and traces. + +## Logs + +The log verbosity and encoding format for the Kyverno controllers can be configured. + +```yaml +logging: + level: 2 + encoding: text +``` + +For more information, check Kyverno documentation for [logs](https://kyverno.io/docs/installation/customization/#container-flags). + +## Metrics + +The Kyverno controllers expose Prometheus metrics by default. This package comes pre-configured with the necessary annotations to let Prometheus scrape metrics automatically from the Kyverno controllers. + +If you want to export metrics based on the OpenTelemetry format rather than Prometheus, you need to configure the OpenTelemetry endpoint where the controllers will push the metrics using gRPC. + +```yaml +metrics: + type: grpc + collector: opentelemetrycollector.kyverno.svc.cluster.local:4317 +``` + +For more information, check the Kyverno documentation for [metrics](https://kyverno.io/docs/monitoring/). + +## Traces + +OpenTelemetry instrumentation is provided for Kyverno. By default, the instrumentation is disabled. You can enable the generation of traces and configure how they are exported to an OpenTelemetry-compatible distributed tracing backend. + +```yaml +tracing: + enabled: true + endpoint: opentelemetrycollector.kyverno.svc.cluster.local + port: 4317 +``` + +For more information, check the Kyverno documentation for [traces](https://kyverno.io/docs/tracing/). + +## Dashboards + +If you use the Grafana observability stack, you can refer to this [dashboard](https://kyverno.io/docs/monitoring/bonus-grafana-dashboard/) as a foundation to build your own. diff --git a/docs/verify-release.md b/docs/verify-release.md new file mode 100644 index 0000000..8371445 --- /dev/null +++ b/docs/verify-release.md @@ -0,0 +1,37 @@ +# Verifying the Package Release + +This package is published as an OCI artifact, signed with Sigstore [Cosign](https://docs.sigstore.dev/cosign/overview), and associated with a [SLSA Provenance](https://slsa.dev/provenance) attestation. + +Using `cosign`, you can display the supply chain security related artifacts for the `ghcr.io/kadras-io/package-for-kyverno` images. Use the specific digest you'd like to verify. + +```shell +cosign tree ghcr.io/kadras-io/package-for-kyverno +``` + +The result: + +```shell +📦 Supply Chain Security Related artifacts for an image: ghcr.io/kadras-io/package-for-kyverno +└── 💾 Attestations for an image tag: ghcr.io/kadras-io/package-for-kyverno:sha256-b7b13bbf52581f722c23819000aa3cfe01f78d59038d7069af25bbfe4a5491be.att + └── 🍒 sha256:0710c13e9738b2a9c718eb7646c4fa9e3fc0a905a6992461b62b703ccae66974 +└── 🔐 Signatures for an image tag: ghcr.io/kadras-io/package-for-kyverno:sha256-b7b13bbf52581f722c23819000aa3cfe01f78d59038d7069af25bbfe4a5491be.sig + └── 🍒 sha256:3f3b64a6f63c382ec1776b5962d74411fa51669e148f073ab28700cf5e10eab4 +``` + +You can verify the signature and its claims: + +```shell +cosign verify \ + --certificate-identity-regexp https://github.com/kadras-io \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + ghcr.io/kadras-io/package-for-kyverno | jq +``` + +You can also verify the SLSA Provenance attestation associated with the image. + +```shell +cosign verify-attestation --type slsaprovenance \ + --certificate-identity-regexp https://github.com/slsa-framework \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + ghcr.io/kadras-io/package-for-kyverno | jq .payload -r | base64 --decode | jq +``` diff --git a/package/config/kapp-config.yml b/package/config/kapp-config.yml index 0b2e394..e7e7734 100644 --- a/package/config/kapp-config.yml +++ b/package/config/kapp-config.yml @@ -8,12 +8,12 @@ minimumRequiredVersion: 0.50.0 #! See more about the resource merge method: https://carvel.dev/kapp/docs/latest/merge-method. rebaseRules: -#! This webhook configuration is populated by Kyverno. -#! We want to persist those changes between upgrades. +#! This configuration is populated by Kyverno at deploy-time. +#! We want to persist those changes between upgrades +#! if it's not defined in the provided manifests. - paths: - [spec, conversion, strategy] - - [status] type: copy - sources: [existing] + sources: [new, existing] resourceMatchers: - apiVersionKindMatcher: {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition} diff --git a/package/config/overlays/ca-certificates.yml b/package/config/overlays/ca-certificates.yml new file mode 100644 index 0000000..ee517f5 --- /dev/null +++ b/package/config/overlays/ca-certificates.yml @@ -0,0 +1,46 @@ +#@ load("@ytt:data", "data") +#@ load("@ytt:overlay", "overlay") + +#! List of Kyverno Deployments interacting with OCI registries. +#@ deployments = ["kyverno-admission-controller", "kyverno-background-controller", "kyverno-reports-controller"] + +#@ if data.values.ca_cert_data != "": + +--- +apiVersion: v1 +kind: Secret +metadata: + name: kyverno-certs + namespace: kyverno +stringData: + #@yaml/text-templated-strings + ca-certificates: | + (@= data.values.ca_cert_data @) + +#@ for deployment in deployments: + +#@overlay/match by=overlay.subset({"kind":"Deployment", "metadata":{"name":deployment}}), expects="1+" +--- +spec: + template: + spec: + containers: + #@overlay/match by=overlay.all, expects=1 + - + #@overlay/match missing_ok=True + volumeMounts: + - name: ca-certificates + mountPath: /etc/ssl/certs/ca-certificates.crt + subPath: ca-certificates.crt + #@overlay/match missing_ok=True + volumes: + - name: ca-certificates + secret: + secretName: kyverno-certs + items: + - key: ca-certificates + path: ca-certificates.crt + +#@ end + +#@ end diff --git a/package/config/overlays/configmap-kyverno.yml b/package/config/overlays/configmap-kyverno.yml new file mode 100644 index 0000000..0b06a14 --- /dev/null +++ b/package/config/overlays/configmap-kyverno.yml @@ -0,0 +1,18 @@ +#@ load("@ytt:data", "data") +#@ load("@ytt:overlay", "overlay") + +#@overlay/match by=overlay.subset({"kind":"ConfigMap", "metadata":{"name":"kyverno"}}) +#@overlay/match-child-defaults missing_ok=True +--- +data: + webhooks: | + - namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - kyverno + webhookAnnotations: | + # On AKS, we disable the Admissions Enforcer so that it doesn't interfere with the webhooks included in this package. + # See: https://learn.microsoft.com/en-us/azure/aks/faq#can-admission-controller-webhooks-impact-kube-system-and-internal-aks-namespaces + admissions.enforcer/disabled: "true" diff --git a/package/config/overlays/controller-admission.yml b/package/config/overlays/controller-admission.yml new file mode 100644 index 0000000..21854e3 --- /dev/null +++ b/package/config/overlays/controller-admission.yml @@ -0,0 +1,37 @@ +#@ load("@ytt:data", "data") +#@ load("@ytt:overlay", "overlay") + +#@overlay/match by=overlay.subset({"kind":"Deployment", "metadata":{"name":"kyverno-admission-controller"}}) +--- +spec: + #@overlay/match missing_ok=True + replicas: #@ data.values.admission_controller.replicas + template: + spec: + initContainers: + #@overlay/match by="name" + - name: kyverno-pre + args: + #@overlay/match by=overlay.subset("--loggingFormat=text") + - #@ "--loggingFormat={}".format(data.values.logging.encoding) + #@overlay/match by=overlay.subset("--v=2") + - #@ "--v={}".format(data.values.logging.level) + +#@ if/end data.values.admission_controller.replicas >= 3: +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: kyverno-admission-controller-pdb + namespace: kyverno + labels: + app.kubernetes.io/component: admission-controller + app.kubernetes.io/instance: kyverno + app.kubernetes.io/part-of: kyverno +spec: + minAvailable: 50% + selector: + matchLabels: + app.kubernetes.io/component: admission-controller + app.kubernetes.io/instance: kyverno + app.kubernetes.io/part-of: kyverno diff --git a/package/config/overlays/controller-background.yml b/package/config/overlays/controller-background.yml new file mode 100644 index 0000000..add302a --- /dev/null +++ b/package/config/overlays/controller-background.yml @@ -0,0 +1,27 @@ +#@ load("@ytt:data", "data") +#@ load("@ytt:overlay", "overlay") + +#@overlay/match by=overlay.subset({"kind":"Deployment", "metadata":{"name":"kyverno-background-controller"}}) +--- +spec: + #@overlay/match missing_ok=True + replicas: #@ data.values.background_controller.replicas + +#@ if/end data.values.background_controller.replicas > 1: +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: kyverno-background-controller-pdb + namespace: kyverno + labels: + app.kubernetes.io/component: background-controller + app.kubernetes.io/instance: kyverno + app.kubernetes.io/part-of: kyverno +spec: + minAvailable: 50% + selector: + matchLabels: + app.kubernetes.io/component: background-controller + app.kubernetes.io/instance: kyverno + app.kubernetes.io/part-of: kyverno diff --git a/package/config/overlays/controller-cleanup.yml b/package/config/overlays/controller-cleanup.yml new file mode 100644 index 0000000..c67f12f --- /dev/null +++ b/package/config/overlays/controller-cleanup.yml @@ -0,0 +1,27 @@ +#@ load("@ytt:data", "data") +#@ load("@ytt:overlay", "overlay") + +#@overlay/match by=overlay.subset({"kind":"Deployment", "metadata":{"name":"kyverno-cleanup-controller"}}) +--- +spec: + #@overlay/match missing_ok=True + replicas: #@ data.values.cleanup_controller.replicas + +#@ if/end data.values.cleanup_controller.replicas > 1: +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: kyverno-cleanup-controller-pdb + namespace: kyverno + labels: + app.kubernetes.io/component: cleanup-controller + app.kubernetes.io/instance: kyverno + app.kubernetes.io/part-of: kyverno +spec: + minAvailable: 50% + selector: + matchLabels: + app.kubernetes.io/component: cleanup-controller + app.kubernetes.io/instance: kyverno + app.kubernetes.io/part-of: kyverno diff --git a/package/config/overlays/controller-reports.yml b/package/config/overlays/controller-reports.yml new file mode 100644 index 0000000..4d6e8da --- /dev/null +++ b/package/config/overlays/controller-reports.yml @@ -0,0 +1,27 @@ +#@ load("@ytt:data", "data") +#@ load("@ytt:overlay", "overlay") + +#@overlay/match by=overlay.subset({"kind":"Deployment", "metadata":{"name":"kyverno-reports-controller"}}) +--- +spec: + #@overlay/match missing_ok=True + replicas: #@ data.values.reports_controller.replicas + +#@ if/end data.values.reports_controller.replicas > 1: +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: kyverno-reports-controller-pdb + namespace: kyverno + labels: + app.kubernetes.io/component: reports-controller + app.kubernetes.io/instance: kyverno + app.kubernetes.io/part-of: kyverno +spec: + minAvailable: 50% + selector: + matchLabels: + app.kubernetes.io/component: reports-controller + app.kubernetes.io/instance: kyverno + app.kubernetes.io/part-of: kyverno diff --git a/package/config/overlays/corporate-proxy.yml b/package/config/overlays/corporate-proxy.yml new file mode 100644 index 0000000..1c57d92 --- /dev/null +++ b/package/config/overlays/corporate-proxy.yml @@ -0,0 +1,20 @@ +#@ load("@ytt:data", "data") +#@ load("@ytt:overlay", "overlay") + +#@overlay/match by=overlay.subset({"kind":"Deployment"}), expects="1+" +--- +spec: + template: + spec: + containers: + #@overlay/match by=overlay.all, expects=1 + - env: + #@ if/end data.values.proxy.http_proxy != "": + - name: HTTP_PROXY + value: #@ data.values.proxy.http_proxy + #@ if/end data.values.proxy.https_proxy != "": + - name: HTTPS_PROXY + value: #@ data.values.proxy.https_proxy + #@ if/end data.values.proxy.no_proxy != "": + - name: NO_PROXY + value: #@ data.values.proxy.no_proxy diff --git a/package/config/overlays/logging.yml b/package/config/overlays/logging.yml new file mode 100644 index 0000000..02ab35c --- /dev/null +++ b/package/config/overlays/logging.yml @@ -0,0 +1,15 @@ +#@ load("@ytt:data", "data") +#@ load("@ytt:overlay", "overlay") + +#@overlay/match by=overlay.subset({"kind":"Deployment"}), expects="1+" +--- +spec: + template: + spec: + containers: + #@overlay/match by=overlay.all, expects=1 + - args: + #@overlay/match by=overlay.subset("--loggingFormat=text") + - #@ "--loggingFormat={}".format(data.values.logging.encoding) + #@overlay/match by=overlay.subset("--v=2") + - #@ "--v={}".format(data.values.logging.level) diff --git a/package/config/overlays/metrics.yml b/package/config/overlays/metrics.yml new file mode 100644 index 0000000..4b64396 --- /dev/null +++ b/package/config/overlays/metrics.yml @@ -0,0 +1,52 @@ +#@ load("@ytt:data", "data") +#@ load("@ytt:overlay", "overlay") + +#@ metrics_services = ["kyverno-svc-metrics", "kyverno-background-controller-metrics", "kyverno-cleanup-controller-metrics", "kyverno-reports-controller-metrics"] + +#@overlay/match by=overlay.subset({"kind":"Deployment"}), expects="1+" +--- +spec: + template: + spec: + containers: + #@overlay/match by=overlay.all, expects=1 + - args: + #@overlay/match by=overlay.subset("--otelConfig=prometheus") + - #@ "--otelConfig={}".format(data.values.metrics.type) + #@ if/end data.values.metrics.collector != "": + - #@ "--otelCollector={}".format(data.values.metrics.collector) + +#! If Prometheus is chosen as the format for the metrics, +#! add annotations to let Prometheus scrape metrics from Kyverno. + +#@ if data.values.metrics.type == "prometheus": + +#@ for metric_service in metrics_services: + +#@overlay/match by=overlay.subset({"kind":"Service", "metadata":{"name":metric_service}}), expects="1+" +--- +#@overlay/match-child-defaults missing_ok=True +metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/scheme: "http" + prometheus.io/port: "metrics-port" + +#@ end + +#@ end + +#! If OpenTelemetry/gRPC is chosen as the format for the metrics, +#! remove the Services only used when exposing Prometheus metrics. + +#@ if data.values.metrics.type == "grpc": + +#@ for metric_service in metrics_services: + +#@overlay/remove +#@overlay/match by=overlay.subset({"kind":"Service", "metadata":{"name":metric_service}}), expects="1+" +--- + +#@ end + +#@ end diff --git a/package/config/overlays/optional-components.yml b/package/config/overlays/optional-components.yml new file mode 100644 index 0000000..ed1e60f --- /dev/null +++ b/package/config/overlays/optional-components.yml @@ -0,0 +1,20 @@ +#@ load("@ytt:data", "data") +#@ load("@ytt:overlay", "overlay") + +#@ if/end not data.values.optional_components.background_controller: + +#@overlay/remove +#@overlay/match by=overlay.subset({"metadata":{"labels":{"app.kubernetes.io/component": "background-controller"}}}), expects="1+" +--- + +#@ if/end not data.values.optional_components.cleanup_controller: + +#@overlay/remove +#@overlay/match by=overlay.subset({"metadata":{"labels":{"app.kubernetes.io/component": "cleanup-controller"}}}), expects="1+" +--- + +#@ if/end not data.values.optional_components.reports_controller: + +#@overlay/remove +#@overlay/match by=overlay.subset({"metadata":{"labels":{"app.kubernetes.io/component": "reports-controller"}}}), expects="1+" +--- diff --git a/package/config/overlays/registry-credentials.yml b/package/config/overlays/registry-credentials.yml index a6273cc..90e43d2 100644 --- a/package/config/overlays/registry-credentials.yml +++ b/package/config/overlays/registry-credentials.yml @@ -19,7 +19,7 @@ type: kubernetes.io/dockerconfigjson data: .dockerconfigjson: e30K -#@overlay/match by=overlay.subset({"kind":"Deployment"}), expects="2+" +#@overlay/match by=overlay.subset({"kind":"Deployment"}), expects="1+" --- spec: template: @@ -27,3 +27,20 @@ spec: #@overlay/match missing_ok=True imagePullSecrets: - name: #@ image_pull_secret_name + +#! List of Kyverno Deployments interacting with OCI registries. +#@ deployments = ["kyverno-admission-controller", "kyverno-background-controller", "kyverno-reports-controller"] + +#@ for deployment in deployments: + +#@overlay/match by=overlay.subset({"kind":"Deployment", "metadata":{"name":deployment}}), expects="1+" +--- +spec: + template: + spec: + containers: + #@overlay/match by=overlay.all, expects=1 + - args: + - #@ "--imagePullSecrets={}".format(image_pull_secret_name) + +#@ end diff --git a/package/config/overlays/tracing.yml b/package/config/overlays/tracing.yml new file mode 100644 index 0000000..517e276 --- /dev/null +++ b/package/config/overlays/tracing.yml @@ -0,0 +1,18 @@ +#@ load("@ytt:data", "data") +#@ load("@ytt:overlay", "overlay") + +#@ if/end data.values.tracing.enabled: + +#@overlay/match by=overlay.subset({"kind":"Deployment"}), expects="1+" +--- +spec: + template: + spec: + containers: + #@overlay/match by=overlay.all, expects=1 + - args: + - --enableTracing + - #@ "--tracingAddress={}".format(data.values.tracing.endpoint) + - #@ "--tracingPort={}".format(data.values.tracing.port) + #@ if/end data.values.tracing.ca_cert_secret != "": + - #@ "--tracingCreds={}".format(data.values.tracing.ca_cert_secret) diff --git a/package/config/values-schema.yml b/package/config/values-schema.yml new file mode 100644 index 0000000..9e21345 --- /dev/null +++ b/package/config/values-schema.yml @@ -0,0 +1,81 @@ +#@data/values-schema +--- + +#@schema/desc "Settings for which Kyverno optional components to deploy." +optional_components: + #@schema/desc "Whether to deploy the Background Controller, responsible for processing of generate and mutate-existing rules." + background_controller: true + #@schema/desc "Whether to deploy the Cleanup Controller, responsible for processing `CleanupPolicy` resources." + cleanup_controller: true + #@schema/desc "Whether to deploy the Reports Controller, responsible for handling `PolicyReport` resources." + reports_controller: true + +#@schema/desc "PEM-encoded certificate data to trust TLS connections with a custom CA." +ca_cert_data: "" + +#@schema/desc "Settings for the Admission Controller component." +admission_controller: + #@schema/desc "The number of replicas for the Admission Controller. In order to enable high availability, it should be at least 3 (2 is not permitted)." + #@schema/validation min=1 + replicas: 1 + +#@schema/desc "Settings for the Background Controller component." +background_controller: + #@schema/desc "The number of replicas for the Background Controller. In order to enable high availability, it should be greater than 1." + #@schema/validation min=1 + replicas: 1 + +#@schema/desc "Settings for the Cleanup Controller component." +cleanup_controller: + #@schema/desc "The number of replicas for the Cleanup Controller. In order to enable high availability, it should be greater than 1." + #@schema/validation min=1 + replicas: 1 + +#@schema/desc "Settings for the Reports Controller component." +reports_controller: + #@schema/desc "The number of replicas for the Reports Controller. In order to enable high availability, it should be greater than 1." + #@schema/validation min=1 + replicas: 1 + +#@schema/desc "Settings for logging." +logging: + #@schema/desc "Number of the log level verbosity (from `1` to `6`)." + #@schema/validation min=1,max=6 + level: 2 + #@schema/desc "Log encoding format. Options: `text`, `json`." + #@schema/validation one_of=["text", "json"] + encoding: text + +#@schema/desc "Settings for metrics via Prometheus or OpenTelemetry." +metrics: + #@schema/desc "Whether to use OpenTelemetry (`grpc`) or Prometheus (`prometheus`) for exporting metrics." + #@schema/validation one_of=["grpc", "prometheus"] + type: prometheus + #@schema/desc "The endpoint where the OpenTelemetry-based collector receives telemetry data." + #@schema/example ("The service endpoint of the OpenTelemetry-based collector", "opentelemetrycollector.kyverno.svc.cluster.local:4317") + #@schema/validation min_len=2, when=lambda _, ctx: ctx.parent["type"] == "grpc" + collector: "" + +#@schema/desc "Settings for tracing via OpenTelemetry." +tracing: + #@schema/desc "Whether to configure Kyverno to export OpenTelemetry traces to a distributed tracing backend." + enabled: false + #@schema/desc "The endpoint where the distributed tracing backend accepts OpenTelemetry traces." + #@schema/example ("The service endpoint of the OpenTelemetry Collector", "opentelemetrycollector.kyverno.svc.cluster.local") + #@schema/validation min_len=2, when=lambda _, ctx: ctx.parent["enabled"] + endpoint: "" + #@schema/desc "The port exposed by the distributed tracing backend to accept OpenTelemetry traces." + #@schema/validation min=80, when=lambda _, ctx: ctx.parent["enabled"] + port: 4317 + #@schema/desc "The Secret containing the certificate which is used by the Opentelemetry Tracing Client. If empty string is set, an insecure connection will be used." + ca_cert_secret: "" + +#@schema/desc "Settings for the corporate proxy." +proxy: + #@schema/desc "The HTTPS proxy to use for network traffic." + https_proxy: "" + #@schema/desc "The HTTP proxy to use for network traffic." + http_proxy: "" + #@schema/desc "A comma-separated list of hostnames, IP addresses, or IP ranges in CIDR format that should not use the proxy." + #@schema/example ("Allow Kyverno controllers to talk to each other", ".cluster.local.,.cluster.local,.svc") + no_proxy: "" diff --git a/test/setup/kind/v1.25/kind-config.yml b/test/setup/kind/v1.28/kind-config.yml similarity index 61% rename from test/setup/kind/v1.25/kind-config.yml rename to test/setup/kind/v1.28/kind-config.yml index 61248f9..766d7ed 100644 --- a/test/setup/kind/v1.25/kind-config.yml +++ b/test/setup/kind/v1.28/kind-config.yml @@ -3,6 +3,6 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane - image: kindest/node:v1.25.11 + image: kindest/node:v1.28.0 - role: worker - image: kindest/node:v1.25.11 + image: kindest/node:v1.28.0