cherry-pick kyverno#10382

Signed-off-by: ShutingZhao <[email protected]>

charts/kyverno/Chart.yaml

@@ -34,7 +34,7 @@ annotations:
   # valid kinds are: added, changed, deprecated, removed, fixed and security
   artifacthub.io/changes: |
     - kind: added
-      description: make webhook pod annotations configurable
+      description: Add a key to preserve configmap settings during upgrade
 dependencies:
   - name: grafana
     version: 3.2.5

charts/kyverno/README.md

@@ -284,6 +284,7 @@ The chart values are organised per component.
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | config.create | bool | `true` | Create the configmap. |
+| config.preserve | bool | `true` | Preserve the configmap settings during upgrade. |
 | config.name | string | `nil` | The configmap name (required if `create` is `false`). |
 | config.annotations | object | `{}` | Additional annotations to add to the configmap. |
 | config.enableDefaultRegistryMutation | bool | `true` | Enable registry mutation for container images. Enabled by default. |

charts/kyverno/templates/config/configmap.yaml

@@ -6,10 +6,13 @@ metadata:
   namespace: {{ template "kyverno.namespace" . }}
   labels:
     {{- include "kyverno.config.labels" . | nindent 4 }}
-  {{- with .Values.config.annotations }}
   annotations:
+    {{- with .Values.annotations }}
     {{- toYaml . | nindent 4 }}
-  {{- end }}
+    {{- end }}
+    {{- if .Values.config.preserve }}
+    helm.sh/resource-policy: "keep"
+    {{- end }}
 data:
   enableDefaultRegistryMutation: {{ .Values.config.enableDefaultRegistryMutation | quote }}
   {{- with .Values.config.defaultRegistry }}

charts/kyverno/templates/hooks/pre-delete-configmap.yaml

@@ -0,0 +1,129 @@
+{{- if .Values.config.preserve -}}
+{{- if not .Values.templating.enabled -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ template "kyverno.fullname" . }}:remove-configmap
+  namespace: {{ template "kyverno.namespace" . }}
+  labels:
+    {{- include "kyverno.hooks.labels" . | nindent 4 }}
+  annotations:
+    helm.sh/hook: pre-delete
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed
+    helm.sh/hook-weight: "0"
+rules:
+  - apiGroups:
+    - ""
+    resources:
+    - configmaps
+    verbs:
+    - list
+    - get
+    - delete
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ template "kyverno.fullname" . }}:remove-configmap
+  namespace: {{ template "kyverno.namespace" . }}
+  labels:
+    {{- include "kyverno.hooks.labels" . | nindent 4 }}
+  annotations:
+    helm.sh/hook: pre-delete
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed
+    helm.sh/hook-weight: "0"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ template "kyverno.fullname" . }}:remove-configmap
+  namespace: {{ template "kyverno.namespace" . }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ template "kyverno.fullname" . }}-remove-configmap
+    namespace: {{ template "kyverno.namespace" . }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ template "kyverno.fullname" . }}-remove-configmap
+  namespace: {{ template "kyverno.namespace" . }}
+  labels:
+    {{- include "kyverno.hooks.labels" . | nindent 4 }}
+  annotations:
+    helm.sh/hook: pre-delete
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+    helm.sh/hook-weight: "0"
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ template "kyverno.fullname" . }}-remove-configmap
+  namespace: {{ template "kyverno.namespace" . }}
+  labels:
+    {{- include "kyverno.hooks.labels" . | nindent 4 }}
+  annotations:
+    helm.sh/hook: pre-delete
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed
+    helm.sh/hook-weight: "10"
+spec:
+  backoffLimit: 2
+  template:
+    metadata:
+      {{- with .Values.webhooksCleanup.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.webhooksCleanup.podLabels }}
+      labels:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+    spec:
+      serviceAccount: {{ template "kyverno.fullname" . }}-remove-configmap
+      {{- with .Values.webhooksCleanup.podSecurityContext }}
+      securityContext:
+        {{- tpl (toYaml .) $ | nindent 8 }}
+      {{- end }}
+      restartPolicy: Never
+      {{- with .Values.webhooksCleanup.imagePullSecrets }}
+      imagePullSecrets: 
+        {{- tpl (toYaml .) $ | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: kubectl
+          image: {{ (include "kyverno.image" (dict "globalRegistry" .Values.global.image.registry "image" .Values.webhooksCleanup.image "defaultTag" (default .Chart.AppVersion .Values.webhooksCleanup.image.tag))) | quote }}
+          imagePullPolicy: {{ .Values.webhooksCleanup.image.pullPolicy }}
+          command:
+            - /bin/bash
+            - '-c'
+            - |-
+              set -euo pipefail
+              kubectl delete cm -n {{ template "kyverno.namespace" . }} {{ template "kyverno.config.configMapName" . }}
+          {{- with .Values.webhooksCleanup.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+      {{- with .Values.webhooksCleanup.tolerations }}
+      tolerations:
+        {{- tpl (toYaml .) $ | nindent 8 }}
+      {{- end }}
+      {{- with .Values.webhooksCleanup.nodeSelector | default .Values.global.nodeSelector }}
+      nodeSelector:
+        {{- tpl (toYaml .) $ | nindent 8 }}
+      {{- end }}
+      {{- if or .Values.webhooksCleanup.podAntiAffinity .Values.webhooksCleanup.podAffinity .Values.webhooksCleanup.nodeAffinity }}
+      affinity:
+        {{- with .Values.webhooksCleanup.podAntiAffinity }}
+        podAntiAffinity:
+          {{- tpl (toYaml .) $ | nindent 10 }}
+        {{- end }}
+        {{- with .Values.webhooksCleanup.podAffinity }}
+        podAffinity:
+          {{- tpl (toYaml .) $ | nindent 10 }}
+        {{- end }}
+        {{- with .Values.webhooksCleanup.nodeAffinity }}
+        nodeAffinity:
+          {{- tpl (toYaml .) $ | nindent 10 }}
+        {{- end }}
+      {{- end }}
+{{- end -}}
+{{- end -}}

charts/kyverno/templates/hooks/pre-delete-scale-to-zero.yaml

@@ -10,6 +10,7 @@ metadata:
   annotations:
     helm.sh/hook: pre-delete
     helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed
+    helm.sh/hook-weight: "100"
 spec:
   backoffLimit: 2
   template:

charts/kyverno/values.yaml

@@ -174,6 +174,9 @@ config:
   # -- Create the configmap.
   create: true
 
+  # -- Preserve the configmap settings during upgrade.
+  preserve: true
+
   # -- (string) The configmap name (required if `create` is `false`).
   name: ~
 

cmd/background-controller/main.go

@@ -26,6 +26,7 @@ import (
 	"github.com/kyverno/kyverno/pkg/logging"
 	"github.com/kyverno/kyverno/pkg/metrics"
 	"github.com/kyverno/kyverno/pkg/policy"
+	"github.com/kyverno/kyverno/pkg/utils/generator"
 	kubeutils "github.com/kyverno/kyverno/pkg/utils/kube"
 	apiserver "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	kubeinformers "k8s.io/client-go/informers"
@@ -52,6 +53,7 @@ func createrLeaderControllers(
 	eventGenerator event.Interface,
 	jp jmespath.Interface,
 	backgroundScanInterval time.Duration,
+	urGenerator generator.UpdateRequestGenerator,
 ) ([]internal.Controller, error) {
 	policyCtrl, err := policy.NewPolicyController(
 		kyvernoClient,
@@ -67,6 +69,7 @@ func createrLeaderControllers(
 		backgroundScanInterval,
 		metricsConfig,
 		jp,
+		urGenerator,
 	)
 	if err != nil {
 		return nil, err
@@ -117,6 +120,7 @@ func main() {
 		internal.WithKyvernoDynamicClient(),
 		internal.WithEventsClient(),
 		internal.WithApiServerClient(),
+		internal.WithMetadataClient(),
 		internal.WithFlagSets(flagset),
 	)
 	// parse flags
@@ -156,6 +160,7 @@ func main() {
 			eventGenerator,
 			event.Workers,
 		)
+		urGenerator := generator.NewUpdateRequestGenerator(setup.Configuration, setup.MetadataClient)
 		gcstore := store.New()
 		gceController := internal.NewController(
 			globalcontextcontroller.ControllerName,
@@ -222,6 +227,7 @@ func main() {
 					eventGenerator,
 					setup.Jp,
 					bgscanInterval,
+					urGenerator,
 				)
 				if err != nil {
 					logger.Error(err, "failed to create leader controllers")

cmd/kyverno/main.go

@@ -34,6 +34,7 @@ import (
 	"github.com/kyverno/kyverno/pkg/policycache"
 	"github.com/kyverno/kyverno/pkg/tls"
 	"github.com/kyverno/kyverno/pkg/toggle"
+	"github.com/kyverno/kyverno/pkg/utils/generator"
 	kubeutils "github.com/kyverno/kyverno/pkg/utils/kube"
 	runtimeutils "github.com/kyverno/kyverno/pkg/utils/runtime"
 	"github.com/kyverno/kyverno/pkg/validation/exception"
@@ -290,6 +291,7 @@ func main() {
 		internal.WithKyvernoDynamicClient(),
 		internal.WithEventsClient(),
 		internal.WithApiServerClient(),
+		internal.WithMetadataClient(),
 		internal.WithFlagSets(flagset),
 	)
 	// parse flags
@@ -499,10 +501,12 @@ func main() {
 			setup.Logger.Error(err, "failed to initialize leader election")
 			os.Exit(1)
 		}
+		urGenerator := generator.NewUpdateRequestGenerator(setup.Configuration, setup.MetadataClient)
 		// create webhooks server
 		urgen := webhookgenerate.NewGenerator(
 			setup.KyvernoClient,
 			kyvernoInformer.Kyverno().V1beta1().UpdateRequests(),
+			urGenerator,
 		)
 		policyHandlers := webhookspolicy.NewHandlers(
 			setup.KyvernoDynamicClient,

config/install-latest-testing.yaml

@@ -72,6 +72,8 @@ metadata:
     app.kubernetes.io/instance: kyverno
     app.kubernetes.io/part-of: kyverno
     app.kubernetes.io/version: latest
+  annotations:
+    helm.sh/resource-policy: "keep"
 data:
   enableDefaultRegistryMutation: "true"
   defaultRegistry: "docker.io"

pkg/config/config.go

@@ -96,8 +96,11 @@ const (
 	webhookAnnotations            = "webhookAnnotations"
 	webhookLabels                 = "webhookLabels"
 	matchConditions               = "matchConditions"
+	updateRequestThreshold        = "updateRequestThreshold"
 )
 
+const UpdateRequestThreshold = 1000
+
 var (
 	// kyvernoNamespace is the Kyverno namespace
 	kyvernoNamespace = osutils.GetEnvWithFallback("KYVERNO_NAMESPACE", "kyverno")
@@ -177,6 +180,8 @@ type Configuration interface {
 	Load(*corev1.ConfigMap)
 	// OnChanged adds a callback to be invoked when the configuration is reloaded
 	OnChanged(func())
+	// GetUpdateRequestThreshold gets the threshold limit for the total number of updaterequests
+	GetUpdateRequestThreshold() int64
 }
 
 // configuration stores the configuration
@@ -194,6 +199,7 @@ type configuration struct {
 	matchConditions               []admissionregistrationv1.MatchCondition
 	mux                           sync.RWMutex
 	callbacks                     []func()
+	updateRequestThreshold        int64
 }
 
 type match struct {
@@ -322,6 +328,12 @@ func (cd *configuration) GetMatchConditions() []admissionregistrationv1.MatchCon
 	return cd.matchConditions
 }
 
+func (cd *configuration) GetUpdateRequestThreshold() int64 {
+	cd.mux.RLock()
+	defer cd.mux.RUnlock()
+	return cd.updateRequestThreshold
+}
+
 func (cd *configuration) Load(cm *corev1.ConfigMap) {
 	if cm != nil {
 		cd.load(cm)
@@ -352,6 +364,7 @@ func (cd *configuration) load(cm *corev1.ConfigMap) {
 	cd.matchConditions = nil
 	// load filters
 	cd.filters = parseKinds(data[resourceFilters])
+	cd.updateRequestThreshold = UpdateRequestThreshold
 	logger.Info("filters configured", "filters", cd.filters)
 	// load defaultRegistry
 	defaultRegistry, ok := data[defaultRegistry]
@@ -482,6 +495,19 @@ func (cd *configuration) load(cm *corev1.ConfigMap) {
 			logger.Info("matchConditions configured")
 		}
 	}
+	threshold, ok := data[updateRequestThreshold]
+	if !ok {
+		logger.Info("enableDefaultRegistryMutation not set")
+	} else {
+		logger := logger.WithValues("enableDefaultRegistryMutation", enableDefaultRegistryMutation)
+		urThreshold, err := strconv.ParseInt(threshold, 10, 64)
+		if err != nil {
+			logger.Error(err, "enableDefaultRegistryMutation is not a boolean")
+		} else {
+			cd.updateRequestThreshold = urThreshold
+			logger.Info("enableDefaultRegistryMutation configured")
+		}
+	}
 }
 
 func (cd *configuration) unload() {

pkg/policy/generate.go

@@ -118,11 +118,15 @@ func (pc *policyController) syncDataRulechanges(policy kyvernov1.PolicyInterface
 		labels := downstream.GetLabels()
 		trigger := generateutils.TriggerFromLabels(labels)
 		ur := newUR(policy, trigger, rule.Name, kyvernov1beta1.Generate, deleteDownstream)
-		created, err := pc.kyvernoClient.KyvernoV1beta1().UpdateRequests(config.KyvernoNamespace()).Create(context.TODO(), ur, metav1.CreateOptions{})
+
+		created, err := pc.urGenerator.Generate(context.TODO(), pc.kyvernoClient, ur, pc.log)
 		if err != nil {
 			errorList = append(errorList, err)
 			continue
 		}
+		if created == nil {
+			continue
+		}
 		updated := created.DeepCopy()
 		updated.Status = newURStatus(downstream)
 		_, err = pc.kyvernoClient.KyvernoV1beta1().UpdateRequests(config.KyvernoNamespace()).UpdateStatus(context.TODO(), updated, metav1.UpdateOptions{})