feat: Watch Namespaces based on labels and label selectors

.github/workflows/pr-kubernetes-tests.yaml

@@ -78,7 +78,7 @@ jobs:
         # September 16, 2024: 17 minutes
         - cluster-name: 'cluster-five'
           go-test-args: '-v -timeout=25m'
-          go-test-run-regex: '^TestValidationStrict$$|^TestValidationAlwaysAccept$$|^TestTransformationValidationDisabled$$|^TestGloomtlsGatewayEdgeGateway$$|^TestHelm$$'
+          go-test-run-regex: '^TestValidationStrict$$|^TestValidationAlwaysAccept$$|^TestTransformationValidationDisabled$$|^TestGloomtlsGatewayEdgeGateway$$|^TestHelm$$|^TestWatchNamespaceSelector$$'
 
         # In our PR tests, we run the suite of tests using the upper ends of versions that we claim to support
         # The versions should mirror: https://docs.solo.io/gloo-edge/latest/reference/support/

changelog/v1.18.0-beta23/watch-namespace-selectors.yaml

@@ -0,0 +1,14 @@
+changelog:
+- type: NEW_FEATURE
+  issueLink: https://github.com/solo-io/gloo/issues/9274
+  resolvesIssue: false
+  description: Adds a new field `watchNamespaceSelectors` to the settings CR. This allows users to specify namespaces to watch based on label selectors. The `watchNamespaces` field will override this if specified.
+- type: HELM
+  issueLink: https://github.com/solo-io/gloo/issues/9274
+  resolvesIssue: false
+  description: Adds a new helm value `settings.watchNamespaceSelectors`. This allows users to specify namespaces to watch based on label selectors that can be specified via `settings.watchNamespaceSelectors.matchLabels` for an exact label match and `settings.watchNamespaceSelectors.matchExpressions` for more generic requirements.  The `settings.watchNamespaces` field will override this if specified.
+- type: DEPENDENCY_BUMP
+  dependencyOwner: solo-io
+  dependencyRepo: solo-kit
+  dependencyTag: v0.35.4
+  issueLink: https://github.com/solo-io/gloo/issues/9274

docs/content/reference/values.txt

@@ -172,7 +172,11 @@
 |kubeGateway.gatewayParameters.glooGateway.aiExtension.ports[].protocol|string|||
 |kubeGateway.gatewayParameters.glooGateway.aiExtension.ports[].hostIP|string|||
 |kubeGateway.gatewayParameters.glooGateway.floatingUserId|bool||If true, allows the cluster to dynamically assign a user ID for the processes running in the container. Default is false.|
-|settings.watchNamespaces[]|string||whitelist of namespaces for Gloo Edge to watch for services and CRDs. Empty list means all namespaces|
+|settings.watchNamespaces[]|string||whitelist of namespaces for Gloo Edge to watch for services and CRDs. Empty list means all namespaces. If this and WatchNamespaceSelectors are specified, this takes precedence and WatchNamespaceSelectors is ignored|
+|settings.watchNamespaceSelectors[].match_labels.NAME|string|||
+|settings.watchNamespaceSelectors[].match_expressions[].key|string|||
+|settings.watchNamespaceSelectors[].match_expressions[].operator|string|||
+|settings.watchNamespaceSelectors[].match_expressions[].values[]|string|||
 |settings.writeNamespace|string||namespace where intermediary CRDs will be written to, e.g. Upstreams written by Gloo Edge Discovery.|
 |settings.integrations.knative.enabled|bool|false|enabled knative components|
 |settings.integrations.knative.version|string|0.10.0|the version of knative installed to the cluster. if using version < 0.8.0, Gloo Edge will use Knative's ClusterIngress API for configuration rather than the namespace-scoped Ingress|

docs/data/ProtoMap.yaml

@@ -872,6 +872,12 @@ apis:
   gloo.solo.io.KubernetesServiceDestination:
     relativepath: reference/api/github.com/solo-io/gloo/projects/gloo/api/v1/proxy.proto.sk/#KubernetesServiceDestination
     package: gloo.solo.io
+  gloo.solo.io.LabelSelector:
+    relativepath: reference/api/github.com/solo-io/gloo/projects/gloo/api/v1/settings.proto.sk/#LabelSelector
+    package: gloo.solo.io
+  gloo.solo.io.LabelSelectorRequirement:
+    relativepath: reference/api/github.com/solo-io/gloo/projects/gloo/api/v1/settings.proto.sk/#LabelSelectorRequirement
+    package: gloo.solo.io
   gloo.solo.io.LbEndpoint:
     relativepath: reference/api/github.com/solo-io/gloo/projects/gloo/api/v1/failover.proto.sk/#LbEndpoint
     package: gloo.solo.io

go.mod

@@ -58,7 +58,7 @@ require (
 	// Ref: https://github.com/solo-io/gloo/pull/9463/files#r1594409655 && https://solo-io-corp.slack.com/archives/C03MFATU265/p1716913420716729?thread_ts=1716476992.938679&cid=C03MFATU265
 	// as to why it is now based off `gloo-main` and not `gloo-repo-branch`
 	github.com/solo-io/solo-apis v0.0.0-20240528173540-7879b7d12cb9
-	github.com/solo-io/solo-kit v0.35.3
+	github.com/solo-io/solo-kit v0.35.4
 	github.com/spf13/afero v1.9.2
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5

go.sum

@@ -2013,8 +2013,8 @@ github.com/solo-io/skv2 v0.39.1 h1:dWaZTWgntAsvh4lTlojd3xE+g7NF4oYNonfcdpy0nXE=
 github.com/solo-io/skv2 v0.39.1/go.mod h1:Zsnl+OYmOkj+6KeaMfkzYIxHTVMC0w2gVApzNJRadM8=
 github.com/solo-io/solo-apis v0.0.0-20240528173540-7879b7d12cb9 h1:4DNulNBJdaVz+fOaMQes1MqVCIO/db2vmwz7bVe3iIU=
 github.com/solo-io/solo-apis v0.0.0-20240528173540-7879b7d12cb9/go.mod h1:fA+jJC7TXNM+i3uXvq7fpVrJ6JpNu1BgBxc4U9ntUW0=
-github.com/solo-io/solo-kit v0.35.3 h1:cm+uHB5cFFlAnghwKdboaKI0Ki+t1ECA0Db3TnTL6KU=
-github.com/solo-io/solo-kit v0.35.3/go.mod h1:KBCEfl59/wE0K68s90aDcrTc36gKR5L97TbVelwL8n4=
+github.com/solo-io/solo-kit v0.35.4 h1:Q6Iai6Us4JYF6fGxED797ygVlytk1uufrSQsPw3bWHk=
+github.com/solo-io/solo-kit v0.35.4/go.mod h1:KBCEfl59/wE0K68s90aDcrTc36gKR5L97TbVelwL8n4=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=

install/helm/gloo/crds/gloo.solo.io_v1_Settings.yaml

@@ -1057,6 +1057,28 @@ spec:
                   token:
                     type: string
                 type: object
+              watchNamespaceSelectors:
+                items:
+                  properties:
+                    matchExpressions:
+                      items:
+                        properties:
+                          key:
+                            type: string
+                          operator:
+                            type: string
+                          values:
+                            items:
+                              type: string
+                            type: array
+                        type: object
+                      type: array
+                    matchLabels:
+                      additionalProperties:
+                        type: string
+                      type: object
+                  type: object
+                type: array
               watchNamespaces:
                 items:
                   type: string

install/helm/gloo/generate/values.go

@@ -1,6 +1,7 @@
 package generate
 
 import (
+	v1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1"
 	corev1 "k8s.io/api/core/v1"
 )
 
@@ -214,7 +215,8 @@ type KnativeProxyInternal struct {
 }
 
 type Settings struct {
-	WatchNamespaces               []string                `json:"watchNamespaces,omitempty" desc:"whitelist of namespaces for Gloo Edge to watch for services and CRDs. Empty list means all namespaces"`
+	WatchNamespaces               []string                `json:"watchNamespaces,omitempty" desc:"whitelist of namespaces for Gloo Edge to watch for services and CRDs. Empty list means all namespaces. If this and WatchNamespaceSelectors are specified, this takes precedence and WatchNamespaceSelectors is ignored"`
+	WatchNamespaceSelectors       []*v1.LabelSelector     `json:"watchNamespaceSelectors,omitempty" desc:"A list of Kubernetes selectors that specify the set of namespaces to restrict the namespaces that Gloo controllers take into consideration when watching for resources. Elements in the list are disjunctive (OR semantics), i.e. a namespace will be included if it matches any selector. An empty list means all namespaces. If this and WatchNamespaces are specified, WatchNamespaces takes precedence and this is ignored"`
 	WriteNamespace                *string                 `json:"writeNamespace,omitempty" desc:"namespace where intermediary CRDs will be written to, e.g. Upstreams written by Gloo Edge Discovery."`
 	Integrations                  *Integrations           `json:"integrations,omitempty"`
 	Create                        *bool                   `json:"create,omitempty" desc:"create a Settings CRD which provides bootstrap configuration to Gloo Edge controllers"`

install/helm/gloo/templates/18-settings.yaml

@@ -193,8 +193,13 @@ spec:
   {{- range . }}
   - {{ . }}
   {{- end }}
-{{- end }}
-{{- end }}
+{{- end }} {{/* with .Values.settings.watchNamespaces */}}
+{{- end }} {{/* if .Values.settings.singleNamespace */}}
+
+{{- if .Values.settings.watchNamespaceSelectors }}
+  watchNamespaceSelectors:
+  {{ toYaml .Values.settings.watchNamespaceSelectors | nindent 4 }}
+{{- end }} {{/* if .Values.settings.watchNamespaceSelectors */}}
 
 {{- end }} {{/* if .Values.settings.create  */}}
 {{- end }} {{/* define "settings.settingsSpec" */}}

install/helm/gloo/templates/5-gateway-validation-webhook-configuration.yaml

@@ -58,6 +58,13 @@ specific resources, we will manage the resources that the webhook receives via t
     apiGroups: [""]{{/* We do not have internal secret CRDs. We want to validate the deletion of secrets such as TLS, so we add "" which refers to Kubernetes' core APIs. */}}
     apiVersions: ["v1"]
     resources: ["secrets"]
+{{- end }}
+{{- if and (not (has "*" .Values.gateway.validation.webhook.skipDeleteValidationResources)) (not (has "namespaces" .Values.gateway.validation.webhook.skipDeleteValidationResources)) }}
+{{- /* A namespace with an updated to a label can cause it to no longer be watched, equivalent to deleting it from the controller's perspective */}}
+  - operations: [ "UPDATE", "DELETE"]
+    apiGroups: [""]{{/* We do not have internal namespace CRDs. We want to validate the updation and deletion of namespaces that are watched. */}}
+    apiVersions: ["v1"]
+    resources: ["namespaces"]
 {{- end }}
   - operations: {{ include "gloo.webhookvalidation.operationsForResource" (list "ratelimitconfigs" .Values.gateway.validation.webhook.skipDeleteValidationResources) }}
     apiGroups: ["ratelimit.solo.io"]

install/test/5-gateway-validation-webhook-configuration_test.go

@@ -43,7 +43,7 @@ var _ = Describe("WebhookValidationConfiguration helm test", func() {
 			timeoutSeconds := 5
 
 			// Count the "DELETES" as a sanity check.
-			expectedDeletes := 5 - expectedRemoved
+			expectedDeletes := 6 - expectedRemoved
 			expectedChart := generateExpectedChart(timeoutSeconds, resources, expectedDeletes)
 
 			prepareMakefile(namespace, glootestutils.HelmValues{
@@ -59,10 +59,11 @@ var _ = Describe("WebhookValidationConfiguration helm test", func() {
 			Entry("gateways", []string{"gateways"}, 0),
 			Entry("upstreams", []string{"upstreams"}, 1),
 			Entry("secrets", []string{"secrets"}, 1),
+			Entry("namespaces", []string{"namespaces"}, 1),
 			Entry("ratelimitconfigs", []string{"ratelimitconfigs"}, 1),
 			Entry("mutiple", []string{"virtualservices", "routetables", "secrets"}, 3),
 			Entry("mutiple (with removal)", []string{"ratelimitconfigs", "secrets"}, 2),
-			Entry("all", []string{"*"}, 5),
+			Entry("all", []string{"*"}, 6),
 			Entry("empty", []string{}, 0),
 		)
 
@@ -253,6 +254,12 @@ func generateRules(skipDeleteReources []string) string {
 			"apiVersions": {"v1"},
 			"resources":   {"secrets"},
 		},
+		{
+			"operations":  {"UPDATE", "DELETE"},
+			"apiGroups":   {""},
+			"apiVersions": {"v1"},
+			"resources":   {"namespaces"},
+		},
 		{
 			"operations":  {"CREATE", "UPDATE", "DELETE"},
 			"apiGroups":   {"ratelimit.solo.io"},
@@ -261,17 +268,23 @@ func generateRules(skipDeleteReources []string) string {
 		},
 	}
 
+	finalRules := []map[string][]string{}
 	for i, rule := range rules {
 		if stringutils.ContainsAny([]string{rule["resources"][0], "*"}, skipDeleteReources) {
 			rule["operations"] = gloostringutils.DeleteOneByValue(rule["operations"], "DELETE")
+			// A namespace with an update to a label can cause it to no longer be watched,
+			// equivalent to deleting it from the controller's perspective
+			if rule["resources"][0] == "namespaces" {
+				rule["operations"] = gloostringutils.DeleteOneByValue(rule["operations"], "UPDATE")
+			}
 		}
 
-		if len(rule["operations"]) == 0 {
-			rules = append(rules[:i], rules[i+1:]...)
+		if len(rule["operations"]) != 0 {
+			finalRules = append(finalRules, rules[i])
 		}
 	}
 
-	str, err := yaml.Marshal(rules)
+	str, err := yaml.Marshal(finalRules)
 	Expect(err).NotTo(HaveOccurred())
 	return string(str)
 }

pkg/utils/utils_suite_test.go → pkg/utils/namespaces/namesaces_suite_test.go

@@ -1,4 +1,4 @@
-package utils_test
+package namespaces_test
 
 import (
 	"testing"
@@ -9,5 +9,5 @@ import (
 
 func TestUtils(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "Utils Suite")
+	RunSpecs(t, "Namespace Suite")
 }