feat: Watch Namespaces based on labels and label selectors

changelog/v1.18.0-beta19/watch-namespace-selectors.yaml

@@ -0,0 +1,6 @@
+changelog:
+- type: NEW_FEATURE
+  issueLink: https://github.com/solo-io/gloo/issues/9274
+  resolvesIssue: false
+  description: Adds a new field `watchNamespaceSelectors` to the settings CR. This allows users to specify namespaces to watch based on label selectors. The `watchNamespaces` field will override this if specified.
+

docs/content/reference/values.txt

@@ -332,6 +332,10 @@
 |settings.secretOptions.sources[].vault.aws.leaseIncrement|uint32||The time increment, in seconds, used in renewing the lease of the Vault token. See: https://developer.hashicorp.com/vault/docs/concepts/lease#lease-durations-and-renewal. Defaults to 0, which causes the default TTL to be used.|
 |settings.secretOptions.sources[].directory.directory|string||Directory to read secrets from.|
 |settings.kubeResourceOverride.NAME|interface||override fields in the generated resource by specifying the yaml structure to override under the top-level key.|
+|settings.watchNamespaceSelectors[].matchLabels.NAME|string|||
+|settings.watchNamespaceSelectors[].matchExpressions[].key|string|||
+|settings.watchNamespaceSelectors[].matchExpressions[].operator|string|||
+|settings.watchNamespaceSelectors[].matchExpressions[].values[]|string|||
 |gloo.deployment.xdsPort|int|9977|port where gloo serves xDS API to Envoy.|
 |gloo.deployment.restXdsPort|uint32|9976|port where gloo serves REST xDS API to Envoy.|
 |gloo.deployment.validationPort|int|9988|port where gloo serves gRPC Proxy Validation to Gateway.|

docs/data/ProtoMap.yaml

@@ -872,6 +872,12 @@ apis:
   gloo.solo.io.KubernetesServiceDestination:
     relativepath: reference/api/github.com/solo-io/gloo/projects/gloo/api/v1/proxy.proto.sk/#KubernetesServiceDestination
     package: gloo.solo.io
+  gloo.solo.io.LabelSelector:
+    relativepath: reference/api/github.com/solo-io/gloo/projects/gloo/api/v1/settings.proto.sk/#LabelSelector
+    package: gloo.solo.io
+  gloo.solo.io.LabelSelectorRequirement:
+    relativepath: reference/api/github.com/solo-io/gloo/projects/gloo/api/v1/settings.proto.sk/#LabelSelectorRequirement
+    package: gloo.solo.io
   gloo.solo.io.LbEndpoint:
     relativepath: reference/api/github.com/solo-io/gloo/projects/gloo/api/v1/failover.proto.sk/#LbEndpoint
     package: gloo.solo.io

install/helm/gloo/crds/gloo.solo.io_v1_Settings.yaml

@@ -1054,6 +1054,28 @@ spec:
                   token:
                     type: string
                 type: object
+              watchNamespaceSelectors:
+                items:
+                  properties:
+                    matchExpressions:
+                      items:
+                        properties:
+                          key:
+                            type: string
+                          operator:
+                            type: string
+                          values:
+                            items:
+                              type: string
+                            type: array
+                        type: object
+                      type: array
+                    matchLabels:
+                      additionalProperties:
+                        type: string
+                      type: object
+                  type: object
+                type: array
               watchNamespaces:
                 items:
                   type: string

install/helm/gloo/generate/values.go

@@ -1,6 +1,7 @@
 package generate
 
 import (
+	v1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1"
 	corev1 "k8s.io/api/core/v1"
 )
 
@@ -234,6 +235,7 @@ type Settings struct {
 	DevMode       *bool         `json:"devMode,omitempty" desc:"Whether or not to enable dev mode. Defaults to false. Setting to true at install time will expose the gloo dev admin endpoint on port 10010. Not recommended for production. Warning: this value is deprecated as of 1.17 and will be removed in a future release."`
 	SecretOptions SecretOptions `json:"secretOptions,omitempty" desc:"Options for how Gloo Edge should handle secrets."`
 	*KubeResourceOverride
+	WatchNamespaceSelectors []*v1.LabelSelector `json:"watchNamespaceSelectors,omitempty" desc:"A list of Kubernetes selectors that specify the set of namespaces to restrict the namespaces that Gloo controllers take into consideration when watching for resources. Elements in the list are disjunctive (OR semantics), i.e. a namespace will be included if it matches any selector."`
 }
 
 type AwsSettings struct {

install/helm/gloo/templates/18-settings.yaml

@@ -192,8 +192,13 @@ spec:
   {{- range . }}
   - {{ . }}
   {{- end }}
-{{- end }}
-{{- end }}
+{{- end }} {{/* with .Values.settings.watchNamespaces */}}
+{{- end }} {{/* if .Values.settings.singleNamespace */}}
+
+{{- if .Values.settings.watchNamespaceSelectors }}
+  watchNamespaceSelectors:
+  {{ toYaml .Values.settings.watchNamespaceSelectors | nindent 4 }}
+{{- end }} {{/* if .Values.settings.watchNamespaceSelectors */}}
 
 {{- end }} {{/* if .Values.settings.create  */}}
 {{- end }} {{/* define "settings.settingsSpec" */}}

pkg/utils/settingsutil/settings.go

@@ -2,13 +2,37 @@ package settingsutil
 
 import (
 	"context"
+	"fmt"
+	"slices"
+	"sync"
 
+	"github.com/solo-io/gloo/pkg/utils"
+	"github.com/solo-io/gloo/projects/gloo/cli/pkg/helpers"
 	v1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1"
+	"github.com/solo-io/gloo/projects/gloo/pkg/defaults"
+	"github.com/solo-io/solo-kit/pkg/api/external/kubernetes/namespace"
+	"github.com/solo-io/solo-kit/pkg/api/v1/clients"
+	"github.com/solo-io/solo-kit/pkg/api/v1/clients/kube/cache"
+	"github.com/solo-io/solo-kit/pkg/api/v1/resources/common/kubernetes"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/selection"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/utils/lru"
 )
 
 type settingsKeyStruct struct{}
 
-var settingsKey = settingsKeyStruct{}
+var (
+	mu sync.Mutex
+
+	// Setting a cache size of 2 should suffice for :
+	// - The current translation loop
+	// - The new loop about to run when the settings have changed
+	namespacesToWatchCache = lru.New(2)
+
+	settingsKey = settingsKeyStruct{}
+)
 
 func WithSettings(ctx context.Context, settings *v1.Settings) context.Context {
 	return context.WithValue(ctx, settingsKey, settings)
@@ -37,11 +61,11 @@ func MaybeFromContext(ctx context.Context) *v1.Settings {
 	return nil
 }
 
-func IsAllNamespacesFromSettings(s *v1.Settings) bool {
-	if s == nil {
+func IsAllNamespacesFromSettings(settings *v1.Settings) bool {
+	if settings == nil {
 		return false
 	}
-	return IsAllNamespaces(s.GetWatchNamespaces())
+	return IsAllNamespaces(GetNamespacesToWatch(settings))
 }
 
 func IsAllNamespaces(watchNs []string) bool {
@@ -54,3 +78,148 @@ func IsAllNamespaces(watchNs []string) bool {
 		return false
 	}
 }
+
+// GenerateNamespacesToWatch generates the list of namespaces to watch based on :
+// - If `watchNamespaces` is defined, return it and do not consider `watchNamespaceSelectors`
+// - If `watchNamespaces` and `watchNamespaceSelectors` are not defined, return all namespaces
+// - If `watchNamespaces` is not defined and `watchNamespaceSelectors` is defined, return all namespaces that match the `watchNamespaceSelectors`
+// In every case, the `discoveryNamespace` (defaults to `gloo-system`) is appended to the list of namespaces
+func GenerateNamespacesToWatch(settings *v1.Settings, namespaces kubernetes.KubeNamespaceList) ([]string, error) {
+	writeNamespace := settings.GetDiscoveryNamespace()
+	if writeNamespace == "" {
+		writeNamespace = defaults.GlooSystem
+	}
+
+	if len(settings.GetWatchNamespaces()) != 0 {
+		// Prevent an error where the controller can not read resources written by discovery if the
+		// install or discovery namespace is not watched
+		return utils.ProcessWatchNamespaces(settings.GetWatchNamespaces(), writeNamespace), nil
+	}
+
+	// Watch all namespaces if `watchNamespaces` or `watchNamespaceSelectors` is not specified
+	if len(settings.GetWatchNamespaceSelectors()) == 0 {
+		return []string{""}, nil
+	}
+
+	var selectors []labels.Selector
+	selectedNamespaces := sets.NewString()
+
+	for _, selector := range settings.GetWatchNamespaceSelectors() {
+		ls, err := labelSelectorAsSelector(selector)
+		if err != nil {
+			return nil, err
+		}
+		selectors = append(selectors, ls)
+	}
+
+	for _, ns := range namespaces {
+		for _, selector := range selectors {
+			if selector.Matches(labels.Set(ns.Labels)) {
+				selectedNamespaces.Insert(ns.Name)
+				break
+			}
+		}
+	}
+
+	// Prevent an error where the controller can not read resources written by discovery if the
+	// install or discovery namespace is not watched
+	selectedNamespaces.Insert(writeNamespace)
+
+	return selectedNamespaces.List(), nil
+}
+
+func setNamespacesToWatch(settings *v1.Settings, namespaces []string) {
+	namespacesToWatchCache.Add(settings.MustHash(), namespaces)
+}
+
+// UpdateNamespacesToWatch generates and updated the list of namespaces to watch and returns true if updated
+func UpdateNamespacesToWatch(settings *v1.Settings, namespaces kubernetes.KubeNamespaceList) (bool, error) {
+	// Run this method synchronously to prevent any issues with caching the namespaces to watch
+	mu.Lock()
+	defer mu.Unlock()
+
+	newNamespacesToWatch, err := GenerateNamespacesToWatch(settings, namespaces)
+	if err != nil {
+		return false, err
+	}
+
+	ns, ok := namespacesToWatchCache.Get(settings.MustHash())
+	if ok {
+		currentNamespacesToWatch, ok := ns.([]string)
+		if ok && slices.Equal(newNamespacesToWatch, currentNamespacesToWatch) {
+			return false, nil
+		}
+	}
+
+	setNamespacesToWatch(settings, newNamespacesToWatch)
+	return true, nil
+}
+
+func getAllNamespaces() (kubernetes.KubeNamespaceList, error) {
+	kubeClient := helpers.MustKubeClient()
+	kubeCache, _ := cache.NewKubeCoreCache(context.TODO(), kubeClient)
+	nsClient := namespace.NewNamespaceClient(kubeClient, kubeCache)
+
+	return nsClient.List(clients.ListOpts{})
+}
+
+// GetNamespacesToWatch returns the list of namespaces to watch based on the last run of `GenerateNamespacesToWatch`
+func GetNamespacesToWatch(settings *v1.Settings) []string {
+	ns, ok := namespacesToWatchCache.Get(settings.MustHash())
+	if ok {
+		currentNamespacesToWatch, ok := ns.([]string)
+		if ok {
+			return currentNamespacesToWatch
+		}
+	}
+
+	// Fallback to fetching all namespaces and updating the cache if not found
+	allNamespaces, err := getAllNamespaces()
+	if err != nil {
+		panic("Unable to fetch namespaces")
+	}
+	UpdateNamespacesToWatch(settings, allNamespaces)
+	ns, _ = namespacesToWatchCache.Get(settings.MustHash())
+	return ns.([]string)
+}
+
+// TODO: remove this and use k8s.io/apimachinery to do this once the settings proto uses the predefined fields
+func labelSelectorAsSelector(ps *v1.LabelSelector) (labels.Selector, error) {
+	if ps == nil {
+		return labels.Nothing(), nil
+	}
+	if len(ps.GetMatchLabels())+len(ps.GetMatchExpressions()) == 0 {
+		return labels.Everything(), nil
+	}
+	requirements := make([]labels.Requirement, 0, len(ps.GetMatchLabels())+len(ps.GetMatchExpressions()))
+	for k, v := range ps.GetMatchLabels() {
+		r, err := labels.NewRequirement(k, selection.Equals, []string{v})
+		if err != nil {
+			return nil, err
+		}
+		requirements = append(requirements, *r)
+	}
+	for _, expr := range ps.GetMatchExpressions() {
+		var op selection.Operator
+		switch metav1.LabelSelectorOperator(expr.GetOperator()) {
+		case metav1.LabelSelectorOpIn:
+			op = selection.In
+		case metav1.LabelSelectorOpNotIn:
+			op = selection.NotIn
+		case metav1.LabelSelectorOpExists:
+			op = selection.Exists
+		case metav1.LabelSelectorOpDoesNotExist:
+			op = selection.DoesNotExist
+		default:
+			return nil, fmt.Errorf("%q is not a valid label selector operator", expr.GetOperator())
+		}
+		r, err := labels.NewRequirement(expr.GetKey(), op, append([]string(nil), expr.GetValues()...))
+		if err != nil {
+			return nil, err
+		}
+		requirements = append(requirements, *r)
+	}
+	selector := labels.NewSelector()
+	selector = selector.Add(requirements...)
+	return selector, nil
+}