From d3d0e702b16d2675c96a92901bc3d1d8a30b8052 Mon Sep 17 00:00:00 2001 From: tesla59 Date: Tue, 10 Sep 2024 23:24:57 +0000 Subject: [PATCH 1/9] config: reload global config when config file is updated Signed-off-by: tesla59 --- KubeArmor/config/config.go | 18 ++++++++++++++++-- KubeArmor/go.mod | 2 +- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/KubeArmor/config/config.go b/KubeArmor/config/config.go index caef760cf4..9284300e09 100644 --- a/KubeArmor/config/config.go +++ b/KubeArmor/config/config.go @@ -6,6 +6,7 @@ package config import ( "fmt" + "github.com/fsnotify/fsnotify" "os" "strings" @@ -238,6 +239,7 @@ func LoadConfig() error { if cfgfile == "" { cfgfile = "kubearmor.yaml" } + if _, err := os.Stat(cfgfile); err == nil { kg.Printf("setting config from file [%s]", cfgfile) viper.SetConfigFile(cfgfile) @@ -247,6 +249,20 @@ func LoadConfig() error { } } + viper.OnConfigChange(func(in fsnotify.Event) { + kg.Printf("Hot reloading config file [%s]", cfgfile) + if err := loadGlobalConfig(); err != nil { + kg.Err(err.Error()) + } + }) + viper.WatchConfig() + + return loadGlobalConfig() +} + +func loadGlobalConfig() error { + kg.Printf("Configuration [%+v]", GlobalCfg) + GlobalCfg.Cluster = viper.GetString(ConfigCluster) GlobalCfg.Host = viper.GetString(ConfigHost) if hostname, err := os.Hostname(); GlobalCfg.Host == "" && err == nil { @@ -286,8 +302,6 @@ func LoadConfig() error { GlobalCfg.HostDefaultNetworkPosture = viper.GetString(ConfigHostDefaultNetworkPosture) GlobalCfg.HostDefaultCapabilitiesPosture = viper.GetString(ConfigHostDefaultCapabilitiesPosture) - kg.Printf("Configuration [%+v]", GlobalCfg) - if GlobalCfg.KVMAgent { GlobalCfg.Policy = false GlobalCfg.HostPolicy = true diff --git a/KubeArmor/go.mod b/KubeArmor/go.mod index 583eaa8103..76362e622b 100644 --- a/KubeArmor/go.mod +++ b/KubeArmor/go.mod @@ -30,6 +30,7 @@ require ( github.com/containerd/containerd v1.7.13 github.com/containerd/typeurl/v2 v2.1.1 github.com/docker/docker v25.0.5+incompatible + github.com/fsnotify/fsnotify v1.7.0 github.com/golang/protobuf v1.5.4 github.com/google/uuid v1.6.0 github.com/kubearmor/KubeArmor/pkg/KubeArmorController v0.0.0-20240110164432-c2c1b121cd94 @@ -64,7 +65,6 @@ require ( github.com/emicklei/go-restful/v3 v3.11.2 // indirect github.com/evanphx/json-patch/v5 v5.7.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.20.2 // indirect From 947b3eb804c49abf72b708a8578db8b6995402f0 Mon Sep 17 00:00:00 2001 From: tesla59 Date: Sat, 21 Sep 2024 16:39:21 +0000 Subject: [PATCH 2/9] core: move configWatcher to WatchConfigChanges() Signed-off-by: tesla59 --- KubeArmor/config/config.go | 32 +++++++++-------------- KubeArmor/core/kubeArmor.go | 2 ++ KubeArmor/core/unorchestratedUpdates.go | 34 +++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 20 deletions(-) diff --git a/KubeArmor/config/config.go b/KubeArmor/config/config.go index 9284300e09..5495bb0407 100644 --- a/KubeArmor/config/config.go +++ b/KubeArmor/config/config.go @@ -6,7 +6,6 @@ package config import ( "fmt" - "github.com/fsnotify/fsnotify" "os" "strings" @@ -249,18 +248,6 @@ func LoadConfig() error { } } - viper.OnConfigChange(func(in fsnotify.Event) { - kg.Printf("Hot reloading config file [%s]", cfgfile) - if err := loadGlobalConfig(); err != nil { - kg.Err(err.Error()) - } - }) - viper.WatchConfig() - - return loadGlobalConfig() -} - -func loadGlobalConfig() error { kg.Printf("Configuration [%+v]", GlobalCfg) GlobalCfg.Cluster = viper.GetString(ConfigCluster) @@ -294,13 +281,7 @@ func loadGlobalConfig() error { GlobalCfg.Debug = viper.GetBool(ConfigDebug) - GlobalCfg.DefaultFilePosture = viper.GetString(ConfigDefaultFilePosture) - GlobalCfg.DefaultNetworkPosture = viper.GetString(ConfigDefaultNetworkPosture) - GlobalCfg.DefaultCapabilitiesPosture = viper.GetString(ConfigDefaultCapabilitiesPosture) - - GlobalCfg.HostDefaultFilePosture = viper.GetString(ConfigHostDefaultFilePosture) - GlobalCfg.HostDefaultNetworkPosture = viper.GetString(ConfigHostDefaultNetworkPosture) - GlobalCfg.HostDefaultCapabilitiesPosture = viper.GetString(ConfigHostDefaultCapabilitiesPosture) + SetDefaultPosture() if GlobalCfg.KVMAgent { GlobalCfg.Policy = false @@ -340,3 +321,14 @@ func loadGlobalConfig() error { return nil } + +// SetDefaultPosture Set default file, network and capabilities posture +func SetDefaultPosture() { + GlobalCfg.DefaultFilePosture = viper.GetString(ConfigDefaultFilePosture) + GlobalCfg.DefaultNetworkPosture = viper.GetString(ConfigDefaultNetworkPosture) + GlobalCfg.DefaultCapabilitiesPosture = viper.GetString(ConfigDefaultCapabilitiesPosture) + + GlobalCfg.HostDefaultFilePosture = viper.GetString(ConfigHostDefaultFilePosture) + GlobalCfg.HostDefaultNetworkPosture = viper.GetString(ConfigHostDefaultNetworkPosture) + GlobalCfg.HostDefaultCapabilitiesPosture = viper.GetString(ConfigHostDefaultCapabilitiesPosture) +} diff --git a/KubeArmor/core/kubeArmor.go b/KubeArmor/core/kubeArmor.go index 9951150ba4..7f2106270e 100644 --- a/KubeArmor/core/kubeArmor.go +++ b/KubeArmor/core/kubeArmor.go @@ -413,6 +413,8 @@ func KubeArmor() { dm.Node.KernelVersion = kl.GetCommandOutputWithoutErr("uname", []string{"-r"}) dm.Node.KernelVersion = strings.TrimSuffix(dm.Node.KernelVersion, "\n") + dm.WatchConfigChanges() + dm.NodeLock.Unlock() } else if cfg.GlobalCfg.K8sEnv { diff --git a/KubeArmor/core/unorchestratedUpdates.go b/KubeArmor/core/unorchestratedUpdates.go index 8608c263d3..db5bcabe03 100644 --- a/KubeArmor/core/unorchestratedUpdates.go +++ b/KubeArmor/core/unorchestratedUpdates.go @@ -5,6 +5,8 @@ package core import ( "encoding/json" + "github.com/fsnotify/fsnotify" + "github.com/spf13/viper" "os" "regexp" "sort" @@ -38,6 +40,38 @@ func (dm *KubeArmorDaemon) SetContainerNSVisibility() { dm.UpdateVisibility("ADDED", "container_namespace", visibility) } +// =================== // +// == Config Update == // +// =================== // + +// WatchConfigChanges watches for configuration changes and updates the default posture +func (dm *KubeArmorDaemon) WatchConfigChanges() { + viper.OnConfigChange(func(e fsnotify.Event) { + dm.Logger.Printf("Config file changed: %s", e.Name) + cfg.SetDefaultPosture() + + // Update the default posture + globalPosture := tp.DefaultPosture{ + FileAction: cfg.GlobalCfg.DefaultFilePosture, + NetworkAction: cfg.GlobalCfg.DefaultNetworkPosture, + CapabilitiesAction: cfg.GlobalCfg.DefaultCapabilitiesPosture, + } + + // Log the current global posture + dm.Logger.Printf("Updating Global Posture to %v", globalPosture) + + // Apply the changes to the daemon + dm.UpdateGlobalPosture(globalPosture) + + // Update default posture for endpoints + for _, ep := range dm.EndPoints { + dm.Logger.Printf("Updating Default Posture for endpoint %s", ep.EndPointName) + dm.UpdateDefaultPosture("MODIFIED", ep.NamespaceName, globalPosture, false) + } + }) + viper.WatchConfig() +} + // ====================================== // // == Container Security Policy Update == // // ====================================== // From 3880c40894c4ec631db42e8a5af417f5052c5168 Mon Sep 17 00:00:00 2001 From: tesla59 Date: Sun, 22 Sep 2024 05:09:40 +0000 Subject: [PATCH 3/9] core: update visbility dynamically Signed-off-by: tesla59 --- KubeArmor/config/config.go | 30 ++++++++++++------------- KubeArmor/core/unorchestratedUpdates.go | 13 +++++++++-- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/KubeArmor/config/config.go b/KubeArmor/config/config.go index 5495bb0407..d26e7dd5f8 100644 --- a/KubeArmor/config/config.go +++ b/KubeArmor/config/config.go @@ -271,9 +271,6 @@ func LoadConfig() error { return fmt.Errorf("CRI socket must start with 'unix://' (%s is invalid)", GlobalCfg.CRISocket) } - GlobalCfg.Visibility = viper.GetString(ConfigVisibility) - GlobalCfg.HostVisibility = viper.GetString(ConfigHostVisibility) - GlobalCfg.Policy = viper.GetBool(ConfigKubearmorPolicy) GlobalCfg.HostPolicy = viper.GetBool(ConfigKubearmorHostPolicy) GlobalCfg.KVMAgent = viper.GetBool(ConfigKubearmorVM) @@ -281,21 +278,11 @@ func LoadConfig() error { GlobalCfg.Debug = viper.GetBool(ConfigDebug) - SetDefaultPosture() - if GlobalCfg.KVMAgent { GlobalCfg.Policy = false GlobalCfg.HostPolicy = true } - if GlobalCfg.HostVisibility == "default" { - if GlobalCfg.KVMAgent || (!GlobalCfg.K8sEnv && GlobalCfg.HostPolicy) { - GlobalCfg.HostVisibility = "process,file,network,capabilities" - } else { // k8s - GlobalCfg.HostVisibility = "none" - } - } - GlobalCfg.CoverageTest = viper.GetBool(ConfigCoverageTest) GlobalCfg.ConfigUntrackedNs = strings.Split(viper.GetString(ConfigUntrackedNs), ",") @@ -317,13 +304,15 @@ func LoadConfig() error { GlobalCfg.ThrottleSec = viper.GetInt(ConfigThrottleSec) GlobalCfg.AnnotateResources = viper.GetBool(ConfigAnnotateResources) + LoadDynamicConfig() + kg.Printf("Final Configuration [%+v]", GlobalCfg) return nil } -// SetDefaultPosture Set default file, network and capabilities posture -func SetDefaultPosture() { +// LoadDynamicConfig set dynamic configuration which can be updated at runtime without restarting kubearmor +func LoadDynamicConfig() { GlobalCfg.DefaultFilePosture = viper.GetString(ConfigDefaultFilePosture) GlobalCfg.DefaultNetworkPosture = viper.GetString(ConfigDefaultNetworkPosture) GlobalCfg.DefaultCapabilitiesPosture = viper.GetString(ConfigDefaultCapabilitiesPosture) @@ -331,4 +320,15 @@ func SetDefaultPosture() { GlobalCfg.HostDefaultFilePosture = viper.GetString(ConfigHostDefaultFilePosture) GlobalCfg.HostDefaultNetworkPosture = viper.GetString(ConfigHostDefaultNetworkPosture) GlobalCfg.HostDefaultCapabilitiesPosture = viper.GetString(ConfigHostDefaultCapabilitiesPosture) + + GlobalCfg.Visibility = viper.GetString(ConfigVisibility) + GlobalCfg.HostVisibility = viper.GetString(ConfigHostVisibility) + + if GlobalCfg.HostVisibility == "default" { + if GlobalCfg.KVMAgent || (!GlobalCfg.K8sEnv && GlobalCfg.HostPolicy) { + GlobalCfg.HostVisibility = "process,file,network,capabilities" + } else { // k8s + GlobalCfg.HostVisibility = "none" + } + } } diff --git a/KubeArmor/core/unorchestratedUpdates.go b/KubeArmor/core/unorchestratedUpdates.go index db5bcabe03..3eba6ec045 100644 --- a/KubeArmor/core/unorchestratedUpdates.go +++ b/KubeArmor/core/unorchestratedUpdates.go @@ -48,7 +48,7 @@ func (dm *KubeArmorDaemon) SetContainerNSVisibility() { func (dm *KubeArmorDaemon) WatchConfigChanges() { viper.OnConfigChange(func(e fsnotify.Event) { dm.Logger.Printf("Config file changed: %s", e.Name) - cfg.SetDefaultPosture() + cfg.LoadDynamicConfig() // Update the default posture globalPosture := tp.DefaultPosture{ @@ -56,9 +56,17 @@ func (dm *KubeArmorDaemon) WatchConfigChanges() { NetworkAction: cfg.GlobalCfg.DefaultNetworkPosture, CapabilitiesAction: cfg.GlobalCfg.DefaultCapabilitiesPosture, } + // Update the visibility + visibility := tp.Visibility{ + File: strings.Contains(cfg.ConfigVisibility, "file"), + Process: strings.Contains(cfg.ConfigVisibility, "process"), + Network: strings.Contains(cfg.ConfigVisibility, "network"), + Capabilities: strings.Contains(cfg.ConfigVisibility, "capabilities"), + } - // Log the current global posture + // Log the current global posture and visibility dm.Logger.Printf("Updating Global Posture to %v", globalPosture) + dm.Logger.Printf("Updating Visibility to %v", visibility) // Apply the changes to the daemon dm.UpdateGlobalPosture(globalPosture) @@ -67,6 +75,7 @@ func (dm *KubeArmorDaemon) WatchConfigChanges() { for _, ep := range dm.EndPoints { dm.Logger.Printf("Updating Default Posture for endpoint %s", ep.EndPointName) dm.UpdateDefaultPosture("MODIFIED", ep.NamespaceName, globalPosture, false) + dm.UpdateVisibility("MODIFIED", ep.NamespaceName, visibility) } }) viper.WatchConfig() From 6ba254b8c985e0167c3df7161016d19a4fcc3916 Mon Sep 17 00:00:00 2001 From: tesla59 Date: Sun, 22 Sep 2024 05:23:26 +0000 Subject: [PATCH 4/9] core: validate posture and visiblity before dynamically loading Signed-off-by: tesla59 --- KubeArmor/core/unorchestratedUpdates.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/KubeArmor/core/unorchestratedUpdates.go b/KubeArmor/core/unorchestratedUpdates.go index 3eba6ec045..4e62a28040 100644 --- a/KubeArmor/core/unorchestratedUpdates.go +++ b/KubeArmor/core/unorchestratedUpdates.go @@ -52,16 +52,16 @@ func (dm *KubeArmorDaemon) WatchConfigChanges() { // Update the default posture globalPosture := tp.DefaultPosture{ - FileAction: cfg.GlobalCfg.DefaultFilePosture, - NetworkAction: cfg.GlobalCfg.DefaultNetworkPosture, - CapabilitiesAction: cfg.GlobalCfg.DefaultCapabilitiesPosture, + FileAction: validateGlobalDefaultPosture(cfg.GlobalCfg.DefaultFilePosture), + NetworkAction: validateGlobalDefaultPosture(cfg.GlobalCfg.DefaultNetworkPosture), + CapabilitiesAction: validateGlobalDefaultPosture(cfg.GlobalCfg.DefaultCapabilitiesPosture), } // Update the visibility visibility := tp.Visibility{ - File: strings.Contains(cfg.ConfigVisibility, "file"), - Process: strings.Contains(cfg.ConfigVisibility, "process"), - Network: strings.Contains(cfg.ConfigVisibility, "network"), - Capabilities: strings.Contains(cfg.ConfigVisibility, "capabilities"), + File: dm.validateVisibility("file", cfg.GlobalCfg.Visibility), + Process: dm.validateVisibility("process", cfg.GlobalCfg.Visibility), + Network: dm.validateVisibility("network", cfg.GlobalCfg.Visibility), + Capabilities: dm.validateVisibility("capabilities", cfg.GlobalCfg.Visibility), } // Log the current global posture and visibility From db299d2c010f0ebce8f544d07e0e3b3851cc1be6 Mon Sep 17 00:00:00 2001 From: tesla59 Date: Wed, 25 Sep 2024 11:56:27 +0000 Subject: [PATCH 5/9] core: (unorchestratedUpdates) update alert and throttling configs with other dynamic configs Signed-off-by: tesla59 --- KubeArmor/config/config.go | 7 ++++--- KubeArmor/core/unorchestratedUpdates.go | 7 +++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/KubeArmor/config/config.go b/KubeArmor/config/config.go index d26e7dd5f8..461b847bc6 100644 --- a/KubeArmor/config/config.go +++ b/KubeArmor/config/config.go @@ -299,9 +299,6 @@ func LoadConfig() error { GlobalCfg.StateAgent = viper.GetBool(ConfigStateAgent) - GlobalCfg.AlertThrottling = viper.GetBool(ConfigAlertThrottling) - GlobalCfg.MaxAlertPerSec = viper.GetInt(ConfigMaxAlertPerSec) - GlobalCfg.ThrottleSec = viper.GetInt(ConfigThrottleSec) GlobalCfg.AnnotateResources = viper.GetBool(ConfigAnnotateResources) LoadDynamicConfig() @@ -331,4 +328,8 @@ func LoadDynamicConfig() { GlobalCfg.HostVisibility = "none" } } + + GlobalCfg.AlertThrottling = viper.GetBool(ConfigAlertThrottling) + GlobalCfg.MaxAlertPerSec = viper.GetInt(ConfigMaxAlertPerSec) + GlobalCfg.ThrottleSec = viper.GetInt(ConfigThrottleSec) } diff --git a/KubeArmor/core/unorchestratedUpdates.go b/KubeArmor/core/unorchestratedUpdates.go index 4e62a28040..010e2137e1 100644 --- a/KubeArmor/core/unorchestratedUpdates.go +++ b/KubeArmor/core/unorchestratedUpdates.go @@ -64,10 +64,6 @@ func (dm *KubeArmorDaemon) WatchConfigChanges() { Capabilities: dm.validateVisibility("capabilities", cfg.GlobalCfg.Visibility), } - // Log the current global posture and visibility - dm.Logger.Printf("Updating Global Posture to %v", globalPosture) - dm.Logger.Printf("Updating Visibility to %v", visibility) - // Apply the changes to the daemon dm.UpdateGlobalPosture(globalPosture) @@ -77,6 +73,9 @@ func (dm *KubeArmorDaemon) WatchConfigChanges() { dm.UpdateDefaultPosture("MODIFIED", ep.NamespaceName, globalPosture, false) dm.UpdateVisibility("MODIFIED", ep.NamespaceName, visibility) } + + // Update throttling configs + dm.SystemMonitor.UpdateThrottlingConfig() }) viper.WatchConfig() } From 0722c2460a9d1617f6aa9fd1da58b1960ebaf440 Mon Sep 17 00:00:00 2001 From: tesla59 Date: Tue, 22 Oct 2024 08:36:37 +0000 Subject: [PATCH 6/9] config: move EnforcerAlerts and DefaultPostureLogs to dynamic config Signed-off-by: tesla59 --- KubeArmor/config/config.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/KubeArmor/config/config.go b/KubeArmor/config/config.go index 461b847bc6..dc0748a2d9 100644 --- a/KubeArmor/config/config.go +++ b/KubeArmor/config/config.go @@ -291,8 +291,6 @@ func LoadConfig() error { GlobalCfg.BPFFsPath = viper.GetString(BPFFsPath) - GlobalCfg.EnforcerAlerts = viper.GetBool(EnforcerAlerts) - GlobalCfg.DefaultPostureLogs = viper.GetBool(ConfigDefaultPostureLogs) GlobalCfg.InitTimeout = viper.GetString(ConfigInitTimeout) @@ -329,6 +327,9 @@ func LoadDynamicConfig() { } } + GlobalCfg.EnforcerAlerts = viper.GetBool(EnforcerAlerts) + GlobalCfg.DefaultPostureLogs = viper.GetBool(ConfigDefaultPostureLogs) + GlobalCfg.AlertThrottling = viper.GetBool(ConfigAlertThrottling) GlobalCfg.MaxAlertPerSec = viper.GetInt(ConfigMaxAlertPerSec) GlobalCfg.ThrottleSec = viper.GetInt(ConfigThrottleSec) From 1f884a84523a62fdc02101c2e72ee4d8f8c0a1f9 Mon Sep 17 00:00:00 2001 From: tesla59 Date: Wed, 23 Oct 2024 17:55:08 +0000 Subject: [PATCH 7/9] core: update HostSecurity Policy dynamically in unorchaestrated mode Signed-off-by: tesla59 --- KubeArmor/core/unorchestratedUpdates.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/KubeArmor/core/unorchestratedUpdates.go b/KubeArmor/core/unorchestratedUpdates.go index 010e2137e1..6afcf2f389 100644 --- a/KubeArmor/core/unorchestratedUpdates.go +++ b/KubeArmor/core/unorchestratedUpdates.go @@ -76,6 +76,9 @@ func (dm *KubeArmorDaemon) WatchConfigChanges() { // Update throttling configs dm.SystemMonitor.UpdateThrottlingConfig() + + // Update the default posture for the unorchestrated containers + dm.UpdateHostSecurityPolicies() }) viper.WatchConfig() } From b93b7891cd833a4f2ff77d425d5c4d87d5d66714 Mon Sep 17 00:00:00 2001 From: tesla59 Date: Fri, 25 Oct 2024 17:58:39 +0000 Subject: [PATCH 8/9] config: remove redundant assignment of DefaultPostureLogs Signed-off-by: tesla59 --- KubeArmor/config/config.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/KubeArmor/config/config.go b/KubeArmor/config/config.go index dc0748a2d9..a00d49874c 100644 --- a/KubeArmor/config/config.go +++ b/KubeArmor/config/config.go @@ -291,8 +291,6 @@ func LoadConfig() error { GlobalCfg.BPFFsPath = viper.GetString(BPFFsPath) - GlobalCfg.DefaultPostureLogs = viper.GetBool(ConfigDefaultPostureLogs) - GlobalCfg.InitTimeout = viper.GetString(ConfigInitTimeout) GlobalCfg.StateAgent = viper.GetBool(ConfigStateAgent) From 6306430fdef664ec140ecfebfdd8e61fd73d36a0 Mon Sep 17 00:00:00 2001 From: tesla59 Date: Thu, 31 Oct 2024 18:44:20 +0000 Subject: [PATCH 9/9] core(unchestratedupdates): dynamically update host visibility Signed-off-by: tesla59 --- KubeArmor/core/unorchestratedUpdates.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/KubeArmor/core/unorchestratedUpdates.go b/KubeArmor/core/unorchestratedUpdates.go index 6afcf2f389..8ec43dfa4a 100644 --- a/KubeArmor/core/unorchestratedUpdates.go +++ b/KubeArmor/core/unorchestratedUpdates.go @@ -77,7 +77,8 @@ func (dm *KubeArmorDaemon) WatchConfigChanges() { // Update throttling configs dm.SystemMonitor.UpdateThrottlingConfig() - // Update the default posture for the unorchestrated containers + // Update the default posture and visibility for the unorchestrated containers + dm.SystemMonitor.UpdateVisibility() dm.UpdateHostSecurityPolicies() }) viper.WatchConfig()