From b8e29c142645f41f847853329ac9b70c3dc3fd4f Mon Sep 17 00:00:00 2001 From: "rajshekar.chavakula" Date: Thu, 18 Jan 2024 10:44:57 +0530 Subject: [PATCH 1/5] new silenced fields addition to config and start command --- backend/cmd/start.go | 15 +++++++++++++++ backend/config.go | 4 ++++ 2 files changed, 19 insertions(+) diff --git a/backend/cmd/start.go b/backend/cmd/start.go index 1d0d28d04..605628f53 100644 --- a/backend/cmd/start.go +++ b/backend/cmd/start.go @@ -72,6 +72,10 @@ const ( flagLabels = "labels" flagAnnotations = "annotations" + // silenced expiry flags + flagMaxSilencedExpiryTimeAllowed = "max-silenced-expiry-time-allowed" + flagDefaultSilencedExpiryTime = "default-silenced-expiry-time" + // Etcd flag constants flagEtcdClientURLs = "etcd-client-urls" flagEtcdListenClientURLs = "etcd-listen-client-urls" @@ -255,6 +259,9 @@ func StartCommand(initialize InitializeFunc) *cobra.Command { CacheDir: viper.GetString(flagCacheDir), StateDir: viper.GetString(flagStateDir), + DefaultSilencedExpiryTime: viper.GetDuration(flagDefaultSilencedExpiryTime), + MaxSilencedExpiryTimeAllowed: viper.GetDuration(flagMaxSilencedExpiryTimeAllowed), + EtcdAdvertiseClientURLs: viper.GetStringSlice(flagEtcdAdvertiseClientURLs), EtcdListenClientURLs: viper.GetStringSlice(flagEtcdListenClientURLs), EtcdClientURLs: fallbackStringSlice(flagEtcdClientURLs, flagEtcdAdvertiseClientURLs), @@ -448,6 +455,10 @@ func handleConfig(cmd *cobra.Command, arguments []string, server bool) error { viper.SetDefault(flagEventLogBufferSize, 100000) viper.SetDefault(flagEventLogFile, "") viper.SetDefault(flagEventLogParallelEncoders, false) + + // default silenced value are set for 1 day = 86400s + viper.SetDefault(flagMaxSilencedExpiryTimeAllowed, "86400s") + viper.SetDefault(flagDefaultSilencedExpiryTime, "86400s") } // Etcd defaults @@ -583,6 +594,10 @@ func flagSet(server bool) *pflag.FlagSet { flagSet.Duration(flagPlatformMetricsLoggingInterval, viper.GetDuration(flagPlatformMetricsLoggingInterval), "platform metrics logging interval") flagSet.String(flagPlatformMetricsLogFile, viper.GetString(flagPlatformMetricsLogFile), "platform metrics log file path") + // silenced configuration flags + flagSet.Duration(flagDefaultSilencedExpiryTime, viper.GetDuration(flagDefaultSilencedExpiryTime), "Default expiry time for silenced if not set in seconds") + flagSet.Duration(flagMaxSilencedExpiryTimeAllowed, viper.GetDuration(flagMaxSilencedExpiryTimeAllowed), "Maximum expiry time allowed for silenced in seconds") + // Etcd server flags flagSet.StringSlice(flagEtcdPeerURLs, viper.GetStringSlice(flagEtcdPeerURLs), "list of URLs to listen on for peer traffic") _ = flagSet.SetAnnotation(flagEtcdPeerURLs, "categories", []string{"store"}) diff --git a/backend/config.go b/backend/config.go index 1c353f7f9..87f84dfb1 100644 --- a/backend/config.go +++ b/backend/config.go @@ -132,4 +132,8 @@ type Config struct { EventLogBufferWait time.Duration EventLogFile string EventLogParallelEncoders bool + + // expiry setting for silences + DefaultSilencedExpiryTime time.Duration + MaxSilencedExpiryTimeAllowed time.Duration } From 417f5daea84a1a7a2861929aa8047aa256ecc38b Mon Sep 17 00:00:00 2001 From: chavakula Date: Tue, 23 Jan 2024 20:16:58 +0530 Subject: [PATCH 2/5] changes for etcd store and fetching configuration --- backend/backend.go | 4 ++++ backend/cmd/start.go | 10 +++++----- backend/etcd/etcd.go | 10 ++++++++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/backend/backend.go b/backend/backend.go index ab2ae5e95..edd56b029 100644 --- a/backend/backend.go +++ b/backend/backend.go @@ -259,6 +259,10 @@ func newClient(ctx context.Context, config *Config, backend *Backend) (*clientv3 cfg.LogTimestampLayout = config.EtcdLogTimestampLayout cfg.ClientLogLevel = config.EtcdClientLogLevel + // set misc configurations here + cfg.DefaultSilencedExpiryTime = config.DefaultSilencedExpiryTime + cfg.MaxSilencedExpiryTimeAllowed = config.MaxSilencedExpiryTimeAllowed + // Heartbeat interval if config.EtcdHeartbeatInterval > 0 { cfg.TickMs = config.EtcdHeartbeatInterval diff --git a/backend/cmd/start.go b/backend/cmd/start.go index 605628f53..12757449a 100644 --- a/backend/cmd/start.go +++ b/backend/cmd/start.go @@ -456,9 +456,9 @@ func handleConfig(cmd *cobra.Command, arguments []string, server bool) error { viper.SetDefault(flagEventLogFile, "") viper.SetDefault(flagEventLogParallelEncoders, false) - // default silenced value are set for 1 day = 86400s - viper.SetDefault(flagMaxSilencedExpiryTimeAllowed, "86400s") - viper.SetDefault(flagDefaultSilencedExpiryTime, "86400s") + // default silenced value are set for 1 day = 1440m + viper.SetDefault(flagMaxSilencedExpiryTimeAllowed, "1440m") + viper.SetDefault(flagDefaultSilencedExpiryTime, "1440m") } // Etcd defaults @@ -595,8 +595,8 @@ func flagSet(server bool) *pflag.FlagSet { flagSet.String(flagPlatformMetricsLogFile, viper.GetString(flagPlatformMetricsLogFile), "platform metrics log file path") // silenced configuration flags - flagSet.Duration(flagDefaultSilencedExpiryTime, viper.GetDuration(flagDefaultSilencedExpiryTime), "Default expiry time for silenced if not set in seconds") - flagSet.Duration(flagMaxSilencedExpiryTimeAllowed, viper.GetDuration(flagMaxSilencedExpiryTimeAllowed), "Maximum expiry time allowed for silenced in seconds") + flagSet.Duration(flagDefaultSilencedExpiryTime, viper.GetDuration(flagDefaultSilencedExpiryTime), "Default expiry time for silenced if not set in minutes") + flagSet.Duration(flagMaxSilencedExpiryTimeAllowed, viper.GetDuration(flagMaxSilencedExpiryTimeAllowed), "Maximum expiry time allowed for silenced in minutes") // Etcd server flags flagSet.StringSlice(flagEtcdPeerURLs, viper.GetStringSlice(flagEtcdPeerURLs), "list of URLs to listen on for peer traffic") diff --git a/backend/etcd/etcd.go b/backend/etcd/etcd.go index 5dd25bf5c..ecdee81b4 100644 --- a/backend/etcd/etcd.go +++ b/backend/etcd/etcd.go @@ -71,6 +71,13 @@ func init() { logutil.DefaultZapLoggerConfig.EncoderConfig.EncodeLevel = sensuLevelEncoder } +// MiscConfig is struct to hold miscellaneous configurations for etcd +type MiscConfig struct { + DefaultSilencedExpiryTime time.Duration + + MaxSilencedExpiryTimeAllowed time.Duration +} + // Config is a configuration for the embedded etcd type Config struct { DataDir string @@ -107,6 +114,9 @@ type Config struct { LogTimestampLayout string UnsafeNoFsync bool + + // holding store specific configuration + MiscConfig } // TLSInfo wraps etcd transport TLSInfo From e472752dc091ab2d01dc7faf05b1eaa84023c080 Mon Sep 17 00:00:00 2001 From: chavakula Date: Thu, 25 Jan 2024 17:29:04 +0530 Subject: [PATCH 3/5] changes to check silenced configuration with user values --- backend/backend.go | 7 +++++++ backend/store/etcd/silenced_store.go | 18 ++++++++++++++++++ backend/store/etcd/store.go | 11 +++++++++++ 3 files changed, 36 insertions(+) diff --git a/backend/backend.go b/backend/backend.go index edd56b029..7a10b3ebb 100644 --- a/backend/backend.go +++ b/backend/backend.go @@ -332,6 +332,13 @@ func Initialize(ctx context.Context, config *Config) (*Backend, error) { // Create the store, which lives on top of etcd stor := etcdstore.NewStore(b.Client, config.EtcdName) + + // set config details + scfg := etcdstore.Config{} + scfg.DefaultSilencedExpiryTime = config.DefaultSilencedExpiryTime + scfg.MaxSilencedExpiryTimeAllowed = config.MaxSilencedExpiryTimeAllowed + etcdstore.SetConfig(scfg, stor) + b.Store = stor storv2 := etcdstorev2.NewStore(b.Client) var storev2Proxy storev2.Proxy diff --git a/backend/store/etcd/silenced_store.go b/backend/store/etcd/silenced_store.go index 7c9127f91..0a8b76ed3 100644 --- a/backend/store/etcd/silenced_store.go +++ b/backend/store/etcd/silenced_store.go @@ -207,6 +207,15 @@ func (s *Store) UpdateSilencedEntry(ctx context.Context, silenced *corev2.Silenc if err := silenced.Validate(); err != nil { return &store.ErrNotValid{Err: err} } + allowedMaxTime := time.Now().Add(s.cfg.MaxSilencedExpiryTimeAllowed).Unix() + + logger.Info("silenced ExpireAt : ", silenced.ExpireAt, " Setting: ", allowedMaxTime) + + if silenced.ExpireAt > 0 && (silenced.ExpireAt > allowedMaxTime) { + logger.Info("Silenced duration crossed maximum") + err := errors.New("silenced duration crossed maximum") + return &store.ErrNotValid{Err: err} + } if silenced.ExpireAt == 0 && silenced.Expire > 0 { start := time.Now() @@ -216,6 +225,15 @@ func (s *Store) UpdateSilencedEntry(ctx context.Context, silenced *corev2.Silenc silenced.ExpireAt = start.Add(time.Duration(silenced.Expire) * time.Second).Unix() } + // set default silenced expiry time configured in backend yaml file + if silenced.Expire == 0 && silenced.ExpireAt == 0 { + start := time.Now() + if silenced.Begin > 0 { + start = time.Unix(silenced.Begin, 0) + } + silenced.ExpireAt = start.Add(s.cfg.DefaultSilencedExpiryTime).Unix() + } + silencedBytes, err := proto.Marshal(silenced) if err != nil { return &store.ErrEncode{Err: err} diff --git a/backend/store/etcd/store.go b/backend/store/etcd/store.go index f1bd46c22..6e7938469 100644 --- a/backend/store/etcd/store.go +++ b/backend/store/etcd/store.go @@ -7,6 +7,7 @@ import ( "path" "reflect" "strings" + "time" "github.com/gogo/protobuf/proto" "github.com/sensu/sensu-go/backend/store" @@ -22,10 +23,16 @@ const ( EtcdRoot = "/sensu.io" ) +type Config struct { + DefaultSilencedExpiryTime time.Duration + MaxSilencedExpiryTimeAllowed time.Duration +} + // Store is an implementation of the sensu-go/backend/store.Store iface. type Store struct { client *clientv3.Client keepalivesPath string + cfg Config } // NewStore creates a new Store. @@ -38,6 +45,10 @@ func NewStore(client *clientv3.Client, name string) *Store { return store } +func SetConfig(cfg Config, store *Store) { + store.cfg = cfg +} + // Create the given key with the serialized object. func Create(ctx context.Context, client *clientv3.Client, key, namespace string, object interface{}) error { bytes, err := marshal(object) From 0c22dfa309a44d17afa1fe5733820e87a3abeef8 Mon Sep 17 00:00:00 2001 From: Rajshekar Chavakula Date: Tue, 30 Jan 2024 16:13:02 +0530 Subject: [PATCH 4/5] condition change for silence --- backend/store/etcd/silenced_store.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/backend/store/etcd/silenced_store.go b/backend/store/etcd/silenced_store.go index 0a8b76ed3..cd3cb22a1 100644 --- a/backend/store/etcd/silenced_store.go +++ b/backend/store/etcd/silenced_store.go @@ -209,11 +209,9 @@ func (s *Store) UpdateSilencedEntry(ctx context.Context, silenced *corev2.Silenc } allowedMaxTime := time.Now().Add(s.cfg.MaxSilencedExpiryTimeAllowed).Unix() - logger.Info("silenced ExpireAt : ", silenced.ExpireAt, " Setting: ", allowedMaxTime) - + // check for maximum allowed duration for silenced allowed if silenced.ExpireAt > 0 && (silenced.ExpireAt > allowedMaxTime) { - logger.Info("Silenced duration crossed maximum") - err := errors.New("silenced duration crossed maximum") + err := errors.New("silenced crossed maximum duration configured") return &store.ErrNotValid{Err: err} } @@ -226,7 +224,7 @@ func (s *Store) UpdateSilencedEntry(ctx context.Context, silenced *corev2.Silenc } // set default silenced expiry time configured in backend yaml file - if silenced.Expire == 0 && silenced.ExpireAt == 0 { + if silenced.Expire <= 0 && silenced.ExpireAt == 0 { start := time.Now() if silenced.Begin > 0 { start = time.Unix(silenced.Begin, 0) From 126dc56f730e1f3a79199d2bb254442860b95586 Mon Sep 17 00:00:00 2001 From: Rajshekar Chavakula Date: Wed, 31 Jan 2024 16:20:27 +0530 Subject: [PATCH 5/5] test case changes and addition Signed-off-by: Rajshekar Chavakula --- backend/store/etcd/silenced_store.go | 4 +-- backend/store/etcd/silenced_store_test.go | 39 ++++++++++++++++++++--- backend/store/etcd/store_test.go | 13 ++++++-- backend/store/store.go | 8 +++++ 4 files changed, 55 insertions(+), 9 deletions(-) diff --git a/backend/store/etcd/silenced_store.go b/backend/store/etcd/silenced_store.go index cd3cb22a1..8c28c5fdf 100644 --- a/backend/store/etcd/silenced_store.go +++ b/backend/store/etcd/silenced_store.go @@ -211,8 +211,8 @@ func (s *Store) UpdateSilencedEntry(ctx context.Context, silenced *corev2.Silenc // check for maximum allowed duration for silenced allowed if silenced.ExpireAt > 0 && (silenced.ExpireAt > allowedMaxTime) { - err := errors.New("silenced crossed maximum duration configured") - return &store.ErrNotValid{Err: err} + err := errors.New("silenced crossed maximum duration allowed") + return &store.ErrThreshold{Err: err} } if silenced.ExpireAt == 0 && silenced.Expire > 0 { diff --git a/backend/store/etcd/silenced_store_test.go b/backend/store/etcd/silenced_store_test.go index 2fd687b0b..c47b9a27e 100644 --- a/backend/store/etcd/silenced_store_test.go +++ b/backend/store/etcd/silenced_store_test.go @@ -67,8 +67,9 @@ func TestSilencedStorage(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, entry) assert.Equal(t, "subscription:*", entry.Name) - // Entries without expirations should return -1 - assert.Equal(t, int64(-1), entry.Expire) + + // check entries without -1 expiry + assert.NotEqual(t, int64(-1), entry.Expire) // Delete silenced entry by name err = store.DeleteSilencedEntryByName(ctx, silenced.Name) @@ -76,6 +77,7 @@ func TestSilencedStorage(t *testing.T) { // Update a silenced entry's expire time silenced.Expire = 2 + silenced.ExpireAt = 0 err = store.UpdateSilencedEntry(ctx, silenced) assert.NoError(t, err) @@ -100,6 +102,7 @@ func TestSilencedStorageWithExpire(t *testing.T) { silenced := types.FixtureSilenced("subscription:checkname") silenced.Namespace = "default" silenced.Expire = 15 + silenced.ExpireAt = 0 ctx := context.WithValue(context.Background(), types.NamespaceKey, silenced.Namespace) err := store.UpdateSilencedEntry(ctx, silenced) @@ -120,9 +123,9 @@ func TestSilencedStorageWithBegin(t *testing.T) { silenced := types.FixtureSilenced("subscription:checkname") silenced.Namespace = "default" // set a begin time in the future - silenced.Begin = time.Date(1970, 01, 01, 01, 00, 00, 00, time.UTC).Unix() + silenced.Begin = time.Now().Add(time.Duration(1) * time.Second).Unix() // current time is before the start time - currentTime := time.Date(1970, 01, 01, 00, 00, 00, 00, time.UTC).Unix() + currentTime := time.Now().Unix() ctx := context.WithValue(context.Background(), types.NamespaceKey, silenced.Namespace) err := store.UpdateSilencedEntry(ctx, silenced) @@ -137,8 +140,11 @@ func TestSilencedStorageWithBegin(t *testing.T) { require.NotNil(t, entry) assert.False(t, entry.Begin < currentTime) + // Wait for begin time to elapse current time. i.e let silencing begin + time.Sleep(3 * time.Second) + // reset current time to be ahead of begin time - currentTime = time.Date(1970, 01, 01, 02, 00, 00, 00, time.UTC).Unix() + currentTime = time.Now().Unix() assert.True(t, entry.Begin < currentTime) }) } @@ -168,3 +174,26 @@ func TestSilencedStorageWithBeginAndExpire(t *testing.T) { assert.Equal(t, entry.Expire, int64(15)) }) } + +func TestSilencedStorageWithMaxAllowedThresholdExpiry(t *testing.T) { + testWithEtcd(t, func(store store.Store) { + silenced := types.FixtureSilenced("subscription:checkname") + silenced.Namespace = "default" + silenced.ExpireAt = time.Now().Add(time.Duration(30000) * time.Second).Unix() + // set a begin time + silenced.Begin = time.Now().Unix() + ctx := context.WithValue(context.Background(), types.NamespaceKey, silenced.Namespace) + + err := store.UpdateSilencedEntry(ctx, silenced) + + // assert that error is thrown for breaching max expiry time allowed + assert.Error(t, err) + + entry, err := store.GetSilencedEntryByName(ctx, silenced.Name) + + // assert that entry is nil + assert.NoError(t, err) + assert.Nil(t, entry) + + }) +} diff --git a/backend/store/etcd/store_test.go b/backend/store/etcd/store_test.go index 58b729c0f..615a2e6c3 100644 --- a/backend/store/etcd/store_test.go +++ b/backend/store/etcd/store_test.go @@ -7,8 +7,6 @@ import ( "context" "encoding/json" "fmt" - "testing" - "github.com/gogo/protobuf/proto" corev2 "github.com/sensu/core/v2" "github.com/sensu/sensu-go/backend/etcd" @@ -18,6 +16,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/etcd/client/v3" + "testing" + "time" ) func testWithEtcd(t *testing.T, f func(store.Store)) { @@ -28,6 +28,9 @@ func testWithEtcd(t *testing.T, f func(store.Store)) { s := NewStore(client, e.Name()) + s.cfg.MaxSilencedExpiryTimeAllowed = time.Duration(3000 * time.Second) + s.cfg.DefaultSilencedExpiryTime = time.Duration(3000 * time.Second) + // Mock a default namespace require.NoError(t, s.CreateNamespace(context.Background(), types.FixtureNamespace("default"))) @@ -42,6 +45,9 @@ func testWithEtcdStore(t *testing.T, f func(*Store)) { s := NewStore(client, e.Name()) + s.cfg.MaxSilencedExpiryTimeAllowed = time.Duration(3000 * time.Second) + s.cfg.DefaultSilencedExpiryTime = time.Duration(3000 * time.Second) + // Mock a default namespace require.NoError(t, s.CreateNamespace(context.Background(), types.FixtureNamespace("default"))) @@ -56,6 +62,9 @@ func testWithEtcdClient(t *testing.T, f func(store.Store, *clientv3.Client)) { s := NewStore(client, e.Name()) + s.cfg.MaxSilencedExpiryTimeAllowed = time.Duration(3000 * time.Second) + s.cfg.DefaultSilencedExpiryTime = time.Duration(3000 * time.Second) + // Mock a default namespace require.NoError(t, s.CreateNamespace(context.Background(), types.FixtureNamespace("default"))) diff --git a/backend/store/store.go b/backend/store/store.go index 8953242c8..e845c29be 100644 --- a/backend/store/store.go +++ b/backend/store/store.go @@ -65,6 +65,14 @@ type ErrNotValid struct { Err error } +type ErrThreshold struct { + Err error +} + +func (e *ErrThreshold) Error() string { + return fmt.Sprintf("Threshold reached: %s", e.Err.Error()) +} + func (e *ErrNotValid) Error() string { return fmt.Sprintf("resource is invalid: %s", e.Err.Error()) }