From 52558eaac41e4430f6424dddd8cde25ff969b7a9 Mon Sep 17 00:00:00 2001 From: Pavel Tisnovsky Date: Thu, 30 Nov 2023 12:08:16 +0100 Subject: [PATCH 1/7] Storage backends --- types/storage_backends.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 types/storage_backends.go diff --git a/types/storage_backends.go b/types/storage_backends.go new file mode 100644 index 00000000..7a2be917 --- /dev/null +++ b/types/storage_backends.go @@ -0,0 +1,23 @@ +// Copyright 2023 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +const ( + // OCPRecommendationsStorage repesents OCP recommendations database schema + OCPRecommendationsStorage = "ocp_recommendations" + + // DVORecommendationsStorage repesents DVO recommendations database schema + DVORecommendationsStorage = "dvo_recommendations" +) From 91f3b22155284cc368cfa5ceb2bbc4d5dc8653bb Mon Sep 17 00:00:00 2001 From: Pavel Tisnovsky Date: Thu, 30 Nov 2023 12:08:37 +0100 Subject: [PATCH 2/7] Common storage interface --- storage/storage.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 storage/storage.go diff --git a/storage/storage.go b/storage/storage.go new file mode 100644 index 00000000..e8d82c27 --- /dev/null +++ b/storage/storage.go @@ -0,0 +1,28 @@ +/* +Copyright © 2023 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +*/ + +package storage + +import "database/sql" + +// PostgreSQL database driver +// SQLite database driver + +// Storage represents an interface to almost any database or storage system +type Storage interface { + Init() error + Close() error + GetConnection() *sql.DB +} From 4587388e42ce61cd7057e171ca56c241d2f4e9a9 Mon Sep 17 00:00:00 2001 From: Pavel Tisnovsky Date: Thu, 30 Nov 2023 12:09:31 +0100 Subject: [PATCH 3/7] Preliminary support for two storages --- aggregator.go | 35 ++++++++++++++++++++++++++--------- consumer.go | 3 ++- server.go | 3 ++- 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/aggregator.go b/aggregator.go index f1a2a6a4..c8163f2f 100644 --- a/aggregator.go +++ b/aggregator.go @@ -111,28 +111,44 @@ func fillInInfoParams(params map[string]string) { // createStorage function initializes connection to preconfigured storage, // usually SQLite, PostgreSQL, or AWS RDS. -func createStorage() (storage.OCPRecommendationsStorage, error) { +func createStorage() (storage.OCPRecommendationsStorage, storage.DVORecommendationsStorage, error) { storageCfg := conf.GetOCPRecommendationsStorageConfiguration() redisCfg := conf.GetRedisConfiguration() // fill-in the missing sub-structure to have the whole Storage // configuration represented as one data structure storageCfg.RedisConfiguration = redisCfg + var ocpStorage storage.OCPRecommendationsStorage = nil + var dvoStorage storage.DVORecommendationsStorage = nil + var err error + log.Info().Str("type", storageCfg.Type).Msg("Storage type") // try to initialize connection to storage - dbStorage, err := storage.NewOCPRecommendationsStorage(storageCfg) - if err != nil { - log.Error().Err(err).Msg("storage.New") - return nil, err + backend := conf.GetStorageBackendConfiguration().Use + switch backend { + case types.OCPRecommendationsStorage: + ocpStorage, err = storage.NewOCPRecommendationsStorage(storageCfg) + if err != nil { + log.Error().Err(err).Msg("storage.NewOCPRecommendationsStorage") + return nil, nil, err + } + case types.DVORecommendationsStorage: + dvoStorage, err = storage.NewDVORecommendationsStorage(storageCfg) + if err != nil { + log.Error().Err(err).Msg("storage.NewDVORecommendationsStorage") + return nil, nil, err + } + default: + return nil, nil, fmt.Errorf("Unknown storage backend %s", backend) } - return dbStorage, nil + return ocpStorage, dvoStorage, nil } // closeStorage function closes specified DBStorage with proper error checking // whether the close operation was successful or not. -func closeStorage(storage storage.OCPRecommendationsStorage) { +func closeStorage(storage storage.Storage) { err := storage.Close() if err != nil { // TODO: error state might be returned from this function @@ -174,7 +190,7 @@ func prepareDBMigrations(dbStorage storage.OCPRecommendationsStorage) int { // prepareDB function opens a connection to database and loads all available // rule content into it. func prepareDB() int { - dbStorage, err := createStorage() + dbStorage, _, err := createStorage() if err != nil { log.Error().Err(err).Msg("Error creating storage") return ExitStatusPrepareDbError @@ -357,7 +373,8 @@ func printEnv() int { // migrations. Non-OK exit code is returned as the last return value in case // of an error. Otherwise, database and connection pointers are returned. func getDBForMigrations() (storage.OCPRecommendationsStorage, *sql.DB, int) { - db, err := createStorage() + // use OCP recommendations storage only, unless migrations will be available for other storage(s) too + db, _, err := createStorage() if err != nil { log.Error().Err(err).Msg("Unable to prepare DB for migrations") return nil, nil, ExitStatusPrepareDbError diff --git a/consumer.go b/consumer.go index 146d2397..211d6a8d 100644 --- a/consumer.go +++ b/consumer.go @@ -33,7 +33,8 @@ var ( func startConsumer(brokerConf broker.Configuration) error { defer finishConsumerInstanceInitialization() - dbStorage, err := createStorage() + // right now just the OCP recommendation storage is handled by consumer + dbStorage, _, err := createStorage() if err != nil { return err } diff --git a/server.go b/server.go index 9814fec5..e615cf59 100644 --- a/server.go +++ b/server.go @@ -35,7 +35,8 @@ var ( func startServer() error { defer finishServerInstanceInitialization() - dbStorage, err := createStorage() + // right now, just the OCP recommendations storage are handled properly + dbStorage, _, err := createStorage() if err != nil { return err } From 456f6da8776940dc79475cbc19a1453ca1b5fc80 Mon Sep 17 00:00:00 2001 From: Pavel Tisnovsky Date: Thu, 30 Nov 2023 12:12:01 +0100 Subject: [PATCH 4/7] Fixed tests --- aggregator_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aggregator_test.go b/aggregator_test.go index b5e9a98e..831a98e9 100644 --- a/aggregator_test.go +++ b/aggregator_test.go @@ -65,7 +65,7 @@ func TestCreateStorage(t *testing.T) { os.Clearenv() mustLoadConfiguration("tests/config1") - _, err := main.CreateStorage() + _, _, err := main.CreateStorage() helpers.FailOnError(t, err) } @@ -116,7 +116,7 @@ func TestCreateStorage_BadDriver(t *testing.T) { "INSIGHTS_RESULTS_AGGREGATOR__OCP_RECOMMENDATIONS_STORAGE__SQLITE_DATASOURCE": "/non/existing/path", }) - _, err := main.CreateStorage() + _, _, err := main.CreateStorage() assert.EqualError(t, err, "driver non-existing-driver is not supported") } From 219920ebdd0306c8a88b248fd012a97a74167467 Mon Sep 17 00:00:00 2001 From: Pavel Tisnovsky Date: Thu, 30 Nov 2023 12:14:09 +0100 Subject: [PATCH 5/7] Fixed warnings --- aggregator.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aggregator.go b/aggregator.go index c8163f2f..fe31d3a5 100644 --- a/aggregator.go +++ b/aggregator.go @@ -118,8 +118,8 @@ func createStorage() (storage.OCPRecommendationsStorage, storage.DVORecommendati // configuration represented as one data structure storageCfg.RedisConfiguration = redisCfg - var ocpStorage storage.OCPRecommendationsStorage = nil - var dvoStorage storage.DVORecommendationsStorage = nil + var ocpStorage storage.OCPRecommendationsStorage + var dvoStorage storage.DVORecommendationsStorage var err error log.Info().Str("type", storageCfg.Type).Msg("Storage type") From 9615dc59708c06d0bdf113bc9b7b36a111930b69 Mon Sep 17 00:00:00 2001 From: Pavel Tisnovsky Date: Thu, 30 Nov 2023 12:27:36 +0100 Subject: [PATCH 6/7] Code prepared to support DVO storage when needed --- aggregator.go | 12 +++++++----- consumer.go | 8 +++++--- server.go | 8 ++++---- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/aggregator.go b/aggregator.go index fe31d3a5..7709b388 100644 --- a/aggregator.go +++ b/aggregator.go @@ -190,27 +190,29 @@ func prepareDBMigrations(dbStorage storage.OCPRecommendationsStorage) int { // prepareDB function opens a connection to database and loads all available // rule content into it. func prepareDB() int { - dbStorage, _, err := createStorage() + // TODO: when migrations for DVO will be available, update the code below + + ocpRecommendationsStorage, _, err := createStorage() if err != nil { log.Error().Err(err).Msg("Error creating storage") return ExitStatusPrepareDbError } - defer closeStorage(dbStorage) + defer closeStorage(ocpRecommendationsStorage) // Ensure that the DB is at the latest migration version. - if exitCode := prepareDBMigrations(dbStorage); exitCode != ExitStatusOK { + if exitCode := prepareDBMigrations(ocpRecommendationsStorage); exitCode != ExitStatusOK { return exitCode } // Initialize the database. - err = dbStorage.Init() + err = ocpRecommendationsStorage.Init() if err != nil { log.Error().Err(err).Msg("DB initialization error") return ExitStatusPrepareDbError } // temporarily print some information from DB because of limited access to DB - dbStorage.PrintRuleDisableDebugInfo() + ocpRecommendationsStorage.PrintRuleDisableDebugInfo() return ExitStatusOK } diff --git a/consumer.go b/consumer.go index 211d6a8d..7f4de5fa 100644 --- a/consumer.go +++ b/consumer.go @@ -34,14 +34,16 @@ func startConsumer(brokerConf broker.Configuration) error { defer finishConsumerInstanceInitialization() // right now just the OCP recommendation storage is handled by consumer - dbStorage, _, err := createStorage() + ocpRecommendationsStorage, _, err := createStorage() if err != nil { return err } - defer closeStorage(dbStorage) + defer closeStorage(ocpRecommendationsStorage) - consumerInstance, err = consumer.NewOCPRulesConsumer(brokerConf, dbStorage) + // when DVO consumer will be made, it will need to use DVO storage + // (see line that calls createStorage()) + consumerInstance, err = consumer.NewOCPRulesConsumer(brokerConf, ocpRecommendationsStorage) if err != nil { log.Error().Err(err).Msg("Broker initialization error") return err diff --git a/server.go b/server.go index e615cf59..e3b5936e 100644 --- a/server.go +++ b/server.go @@ -36,15 +36,15 @@ func startServer() error { defer finishServerInstanceInitialization() // right now, just the OCP recommendations storage are handled properly - dbStorage, _, err := createStorage() + ocpRecommendationsStorage, _, err := createStorage() if err != nil { return err } - defer closeStorage(dbStorage) + defer closeStorage(ocpRecommendationsStorage) serverCfg := conf.GetServerConfiguration() - serverInstance = server.New(serverCfg, dbStorage) + serverInstance = server.New(serverCfg, ocpRecommendationsStorage) // fill-in additional info used by /info endpoint handler fillInInfoParams(serverInstance.InfoParams) @@ -55,7 +55,7 @@ func startServer() error { if conf.GetOCPRecommendationsStorageConfiguration().Type == types.SQLStorage { // migration and DB versioning is now supported for SQL // databases only - currentVersion, err := migration.GetDBVersion(dbStorage.GetConnection()) + currentVersion, err := migration.GetDBVersion(ocpRecommendationsStorage.GetConnection()) if err != nil { const msg = "Unable to retrieve DB migration version" log.Error().Err(err).Msg(msg) From 181596a7166cb3beab17b844fe40bd5e0b913abd Mon Sep 17 00:00:00 2001 From: Pavel Tisnovsky Date: Thu, 30 Nov 2023 12:45:19 +0100 Subject: [PATCH 7/7] Fixed typo --- types/storage_backends.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/types/storage_backends.go b/types/storage_backends.go index 7a2be917..e327f0e9 100644 --- a/types/storage_backends.go +++ b/types/storage_backends.go @@ -15,9 +15,9 @@ package types const ( - // OCPRecommendationsStorage repesents OCP recommendations database schema + // OCPRecommendationsStorage represents OCP recommendations database schema OCPRecommendationsStorage = "ocp_recommendations" - // DVORecommendationsStorage repesents DVO recommendations database schema + // DVORecommendationsStorage represents DVO recommendations database schema DVORecommendationsStorage = "dvo_recommendations" )