Merge pull request #1151 from shekhar316/db_tables

Adding Datasource and Metadata Tables

manifests/crc/default-db-included-installation/minikube/kruize-crc-minikube.yaml

@@ -111,6 +111,7 @@ data:
       "savetodb": "true",
       "dbdriver": "jdbc:postgresql://",
       "plots": "false",
+      "local": "false",
       "hibernate": {
         "dialect": "org.hibernate.dialect.PostgreSQLDialect",
         "driver": "org.postgresql.Driver",

migrations/kruize_local_ddl.sql

@@ -0,0 +1,3 @@
+create table IF NOT EXISTS kruize_datasources (version varchar(255), name varchar(255), provider varchar(255), serviceName varchar(255), namespace varchar(255), url varchar(255), primary key (name));
+create table IF NOT EXISTS kruize_dsmetadata (id serial, version varchar(255), cluster_group_name varchar(255), cluster_name varchar(255), namespace varchar(255), workload_type varchar(255), workload_name varchar(255), container_name varchar(255), container_image_name varchar(255), primary key (id));
+alter table kruize_experiments add column metadata_id bigint references kruize_metadata(id), alter column datasource type varchar(255);

src/main/java/com/autotune/analyzer/Analyzer.java

@@ -18,6 +18,7 @@
 import com.autotune.analyzer.experiment.Experimentator;
 import com.autotune.analyzer.performanceProfiles.PerformanceProfilesDeployment;
 import com.autotune.analyzer.services.*;
+import com.autotune.operator.KruizeDeploymentInfo;
 import com.autotune.operator.KruizeOperator;
 import com.autotune.utils.ServerContext;
 import org.eclipse.jetty.servlet.ServletContextHandler;

src/main/java/com/autotune/analyzer/serviceObjects/CreateExperimentAPIObject.java

@@ -45,6 +45,8 @@ public class CreateExperimentAPIObject extends BaseSO {
     private TrialSettings trialSettings;
     @SerializedName(KruizeConstants.JSONKeys.RECOMMENDATION_SETTINGS)
     private RecommendationSettings recommendationSettings;
+    @SerializedName(KruizeConstants.JSONKeys.DATASOURCE) //TODO: to be used in future
+    private String datasource;
     private AnalyzerConstants.ExperimentStatus status;
     private String experiment_id;   // this id is UUID and getting set at createExperiment API
     private ValidationOutputData validationData;  // This object indicates if this API object is valid or invalid
@@ -137,6 +139,14 @@ public void setValidationData(ValidationOutputData validationData) {
         this.validationData = validationData;
     }
 
+    public String getDatasource() {
+        return datasource;
+    }
+
+    public void setDatasource(String datasource) {
+        this.datasource = datasource;
+    }
+
     @Override
     public String toString() {
         return "CreateExperimentAPIObject{" +

src/main/java/com/autotune/database/dao/ExperimentDAO.java

@@ -4,10 +4,7 @@
 import com.autotune.analyzer.serviceObjects.KubernetesAPIObject;
 import com.autotune.analyzer.utils.AnalyzerConstants;
 import com.autotune.common.data.ValidationOutputData;
-import com.autotune.database.table.KruizeExperimentEntry;
-import com.autotune.database.table.KruizePerformanceProfileEntry;
-import com.autotune.database.table.KruizeRecommendationEntry;
-import com.autotune.database.table.KruizeResultsEntry;
+import com.autotune.database.table.*;
 
 import java.sql.Timestamp;
 import java.util.List;
@@ -28,6 +25,9 @@ public interface ExperimentDAO {
     // Add Performance Profile  to DB
     public ValidationOutputData addPerformanceProfileToDB(KruizePerformanceProfileEntry kruizePerformanceProfileEntry);
 
+    // Add DataSource to DB
+    ValidationOutputData addDataSourceToDB(KruizeDataSourceEntry kruizeDataSourceEntry);
+
     // Update experiment status
     public boolean updateExperimentStatus(KruizeObject kruizeObject, AnalyzerConstants.ExperimentStatus status);
 
@@ -49,6 +49,9 @@ public interface ExperimentDAO {
     // Load a single experiment based on experimentName
     List<KruizeExperimentEntry> loadExperimentByName(String experimentName) throws Exception;
 
+    // Load a single data source based on name
+    List<KruizeDataSourceEntry> loadDataSourceByName(String name) throws Exception;
+
     // Load all results for a particular experimentName
 
     List<KruizeResultsEntry> loadResultsByExperimentName(String experimentName, String cluster_name, Timestamp interval_start_time, Timestamp interval_end_time) throws Exception;
@@ -72,4 +75,17 @@ public interface ExperimentDAO {
     public void addPartitions(String tableName, String month, String year, int dayOfTheMonth, String partitionType) throws Exception;
 
     List<KruizeExperimentEntry> loadExperimentFromDBByInputJSON(StringBuilder clusterName, KubernetesAPIObject kubernetesAPIObject) throws Exception;
+
+    // Load all the datasources
+    List<KruizeDataSourceEntry> loadAllDataSources() throws Exception;
+
+    // Load data source cluster group by name
+    List<KruizeDSMetadataEntry> loadDataSourceClusterGroupByName(String clusterGroupName) throws Exception;
+
+    // add metadata
+    ValidationOutputData addMetadataToDB(KruizeDSMetadataEntry kruizeDSMetadataEntry);
+
+    // Load metadata
+    List<KruizeDSMetadataEntry> loadMetadata() throws Exception;
+
 }

src/main/java/com/autotune/database/dao/ExperimentDAOImpl.java

@@ -8,10 +8,7 @@
 import com.autotune.common.data.ValidationOutputData;
 import com.autotune.database.helper.DBConstants;
 import com.autotune.database.init.KruizeHibernateUtil;
-import com.autotune.database.table.KruizeExperimentEntry;
-import com.autotune.database.table.KruizePerformanceProfileEntry;
-import com.autotune.database.table.KruizeRecommendationEntry;
-import com.autotune.database.table.KruizeResultsEntry;
+import com.autotune.database.table.*;
 import com.autotune.utils.KruizeConstants;
 import com.autotune.utils.MetricsConfig;
 import io.micrometer.core.instrument.Timer;
@@ -387,6 +384,65 @@ public ValidationOutputData addPerformanceProfileToDB(KruizePerformanceProfileEn
         return validationOutputData;
     }
 
+    /**
+     * @param kruizeDataSourceEntry
+     * @return validationOutputData contains the status of the DB insert operation
+     */
+    @Override
+    public ValidationOutputData addDataSourceToDB(KruizeDataSourceEntry kruizeDataSourceEntry) {
+        ValidationOutputData validationOutputData = new ValidationOutputData(false, null, null);
+        Transaction tx = null;
+        try (Session session = KruizeHibernateUtil.getSessionFactory().openSession()) {
+            try {
+                tx = session.beginTransaction();
+                session.persist(kruizeDataSourceEntry);
+                tx.commit();
+                validationOutputData.setSuccess(true);
+            } catch (HibernateException e) {
+                LOGGER.error("Not able to save data source due to {}", e.getMessage());
+                if (tx != null) tx.rollback();
+                e.printStackTrace();
+                validationOutputData.setSuccess(false);
+                validationOutputData.setMessage(e.getMessage());
+                //todo save error to API_ERROR_LOG
+            }
+        } catch (Exception e) {
+            LOGGER.error("Not able to save data source due to {}", e.getMessage());
+            validationOutputData.setMessage(e.getMessage());
+        }
+        return validationOutputData;
+    }
+
+    /**
+     * @param kruizeDSMetadataEntry
+     * @return
+     */
+    @Override
+    public ValidationOutputData addMetadataToDB(KruizeDSMetadataEntry kruizeDSMetadataEntry) {
+        ValidationOutputData validationOutputData = new ValidationOutputData(false, null, null);
+        Transaction tx = null;
+        try (Session session = KruizeHibernateUtil.getSessionFactory().openSession()) {
+            try {
+                tx = session.beginTransaction();
+                session.persist(kruizeDSMetadataEntry);
+                tx.commit();
+                validationOutputData.setSuccess(true);
+            } catch (HibernateException e) {
+                LOGGER.error("Not able to save metadata due to {}", e.getMessage());
+                if (tx != null) tx.rollback();
+                e.printStackTrace();
+                validationOutputData.setSuccess(false);
+                validationOutputData.setMessage(e.getMessage());
+                //todo save error to API_ERROR_LOG
+            }
+        } catch (Exception e) {
+            LOGGER.error("Not able to save metadata source due to {}", e.getMessage());
+            validationOutputData.setMessage(e.getMessage());
+        }
+        return validationOutputData;
+    }
+
+
 
     @Override
     public boolean updateExperimentStatus(KruizeObject kruizeObject, AnalyzerConstants.ExperimentStatus status) {
@@ -584,11 +640,12 @@ public List<KruizeExperimentEntry> loadExperimentFromDBByInputJSON(StringBuilder
                 MetricsConfig.timerLoadExpName = MetricsConfig.timerBLoadExpName.tag("status", statusValue).register(MetricsConfig.meterRegistry());
                 timerLoadExpName.stop(MetricsConfig.timerLoadExpName);
             }
-
         }
         return entries;
     }
 
+
+
     @Override
     public List<KruizeResultsEntry> loadResultsByExperimentName(String experimentName, String cluster_name, Timestamp calculated_start_time, Timestamp interval_end_time) throws Exception {
         // TODO: load only experimentStatus=inProgress , playback may not require completed experiments
@@ -751,4 +808,66 @@ public YearMonth buildDateForNextMonth(YearMonth yearMonth) {
         yearMonth = YearMonth.of(year, month);
         return yearMonth;
     }
+
+    /**
+     * @param clusterGroupName
+     * @return
+     */
+    @Override
+    public List<KruizeDSMetadataEntry> loadDataSourceClusterGroupByName(String clusterGroupName) throws Exception {
+        List<KruizeDSMetadataEntry> kruizeMetadataList;
+        try (Session session = KruizeHibernateUtil.getSessionFactory().openSession()) {
+            kruizeMetadataList = session.createQuery(SELECT_FROM_METADATA_BY_CLUSTER_GROUP_NAME, KruizeDSMetadataEntry.class)
+                    .setParameter("cluster_group_name", clusterGroupName).list();
+        } catch (Exception e) {
+            LOGGER.error("Unable to load metadata with clusterGroupName: {} : {}", clusterGroupName, e.getMessage());
+            throw new Exception("Error while loading existing metadata object from database : " + e.getMessage());
+        }
+        return kruizeMetadataList;
+    }
+
+    @Override
+    public List<KruizeDSMetadataEntry> loadMetadata() throws Exception {
+        List<KruizeDSMetadataEntry> kruizeMetadataList;
+        try (Session session = KruizeHibernateUtil.getSessionFactory().openSession()) {
+            kruizeMetadataList = session.createQuery(SELECT_FROM_METADATA, KruizeDSMetadataEntry.class).list();
+        } catch (Exception e) {
+            LOGGER.error("Unable to load metadata : {}", e.getMessage());
+            throw new Exception("Error while loading existing metadata object from database : " + e.getMessage());
+        }
+        return kruizeMetadataList;
+    }
+
+    /**
+     * @param name
+     * @return single element list of datasource after fetching from the DB
+     * @throws Exception
+     */
+    @Override
+    public List<KruizeDataSourceEntry> loadDataSourceByName(String name) throws Exception {
+        List<KruizeDataSourceEntry> kruizeDataSourceList;
+        try (Session session = KruizeHibernateUtil.getSessionFactory().openSession()) {
+            kruizeDataSourceList = session.createQuery(DBConstants.SQLQUERY.SELECT_FROM_DATASOURCE_BY_NAME, KruizeDataSourceEntry.class)
+                    .setParameter("name", name).list();
+        } catch (Exception e) {
+            LOGGER.error("Not able to load datasource: {} : {}", name, e.getMessage());
+            throw new Exception("Error while loading existing datasource from database : " + e.getMessage());
+        }
+        return kruizeDataSourceList;
+    }
+
+    /**
+     * @return list of datasources after fetching from the DB
+     */
+    @Override
+    public List<KruizeDataSourceEntry> loadAllDataSources() throws Exception {
+        List<KruizeDataSourceEntry> entries;
+        try (Session session = KruizeHibernateUtil.getSessionFactory().openSession()) {
+            entries = session.createQuery(SELECT_FROM_DATASOURCE, KruizeDataSourceEntry.class).list();
+        } catch (Exception e) {
+            LOGGER.error("Not able to load datasource: {}", e.getMessage());
+            throw new Exception("Error while loading existing datasources from database: " + e.getMessage());
+        }
+        return entries;
+    }
 }

src/main/java/com/autotune/database/helper/DBConstants.java

@@ -9,6 +9,10 @@ public static final class SQLQUERY {
         public static final String SELECT_FROM_EXPERIMENTS_BY_EXP_NAME = "from KruizeExperimentEntry k WHERE k.experiment_name = :experimentName";
         public static final String SELECT_FROM_RESULTS = "from KruizeResultsEntry";
         public static final String SELECT_FROM_RESULTS_BY_EXP_NAME = "from KruizeResultsEntry k WHERE k.experiment_name = :experimentName";
+        public static final String SELECT_FROM_DATASOURCE = "from KruizeDataSourceEntry";
+        public static final String SELECT_FROM_DATASOURCE_BY_NAME = "from KruizeDataSourceEntry kd WHERE kd.name = :name";
+        public static final String SELECT_FROM_METADATA = "from KruizeDSMetadataEntry";
+        public static final String SELECT_FROM_METADATA_BY_CLUSTER_GROUP_NAME = "from KruizeDSMetadataEntry km WHERE km.cluster_group_name = :clusterGroupName";
         public static final String SELECT_FROM_RESULTS_BY_EXP_NAME_AND_DATE_RANGE_AND_LIMIT =
                 String.format("from KruizeResultsEntry k " +
                                 "WHERE k.experiment_name = :%s and " +

src/main/java/com/autotune/database/init/KruizeHibernateUtil.java

@@ -16,10 +16,7 @@
 package com.autotune.database.init;
 
 
-import com.autotune.database.table.KruizeExperimentEntry;
-import com.autotune.database.table.KruizePerformanceProfileEntry;
-import com.autotune.database.table.KruizeRecommendationEntry;
-import com.autotune.database.table.KruizeResultsEntry;
+import com.autotune.database.table.*;
 import com.autotune.operator.KruizeDeploymentInfo;
 import org.hibernate.Session;
 import org.hibernate.SessionFactory;
@@ -59,6 +56,10 @@ public static void buildSessionFactory() {
             configuration.addAnnotatedClass(KruizeResultsEntry.class);
             configuration.addAnnotatedClass(KruizeRecommendationEntry.class);
             configuration.addAnnotatedClass(KruizePerformanceProfileEntry.class);
+            if (KruizeDeploymentInfo.local) {
+                configuration.addAnnotatedClass(KruizeDataSourceEntry.class);
+                configuration.addAnnotatedClass(KruizeDSMetadataEntry.class);
+            }
             LOGGER.info("DB is trying to connect to {}", connectionURL);
             sfTemp = configuration.buildSessionFactory();
             LOGGER.info("DB build session is successful !");