1 - BulkJobsave - table sql

Signed-off-by: msvinaykumar <[email protected]>

design/KruizeDatabaseDesign.md

@@ -19,6 +19,8 @@ The Kruize Autotune project has the following entities:
 2. kruize_results
 3. kruize_recommendations
 4. kruize_performance_profiles
+5. kruize_jobs
+6. kruize_jobmetadata
 
 ## **kruize_experiments**
 
@@ -819,3 +821,43 @@ curl --location --request POST 'http://127.0.0.1:8080/createPerformanceProfile'
 ```
 insert into kruize_performance_profiles; 
 ```
+
+## **kruize_jobs**
+
+---
+
+This table stores job-level data, including information such as job status, start and end times, notification details, total and processed counts.
+```sql
+CREATE TABLE kruize_jobs (
+    job_id UUID NOT NULL,
+    end_time TIMESTAMP(6),
+    start_time TIMESTAMP(6),
+    notifications JSONB,
+    processed_count INTEGER,
+    status VARCHAR(255),
+    total_count INTEGER,
+    webhook VARCHAR(255),
+    PRIMARY KEY (job_id)
+);
+```
+
+## **kruize_jobmetadata**
+
+---
+This table stores metadata for individual experiments associated with a job. It uses hash-based partitioning on job_id for scalability and performance.
+```sql
+CREATE TABLE kruize_jobmetadata (
+    id BIGSERIAL NOT NULL,
+    experiment_name VARCHAR(255),
+    notification JSONB,
+    recommendation_notifications JSONB,
+    recommendation_status VARCHAR(255),
+    job_id UUID NOT NULL,
+    PRIMARY KEY (job_id, experiment_name)
+) PARTITION BY HASH (job_id);
+
+ALTER TABLE IF EXISTS kruize_jobmetadata
+    ADD CONSTRAINT bulkJobMetaDataConstraint
+    FOREIGN KEY (job_id) REFERENCES kruize_jobs;
+
+```

migrations/kruize_local_ddl.sql

@@ -6,3 +6,10 @@ alter table kruize_lm_experiments  add column metadata_id bigint references krui
 alter table if exists kruize_lm_experiments add constraint UK_lm_experiment_name unique (experiment_name);
 create table IF NOT EXISTS kruize_metric_profiles (api_version varchar(255), kind varchar(255), metadata jsonb, name varchar(255) not null, k8s_type varchar(255), profile_version float(53) not null, slo jsonb, primary key (name));
 create table IF NOT EXISTS kruize_lm_recommendations (interval_end_time timestamp(6) not null, experiment_name varchar(255) not null, cluster_name varchar(255), extended_data jsonb, version varchar(255),experiment_type varchar(255), primary key (experiment_name, interval_end_time)) PARTITION BY RANGE (interval_end_time);
+create table IF NOT EXISTS kruize_jobs (job_id uuid not null, end_time timestamp(6), start_time timestamp(6), notifications jsonb, processed_count integer, status varchar(255), total_count integer, webhook varchar(255), primary key (job_id));
+create table IF NOT EXISTS kruize_jobmetadata (id bigserial not null, experiment_name varchar(255), notification jsonb, recommendation_notifications jsonb, recommendation_status varchar(255), job_id uuid not null, primary key (job_id, experiment_name))  PARTITION BY HASH (job_id);
+alter table if exists kruize_jobmetadata add constraint bulkJobMetaDataConstraint foreign key (job_id) references kruize_jobs;
+
+
+
+

src/main/java/com/autotune/analyzer/services/BulkService.java

@@ -42,6 +42,7 @@
 import java.util.concurrent.Executors;
 
 import static com.autotune.analyzer.utils.AnalyzerConstants.ServiceConstants.*;
+import static com.autotune.operator.KruizeDeploymentInfo.cacheJobInMemory;
 import static com.autotune.utils.KruizeConstants.KRUIZE_BULK_API.*;
 
 /**
@@ -153,7 +154,8 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response)
             // Generate a unique jobID
             String jobID = UUID.randomUUID().toString();
             BulkJobStatus jobStatus = new BulkJobStatus(jobID, IN_PROGRESS, Instant.now());
-            jobStatusMap.put(jobID, jobStatus);
+            if(cacheJobInMemory)
+                jobStatusMap.put(jobID, jobStatus);
             // Submit the job to be processed asynchronously
             executorService.submit(new BulkJobManager(jobID, jobStatus, payload));
 

src/main/java/com/autotune/database/init/KruizeHibernateUtil.java

@@ -17,6 +17,8 @@
 
 
 import com.autotune.database.table.*;
+import com.autotune.database.table.lm.Jobs;
+import com.autotune.database.table.lm.JobMetaData;
 import com.autotune.database.table.lm.KruizeLMExperimentEntry;
 import com.autotune.database.table.lm.KruizeLMRecommendationEntry;
 import com.autotune.operator.KruizeDeploymentInfo;
@@ -65,6 +67,8 @@ public static void buildSessionFactory() {
                 configuration.addAnnotatedClass(KruizeDSMetadataEntry.class);
                 configuration.addAnnotatedClass(KruizeMetricProfileEntry.class);
                 configuration.addAnnotatedClass(KruizeAuthenticationEntry.class);
+                configuration.addAnnotatedClass(Jobs.class);
+                configuration.addAnnotatedClass(JobMetaData.class);
             }
             LOGGER.info("DB is trying to connect to {}", connectionURL);
             sfTemp = configuration.buildSessionFactory();

src/main/java/com/autotune/database/table/lm/JobMetaData.java

@@ -0,0 +1,29 @@
+package com.autotune.database.table.lm;
+
+import jakarta.persistence.*;
+
+@Entity
+@Table(name = "kruize_jobmetadata")
+public class JobMetaData {
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    private Long id;
+
+    @ManyToOne
+    @JoinColumn(name = "job_id", nullable = false)
+    private Jobs job;
+
+    @Column(name = "experiment_name")
+    private String experimentName;
+
+    @Column(columnDefinition = "jsonb")
+    private String notification;
+
+    @Column(name = "recommendation_status")
+    private String recommendationsStatus;
+
+    @Column(columnDefinition = "jsonb" , name="recommendation_notifications")
+    private String recommendationsNotifications;
+
+    // Getters and Setters
+}

src/main/java/com/autotune/database/table/lm/Jobs.java

@@ -0,0 +1,35 @@
+package com.autotune.database.table.lm;
+
+import jakarta.persistence.*;
+
+import java.sql.Timestamp;
+import java.util.List;
+import java.util.UUID;
+
+
+@Entity
+@Table(name = "kruize_jobs")
+public class Jobs {
+    @Id
+    @Column(name = "job_id")
+    private UUID jobId;
+    private String status;
+    @Column(name = "total_count")
+    private int totalExperiments;
+    @Column(name = "processed_count")
+    private int processedExperiments;
+    @Column(name = "start_time")
+    private Timestamp jobStartTime;
+    @Column(name = "end_time")
+    private Timestamp jobEndTime;
+    private String webhook;
+
+    @Column(columnDefinition = "jsonb")
+    private String notifications; // Stored as JSON string
+
+    @OneToMany(mappedBy = "job", cascade = CascadeType.ALL, fetch = FetchType.LAZY)
+    @Column(name = "meta_data")
+    private List<JobMetaData> jobMetaData;
+
+    // Getters and Setters
+}

src/main/java/com/autotune/operator/KruizeDeploymentInfo.java

@@ -66,7 +66,7 @@ public class KruizeDeploymentInfo {
     public static String database_admin_username;
     public static String database_admin_password;
     public static String database_ssl_mode;
-
+    public static boolean cacheJobInMemory = true;
     public static String cloudwatch_logs_access_key_id;
     public static String cloudwatch_logs_secret_access_key;
     public static String cloudwatch_logs_log_group;