Skip to content

Commit

Permalink
Merge pull request #1152 from khansaad/listExp-new-params
Browse files Browse the repository at this point in the history 
Update List Experiments API to accept input JSON as params
  • Loading branch information
@dinogun
dinogun authored Apr 12, 2024
2 parents da0313a + c5ef2cc commit 33d6f96
Show file tree
Hide file tree
Showing 6 changed files with 205 additions and 48 deletions.
34 changes: 34 additions & 0 deletions design/MonitoringModeAPI.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1655,6 +1655,40 @@ name parameter**
`curl -H 'Accept: application/json' http://<URL>:<PORT>/listExperiments?recommendations=true&results=true&latest=false`

Returns all the recommendations and all the results of the specified experiment.
<br><br>
**List Experiments also allows the user to send a request body to fetch the records based on `cluster_name` and `kubernetes_object`.**
<br><br>
*Note: This request body can be sent along with other query params which are mentioned above.*

`curl -H 'Accept: application/json' -X POST --data 'copy paste below JSON' http://<URL>:<PORT>/listExperiments`

<details>

<summary><b>Example Request</b></summary>

### Example Request

```json
{
"cluster_name": "cluster-one-division-bell",
"kubernetes_objects": [
{
"type": "deployment",
"name": "tfb-qrh-deployment",
"namespace": "default",
"containers": [
{
"container_image_name": "kruize/tfb-db:1.15",
"container_name": "tfb-server-1"
}
]
}
]
}
```

</details>
Returns all the experiments matching the input JSON data.

<a name="list-recommendations-api"></a>

Expand Down
138 changes: 98 additions & 40 deletions src/main/java/com/autotune/analyzer/services/ListExperiments.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,7 @@

package com.autotune.analyzer.services;

import com.autotune.analyzer.exceptions.InvalidValueException;
import com.autotune.analyzer.experiment.KruizeExperiment;
import com.autotune.analyzer.experiment.RunExperiment;
import com.autotune.analyzer.kruizeObject.KruizeObject;
import com.autotune.analyzer.serviceObjects.ContainerAPIObject;
import com.autotune.analyzer.serviceObjects.Converters;
Expand All @@ -35,18 +33,13 @@
import com.autotune.common.target.kubernetes.service.KubernetesServices;
import com.autotune.common.trials.ExperimentTrial;
import com.autotune.database.service.ExperimentDBService;
import com.autotune.experimentManager.exceptions.IncompatibleInputJSONException;
import com.autotune.utils.KruizeConstants;
import com.autotune.utils.KruizeSupportedTypes;
import com.autotune.utils.MetricsConfig;
import com.autotune.utils.TrialHelpers;
import com.google.gson.ExclusionStrategy;
import com.google.gson.FieldAttributes;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.*;
import io.micrometer.core.instrument.Timer;
import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -56,14 +49,12 @@
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.sql.Timestamp;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;

import static com.autotune.analyzer.experiment.Experimentator.experimentsMap;
import static com.autotune.analyzer.utils.AnalyzerConstants.ServiceConstants.*;
import static com.autotune.utils.TrialHelpers.updateExperimentTrial;

/**
* Rest API used to list experiments.
Expand Down Expand Up @@ -110,9 +101,13 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) t
response.setCharacterEncoding(CHARACTER_ENCODING);
String gsonStr;
String results = request.getParameter(KruizeConstants.JSONKeys.RESULTS);
String latest = request.getParameter(AnalyzerConstants.ServiceConstants.LATEST);
String latest = request.getParameter(LATEST);
String recommendations = request.getParameter(KruizeConstants.JSONKeys.RECOMMENDATIONS);
String experimentName = request.getParameter(AnalyzerConstants.ServiceConstants.EXPERIMENT_NAME);
String experimentName = request.getParameter(EXPERIMENT_NAME);
String requestBody = request.getReader().lines().collect(Collectors.joining(System.lineSeparator()));
StringBuilder clusterName = new StringBuilder();
List<KubernetesAPIObject> kubernetesAPIObjectList = new ArrayList<>();
boolean isJSONValid = true;
Map<String, KruizeObject> mKruizeExperimentMap = new ConcurrentHashMap<>();
boolean error = false;
// validate Query params
Expand All @@ -133,36 +128,60 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) t
latest = "true";
// Validate query parameter values
if (isValidBooleanValue(results) && isValidBooleanValue(recommendations) && isValidBooleanValue(latest)) {
try {
// Fetch experiments data from the DB and check if the requested experiment exists
loadExperimentsFromDatabase(mKruizeExperimentMap, experimentName);
// Check if experiment exists
if (experimentName != null && !mKruizeExperimentMap.containsKey(experimentName)) {
error = true;
sendErrorResponse(
response,
new Exception(AnalyzerErrorConstants.APIErrors.ListRecommendationsAPI.INVALID_EXPERIMENT_NAME_EXCPTN),
HttpServletResponse.SC_BAD_REQUEST,
String.format(AnalyzerErrorConstants.APIErrors.ListRecommendationsAPI.INVALID_EXPERIMENT_NAME_MSG, experimentName)
);
}
if (!error) {
// create Gson Object
Gson gsonObj = createGsonObject();

// Modify the JSON response here based on query params.
gsonStr = buildResponseBasedOnQuery(mKruizeExperimentMap, gsonObj, results, recommendations, latest, experimentName);
if (gsonStr.isEmpty()) {
gsonStr = generateDefaultResponse();
// Check if JSON input is provided in the request body and validate it
if (!requestBody.isEmpty()) {
isJSONValid = validateInputJSON(requestBody);
}
if (isJSONValid) {
try {
// Fetch experiments data based on request body input, if it's present
if (!requestBody.isEmpty()) {
// parse the requestBody JSON into corresponding classes
parseInputJSON(requestBody, clusterName, kubernetesAPIObjectList);
try {
new ExperimentDBService().loadExperimentFromDBByInputJSON(mKruizeExperimentMap, clusterName, kubernetesAPIObjectList);
} catch (Exception e) {
LOGGER.error("Failed to load saved experiment data: {} ", e.getMessage());
}
} else {
// Fetch experiments data from the DB and check if the requested experiment exists
loadExperimentsFromDatabase(mKruizeExperimentMap, experimentName);
}
response.getWriter().println(gsonStr);
response.getWriter().close();
statusValue = "success";
// Check if experiment exists
if (experimentName != null && !mKruizeExperimentMap.containsKey(experimentName)) {
error = true;
sendErrorResponse(
response,
new Exception(AnalyzerErrorConstants.APIErrors.ListRecommendationsAPI.INVALID_EXPERIMENT_NAME_EXCPTN),
HttpServletResponse.SC_BAD_REQUEST,
String.format(AnalyzerErrorConstants.APIErrors.ListRecommendationsAPI.INVALID_EXPERIMENT_NAME_MSG, experimentName)
);
}
if (!error) {
// create Gson Object
Gson gsonObj = createGsonObject();

// Modify the JSON response here based on query params.
gsonStr = buildResponseBasedOnQuery(mKruizeExperimentMap, gsonObj, results, recommendations, latest, experimentName);
if (gsonStr.isEmpty()) {
gsonStr = generateDefaultResponse();
}
response.getWriter().println(gsonStr);
response.getWriter().close();
statusValue = "success";
}
} catch (Exception e) {
LOGGER.error("Exception: " + e.getMessage());
e.printStackTrace();
sendErrorResponse(response, e, HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.getMessage());
}
} catch (Exception e) {
LOGGER.error("Exception: " + e.getMessage());
e.printStackTrace();
sendErrorResponse(response, e, HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.getMessage());
} else {
sendErrorResponse(
response,
new Exception(AnalyzerErrorConstants.AutotuneObjectErrors.JSON_PARSING_ERROR),
HttpServletResponse.SC_BAD_REQUEST,
String.format(AnalyzerErrorConstants.AutotuneObjectErrors.JSON_PARSING_ERROR)
);
}
} else {
sendErrorResponse(
Expand All @@ -188,6 +207,45 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) t
}
}

private void parseInputJSON(String requestBody, StringBuilder clusterName, List<KubernetesAPIObject> kubernetesAPIObjectList) {
// Parse the JSON string into a JsonObject
JsonObject jsonObject = new Gson().fromJson(requestBody, JsonObject.class);

// Extract cluster name
clusterName.append(jsonObject.get(KruizeConstants.JSONKeys.CLUSTER_NAME).getAsString());

// Extract Kubernetes objects
JsonArray kubernetesObjectsArray = jsonObject.getAsJsonArray(KruizeConstants.JSONKeys.KUBERNETES_OBJECTS);
for (JsonElement element : kubernetesObjectsArray) {
JsonObject kubernetesObjectJson = element.getAsJsonObject();
String type = kubernetesObjectJson.get(KruizeConstants.JSONKeys.TYPE).getAsString();
String name = kubernetesObjectJson.get(KruizeConstants.JSONKeys.NAME).getAsString();
String namespace = kubernetesObjectJson.get(KruizeConstants.JSONKeys.NAMESPACE).getAsString();
List<ContainerAPIObject> containerAPIObjects = extractContainersFromJson(kubernetesObjectJson);
KubernetesAPIObject kubernetesAPIObject = new KubernetesAPIObject(name, type, namespace);
kubernetesAPIObject.setContainerAPIObjects(containerAPIObjects);
kubernetesAPIObjectList.add(kubernetesAPIObject);
}
}

public List<ContainerAPIObject> extractContainersFromJson(JsonObject jsonObject) {
JsonArray containersArray = jsonObject.getAsJsonArray(KruizeConstants.JSONKeys.CONTAINERS);
List<ContainerAPIObject> containerAPIObjects = new ArrayList<>();
for (JsonElement element : containersArray) {
JsonObject containerJson = element.getAsJsonObject();
String containerName = containerJson.get(KruizeConstants.JSONKeys.CONTAINER_NAME).getAsString();
String containerImageName = containerJson.get(KruizeConstants.JSONKeys.CONTAINER_IMAGE_NAME).getAsString();
ContainerAPIObject containerAPIObject = new ContainerAPIObject(containerName, containerImageName, null, null);
containerAPIObjects.add(containerAPIObject);
}
return containerAPIObjects;
}

private boolean validateInputJSON(String requestBody) {
//TODO: add validations for the requestBody
return true;
}

private boolean isValidBooleanValue(String value) {
return value != null && (value.equals("true") || value.equals("false"));
}
Expand Down
2 changes: 2 additions & 0 deletions src/main/java/com/autotune/database/dao/ExperimentDAO.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.autotune.database.dao;

import com.autotune.analyzer.kruizeObject.KruizeObject;
import com.autotune.analyzer.serviceObjects.KubernetesAPIObject;
import com.autotune.analyzer.utils.AnalyzerConstants;
import com.autotune.common.data.ValidationOutputData;
import com.autotune.database.table.KruizeExperimentEntry;
Expand Down Expand Up @@ -70,4 +71,5 @@ public interface ExperimentDAO {

public void addPartitions(String tableName, String month, String year, int dayOfTheMonth, String partitionType) throws Exception;

List<KruizeExperimentEntry> loadExperimentFromDBByInputJSON(StringBuilder clusterName, KubernetesAPIObject kubernetesAPIObject) throws Exception;
}
45 changes: 41 additions & 4 deletions src/main/java/com/autotune/database/dao/ExperimentDAOImpl.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package com.autotune.database.dao;

import com.autotune.analyzer.kruizeObject.KruizeObject;
import com.autotune.analyzer.serviceObjects.ContainerAPIObject;
import com.autotune.analyzer.serviceObjects.KubernetesAPIObject;
import com.autotune.analyzer.utils.AnalyzerConstants;
import com.autotune.analyzer.utils.AnalyzerErrorConstants;
import com.autotune.common.data.ValidationOutputData;
Expand Down Expand Up @@ -28,10 +30,7 @@
import java.time.LocalDateTime;
import java.time.YearMonth;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.*;
import java.util.stream.IntStream;

import static com.autotune.database.helper.DBConstants.DB_MESSAGES.DUPLICATE_KEY;
Expand Down Expand Up @@ -552,6 +551,44 @@ public List<KruizeExperimentEntry> loadExperimentByName(String experimentName) t
return entries;
}

/**
* @param clusterName
* @param kubernetesAPIObject
* @return list of experiments from the DB matching the input params
*/
@Override
public List<KruizeExperimentEntry> loadExperimentFromDBByInputJSON(StringBuilder clusterName, KubernetesAPIObject kubernetesAPIObject) throws Exception {
//todo load only experimentStatus=inprogress , playback may not require completed experiments
List<KruizeExperimentEntry> entries;
String statusValue = "failure";
Timer.Sample timerLoadExpName = Timer.start(MetricsConfig.meterRegistry());
try (Session session = KruizeHibernateUtil.getSessionFactory().openSession()) {
// assuming there will be only one container
ContainerAPIObject containerAPIObject = kubernetesAPIObject.getContainerAPIObjects().get(0);
// Set parameters for KubernetesObject and Container
Query<KruizeExperimentEntry> query = session.createNativeQuery(SELECT_FROM_EXPERIMENTS_BY_INPUT_JSON, KruizeExperimentEntry.class);
query.setParameter(CLUSTER_NAME, clusterName.toString());
query.setParameter(KruizeConstants.JSONKeys.NAME, kubernetesAPIObject.getName());
query.setParameter(KruizeConstants.JSONKeys.NAMESPACE, kubernetesAPIObject.getNamespace());
query.setParameter(KruizeConstants.JSONKeys.TYPE, kubernetesAPIObject.getType());
query.setParameter(KruizeConstants.JSONKeys.CONTAINER_NAME, containerAPIObject.getContainer_name());
query.setParameter(KruizeConstants.JSONKeys.CONTAINER_IMAGE_NAME, containerAPIObject.getContainer_image_name());

entries = query.getResultList();
statusValue = "success";
} catch (Exception e) {
LOGGER.error("Error fetching experiment data: {}", e.getMessage());
throw new Exception("Error while fetching experiment data from database: " + e.getMessage());
} finally {
if (null != timerLoadExpName) {
MetricsConfig.timerLoadExpName = MetricsConfig.timerBLoadExpName.tag("status", statusValue).register(MetricsConfig.meterRegistry());
timerLoadExpName.stop(MetricsConfig.timerLoadExpName);
}

}
return entries;
}

@Override
public List<KruizeResultsEntry> loadResultsByExperimentName(String experimentName, String cluster_name, Timestamp calculated_start_time, Timestamp interval_end_time) throws Exception {
// TODO: load only experimentStatus=inProgress , playback may not require completed experiments
Expand Down
10 changes: 10 additions & 0 deletions src/main/java/com/autotune/database/helper/DBConstants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,16 @@ public static final class SQLQUERY {
public static final String DB_PARTITION_DATERANGE = "CREATE TABLE IF NOT EXISTS %s_%s%s%s PARTITION OF %s FOR VALUES FROM ('%s-%s-%s 00:00:00.000') TO ('%s-%s-%s 23:59:59');";
public static final String SELECT_ALL_KRUIZE_TABLES = "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' " +
"and (table_name like 'kruize_results_%' or table_name like 'kruize_recommendations_%') ";
public static final String SELECT_FROM_EXPERIMENTS_BY_INPUT_JSON = "SELECT * FROM kruize_experiments WHERE cluster_name = :cluster_name " +
"AND EXISTS (SELECT 1 FROM jsonb_array_elements(extended_data->'kubernetes_objects') AS kubernetes_object" +
" WHERE kubernetes_object->>'name' = :name " +
" AND kubernetes_object->>'namespace' = :namespace " +
" AND kubernetes_object->>'type' = :type " +
" AND EXISTS (SELECT 1 FROM jsonb_array_elements(kubernetes_object->'containers') AS container" +
" WHERE container->>'container_name' = :container_name" +
" AND container->>'container_image_name' = :container_image_name" +
" ))";

}

public static final class TABLE_NAMES {
Expand Down
Loading

0 comments on commit 33d6f96

Please sign in to comment.