Skip to content

Commit

Permalink
updates added for more validation
Browse files Browse the repository at this point in the history 
Signed-off-by: Saad Khan <[email protected]>
  • Loading branch information
@khansaad
khansaad committed Nov 27, 2023
1 parent ecf8b21 commit 31ff3d8
Show file tree
Hide file tree
Showing 4 changed files with 86 additions and 14 deletions.
4 changes: 2 additions & 2 deletions src/main/java/com/autotune/analyzer/performanceProfiles/utils/PerformanceProfileUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ public static ValidationOutputData validateAndAddProfile(Map<String, Performance
* @param updateResultsAPIObject
* @return
*/
public static String validateResults(PerformanceProfile performanceProfile, UpdateResultsAPIObject updateResultsAPIObject) {
public static List<String> validateResults(PerformanceProfile performanceProfile, UpdateResultsAPIObject updateResultsAPIObject) {

List<String> errorReasons = new ArrayList<>();
String errorMsg = "";
Expand Down Expand Up @@ -153,7 +153,7 @@ public static String validateResults(PerformanceProfile performanceProfile, Upda
}
}
}
return errorReasons.toString();
return errorReasons;
}

public static void addPerformanceProfile(Map<String, PerformanceProfile> performanceProfileMap, PerformanceProfile performanceProfile) {
Expand Down
75 changes: 72 additions & 3 deletions ...autotune/analyzer/serviceObjects/verification/validators/KubernetesElementsValidator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,19 +16,21 @@
package com.autotune.analyzer.serviceObjects.verification.validators;

import com.autotune.analyzer.kruizeObject.KruizeObject;
import com.autotune.analyzer.performanceProfiles.PerformanceProfile;
import com.autotune.analyzer.serviceObjects.Converters;
import com.autotune.analyzer.serviceObjects.UpdateResultsAPIObject;
import com.autotune.analyzer.serviceObjects.verification.annotators.KubernetesElementsCheck;
import com.autotune.analyzer.services.UpdateResults;
import com.autotune.common.data.result.ContainerData;
import com.autotune.common.data.result.ExperimentResultData;
import com.autotune.common.k8sObjects.K8sObject;
import jakarta.validation.ConstraintValidator;
import jakarta.validation.ConstraintValidatorContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

public class KubernetesElementsValidator implements ConstraintValidator<KubernetesElementsCheck, UpdateResultsAPIObject> {
private static final Logger LOGGER = LoggerFactory.getLogger(KubernetesElementsValidator.class);
Expand All @@ -45,7 +47,6 @@ public boolean isValid(UpdateResultsAPIObject updateResultsAPIObject, Constraint
String errorMessage = "";
try {
KruizeObject kruizeObject = updateResultsAPIObject.getKruizeObject();
PerformanceProfile performanceProfile = UpdateResults.performanceProfilesMap.get(kruizeObject.getPerformanceProfile());
ExperimentResultData resultData = Converters.KruizeObjectConverters.convertUpdateResultsAPIObjToExperimentResultData(updateResultsAPIObject);
String expName = kruizeObject.getExperimentName();
String errorMsg = "";
Expand Down Expand Up @@ -96,6 +97,55 @@ public boolean isValid(UpdateResultsAPIObject updateResultsAPIObject, Constraint
));
}

// Validate container names
if(kruizeObject.getKubernetes_objects().size() == resultData.getKubernetes_objects().size() &&
IntStream.range(0, kruizeObject.getKubernetes_objects().size())
.allMatch(i -> {
K8sObject kruizeK8sObject = kruizeObject.getKubernetes_objects().get(i);
K8sObject resultDataK8sObject = resultData.getKubernetes_objects().get(i);
return kruizeK8sObject.getName().equals(resultDataK8sObject.getName()) && compareContainerNames(kruizeK8sObject, resultDataK8sObject);
})) {
kubeObjsMisMatch = true;
errorMsg = errorMsg.concat(
String.format(
"Kubernetes Object Container names MisMatched. Expected names: %s, Found: %s in Results for experiment: %s \n",
kruizeObject.getKubernetes_objects().stream()
.flatMap(k8sObject -> k8sObject.getContainerDataMap().values().stream())
.map(ContainerData::getContainer_name)
.collect(Collectors.toList()),
resultData.getKubernetes_objects().stream()
.flatMap(k8sObject -> k8sObject.getContainerDataMap().values().stream())
.map(ContainerData::getContainer_name)
.collect(Collectors.toList()),
expName
));
}

// Validate container image names
if (kruizeObject.getKubernetes_objects().size() == resultData.getKubernetes_objects().size() &&
IntStream.range(0, kruizeObject.getKubernetes_objects().size())
.allMatch(i -> {
K8sObject kruizeK8sObject = kruizeObject.getKubernetes_objects().get(i);
K8sObject resultDataK8sObject = resultData.getKubernetes_objects().get(i);
return kruizeK8sObject.getName().equals(resultDataK8sObject.getName()) && compareContainerImageNames(kruizeK8sObject, resultDataK8sObject);
})) {
kubeObjsMisMatch = true;
errorMsg = errorMsg.concat(
String.format(
"Kubernetes Object Container names MisMatched. Expected names: %s, Found: %s in Results for experiment: %s \n",
kruizeObject.getKubernetes_objects().stream()
.flatMap(k8sObject -> k8sObject.getContainerDataMap().values().stream())
.map(ContainerData::getContainer_image_name)
.collect(Collectors.toList()),
resultData.getKubernetes_objects().stream()
.flatMap(k8sObject -> k8sObject.getContainerDataMap().values().stream())
.map(ContainerData::getContainer_image_name)
.collect(Collectors.toList()),
expName
));
}


if (kubeObjsMisMatch) {
context.disableDefaultConstraintViolation();
context.buildConstraintViolationWithTemplate(errorMsg)
Expand Down Expand Up @@ -126,4 +176,23 @@ public boolean isValid(UpdateResultsAPIObject updateResultsAPIObject, Constraint
LOGGER.debug("KubernetesElementsValidator success : {}", success);
return success;
}

private boolean compareContainerImageNames(K8sObject kruizeK8sObject, K8sObject resultDataK8sObject) {
return kruizeK8sObject.getContainerDataMap().size() == resultDataK8sObject.getContainerDataMap().size() &&
kruizeK8sObject.getContainerDataMap().values().stream()
.allMatch(containerData ->
resultDataK8sObject.getContainerDataMap().values().stream()
.anyMatch(data ->
data.getContainer_image_name().equals(containerData.getContainer_image_name())
)
);
}

private boolean compareContainerNames(K8sObject kruizeK8sObject, K8sObject resultDataK8sObject) {
return kruizeK8sObject.getContainerDataMap().size() == resultDataK8sObject.getContainerDataMap().size() &&
kruizeK8sObject.getContainerDataMap().keySet().stream()
.allMatch(containerData ->
resultDataK8sObject.getContainerDataMap().containsKey(containerData)
);
}
}
7 changes: 4 additions & 3 deletions ...autotune/analyzer/serviceObjects/verification/validators/PerformanceProfileValidator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@

import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;

import static com.autotune.analyzer.utils.AnalyzerErrorConstants.AutotuneObjectErrors.MISSING_PERF_PROFILE;
Expand Down Expand Up @@ -63,12 +64,12 @@ public boolean isValid(UpdateResultsAPIObject updateResultsAPIObject, Constraint
}

// validate the results value present in the updateResultsAPIObject
String errorMsg = PerformanceProfileUtil.validateResults(performanceProfile, updateResultsAPIObject);
if (null == errorMsg || errorMsg.isEmpty()) {
List<String> errorMsg = PerformanceProfileUtil.validateResults(performanceProfile, updateResultsAPIObject);
if (errorMsg.isEmpty()) {
success = true;
} else {
context.disableDefaultConstraintViolation();
context.buildConstraintViolationWithTemplate(errorMsg)
context.buildConstraintViolationWithTemplate(errorMsg.toString())
.addPropertyNode("Performance profile")
.addConstraintViolation();
}
Expand Down
14 changes: 8 additions & 6 deletions tests/scripts/remote_monitoring_tests/rest_apis/test_update_results.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,11 @@
]

missing_metrics = [
("Missing_metrics_single_res_single_container", "../json_files/missing_metrics_jsons/update_results_missing_metrics_single_container.json", "Out of a total of 1 records, 1 failed to save", "kubernetes_objects : Metric data is not present for container : tfb-server-0 for experiment: quarkus-resteasy-kruize-min-http-response-time-db"),
("Missing_metrics_single_res_all_containers", "../json_files/missing_metrics_jsons/update_results_missing_metrics_all_containers.json", "Out of a total of 1 records, 1 failed to save", "kubernetes_objects : Metric data is not present for container : tfb-server-0 for experiment: quarkus-resteasy-kruize-min-http-response-time-db. Metric data is not present for container : tfb-server-1 for experiment: quarkus-resteasy-kruize-min-http-response-time-db"),
("Missing_metrics_bulk_res_single_container", "../json_files/missing_metrics_jsons/bulk_update_results_missing_metrics_single_container.json", "Out of a total of 100 records, 1 failed to save", "kubernetes_objects : Metric data is not present for container : tfb-server-1 for experiment: quarkus-resteasy-kruize-min-http-response-time-db"),
("Missing_metrics_bulk_res_few_containers", "../json_files/missing_metrics_jsons/bulk_update_results_missing_metrics_few_containers.json", "Out of a total of 100 records, 2 failed to save", "kubernetes_objects : Metric data is not present for container : tfb-server-1 for experiment: quarkus-resteasy-kruize-min-http-response-time-db"),
("Missing_metrics_bulk_res_few_containers_few_individual_metrics_missing", "../json_files/missing_metrics_jsons/bulk_update_results_missing_metrics_few_containers_few_individual_metrics_missing.json", "Out of a total of 100 records, 4 failed to save", "Metric data is not present for container")
("Missing_metrics_single_res_single_container", "../json_files/missing_metrics_jsons/update_results_missing_metrics_single_container.json", "Out of a total of 1 records, 1 failed to save", "Performance profile: [Metric data is not present for container : tfb-server-0 for experiment: quarkus-resteasy-kruize-min-http-response-time-db"),
("Missing_metrics_single_res_all_containers", "../json_files/missing_metrics_jsons/update_results_missing_metrics_all_containers.json", "Out of a total of 1 records, 1 failed to save", "Performance profile: [Metric data is not present for container : tfb-server-0 for experiment: quarkus-resteasy-kruize-min-http-response-time-db. , Metric data is not present for container : tfb-server-1 for experiment: quarkus-resteasy-kruize-min-http-response-time-db"),
("Missing_metrics_bulk_res_single_container", "../json_files/missing_metrics_jsons/bulk_update_results_missing_metrics_single_container.json", "Out of a total of 100 records, 1 failed to save", "Performance profile: [Metric data is not present for container : tfb-server-1 for experiment: quarkus-resteasy-kruize-min-http-response-time-db"),
("Missing_metrics_bulk_res_few_containers", "../json_files/missing_metrics_jsons/bulk_update_results_missing_metrics_few_containers.json", "Out of a total of 100 records, 2 failed to save", "Performance profile: [Metric data is not present for container : tfb-server-1 for experiment: quarkus-resteasy-kruize-min-http-response-time-db"),
("Missing_metrics_bulk_res_few_containers_few_individual_metrics_missing", "../json_files/missing_metrics_jsons/bulk_update_results_missing_metrics_few_containers_few_individual_metrics_missing.json", "Out of a total of 100 records, 2 failed to save", "Metric data is not present for container")
]


Expand Down Expand Up @@ -180,7 +180,9 @@ def test_update_results_with_missing_metrics_section(test_name, result_json_file
for err in error_data:
actual_error_message = err["message"]
if test_name == "Missing_metrics_bulk_res_few_containers" and d["interval_end_time"] == "2023-04-13T23:29:20.982Z":
error_message = "kubernetes_objects : Metric data is not present for container : tfb-server-1 for experiment: quarkus-resteasy-kruize-min-http-response-time-db. Metric data is not present for container : tfb-server-0 for experiment: quarkus-resteasy-kruize-min-http-response-time-db"
error_message = "Performance profile: [Metric data is not present for container : tfb-server-1 for experiment: quarkus-resteasy-kruize-min-http-response-time-db. ] , " \
"Performance profile: [Metric data is not present for container : tfb-server-1 for experiment: quarkus-resteasy-kruize-min-http-response-time-db. , " \
"Metric data is not present for container : tfb-server-0 for experiment: quarkus-resteasy-kruize-min-http-response-time-db"
if test_name == "Missing_metrics_bulk_res_few_containers_few_individual_metrics_missing" and d["interval_end_time"] == "2023-04-13T23:44:20.982Z" or d["interval_end_time"] == "2023-04-13T23:59:20.982Z":
error_message = "Performance profile: Missing one of the following mandatory parameters for experiment - quarkus-resteasy-kruize-min-http-response-time-db : [cpuUsage, memoryUsage, memoryRSS]"
assert error_message in actual_error_message
Expand Down

0 comments on commit 31ff3d8

Please sign in to comment.