From 7fb8c41fa3340427ff4e7917b1ac4f5c323c3da1 Mon Sep 17 00:00:00 2001 From: Mitali Salvi Date: Sat, 28 Oct 2023 23:24:40 -0400 Subject: [PATCH 1/7] Validating logs --- generator/test_case_generator.go | 26 ++++++------ .../default_amazon_cloudwatch_agent.json | 3 +- .../test_schemas/cluster_daemonset.json | 33 +++++++++++++++ .../test_schemas/cluster_deployment.json | 33 +++++++++++++++ .../eks_resources/test_schemas/container.json | 3 ++ .../test_schemas/control_plane.json | 42 +++++++++++++++++++ .../eks_resources/test_schemas/pod.json | 11 +++++ .../eks_resources/util.go | 31 +++++++++----- 8 files changed, 157 insertions(+), 25 deletions(-) create mode 100644 test/metric_value_benchmark/eks_resources/test_schemas/cluster_daemonset.json create mode 100644 test/metric_value_benchmark/eks_resources/test_schemas/cluster_deployment.json create mode 100644 test/metric_value_benchmark/eks_resources/test_schemas/control_plane.json diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index 3c9a0ed91..1b4551c68 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -174,19 +174,19 @@ var testTypeToTestConfig = map[string][]testConfig{ testDir: "./test/metric_value_benchmark", targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, }, - { - testDir: "./test/statsd", terraformDir: "terraform/eks/daemon/statsd", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - }, - { - testDir: "./test/emf", terraformDir: "terraform/eks/daemon/emf", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - }, - { - testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/d", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - }, - {testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/bit"}, + //{ + // testDir: "./test/statsd", terraformDir: "terraform/eks/daemon/statsd", + // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + //}, + //{ + // testDir: "./test/emf", terraformDir: "terraform/eks/daemon/emf", + // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + //}, + //{ + // testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/d", + // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + //}, + //{testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/bit"}, }, "eks_deployment": { {testDir: "./test/metric_value_benchmark"}, diff --git a/terraform/eks/daemon/default_resources/default_amazon_cloudwatch_agent.json b/terraform/eks/daemon/default_resources/default_amazon_cloudwatch_agent.json index 01d141110..2c6ee84b2 100644 --- a/terraform/eks/daemon/default_resources/default_amazon_cloudwatch_agent.json +++ b/terraform/eks/daemon/default_resources/default_amazon_cloudwatch_agent.json @@ -2,7 +2,8 @@ "logs": { "metrics_collected": { "kubernetes": { - "metrics_collection_interval": 30 + "metrics_collection_interval": 30, + "enhanced_container_insights": true } }, "force_flush_interval": 5 diff --git a/test/metric_value_benchmark/eks_resources/test_schemas/cluster_daemonset.json b/test/metric_value_benchmark/eks_resources/test_schemas/cluster_daemonset.json new file mode 100644 index 000000000..56377ee2a --- /dev/null +++ b/test/metric_value_benchmark/eks_resources/test_schemas/cluster_daemonset.json @@ -0,0 +1,33 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "structured log schema", + "description": "json schema for the cloudwatch agent k8s structured log", + "type": "object", + "properties": { + "ClusterName":{}, + "Type":{}, + "Sources":{}, + "Timestamp":{}, + "Version":{}, + "AutoScalingGroupName":{}, + "Namespace": {}, + "NodeName": {}, + "PodName": {}, + "replicas_desired": {}, + "replicas_ready": {}, + "status_replicas_available": {}, + "status_replicas_unavailable": {} + }, + "required": [ + "ClusterName", + "Type", + "Sources", + "Timestamp", + "Version", + "AutoScalingGroupName", + "NodeName", + "PodName", + "Namespace" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/test/metric_value_benchmark/eks_resources/test_schemas/cluster_deployment.json b/test/metric_value_benchmark/eks_resources/test_schemas/cluster_deployment.json new file mode 100644 index 000000000..56377ee2a --- /dev/null +++ b/test/metric_value_benchmark/eks_resources/test_schemas/cluster_deployment.json @@ -0,0 +1,33 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "structured log schema", + "description": "json schema for the cloudwatch agent k8s structured log", + "type": "object", + "properties": { + "ClusterName":{}, + "Type":{}, + "Sources":{}, + "Timestamp":{}, + "Version":{}, + "AutoScalingGroupName":{}, + "Namespace": {}, + "NodeName": {}, + "PodName": {}, + "replicas_desired": {}, + "replicas_ready": {}, + "status_replicas_available": {}, + "status_replicas_unavailable": {} + }, + "required": [ + "ClusterName", + "Type", + "Sources", + "Timestamp", + "Version", + "AutoScalingGroupName", + "NodeName", + "PodName", + "Namespace" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/test/metric_value_benchmark/eks_resources/test_schemas/container.json b/test/metric_value_benchmark/eks_resources/test_schemas/container.json index f12b4e22e..aa30c0e36 100644 --- a/test/metric_value_benchmark/eks_resources/test_schemas/container.json +++ b/test/metric_value_benchmark/eks_resources/test_schemas/container.json @@ -46,6 +46,9 @@ "container_last_termination_reason":{}, "number_of_container_restarts":{}, "container_status":{}, + "container_filesystem_usage":{}, + "container_filesystem_available":{}, + "container_filesystem_utilization":{}, "kubernetes":{ "type": "object", "properties": { diff --git a/test/metric_value_benchmark/eks_resources/test_schemas/control_plane.json b/test/metric_value_benchmark/eks_resources/test_schemas/control_plane.json new file mode 100644 index 000000000..80391615c --- /dev/null +++ b/test/metric_value_benchmark/eks_resources/test_schemas/control_plane.json @@ -0,0 +1,42 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "structured log schema", + "description": "json schema for the cloudwatch agent k8s structured log", + "type": "object", + "properties": { + "ClusterName": {}, + "Type": {}, + "Sources": {}, + "Timestamp": {}, + "Version": {}, + "apiserver_storage_size_bytes": {}, + "apiserver_storage_db_total_size_in_bytes": {}, + "etcd_db_total_size_in_bytes": {}, + "apiserver_storage_list_duration_seconds": {}, + "apiserver_longrunning_requests": {}, + "apiserver_request_duration_seconds": {}, + "rest_client_request_duration_seconds": {}, + "apiserver_request_total": {}, + "apiserver_request_total_5xx": {}, + "apiserver_admission_controller_admission_duration_seconds": {}, + "apiserver_admission_step_admission_duration_seconds", + "etcd_request_duration_seconds": {}, + "rest_client_requests_total": {}, + "apiserver_current_inflight_requests": {}, + "apiserver_current_inqueue_requests": {}, + "apiserver_admission_webhook_admission_duration_seconds": {}, + "apiserver_requested_deprecated_apis": {}, + "apiserver_flowcontrol_rejected_requests_total": {}, + "apiserver_flowcontrol_request_concurrency_limit": {}, + "CloudWatchMetrics": {} + }, + "required": [ + "ClusterName", + "Type", + "Sources", + "Timestamp", + "Version", + "CloudWatchMetrics" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/test/metric_value_benchmark/eks_resources/test_schemas/pod.json b/test/metric_value_benchmark/eks_resources/test_schemas/pod.json index 6e78e6b1a..1472e9c61 100644 --- a/test/metric_value_benchmark/eks_resources/test_schemas/pod.json +++ b/test/metric_value_benchmark/eks_resources/test_schemas/pod.json @@ -45,6 +45,17 @@ "pod_number_of_running_containers":{}, "pod_number_of_containers":{}, "pod_number_of_container_restarts":{}, + "pod_container_status_running":{}, + "pod_container_status_terminated":{}, + "pod_container_status_waiting":{}, + "pod_container_status_waiting_reason_crash_loop_back_off":{}, + "pod_container_status_waiting_reason_image_pull_error":{}, + "pod_container_status_waiting_reason_start_error":{}, + "pod_container_status_waiting_reason_create_container_error":{}, + "pod_container_status_waiting_reason_create_container_config_error":{}, + "pod_container_status_terminated_reason_oom_killed":{}, + "pod_interface_network_rx_dropped":{}, + "pod_interface_network_tx_dropped":{}, "pod_status":{}, "kubernetes":{ "type": "object", diff --git a/test/metric_value_benchmark/eks_resources/util.go b/test/metric_value_benchmark/eks_resources/util.go index 537183575..d4e73912a 100644 --- a/test/metric_value_benchmark/eks_resources/util.go +++ b/test/metric_value_benchmark/eks_resources/util.go @@ -8,6 +8,10 @@ import _ "embed" var ( //go:embed test_schemas/cluster.json eksClusterSchema string + //go:embed test_schemas/cluster_daemonset.json + eksClusterDaemonsetSchema string + //go:embed test_schemas/cluster_deployment.json + eksClusterDeploymentSchema string //go:embed test_schemas/cluster_namespace.json eksClusterNamespaceSchema string //go:embed test_schemas/cluster_service.json @@ -16,6 +20,8 @@ var ( eksContainerSchema string //go:embed test_schemas/container_fs.json eksContainerFSSchema string + //go:embed test_schemas/control_plane.json + eksControlPlaneSchema string //go:embed test_schemas/node.json eksNodeSchema string //go:embed test_schemas/node_disk_io.json @@ -30,16 +36,19 @@ var ( eksPodNetSchema string EksClusterValidationMap = map[string]string{ - "Cluster": eksClusterSchema, - "ClusterNamespace": eksClusterNamespaceSchema, - "ClusterService": eksClusterServiceSchema, - "Container": eksContainerSchema, - "ContainerFS": eksContainerFSSchema, - "Node": eksNodeSchema, - "NodeDiskIO": eksNodeDiskIOSchema, - "NodeFS": eksNodeFSSchema, - "NodeNet": eksNodeNetSchema, - "Pod": eksPodSchema, - "PodNet": eksPodNetSchema, + "Cluster": eksClusterSchema, + "ClusterDaemonset": eksClusterDaemonsetSchema, + "ClusterDeployment": eksClusterDeploymentSchema, + "ClusterNamespace": eksClusterNamespaceSchema, + "ClusterService": eksClusterServiceSchema, + "Container": eksContainerSchema, + "ContainerFS": eksContainerFSSchema, + "ControlPlane": eksControlPlaneSchema, + "Node": eksNodeSchema, + "NodeDiskIO": eksNodeDiskIOSchema, + "NodeFS": eksNodeFSSchema, + "NodeNet": eksNodeNetSchema, + "Pod": eksPodSchema, + "PodNet": eksPodNetSchema, } ) From a4056f56af79c59777a7ac0c432cb80d510779e1 Mon Sep 17 00:00:00 2001 From: Mitali Salvi Date: Sat, 28 Oct 2023 23:27:48 -0400 Subject: [PATCH 2/7] Validating logs --- .../eks_daemonset_test.go | 193 +++++++++--------- 1 file changed, 94 insertions(+), 99 deletions(-) diff --git a/test/metric_value_benchmark/eks_daemonset_test.go b/test/metric_value_benchmark/eks_daemonset_test.go index 35e4298a7..78d2f22c6 100644 --- a/test/metric_value_benchmark/eks_daemonset_test.go +++ b/test/metric_value_benchmark/eks_daemonset_test.go @@ -12,22 +12,17 @@ import ( "log" "time" - "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" - "golang.org/x/exp/slices" - "github.com/aws/amazon-cloudwatch-agent-test/environment" - "github.com/aws/amazon-cloudwatch-agent-test/test/metric" - "github.com/aws/amazon-cloudwatch-agent-test/test/metric/dimension" "github.com/aws/amazon-cloudwatch-agent-test/test/metric_value_benchmark/eks_resources" "github.com/aws/amazon-cloudwatch-agent-test/test/status" "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" ) -const containerInsightsNamespace = "ContainerInsights" +//const containerInsightsNamespace = "ContainerInsights" // list of metrics with more dimensions e.g. PodName and Namespace -var metricsWithMoreDimensions = []string{"pod_number_of_container_restarts"} +//var metricsWithMoreDimensions = []string{"pod_number_of_container_restarts"} type EKSDaemonTestRunner struct { test_runner.BaseTestRunner @@ -35,11 +30,11 @@ type EKSDaemonTestRunner struct { } func (e *EKSDaemonTestRunner) Validate() status.TestGroupResult { - metrics := e.GetMeasuredMetrics() + //metrics := e.GetMeasuredMetrics() testResults := make([]status.TestResult, 0) - for _, name := range metrics { - testResults = append(testResults, e.validateInstanceMetrics(name)) - } + //for _, name := range metrics { + // testResults = append(testResults, e.validateInstanceMetrics(name)) + //} testResults = append(testResults, e.validateLogs(e.env)) return status.TestGroupResult{ @@ -48,66 +43,66 @@ func (e *EKSDaemonTestRunner) Validate() status.TestGroupResult { } } -func (e *EKSDaemonTestRunner) validateInstanceMetrics(name string) status.TestResult { - testResult := status.TestResult{ - Name: name, - Status: status.FAILED, - } - - dims, failed := e.DimensionFactory.GetDimensions([]dimension.Instruction{ - { - Key: "ClusterName", - Value: dimension.UnknownDimensionValue(), - }, - }) - if len(failed) > 0 { - log.Println("failed to get dimensions") - return testResult - } - - // get list of metrics that has more dimensions for container insights - // this is to avoid adding more dimension provider for non-trivial dimensions e.g. PodName - listFetcher := metric.MetricListFetcher{} - if slices.Contains(metricsWithMoreDimensions, name) { - metrics, err := listFetcher.Fetch(containerInsightsNamespace, name, dims) - if err != nil { - log.Println("failed to fetch metric list", err) - return testResult - } - - if len(metrics) < 1 { - log.Println("metric list is empty") - return testResult - } - - // just verify 1 of returned metrics for values - for _, dim := range metrics[0].Dimensions { - // skip since it's provided by dimension provider - if *dim.Name == "ClusterName" { - continue - } - - dims = append(dims, types.Dimension{ - Name: dim.Name, - Value: dim.Value, - }) - } - } - - valueFetcher := metric.MetricValueFetcher{} - values, err := valueFetcher.Fetch(containerInsightsNamespace, name, dims, metric.AVERAGE, metric.HighResolutionStatPeriod) - if err != nil { - log.Println("failed to fetch metrics", err) - return testResult - } - - if !metric.IsAllValuesGreaterThanOrEqualToExpectedValue(name, values, 0) { - return testResult - } - - testResult.Status = status.SUCCESSFUL - return testResult -} +//func (e *EKSDaemonTestRunner) validateInstanceMetrics(name string) status.TestResult { +// testResult := status.TestResult{ +// Name: name, +// Status: status.FAILED, +// } +// +// dims, failed := e.DimensionFactory.GetDimensions([]dimension.Instruction{ +// { +// Key: "ClusterName", +// Value: dimension.UnknownDimensionValue(), +// }, +// }) +// if len(failed) > 0 { +// log.Println("failed to get dimensions") +// return testResult +// } +// +// // get list of metrics that has more dimensions for container insights +// // this is to avoid adding more dimension provider for non-trivial dimensions e.g. PodName +// listFetcher := metric.MetricListFetcher{} +// if slices.Contains(metricsWithMoreDimensions, name) { +// metrics, err := listFetcher.Fetch(containerInsightsNamespace, name, dims) +// if err != nil { +// log.Println("failed to fetch metric list", err) +// return testResult +// } +// +// if len(metrics) < 1 { +// log.Println("metric list is empty") +// return testResult +// } +// +// // just verify 1 of returned metrics for values +// for _, dim := range metrics[0].Dimensions { +// // skip since it's provided by dimension provider +// if *dim.Name == "ClusterName" { +// continue +// } +// +// dims = append(dims, types.Dimension{ +// Name: dim.Name, +// Value: dim.Value, +// }) +// } +// } +// +// valueFetcher := metric.MetricValueFetcher{} +// values, err := valueFetcher.Fetch(containerInsightsNamespace, name, dims, metric.AVERAGE, metric.HighResolutionStatPeriod) +// if err != nil { +// log.Println("failed to fetch metrics", err) +// return testResult +// } +// +// if !metric.IsAllValuesGreaterThanOrEqualToExpectedValue(name, values, 0) { +// return testResult +// } +// +// testResult.Status = status.SUCCESSFUL +// return testResult +//} func (e *EKSDaemonTestRunner) validateLogs(env *environment.MetaData) status.TestResult { testResult := status.TestResult{ @@ -174,34 +169,34 @@ func (e *EKSDaemonTestRunner) GetAgentRunDuration() time.Duration { return time.Minute * 3 } -func (e *EKSDaemonTestRunner) GetMeasuredMetrics() []string { - return []string{ - "cluster_failed_node_count", - "cluster_node_count", - "namespace_number_of_running_pods", - "node_cpu_limit", - "node_cpu_reserved_capacity", - "node_cpu_usage_total", - "node_cpu_utilization", - "node_filesystem_utilization", - "node_memory_limit", - "node_memory_reserved_capacity", - "node_memory_utilization", - "node_memory_working_set", - "node_network_total_bytes", - "node_number_of_running_containers", - "node_number_of_running_pods", - "pod_cpu_reserved_capacity", - "pod_cpu_utilization", - "pod_cpu_utilization_over_pod_limit", - "pod_memory_reserved_capacity", - "pod_memory_utilization", - "pod_memory_utilization_over_pod_limit", - "pod_network_rx_bytes", - "pod_network_tx_bytes", - "service_number_of_running_pods", - } -} +//func (e *EKSDaemonTestRunner) GetMeasuredMetrics() []string { +// return []string{ +// "cluster_failed_node_count", +// "cluster_node_count", +// "namespace_number_of_running_pods", +// "node_cpu_limit", +// "node_cpu_reserved_capacity", +// "node_cpu_usage_total", +// "node_cpu_utilization", +// "node_filesystem_utilization", +// "node_memory_limit", +// "node_memory_reserved_capacity", +// "node_memory_utilization", +// "node_memory_working_set", +// "node_network_total_bytes", +// "node_number_of_running_containers", +// "node_number_of_running_pods", +// "pod_cpu_reserved_capacity", +// "pod_cpu_utilization", +// "pod_cpu_utilization_over_pod_limit", +// "pod_memory_reserved_capacity", +// "pod_memory_utilization", +// "pod_memory_utilization_over_pod_limit", +// "pod_network_rx_bytes", +// "pod_network_tx_bytes", +// "service_number_of_running_pods", +// } +//} func (t *EKSDaemonTestRunner) SetAgentConfig(config test_runner.AgentConfig) {} From 4cded07af148abdf8895ea245f2f0d03fd8e7637 Mon Sep 17 00:00:00 2001 From: Mitali Salvi Date: Sun, 29 Oct 2023 15:05:41 -0400 Subject: [PATCH 3/7] testing metrics --- .../eks_daemonset_test.go | 188 +++++++++--------- 1 file changed, 94 insertions(+), 94 deletions(-) diff --git a/test/metric_value_benchmark/eks_daemonset_test.go b/test/metric_value_benchmark/eks_daemonset_test.go index 78d2f22c6..3580568ef 100644 --- a/test/metric_value_benchmark/eks_daemonset_test.go +++ b/test/metric_value_benchmark/eks_daemonset_test.go @@ -19,10 +19,10 @@ import ( "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" ) -//const containerInsightsNamespace = "ContainerInsights" +const containerInsightsNamespace = "ContainerInsights" // list of metrics with more dimensions e.g. PodName and Namespace -//var metricsWithMoreDimensions = []string{"pod_number_of_container_restarts"} +var metricsWithMoreDimensions = []string{"pod_number_of_container_restarts"} type EKSDaemonTestRunner struct { test_runner.BaseTestRunner @@ -30,11 +30,11 @@ type EKSDaemonTestRunner struct { } func (e *EKSDaemonTestRunner) Validate() status.TestGroupResult { - //metrics := e.GetMeasuredMetrics() + metrics := e.GetMeasuredMetrics() testResults := make([]status.TestResult, 0) - //for _, name := range metrics { - // testResults = append(testResults, e.validateInstanceMetrics(name)) - //} + for _, name := range metrics { + testResults = append(testResults, e.validateInstanceMetrics(name)) + } testResults = append(testResults, e.validateLogs(e.env)) return status.TestGroupResult{ @@ -43,66 +43,66 @@ func (e *EKSDaemonTestRunner) Validate() status.TestGroupResult { } } -//func (e *EKSDaemonTestRunner) validateInstanceMetrics(name string) status.TestResult { -// testResult := status.TestResult{ -// Name: name, -// Status: status.FAILED, -// } -// -// dims, failed := e.DimensionFactory.GetDimensions([]dimension.Instruction{ -// { -// Key: "ClusterName", -// Value: dimension.UnknownDimensionValue(), -// }, -// }) -// if len(failed) > 0 { -// log.Println("failed to get dimensions") -// return testResult -// } -// -// // get list of metrics that has more dimensions for container insights -// // this is to avoid adding more dimension provider for non-trivial dimensions e.g. PodName -// listFetcher := metric.MetricListFetcher{} -// if slices.Contains(metricsWithMoreDimensions, name) { -// metrics, err := listFetcher.Fetch(containerInsightsNamespace, name, dims) -// if err != nil { -// log.Println("failed to fetch metric list", err) -// return testResult -// } -// -// if len(metrics) < 1 { -// log.Println("metric list is empty") -// return testResult -// } -// -// // just verify 1 of returned metrics for values -// for _, dim := range metrics[0].Dimensions { -// // skip since it's provided by dimension provider -// if *dim.Name == "ClusterName" { -// continue -// } -// -// dims = append(dims, types.Dimension{ -// Name: dim.Name, -// Value: dim.Value, -// }) -// } -// } -// -// valueFetcher := metric.MetricValueFetcher{} -// values, err := valueFetcher.Fetch(containerInsightsNamespace, name, dims, metric.AVERAGE, metric.HighResolutionStatPeriod) -// if err != nil { -// log.Println("failed to fetch metrics", err) -// return testResult -// } -// -// if !metric.IsAllValuesGreaterThanOrEqualToExpectedValue(name, values, 0) { -// return testResult -// } -// -// testResult.Status = status.SUCCESSFUL -// return testResult -//} +func (e *EKSDaemonTestRunner) validateInstanceMetrics(name string) status.TestResult { + testResult := status.TestResult{ + Name: name, + Status: status.FAILED, + } + + dims, failed := e.DimensionFactory.GetDimensions([]dimension.Instruction{ + { + Key: "ClusterName", + Value: dimension.UnknownDimensionValue(), + }, + }) + if len(failed) > 0 { + log.Println("failed to get dimensions") + return testResult + } + + // get list of metrics that has more dimensions for container insights + // this is to avoid adding more dimension provider for non-trivial dimensions e.g. PodName + listFetcher := metric.MetricListFetcher{} + if slices.Contains(metricsWithMoreDimensions, name) { + metrics, err := listFetcher.Fetch(containerInsightsNamespace, name, dims) + if err != nil { + log.Println("failed to fetch metric list", err) + return testResult + } + + if len(metrics) < 1 { + log.Println("metric list is empty") + return testResult + } + + // just verify 1 of returned metrics for values + for _, dim := range metrics[0].Dimensions { + // skip since it's provided by dimension provider + if *dim.Name == "ClusterName" { + continue + } + + dims = append(dims, types.Dimension{ + Name: dim.Name, + Value: dim.Value, + }) + } + } + + valueFetcher := metric.MetricValueFetcher{} + values, err := valueFetcher.Fetch(containerInsightsNamespace, name, dims, metric.AVERAGE, metric.HighResolutionStatPeriod) + if err != nil { + log.Println("failed to fetch metrics", err) + return testResult + } + + if !metric.IsAllValuesGreaterThanOrEqualToExpectedValue(name, values, 0) { + return testResult + } + + testResult.Status = status.SUCCESSFUL + return testResult +} func (e *EKSDaemonTestRunner) validateLogs(env *environment.MetaData) status.TestResult { testResult := status.TestResult{ @@ -169,34 +169,34 @@ func (e *EKSDaemonTestRunner) GetAgentRunDuration() time.Duration { return time.Minute * 3 } -//func (e *EKSDaemonTestRunner) GetMeasuredMetrics() []string { -// return []string{ -// "cluster_failed_node_count", -// "cluster_node_count", -// "namespace_number_of_running_pods", -// "node_cpu_limit", -// "node_cpu_reserved_capacity", -// "node_cpu_usage_total", -// "node_cpu_utilization", -// "node_filesystem_utilization", -// "node_memory_limit", -// "node_memory_reserved_capacity", -// "node_memory_utilization", -// "node_memory_working_set", -// "node_network_total_bytes", -// "node_number_of_running_containers", -// "node_number_of_running_pods", -// "pod_cpu_reserved_capacity", -// "pod_cpu_utilization", -// "pod_cpu_utilization_over_pod_limit", -// "pod_memory_reserved_capacity", -// "pod_memory_utilization", -// "pod_memory_utilization_over_pod_limit", -// "pod_network_rx_bytes", -// "pod_network_tx_bytes", -// "service_number_of_running_pods", -// } -//} +func (e *EKSDaemonTestRunner) GetMeasuredMetrics() []string { + return []string{ + "cluster_failed_node_count", + "cluster_node_count", + "namespace_number_of_running_pods", + "node_cpu_limit", + "node_cpu_reserved_capacity", + "node_cpu_usage_total", + "node_cpu_utilization", + "node_filesystem_utilization", + "node_memory_limit", + "node_memory_reserved_capacity", + "node_memory_utilization", + "node_memory_working_set", + "node_network_total_bytes", + "node_number_of_running_containers", + "node_number_of_running_pods", + "pod_cpu_reserved_capacity", + "pod_cpu_utilization", + "pod_cpu_utilization_over_pod_limit", + "pod_memory_reserved_capacity", + "pod_memory_utilization", + "pod_memory_utilization_over_pod_limit", + "pod_network_rx_bytes", + "pod_network_tx_bytes", + "service_number_of_running_pods", + } +} func (t *EKSDaemonTestRunner) SetAgentConfig(config test_runner.AgentConfig) {} From a46a5a446e67cce0e6d0e33a13aa1c4e3a17afb2 Mon Sep 17 00:00:00 2001 From: Mitali Salvi Date: Sun, 29 Oct 2023 15:48:04 -0400 Subject: [PATCH 4/7] testing metrics --- test/metric_value_benchmark/eks_daemonset_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/metric_value_benchmark/eks_daemonset_test.go b/test/metric_value_benchmark/eks_daemonset_test.go index 3580568ef..35e4298a7 100644 --- a/test/metric_value_benchmark/eks_daemonset_test.go +++ b/test/metric_value_benchmark/eks_daemonset_test.go @@ -12,7 +12,12 @@ import ( "log" "time" + "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" + "golang.org/x/exp/slices" + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/test/metric" + "github.com/aws/amazon-cloudwatch-agent-test/test/metric/dimension" "github.com/aws/amazon-cloudwatch-agent-test/test/metric_value_benchmark/eks_resources" "github.com/aws/amazon-cloudwatch-agent-test/test/status" "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" From c1ed93841368c85cfccc77a1fd73051b5dd69dc4 Mon Sep 17 00:00:00 2001 From: Mitali Salvi Date: Sun, 29 Oct 2023 16:09:35 -0400 Subject: [PATCH 5/7] testing metrics --- .../eks_resources/test_schemas/cluster.json | 3 +-- .../eks_resources/test_schemas/cluster_daemonset.json | 3 +-- .../eks_resources/test_schemas/cluster_deployment.json | 3 +-- .../eks_resources/test_schemas/cluster_service.json | 3 +-- .../eks_resources/test_schemas/container.json | 3 +-- .../eks_resources/test_schemas/container_fs.json | 3 +-- .../eks_resources/test_schemas/control_plane.json | 3 +-- .../eks_resources/test_schemas/node.json | 3 +-- .../eks_resources/test_schemas/node_disk_io.json | 3 +-- .../eks_resources/test_schemas/node_fs.json | 3 +-- .../eks_resources/test_schemas/node_net.json | 3 +-- .../metric_value_benchmark/eks_resources/test_schemas/pod.json | 3 +-- .../eks_resources/test_schemas/pod_net.json | 3 +-- 13 files changed, 13 insertions(+), 26 deletions(-) diff --git a/test/metric_value_benchmark/eks_resources/test_schemas/cluster.json b/test/metric_value_benchmark/eks_resources/test_schemas/cluster.json index 6041a219f..514e9edb1 100644 --- a/test/metric_value_benchmark/eks_resources/test_schemas/cluster.json +++ b/test/metric_value_benchmark/eks_resources/test_schemas/cluster.json @@ -22,6 +22,5 @@ "Timestamp", "Version", "CloudWatchMetrics" - ], - "additionalProperties": false + ] } \ No newline at end of file diff --git a/test/metric_value_benchmark/eks_resources/test_schemas/cluster_daemonset.json b/test/metric_value_benchmark/eks_resources/test_schemas/cluster_daemonset.json index 56377ee2a..e8aa9420c 100644 --- a/test/metric_value_benchmark/eks_resources/test_schemas/cluster_daemonset.json +++ b/test/metric_value_benchmark/eks_resources/test_schemas/cluster_daemonset.json @@ -28,6 +28,5 @@ "NodeName", "PodName", "Namespace" - ], - "additionalProperties": false + ] } \ No newline at end of file diff --git a/test/metric_value_benchmark/eks_resources/test_schemas/cluster_deployment.json b/test/metric_value_benchmark/eks_resources/test_schemas/cluster_deployment.json index 56377ee2a..e8aa9420c 100644 --- a/test/metric_value_benchmark/eks_resources/test_schemas/cluster_deployment.json +++ b/test/metric_value_benchmark/eks_resources/test_schemas/cluster_deployment.json @@ -28,6 +28,5 @@ "NodeName", "PodName", "Namespace" - ], - "additionalProperties": false + ] } \ No newline at end of file diff --git a/test/metric_value_benchmark/eks_resources/test_schemas/cluster_service.json b/test/metric_value_benchmark/eks_resources/test_schemas/cluster_service.json index 5c8294023..2bc4ecfb4 100644 --- a/test/metric_value_benchmark/eks_resources/test_schemas/cluster_service.json +++ b/test/metric_value_benchmark/eks_resources/test_schemas/cluster_service.json @@ -34,6 +34,5 @@ "Service", "kubernetes", "CloudWatchMetrics" - ], - "additionalProperties": false + ] } \ No newline at end of file diff --git a/test/metric_value_benchmark/eks_resources/test_schemas/container.json b/test/metric_value_benchmark/eks_resources/test_schemas/container.json index aa30c0e36..980c23526 100644 --- a/test/metric_value_benchmark/eks_resources/test_schemas/container.json +++ b/test/metric_value_benchmark/eks_resources/test_schemas/container.json @@ -106,6 +106,5 @@ "PodName", "Namespace", "kubernetes" - ], - "additionalProperties": false + ] } \ No newline at end of file diff --git a/test/metric_value_benchmark/eks_resources/test_schemas/container_fs.json b/test/metric_value_benchmark/eks_resources/test_schemas/container_fs.json index 50625cadc..c13fb7ff7 100644 --- a/test/metric_value_benchmark/eks_resources/test_schemas/container_fs.json +++ b/test/metric_value_benchmark/eks_resources/test_schemas/container_fs.json @@ -84,6 +84,5 @@ "device", "fstype", "kubernetes" - ], - "additionalProperties": false + ] } \ No newline at end of file diff --git a/test/metric_value_benchmark/eks_resources/test_schemas/control_plane.json b/test/metric_value_benchmark/eks_resources/test_schemas/control_plane.json index 80391615c..903122871 100644 --- a/test/metric_value_benchmark/eks_resources/test_schemas/control_plane.json +++ b/test/metric_value_benchmark/eks_resources/test_schemas/control_plane.json @@ -37,6 +37,5 @@ "Timestamp", "Version", "CloudWatchMetrics" - ], - "additionalProperties": false + ] } \ No newline at end of file diff --git a/test/metric_value_benchmark/eks_resources/test_schemas/node.json b/test/metric_value_benchmark/eks_resources/test_schemas/node.json index 3215d0b37..e112b5f12 100644 --- a/test/metric_value_benchmark/eks_resources/test_schemas/node.json +++ b/test/metric_value_benchmark/eks_resources/test_schemas/node.json @@ -78,6 +78,5 @@ "NodeName", "kubernetes", "CloudWatchMetrics" - ], - "additionalProperties": false + ] } \ No newline at end of file diff --git a/test/metric_value_benchmark/eks_resources/test_schemas/node_disk_io.json b/test/metric_value_benchmark/eks_resources/test_schemas/node_disk_io.json index effa56226..5f190b686 100644 --- a/test/metric_value_benchmark/eks_resources/test_schemas/node_disk_io.json +++ b/test/metric_value_benchmark/eks_resources/test_schemas/node_disk_io.json @@ -47,6 +47,5 @@ "NodeName", "kubernetes", "device" - ], - "additionalProperties": false + ] } \ No newline at end of file diff --git a/test/metric_value_benchmark/eks_resources/test_schemas/node_fs.json b/test/metric_value_benchmark/eks_resources/test_schemas/node_fs.json index e514c13da..5417c80f5 100644 --- a/test/metric_value_benchmark/eks_resources/test_schemas/node_fs.json +++ b/test/metric_value_benchmark/eks_resources/test_schemas/node_fs.json @@ -47,6 +47,5 @@ "kubernetes", "device", "fstype" - ], - "additionalProperties": false + ] } \ No newline at end of file diff --git a/test/metric_value_benchmark/eks_resources/test_schemas/node_net.json b/test/metric_value_benchmark/eks_resources/test_schemas/node_net.json index 4ce5aa371..8072909ef 100644 --- a/test/metric_value_benchmark/eks_resources/test_schemas/node_net.json +++ b/test/metric_value_benchmark/eks_resources/test_schemas/node_net.json @@ -45,6 +45,5 @@ "NodeName", "kubernetes", "interface" - ], - "additionalProperties": false + ] } \ No newline at end of file diff --git a/test/metric_value_benchmark/eks_resources/test_schemas/pod.json b/test/metric_value_benchmark/eks_resources/test_schemas/pod.json index 1472e9c61..c202160fc 100644 --- a/test/metric_value_benchmark/eks_resources/test_schemas/pod.json +++ b/test/metric_value_benchmark/eks_resources/test_schemas/pod.json @@ -105,6 +105,5 @@ "Namespace", "kubernetes", "CloudWatchMetrics" - ], - "additionalProperties": false + ] } \ No newline at end of file diff --git a/test/metric_value_benchmark/eks_resources/test_schemas/pod_net.json b/test/metric_value_benchmark/eks_resources/test_schemas/pod_net.json index f47a6091a..b4e79f6d3 100644 --- a/test/metric_value_benchmark/eks_resources/test_schemas/pod_net.json +++ b/test/metric_value_benchmark/eks_resources/test_schemas/pod_net.json @@ -55,6 +55,5 @@ "Namespace", "interface", "kubernetes" - ], - "additionalProperties": false + ] } \ No newline at end of file From 1382e434a88c6a6b051ef44f220010d7a13efddf Mon Sep 17 00:00:00 2001 From: Mitali Salvi Date: Sun, 29 Oct 2023 19:04:44 -0400 Subject: [PATCH 6/7] fixing test-case-generator --- generator/test_case_generator.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index 1b4551c68..3c9a0ed91 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -174,19 +174,19 @@ var testTypeToTestConfig = map[string][]testConfig{ testDir: "./test/metric_value_benchmark", targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, }, - //{ - // testDir: "./test/statsd", terraformDir: "terraform/eks/daemon/statsd", - // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - //}, - //{ - // testDir: "./test/emf", terraformDir: "terraform/eks/daemon/emf", - // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - //}, - //{ - // testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/d", - // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - //}, - //{testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/bit"}, + { + testDir: "./test/statsd", terraformDir: "terraform/eks/daemon/statsd", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + }, + { + testDir: "./test/emf", terraformDir: "terraform/eks/daemon/emf", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + }, + { + testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/d", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + }, + {testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/bit"}, }, "eks_deployment": { {testDir: "./test/metric_value_benchmark"}, From ad99caa9e7474cd71641d781db0bdad25b81fd5a Mon Sep 17 00:00:00 2001 From: Mitali Salvi Date: Mon, 30 Oct 2023 13:37:24 -0400 Subject: [PATCH 7/7] Updating control-plane json --- .../eks_resources/test_schemas/control_plane.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/metric_value_benchmark/eks_resources/test_schemas/control_plane.json b/test/metric_value_benchmark/eks_resources/test_schemas/control_plane.json index 903122871..fe3531447 100644 --- a/test/metric_value_benchmark/eks_resources/test_schemas/control_plane.json +++ b/test/metric_value_benchmark/eks_resources/test_schemas/control_plane.json @@ -19,7 +19,7 @@ "apiserver_request_total": {}, "apiserver_request_total_5xx": {}, "apiserver_admission_controller_admission_duration_seconds": {}, - "apiserver_admission_step_admission_duration_seconds", + "apiserver_admission_step_admission_duration_seconds": {}, "etcd_request_duration_seconds": {}, "rest_client_requests_total": {}, "apiserver_current_inflight_requests": {},