From 93b96ef8089c796f4707c2c01f24e1927123ba70 Mon Sep 17 00:00:00 2001 From: Eddy Decena Date: Wed, 17 Apr 2024 10:05:56 -0400 Subject: [PATCH 1/8] fix: Add tags to Datadog agent --- common.tf | 1 + templates.tf | 1 + templates/datadog-agent.json | 4 ++++ 3 files changed, 6 insertions(+) diff --git a/common.tf b/common.tf index 0b11d38..f4ef2c5 100644 --- a/common.tf +++ b/common.tf @@ -7,6 +7,7 @@ locals { instance_alias = var.instance_name == "" ? "waggledance" : format("waggledance-%s", var.instance_name) remote_metastore_zone_prefix = var.instance_name == "" ? "remote-metastore" : format("remote-metastore-%s", var.instance_name) + datadog_tags = join(" ", formatlist("%s:%s", keys(var.tags), values(var.tags))) } data "aws_caller_identity" "current" {} diff --git a/templates.tf b/templates.tf index fee7052..d145eb3 100644 --- a/templates.tf +++ b/templates.tf @@ -7,6 +7,7 @@ locals { default_exposed_endpoints = "health,info,metrics" exposed_endpoints = var.prometheus_enabled ? join(",", [local.default_exposed_endpoints, "prometheus"]) : local.default_exposed_endpoints + datadog_tags = local.datadog_tags } data "template_file" "endpoints_server_yaml" { diff --git a/templates/datadog-agent.json b/templates/datadog-agent.json index d2f49fb..f107910 100644 --- a/templates/datadog-agent.json +++ b/templates/datadog-agent.json @@ -16,6 +16,10 @@ "name": "DD_API_KEY", "value": "${datadog_secret_key}" }, + { + "name": "DD_TAGS", + "value": "${datadog_tags}" + }, { "name": "ECS_FARGATE", "value": "true" From 602540eebbaa916a884f198f729343e2f520c977 Mon Sep 17 00:00:00 2001 From: Eddy Decena Date: Thu, 18 Apr 2024 09:19:38 -0400 Subject: [PATCH 2/8] fix: Add tags to Datadog agent --- common.tf | 1 - templates.tf | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/common.tf b/common.tf index f4ef2c5..0b11d38 100644 --- a/common.tf +++ b/common.tf @@ -7,7 +7,6 @@ locals { instance_alias = var.instance_name == "" ? "waggledance" : format("waggledance-%s", var.instance_name) remote_metastore_zone_prefix = var.instance_name == "" ? "remote-metastore" : format("remote-metastore-%s", var.instance_name) - datadog_tags = join(" ", formatlist("%s:%s", keys(var.tags), values(var.tags))) } data "aws_caller_identity" "current" {} diff --git a/templates.tf b/templates.tf index d145eb3..4fd5ae1 100644 --- a/templates.tf +++ b/templates.tf @@ -7,7 +7,7 @@ locals { default_exposed_endpoints = "health,info,metrics" exposed_endpoints = var.prometheus_enabled ? join(",", [local.default_exposed_endpoints, "prometheus"]) : local.default_exposed_endpoints - datadog_tags = local.datadog_tags + datadog_tags = join(" ", formatlist("%s:%s", keys(var.tags), values(var.tags))) } data "template_file" "endpoints_server_yaml" { From 262bfc81ba5fae84708e947450ee0005b3448cee Mon Sep 17 00:00:00 2001 From: Eddy Decena Date: Thu, 18 Apr 2024 09:23:33 -0400 Subject: [PATCH 3/8] fix: Add tags to Datadog agent --- templates.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/templates.tf b/templates.tf index 4fd5ae1..b68f641 100644 --- a/templates.tf +++ b/templates.tf @@ -206,5 +206,6 @@ data "template_file" "datadog-agent" { wd_instance_type = var.wd_instance_type metrics_port = var.metrics_port datadog_agent_version = var.datadog_agent_version + datadog_tags = local.datadog_tags } } From 754f6d218dc23c0ea524284ef2d36dce324c6f64 Mon Sep 17 00:00:00 2001 From: Eddy Decena Date: Thu, 18 Apr 2024 13:44:56 -0400 Subject: [PATCH 4/8] fix: Add tags to Datadog agent --- ecs.tf | 9 +- templates.tf | 24 ++-- templates/datadog-agent.json | 35 ------ templates/waggledance.json | 215 ++++++++++++++++++++--------------- 4 files changed, 132 insertions(+), 151 deletions(-) delete mode 100644 templates/datadog-agent.json diff --git a/ecs.tf b/ecs.tf index 1d4ab6d..908d2e0 100644 --- a/ecs.tf +++ b/ecs.tf @@ -18,7 +18,7 @@ resource "aws_ecs_service" "waggledance_service" { task_definition = aws_ecs_task_definition.waggledance[0].arn desired_count = var.wd_ecs_task_count - propagate_tags = "SERVICE" + # propagate_tags = "SERVICE" tags = var.tags network_configuration { @@ -52,12 +52,7 @@ resource "aws_ecs_task_definition" "waggledance" { memory = var.memory cpu = var.cpu requires_compatibilities = ["EC2", "FARGATE"] - container_definitions = < 0 ? chomp(data.external.datadog_key[0].result["api_key"]) : "" - wd_instance_type = var.wd_instance_type metrics_port = var.metrics_port datadog_agent_version = var.datadog_agent_version + include_datadog_agent = var.include_datadog_agent datadog_tags = local.datadog_tags + datadog_secret_key = length(var.datadog_key_secret_name) > 0 ? chomp(data.external.datadog_key[0].result["api_key"]) : "" } -} +} \ No newline at end of file diff --git a/templates/datadog-agent.json b/templates/datadog-agent.json deleted file mode 100644 index f107910..0000000 --- a/templates/datadog-agent.json +++ /dev/null @@ -1,35 +0,0 @@ -, -{ - "name": "datadog-agent", - "image": "public.ecr.aws/datadog/agent:${datadog_agent_version}", - "essential": true, - "logConfiguration": { - "logDriver": "awslogs", - "options": { - "awslogs-group": "${loggroup}", - "awslogs-region": "${region}", - "awslogs-stream-prefix": "/" - } - }, - "environment": [ - { - "name": "DD_API_KEY", - "value": "${datadog_secret_key}" - }, - { - "name": "DD_TAGS", - "value": "${datadog_tags}" - }, - { - "name": "ECS_FARGATE", - "value": "true" - } - ], - "healthCheck": { - "command": ["CMD-SHELL", "curl -f http://localhost:18000/actuator/health || exit 1"], - "interval": 5, - "retries": 3, - "startPeriod": 60, - "timeout": 5 - } -} diff --git a/templates/waggledance.json b/templates/waggledance.json index 84c45d7..0e0e52e 100644 --- a/templates/waggledance.json +++ b/templates/waggledance.json @@ -1,92 +1,123 @@ -{ - "name": "waggledance", - "image": "${docker_image}:${docker_version}", - ${docker_auth} - "essential": true, - "logConfiguration": { - "logDriver": "awslogs", - "options": { - "awslogs-group": "${loggroup}", - "awslogs-region": "${region}", - "awslogs-stream-prefix": "/" - } - }, - "dockerLabels": { - "com.datadoghq.ad.instances": "[{\"openmetrics_endpoint\": \"http://%%host%%:18000/actuator/prometheus\", \"namespace\": \"waggledance\", \"metrics\": [\"metastore_status\",\"counter_*\",\"jvm_*\",\"system_*\",\"timer_*\",\"http_*\",\"process_*\"], \"tags\": { \"region\":\"${region}\", \"instance_type\":\"${wd_instance_type}\"} }]", - "com.datadoghq.ad.check_names": "[\"openmetrics\"]", - "com.datadoghq.ad.init_configs": "[{}]" - }, - "portMappings": [ - { - "containerPort": 48869, - "hostPort": 48869 - } - ], - "environment":[ - { - "name": "HEAPSIZE", - "value": "${heapsize}" - }, - { - "name": "LOGLEVEL", - "value": "${loglevel}" - }, - { - "name": "INVOCATIONLOGLEVEL", - "value": "${invocationloglevel}" - }, - { - "name": "SERVER_YAML", - "value": "${server_yaml}" - }, - { - "name": "FEDERATION_YAML", - "value": "${federation_yaml}" - }, - { - "name": "HIVE_SITE_XML", - "value": "${hive_site_xml}" - }, - { - "name": "BASTION_SSH_KEY_ARN", - "value": "${bastion_ssh_key_arn}" - }, - { - "name": "LOG4J_FORMAT_MSG_NO_LOOKUPS", - "value": "true" - } - ], - "healthCheck": { - "command": ["CMD-SHELL", "curl -f http://localhost:18000/actuator/health || exit 1"], - "interval": 5, - "retries": 3, - "startPeriod": 60, - "timeout": 5 - }, - "ulimits": [ - { - "name": "nofile", - "softLimit": 65536, - "hardLimit": 65536 - }, - { - "name": "nproc", - "softLimit": 65536, - "hardLimit": 65536 - } - ], - "systemControls": [ - { - "namespace": "net.ipv4.tcp_keepalive_time", - "value": "${tcp_keepalive_time}" - }, - { - "namespace": "net.ipv4.tcp_keepalive_intvl", - "value": "${tcp_keepalive_intvl}" - }, - { - "namespace": "net.ipv4.tcp_keepalive_probes", - "value": "${tcp_keepalive_probes}" - } - ] -} +[ + { + "name": "waggledance", + "image": "${docker_image}:${docker_version}", + ${docker_auth} + "essential": true, + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "${loggroup}", + "awslogs-region": "${region}", + "awslogs-stream-prefix": "/" + } + }, + "dockerLabels": { + "com.datadoghq.ad.instances": "[{\"openmetrics_endpoint\": \"http://%%host%%:18000/actuator/prometheus\", \"namespace\": \"waggledance\", \"metrics\": [\"metastore_status\",\"counter_*\",\"jvm_*\",\"system_*\",\"timer_*\",\"http_*\",\"process_*\"], \"tags\": { \"region\":\"${region}\", \"instance_type\":\"${wd_instance_type}\"} }]", + "com.datadoghq.ad.check_names": "[\"openmetrics\"]", + "com.datadoghq.ad.init_configs": "[{}]" + }, + "portMappings": [ + { + "containerPort": 48869, + "hostPort": 48869 + } + ], + "environment": [ + { + "name": "HEAPSIZE", + "value": "${heapsize}" + }, + { + "name": "LOGLEVEL", + "value": "${loglevel}" + }, + { + "name": "INVOCATIONLOGLEVEL", + "value": "${invocationloglevel}" + }, + { + "name": "SERVER_YAML", + "value": "${server_yaml}" + }, + { + "name": "FEDERATION_YAML", + "value": "${federation_yaml}" + }, + { + "name": "HIVE_SITE_XML", + "value": "${hive_site_xml}" + }, + { + "name": "BASTION_SSH_KEY_ARN", + "value": "${bastion_ssh_key_arn}" + }, + { + "name": "LOG4J_FORMAT_MSG_NO_LOOKUPS", + "value": "true" + } + ], + "healthCheck": { + "command": ["CMD-SHELL", "curl -f http://localhost:18000/actuator/health || exit 1"], + "interval": 5, + "retries": 3, + "startPeriod": 60, + "timeout": 5 + }, + "ulimits": [ + { + "name": "nofile", + "softLimit": 65536, + "hardLimit": 65536 + }, + { + "name": "nproc", + "softLimit": 65536, + "hardLimit": 65536 + } + ], + "systemControls": [ + { + "namespace": "net.ipv4.tcp_keepalive_time", + "value": "${tcp_keepalive_time}" + }, + { + "namespace": "net.ipv4.tcp_keepalive_intvl", + "value": "${tcp_keepalive_intvl}" + }, + { + "namespace": "net.ipv4.tcp_keepalive_probes", + "value": "${tcp_keepalive_probes}" + } + ] + } + %{ if datadog_agent_enabled } + ,{ + "name": "datadog-agent", + "image": "public.ecr.aws/datadog/agent:${datadog_agent_version}", + "essential": true, + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "${loggroup}", + "awslogs-region": "${region}", + "awslogs-stream-prefix": "/" + } + }, + "environment": [ + { + "name": "DD_API_KEY", + "value": "${datadog_secret_key}" + }, + { + "name": "ECS_FARGATE", + "value": "true" + }, + { + "name": "DD_TAGS", + "value": "${datadog_tags}" + } + ] + } +%{ endif } +] \ No newline at end of file From 769b76738b4e51b407edfcca586d3137dff628de Mon Sep 17 00:00:00 2001 From: Eddy Decena Date: Thu, 18 Apr 2024 13:56:32 -0400 Subject: [PATCH 5/8] fix: Add tags to Datadog agent --- templates/waggledance.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/waggledance.json b/templates/waggledance.json index 0e0e52e..5c2ab38 100644 --- a/templates/waggledance.json +++ b/templates/waggledance.json @@ -91,7 +91,7 @@ } ] } - %{ if datadog_agent_enabled } + %{ if include_datadog_agent } ,{ "name": "datadog-agent", "image": "public.ecr.aws/datadog/agent:${datadog_agent_version}", From 37dc6a8ba2b7de07e64f2dc408e06d01b0dd64bb Mon Sep 17 00:00:00 2001 From: Eddy Decena Date: Mon, 22 Apr 2024 00:57:12 -0400 Subject: [PATCH 6/8] fix: Add tags to Datadog agent --- ecs.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ecs.tf b/ecs.tf index 908d2e0..3fa4eae 100644 --- a/ecs.tf +++ b/ecs.tf @@ -18,6 +18,8 @@ resource "aws_ecs_service" "waggledance_service" { task_definition = aws_ecs_task_definition.waggledance[0].arn desired_count = var.wd_ecs_task_count + platform_version = "1.4" + # propagate_tags = "SERVICE" tags = var.tags From ca90b1bdf0fd110727d8de668edf3f30fffe9b4a Mon Sep 17 00:00:00 2001 From: Eddy Decena Date: Mon, 22 Apr 2024 01:17:07 -0400 Subject: [PATCH 7/8] fix: Add tags to Datadog agent --- ecs.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs.tf b/ecs.tf index 3fa4eae..cf7164d 100644 --- a/ecs.tf +++ b/ecs.tf @@ -18,7 +18,7 @@ resource "aws_ecs_service" "waggledance_service" { task_definition = aws_ecs_task_definition.waggledance[0].arn desired_count = var.wd_ecs_task_count - platform_version = "1.4" + platform_version = "1.4.0" # propagate_tags = "SERVICE" tags = var.tags From efbf9cfd7dff59f620bb7a92d1c283ffd4f1b624 Mon Sep 17 00:00:00 2001 From: Eddy Decena Date: Mon, 6 May 2024 12:13:42 -0400 Subject: [PATCH 8/8] fix: Add tags to DD agent --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dac8528..320e21a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [4.5.1] - 2024-05-06 +### Added +- Adding tags to the Datadog agent +- Refactor Datadog agent implementation + +## [4.5.0] - 2024-04-25 +### Added +- Added optional `extended_server_config`. For instance to configure Waggle Dance Rate limiting configuration. ## [4.5.0] - 2024-04-25 ### Added