Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: Add tags to Datadog agent #142

Open
wants to merge 9 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,14 @@ All notable changes to this project will be documented in this file.

The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).

## [4.5.1] - 2024-05-06
### Added
- Adding tags to the Datadog agent
- Refactor Datadog agent implementation

## [4.5.0] - 2024-04-25
### Added
- Added optional `extended_server_config`. For instance to configure Waggle Dance Rate limiting configuration.

## [4.5.0] - 2024-04-25
### Added
Expand Down
11 changes: 4 additions & 7 deletions ecs.tf
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,9 @@ resource "aws_ecs_service" "waggledance_service" {
task_definition = aws_ecs_task_definition.waggledance[0].arn
desired_count = var.wd_ecs_task_count

propagate_tags = "SERVICE"
platform_version = "1.4.0"

# propagate_tags = "SERVICE"
tags = var.tags

network_configuration {
Expand Down Expand Up @@ -52,12 +54,7 @@ resource "aws_ecs_task_definition" "waggledance" {
memory = var.memory
cpu = var.cpu
requires_compatibilities = ["EC2", "FARGATE"]
container_definitions = <<DEFINITION
[
${data.template_file.waggledance.rendered}
${var.include_datadog_agent ? data.template_file.datadog-agent.rendered : ""}
]
DEFINITION
container_definitions = data.template_file.waggledance.rendered
tags = var.tags
}

Expand Down
26 changes: 9 additions & 17 deletions templates.tf
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
locals {
default_exposed_endpoints = "health,info,metrics"
exposed_endpoints = var.prometheus_enabled ? join(",", [local.default_exposed_endpoints, "prometheus"]) : local.default_exposed_endpoints
datadog_tags = join(" ", formatlist("%s:%s", keys(var.tags), values(var.tags)))
}

data "template_file" "endpoints_server_yaml" {
Expand Down Expand Up @@ -186,25 +187,16 @@ data "template_file" "waggledance" {
hive_site_xml = length(var.alluxio_endpoints) == 0 ? "" : base64encode(data.template_file.hive_site_xml.rendered)
bastion_ssh_key_arn = var.bastion_ssh_key_secret_name == "" ? "" : join("", data.aws_secretsmanager_secret.bastion_ssh_key.*.arn)
docker_auth = var.docker_registry_auth_secret_name == "" ? "" : format("\"repositoryCredentials\" :{\n \"credentialsParameter\":\"%s\"\n},", join("\",\"", concat(data.aws_secretsmanager_secret.docker_registry.*.arn)))
tcp_keepalive_time = var.tcp_keepalive_time
tcp_keepalive_intvl = var.tcp_keepalive_intvl
tcp_keepalive_probes = var.tcp_keepalive_probes
wd_instance_type = var.wd_instance_type
metrics_port = var.metrics_port
datadog_agent_version = var.datadog_agent_version
include_datadog_agent = var.include_datadog_agent
}
}
wd_instance_type = var.wd_instance_type

data "template_file" "datadog-agent" {
template = file("${path.module}/templates/datadog-agent.json")
tcp_keepalive_time = var.tcp_keepalive_time
tcp_keepalive_intvl = var.tcp_keepalive_intvl
tcp_keepalive_probes = var.tcp_keepalive_probes

vars = {
region = var.aws_region
loggroup = var.wd_instance_type == "ecs" ? join("", aws_cloudwatch_log_group.waggledance_ecs.*.name) : ""
datadog_secret_key = length(var.datadog_key_secret_name) > 0 ? chomp(data.external.datadog_key[0].result["api_key"]) : ""
wd_instance_type = var.wd_instance_type
metrics_port = var.metrics_port
datadog_agent_version = var.datadog_agent_version
include_datadog_agent = var.include_datadog_agent
datadog_tags = local.datadog_tags
datadog_secret_key = length(var.datadog_key_secret_name) > 0 ? chomp(data.external.datadog_key[0].result["api_key"]) : ""
}
}
}
31 changes: 0 additions & 31 deletions templates/datadog-agent.json

This file was deleted.

215 changes: 123 additions & 92 deletions templates/waggledance.json
Original file line number Diff line number Diff line change
@@ -1,92 +1,123 @@
{
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Remove the refactoring, keep it as 4 spaces instead of 2 tabs.

"name": "waggledance",
"image": "${docker_image}:${docker_version}",
${docker_auth}
"essential": true,
"logConfiguration": {
"logDriver": "awslogs",
"options": {
"awslogs-group": "${loggroup}",
"awslogs-region": "${region}",
"awslogs-stream-prefix": "/"
}
},
"dockerLabels": {
"com.datadoghq.ad.instances": "[{\"openmetrics_endpoint\": \"http://%%host%%:18000/actuator/prometheus\", \"namespace\": \"waggledance\", \"metrics\": [\"metastore_status\",\"counter_*\",\"jvm_*\",\"system_*\",\"timer_*\",\"http_*\",\"process_*\"], \"tags\": { \"region\":\"${region}\", \"instance_type\":\"${wd_instance_type}\"} }]",
"com.datadoghq.ad.check_names": "[\"openmetrics\"]",
"com.datadoghq.ad.init_configs": "[{}]"
},
"portMappings": [
{
"containerPort": 48869,
"hostPort": 48869
}
],
"environment":[
{
"name": "HEAPSIZE",
"value": "${heapsize}"
},
{
"name": "LOGLEVEL",
"value": "${loglevel}"
},
{
"name": "INVOCATIONLOGLEVEL",
"value": "${invocationloglevel}"
},
{
"name": "SERVER_YAML",
"value": "${server_yaml}"
},
{
"name": "FEDERATION_YAML",
"value": "${federation_yaml}"
},
{
"name": "HIVE_SITE_XML",
"value": "${hive_site_xml}"
},
{
"name": "BASTION_SSH_KEY_ARN",
"value": "${bastion_ssh_key_arn}"
},
{
"name": "LOG4J_FORMAT_MSG_NO_LOOKUPS",
"value": "true"
}
],
"healthCheck": {
"command": ["CMD-SHELL", "curl -f http://localhost:18000/actuator/health || exit 1"],
"interval": 5,
"retries": 3,
"startPeriod": 60,
"timeout": 5
},
"ulimits": [
{
"name": "nofile",
"softLimit": 65536,
"hardLimit": 65536
},
{
"name": "nproc",
"softLimit": 65536,
"hardLimit": 65536
}
],
"systemControls": [
{
"namespace": "net.ipv4.tcp_keepalive_time",
"value": "${tcp_keepalive_time}"
},
{
"namespace": "net.ipv4.tcp_keepalive_intvl",
"value": "${tcp_keepalive_intvl}"
},
{
"namespace": "net.ipv4.tcp_keepalive_probes",
"value": "${tcp_keepalive_probes}"
}
]
}
[
{
"name": "waggledance",
"image": "${docker_image}:${docker_version}",
${docker_auth}
"essential": true,
"logConfiguration": {
"logDriver": "awslogs",
"options": {
"awslogs-group": "${loggroup}",
"awslogs-region": "${region}",
"awslogs-stream-prefix": "/"
}
},
"dockerLabels": {
"com.datadoghq.ad.instances": "[{\"openmetrics_endpoint\": \"http://%%host%%:18000/actuator/prometheus\", \"namespace\": \"waggledance\", \"metrics\": [\"metastore_status\",\"counter_*\",\"jvm_*\",\"system_*\",\"timer_*\",\"http_*\",\"process_*\"], \"tags\": { \"region\":\"${region}\", \"instance_type\":\"${wd_instance_type}\"} }]",
"com.datadoghq.ad.check_names": "[\"openmetrics\"]",
"com.datadoghq.ad.init_configs": "[{}]"
},
"portMappings": [
{
"containerPort": 48869,
"hostPort": 48869
}
],
"environment": [
{
"name": "HEAPSIZE",
"value": "${heapsize}"
},
{
"name": "LOGLEVEL",
"value": "${loglevel}"
},
{
"name": "INVOCATIONLOGLEVEL",
"value": "${invocationloglevel}"
},
{
"name": "SERVER_YAML",
"value": "${server_yaml}"
},
{
"name": "FEDERATION_YAML",
"value": "${federation_yaml}"
},
{
"name": "HIVE_SITE_XML",
"value": "${hive_site_xml}"
},
{
"name": "BASTION_SSH_KEY_ARN",
"value": "${bastion_ssh_key_arn}"
},
{
"name": "LOG4J_FORMAT_MSG_NO_LOOKUPS",
"value": "true"
}
],
"healthCheck": {
"command": ["CMD-SHELL", "curl -f http://localhost:18000/actuator/health || exit 1"],
"interval": 5,
"retries": 3,
"startPeriod": 60,
"timeout": 5
},
"ulimits": [
{
"name": "nofile",
"softLimit": 65536,
"hardLimit": 65536
},
{
"name": "nproc",
"softLimit": 65536,
"hardLimit": 65536
}
],
"systemControls": [
{
"namespace": "net.ipv4.tcp_keepalive_time",
"value": "${tcp_keepalive_time}"
},
{
"namespace": "net.ipv4.tcp_keepalive_intvl",
"value": "${tcp_keepalive_intvl}"
},
{
"namespace": "net.ipv4.tcp_keepalive_probes",
"value": "${tcp_keepalive_probes}"
}
]
}
%{ if include_datadog_agent }
,{
"name": "datadog-agent",
"image": "public.ecr.aws/datadog/agent:${datadog_agent_version}",
"essential": true,
"logConfiguration": {
"logDriver": "awslogs",
"options": {
"awslogs-group": "${loggroup}",
"awslogs-region": "${region}",
"awslogs-stream-prefix": "/"
}
},
"environment": [
{
"name": "DD_API_KEY",
"value": "${datadog_secret_key}"
},
{
"name": "ECS_FARGATE",
"value": "true"
},
{
"name": "DD_TAGS",
"value": "${datadog_tags}"
}
]
}
%{ endif }
]