Merge pull request #6 from rhythmictech/rds-enhanced-monitoring

Add DataDog enhanced monitoring

.github/workflows/pre-commit.yaml

@@ -2,7 +2,7 @@
 name: pre-commit-check
 on:
   push:
-    branches: -- main
+    branches:
       - master
       - prod
       - develop

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 exclude: ".terraform"
 repos:
   - repo: https://github.com/antonbabenko/pre-commit-terraform
-    rev: v1.62.3
+    rev: v1.81.2
     hooks:
       - id: terraform_docs
         always_run: true
@@ -56,7 +56,7 @@ repos:
         files: \.tf(vars)?$
         exclude: examples
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.1.0
+    rev: v4.4.0
     hooks:
       - id: check-case-conflict
       - id: check-json

.tflint.hcl

@@ -2,12 +2,6 @@ config {
   module     = true
 }
 
-plugin "aws" {
-    enabled = true
-    version = "0.5.0"
-    source  = "github.com/terraform-linters/tflint-ruleset-aws"
-}
-
 rule "terraform_deprecated_interpolation" {
   enabled = true
 }
@@ -42,7 +36,7 @@ rule "terraform_naming_convention" {
 }
 
 rule "terraform_required_version" {
-  enabled = true
+  enabled = false
 }
 
 rule "terraform_required_providers" {

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2022 Rhythmic Technologies, Inc.
+Copyright (c) 2023 Rhythmic Technologies, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -31,35 +31,71 @@ module "datadog" {
 ```
 
 ## About
-A bit about this module
+By default it installs the DataDog log forwarder. Can also optionally install the RDS Enhanced metrics forwarder. 
+
+Example adding metrics forwarding and logging:
+```
+module "datadog" {
+  source  = "rhythmictech/datadog/aws"
+  version = "0.3.0"
+
+  name                                          = "datadog-integration"
+  cspm_resource_collection_enabled              = "true"
+  install_log_forwarder                         = true
+  integration_default_namespace_rules           = var.datadog_metric_namespaces
+  install_rds_enhanced_monitoring_lambda        = var.install_rds_enhanced_monitoring_lambda
+  log_forwarder_sources                         = ["lambda"]
+  tags                                          = local.tags
+  use_cspm_permissions                          = true
+}
+
+resource "aws_lambda_permission" "cloudwatch" {
+
+  statement_id  = "datadog-forwarder-RDSCloudWatchLogsPermission"
+  action        = "lambda:InvokeFunction"
+  function_name = reverse(split(":", module.datadog.lambda_arn_forwarder))[0]
+  principal     = "logs.amazonaws.com"
+  source_arn    = "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:/aws/rds/instance/production-db/postgresql:*"
+}
+resource "aws_cloudwatch_log_subscription_filter" "rds_log_forwarding" {
+  name            = "production-db"
+  log_group_name  = "/aws/rds/instance/production-db/postgresql"
+  filter_pattern  = ""
+  destination_arn = module.datadog.lambda_arn_forwarder
+}
+
+```
+
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Requirements
 
 | Name | Version |
 |------|---------|
-| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 0.14 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 3.74 |
-| <a name="requirement_datadog"></a> [datadog](#requirement\_datadog) | ~>3.8 |
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | ~> 1.1 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 4.10 |
+| <a name="requirement_datadog"></a> [datadog](#requirement\_datadog) | ~>3.11 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | 3.74.2 |
-| <a name="provider_datadog"></a> [datadog](#provider\_datadog) | 3.8.1 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | 4.67.0 |
+| <a name="provider_datadog"></a> [datadog](#provider\_datadog) | 3.28.0 |
 
 ## Modules
 
 | Name | Source | Version |
 |------|--------|---------|
+| <a name="module_rds_enhanced_monitoring_lambda_code"></a> [rds\_enhanced\_monitoring\_lambda\_code](#module\_rds\_enhanced\_monitoring\_lambda\_code) | git::https://github.com/DataDog/datadog-serverless-functions.git | aws-dd-forwarder-3.83.0 |
 | <a name="module_tags"></a> [tags](#module\_tags) | rhythmictech/tags/terraform | ~> 1.1 |
 
 ## Resources
 
 | Name | Type |
 |------|------|
 | [aws_cloudformation_stack.datadog_forwarder](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudformation_stack) | resource |
+| [aws_cloudformation_stack.rds_enhanced_monitoring_lambda](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudformation_stack) | resource |
 | [aws_cloudwatch_event_rule.guardduty](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource |
 | [aws_cloudwatch_event_target.guardduty](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource |
 | [aws_iam_policy.datadog](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
@@ -83,13 +119,17 @@ A bit about this module
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_cloudtrail_buckets"></a> [cloudtrail\_buckets](#input\_cloudtrail\_buckets) | Bucket(s) to collect CloudTrail logs from | `list(string)` | `[]` | no |
+| <a name="input_cspm_resource_collection_enabled"></a> [cspm\_resource\_collection\_enabled](#input\_cspm\_resource\_collection\_enabled) | Whether Datadog collects cloud security posture management resources from your AWS account. This includes additional resources not covered under the general resource\_collection. | `string` | `"false"` | no |
 | <a name="input_datadog_account_id"></a> [datadog\_account\_id](#input\_datadog\_account\_id) | DataDog AWS account ID (should not need changed) | `string` | `"464622532012"` | no |
 | <a name="input_datadog_site_name"></a> [datadog\_site\_name](#input\_datadog\_site\_name) | DataDog site (e.g., datadoghq.com) | `string` | `"datadoghq.com"` | no |
 | <a name="input_enable_guardduty_notifications"></a> [enable\_guardduty\_notifications](#input\_enable\_guardduty\_notifications) | Send GuardDuty notifications to Datadog (`install_log_forwarder` must be true) | `bool` | `true` | no |
 | <a name="input_install_log_forwarder"></a> [install\_log\_forwarder](#input\_install\_log\_forwarder) | controls whether log forwarder lambda should be installed | `bool` | `true` | no |
+| <a name="input_install_rds_enhanced_monitoring_lambda"></a> [install\_rds\_enhanced\_monitoring\_lambda](#input\_install\_rds\_enhanced\_monitoring\_lambda) | Bool to install the RDS Enhanced Monitoring Lambda | `bool` | `true` | no |
+| <a name="input_integration_default_namespace_rules"></a> [integration\_default\_namespace\_rules](#input\_integration\_default\_namespace\_rules) | Set all services to disabled by default. | `map(bool)` | <pre>{<br>  "api_gateway": false,<br>  "application_elb": false,<br>  "apprunner": false,<br>  "appstream": false,<br>  "appsync": false,<br>  "athena": false,<br>  "auto_scaling": false,<br>  "backup": false,<br>  "billing": false,<br>  "bracket": false,<br>  "budgeting": false,<br>  "certificatemanager": false,<br>  "cloud9": false,<br>  "cloudfront": false,<br>  "cloudhsm": false,<br>  "cloudsearch": false,<br>  "cloudwatch_events": false,<br>  "cloudwatch_logs": false,<br>  "codeartifact": false,<br>  "codebuild": false,<br>  "codecommit": false,<br>  "codegurureviewer": false,<br>  "codepipeline": false,<br>  "cognito": false,<br>  "collect_custom_metrics": false,<br>  "comprehend": false,<br>  "config": false,<br>  "connect": false,<br>  "crawl_alarms": false,<br>  "dataexchange": false,<br>  "datapipeline": false,<br>  "directconnect": false,<br>  "dms": false,<br>  "documentdb": false,<br>  "dynamodb": false,<br>  "ebs": false,<br>  "ec2": false,<br>  "ec2api": false,<br>  "ec2spot": false,<br>  "ecr": false,<br>  "ecs": false,<br>  "efs": false,<br>  "eks": false,<br>  "elasticache": false,<br>  "elasticbeanstalk": false,<br>  "elasticinference": false,<br>  "elasticmapreducecontainers": false,<br>  "elastictranscoder": false,<br>  "elb": false,<br>  "emr": false,<br>  "es": false,<br>  "firehose": false,<br>  "forecast": false,<br>  "frauddetector": false,<br>  "fsx": false,<br>  "gamelift": false,<br>  "glacier": false,<br>  "glue": false,<br>  "gluedatabrew": false,<br>  "iam": false,<br>  "inspector": false,<br>  "iot": false,<br>  "iotanalytics": false,<br>  "iotevents": false,<br>  "iotgreengrass": false,<br>  "keyspaces": false,<br>  "kinesis": false,<br>  "kinesis_analytics": false,<br>  "kms": false,<br>  "lambda": false,<br>  "lex": false,<br>  "macie": false,<br>  "mediaconnect": false,<br>  "mediaconvert": false,<br>  "mediapackage": false,<br>  "mediatailor": false,<br>  "ml": false,<br>  "mq": false,<br>  "msk": false,<br>  "mwaa": false,<br>  "nat_gateway": false,<br>  "neptune": false,<br>  "network_elb": false,<br>  "networkfirewall": false,<br>  "opsworks": false,<br>  "organizations": false,<br>  "pinpoint": false,<br>  "polly": false,<br>  "qldb": false,<br>  "ram": false,<br>  "rds": false,<br>  "rdsproxy": false,<br>  "redshift": false,<br>  "rekognition": false,<br>  "resourcegroups": false,<br>  "robomaker": false,<br>  "route53": false,<br>  "route53resolver": false,<br>  "s3": false,<br>  "s3storagelens": false,<br>  "sagemaker": false,<br>  "secretsmanager": false,<br>  "service_quotas": false,<br>  "servicecatalog": false,<br>  "ses": false,<br>  "shield": false,<br>  "sns": false,<br>  "sqs": false,<br>  "ssm": false,<br>  "step_functions": false,<br>  "storage_gateway": false,<br>  "swf": false,<br>  "textract": false,<br>  "transitgateway": false,<br>  "translate": false,<br>  "trusted_advisor": false,<br>  "usage": false,<br>  "vpn": false,<br>  "waf": false,<br>  "wafv2": false,<br>  "workspaces": false,<br>  "xray": false<br>}</pre> | no |
 | <a name="input_integration_excluded_regions"></a> [integration\_excluded\_regions](#input\_integration\_excluded\_regions) | Regions to exclude from DataDog monitoring | `list(string)` | `[]` | no |
 | <a name="input_integration_filter_tags"></a> [integration\_filter\_tags](#input\_integration\_filter\_tags) | Tags to filter EC2 instances on (see https://registry.terraform.io/providers/DataDog/datadog/latest/docs/resources/integration_aws) | `list(string)` | `[]` | no |
 | <a name="input_integration_host_tags"></a> [integration\_host\_tags](#input\_integration\_host\_tags) | Tags to apply to instances (see https://registry.terraform.io/providers/DataDog/datadog/latest/docs/resources/integration_aws) | `list(string)` | `[]` | no |
+| <a name="input_integration_namespace_rules"></a> [integration\_namespace\_rules](#input\_integration\_namespace\_rules) | Map of AWS services to allow in the integration. Defaults to none. | `map(bool)` | `{}` | no |
 | <a name="input_log_forwarder_sources"></a> [log\_forwarder\_sources](#input\_log\_forwarder\_sources) | List of services to automatically ingest all logs from (see https://docs.datadoghq.com/api/latest/aws-logs-integration/#get-list-of-aws-log-ready-services) | `list(string)` | `[]` | no |
 | <a name="input_name"></a> [name](#input\_name) | Moniker to apply to all resources in the module | `string` | n/a | yes |
 | <a name="input_tags"></a> [tags](#input\_tags) | User-Defined tags | `map(string)` | `{}` | no |

bin/install-ubuntu.sh

@@ -15,7 +15,7 @@ cd ..
 rm -rf tmp
 
 curl -L "$(curl -sL https://api.github.com/repos/terraform-linters/tflint/releases/latest | grep -o -E "https://.+?_linux_amd64.zip")" > tflint.zip && unzip tflint.zip && rm tflint.zip && sudo mv tflint /usr/bin/
-env GO111MODULE=on go get -u github.com/liamg/tfsec/cmd/tfsec
+go install github.com/aquasecurity/tfsec/cmd/tfsec@latest
 git clone https://github.com/tfutils/tfenv.git ~/.tfenv || true
 mkdir -p ~/.local/bin/
 . ~/.profile

main.tf

@@ -22,14 +22,15 @@ resource "datadog_api_key" "datadog" {
 
 resource "datadog_integration_aws" "datadog" {
   account_id                       = local.account_id
-  account_specific_namespace_rules = merge( var.integration_default_namespace_rules, var.integration_namespace_rules )
-  cspm_resource_collection_enabled = var.cspm_resource_collection_enabled 
+  account_specific_namespace_rules = merge(var.integration_default_namespace_rules, var.integration_namespace_rules)
+  cspm_resource_collection_enabled = var.cspm_resource_collection_enabled
   excluded_regions                 = var.integration_excluded_regions
   filter_tags                      = var.integration_filter_tags
   host_tags                        = var.integration_host_tags
   role_name                        = "DatadogIntegrationRole"
 }
 
+#tfsec:ignore:aws-ssm-secret-use-customer-key
 resource "aws_secretsmanager_secret" "datadog" {
   name_prefix = "${var.name}-api-key"
   description = "Datadog API Key"

rds_enhanced_monitoring.tf

@@ -0,0 +1,49 @@
+# TODO: allow version specification
+# right now you can't use variables in a module's source so we'll have to use the external provider or git submodules
+module "rds_enhanced_monitoring_lambda_code" {
+  source = "git::https://github.com/DataDog/datadog-serverless-functions.git?ref=aws-dd-forwarder-3.83.0"
+}
+
+##########################################
+# this is basically copied from https://github.com/DataDog/datadog-serverless-functions/blob/master/aws/rds_enhanced_monitoring/rds-enhanced-sam-template.yaml
+# because their documentation says to use the SAM repo which points to an out-of-date version
+# and because we load the API key differently
+##########################################
+
+resource "aws_cloudformation_stack" "rds_enhanced_monitoring_lambda" {
+  count = var.install_rds_enhanced_monitoring_lambda ? 1 : 0
+
+  name         = "${var.name}-rds-enhanced-monitoring-forwarder"
+  capabilities = ["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM", "CAPABILITY_AUTO_EXPAND"]
+  depends_on   = [module.rds_enhanced_monitoring_lambda_code]
+
+  template_body = <<EOF
+AWSTemplateFormatVersion: '2010-09-09'
+Transform: AWS::Serverless-2016-10-31
+Description: 'Pushes RDS Enhanced metrics to Datadog.'
+Resources:
+  rdslambdaddfunction:
+    Type: 'AWS::Serverless::Function'
+    Properties:
+      Description: Pushes RDS Enhanced metrics to Datadog.
+      InlineCode: |
+        ${indent(8, file("${path.module}.rds_enhanced_monitoring_lambda_code/aws/rds_enhanced_monitoring/lambda_function.py"))}
+      Environment:
+        Variables:
+          DD_API_KEY_SECRET_ARN: '${aws_secretsmanager_secret_version.datadog.arn}'
+      Events:
+        RDSEnhancedMetrics:
+          Type: CloudWatchLogs
+          Properties:
+            LogGroupName: RDSOSMetrics
+            FilterPattern: ""
+      Handler: index.lambda_handler
+      MemorySize: 128
+      Runtime: python3.9
+      Policies:
+        - AWSLambdaExecute
+        - AWSSecretsManagerGetSecretValuePolicy:
+            SecretArn: '${aws_secretsmanager_secret_version.datadog.arn}'
+      Timeout: 10
+EOF
+}