diff --git a/operations/template/alert.tf b/operations/template/alert.tf index de9435d36..4d7211cf9 100644 --- a/operations/template/alert.tf +++ b/operations/template/alert.tf @@ -120,6 +120,47 @@ resource "azurerm_monitor_scheduled_query_rules_alert" "database_token_expired_a ] } } +resource "azurerm_monitor_metric_alert" "low_instance_count_alert" { + count = local.non_pr_environment ? 1 : 0 + name = "cdcti-${var.environment}-azure-low-instance-count-alert" + resource_group_name = data.azurerm_resource_group.group.name + scopes = [azurerm_monitor_autoscale_setting.api_autoscale.id] + description = "The instance count in ${var.environment} is too low" + severity = 2 // warning + frequency = "PT1M" // Checks every 1 minute + window_size = "PT15M" // Every Check, looks back 15 minutes in history + + criteria { + metric_namespace = "Microsoft.Insights/autoscalesettings" + metric_name = "ObservedCapacity" + aggregation = "Average" + operator = "LessThanOrEqual" + threshold = azurerm_monitor_autoscale_setting.api_autoscale.profile[0].capacity[0].default - 0.5 + } + + action { + action_group_id = azurerm_monitor_action_group.notify_slack_email[count.index].id + } + + lifecycle { + # Ignore changes to tags because the CDC sets these automagically + ignore_changes = [ + tags["business_steward"], + tags["center"], + tags["environment"], + tags["escid"], + tags["funding_source"], + tags["pii_data"], + tags["security_compliance"], + tags["security_steward"], + tags["support_group"], + tags["system"], + tags["technical_steward"], + tags["zone"] + ] + } +} + resource "azurerm_monitor_scheduled_query_rules_alert" "ti-log-errors-alert" { count = local.non_pr_environment ? 1 : 0 name = "cdcti-${var.environment}-log-errors-alert" diff --git a/operations/template/main.tf b/operations/template/main.tf index ff42970ce..bc083cff3 100644 --- a/operations/template/main.tf +++ b/operations/template/main.tf @@ -8,7 +8,9 @@ locals { rs_domain_prefix = "${local.selected_rs_environment_prefix}${length(local.selected_rs_environment_prefix) == 0 ? "" : "."}" higher_environment_level = var.environment == "stg" || var.environment == "prd" cdc_domain_environment = var.environment == "dev" || var.environment == "stg" || var.environment == "prd" - non_pr_environment = !strcontains(var.environment, "pr") + + // If the environment looks like pr123, regexall will contain matches. If there are no matches, it's a non-pr env + non_pr_environment = length(regexall("^pr\\d+", var.environment)) == 0 } data "azurerm_resource_group" "group" {