Add 4xx Alerts

operations/template/alert.tf

@@ -0,0 +1,45 @@
+data "azurerm_monitor_action_group" "notify_slack_email" {
+  count               = local.non_pr_environment ? 1 : 0
+  resource_group_name = data.azurerm_resource_group.group.name
+  name                = "cdcti${var.environment}-actiongroup"
+}
+
+resource "azurerm_monitor_metric_alert" "azure_4XX_alert" {
+  count               = local.non_pr_environment ? 1 : 0
+  name                = "cdc-rs-sftp-${var.environment}-azure-http-4XX-alert"
+  resource_group_name = data.azurerm_resource_group.group.name
+  scopes              = [azurerm_linux_web_app.sftp.id]
+  description         = "Action will be triggered when Http Status Code 4XX is greater than or equal to 3"
+  frequency           = "PT1M" // Checks every 1 minute
+  window_size         = "PT1H" // Every Check looks back 1 hour for 4xx errors
+
+  criteria {
+    metric_namespace = "Microsoft.Web/sites"
+    metric_name      = "Http4xx"
+    aggregation      = "Count"
+    operator         = "GreaterThanOrEqual"
+    threshold        = 3
+  }
+
+  action {
+    action_group_id = data.azurerm_monitor_action_group.notify_slack_email[count.index].id
+  }
+
+  lifecycle {
+    # Ignore changes to tags because the CDC sets these automagically
+    ignore_changes = [
+      tags["business_steward"],
+      tags["center"],
+      tags["environment"],
+      tags["escid"],
+      tags["funding_source"],
+      tags["pii_data"],
+      tags["security_compliance"],
+      tags["security_steward"],
+      tags["support_group"],
+      tags["system"],
+      tags["technical_steward"],
+      tags["zone"]
+    ]
+  }
+}

operations/template/main.tf

@@ -8,6 +8,7 @@ locals {
   rs_domain_prefix               = "${local.selected_rs_environment_prefix}${length(local.selected_rs_environment_prefix) == 0 ? "" : "."}"
   higher_environment_level       = var.environment == "stg" || var.environment == "prd"
   cdc_domain_environment         = var.environment == "dev" || var.environment == "stg" || var.environment == "prd"
+  non_pr_environment = length(regexall("^pr\\d+", var.environment)) == 0
 }
 
 data "azurerm_resource_group" "group" {

src/cmd/main.go

@@ -77,6 +77,16 @@ func setupLogging() {
 func setupHealthCheck() {
 	slog.Info("Bootstrapping health check")
 
+	http.HandleFunc("/test400", func(response http.ResponseWriter, request *http.Request) {
+		slog.Info("4xx ping", slog.String("method", request.Method), slog.String("path", request.URL.String()))
+
+		response.WriteHeader(400)
+		_, err := io.WriteString(response, "400 Peters are Great")
+		if err != nil {
+			slog.Error("Failed to respond to health check", slog.Any(utils.ErrorKey, err))
+		}
+	})
+
 	http.HandleFunc("/", func(response http.ResponseWriter, request *http.Request) {
 		slog.Info("Health check ping", slog.String("method", request.Method), slog.String("path", request.URL.String()))