fix: [PIDM-56] tuning notifications queue params (#2735)

* tuning notifications queue params * set different percentage threshold for storage account alert queries * restore alert for production env * run pre-commit * fix readme ecommerce-app * chore: update docs --------- Co-authored-by: Gianluca Ciuffa <[email protected]> Co-authored-by: Simone Infante <[email protected]>
pagopa · Jan 22, 2025 · c55a829 · c55a829
1 parent 3f25ced
commit c55a829
Showing 1 changed file with 20 additions and 17 deletions.
diff --git a/src/domains/ecommerce-common/03_storage.tf b/src/domains/ecommerce-common/03_storage.tf
@@ -302,25 +302,28 @@ resource "azurerm_monitor_diagnostic_setting" "ecommerce_transient_queue_diagnos
 locals {
   queue_transient_alert_props = var.env_short == "p" ? [
     {
-      "queue_key"   = "transaction-notifications-queue"
-      "severity"    = 1
-      "time_window" = 30
-      "frequency"   = 15
-      "threshold"   = 10
+      "queue_key"         = "transaction-notifications-queue"
+      "severity"          = 1
+      "time_window"       = 30
+      "frequency"         = 15
+      "threshold"         = 20
+      "dynamic_threshold" = 2.0
     },
     {
-      "queue_key"   = "transactions-close-payment-queue"
-      "severity"    = 1
-      "time_window" = 30
-      "frequency"   = 15
-      "threshold"   = 10
+      "queue_key"         = "transactions-close-payment-queue"
+      "severity"          = 1
+      "time_window"       = 30
+      "frequency"         = 15
+      "threshold"         = 10
+      "dynamic_threshold" = 1.0
     },
     {
-      "queue_key"   = "transactions-refund-queue"
-      "severity"    = 1
-      "time_window" = 30
-      "frequency"   = 15
-      "threshold"   = 10
+      "queue_key"         = "transactions-refund-queue"
+      "severity"          = 1
+      "time_window"       = 30
+      "frequency"         = 15
+      "threshold"         = 10
+      "dynamic_threshold" = 1.0
     }
   ] : []
 }
@@ -338,7 +341,7 @@ resource "azurerm_monitor_scheduled_query_rules_alert" "ecommerce_transient_enqu
     custom_webhook_payload = "{}"
   }
   data_source_id = module.ecommerce_storage_transient.id
-  description    = format("Enqueuing rate for queue %s > 1 percent threshold of 'PutMessage' (at least 10) during last ${each.value.time_window} minutes", replace("${each.value.queue_key}", "-", " "))
+  description    = format("Enqueuing rate for queue %s > ${each.value.dynamic_threshold} percent threshold of 'PutMessage' (at least 10) during last ${each.value.time_window} minutes", replace("${each.value.queue_key}", "-", " "))
   enabled        = true
   query = format(<<-QUERY
     let OpCountForQueue = (operation: string, queueKey: string) {
@@ -365,7 +368,7 @@ resource "azurerm_monitor_scheduled_query_rules_alert" "ecommerce_transient_enqu
         | project PutCount, DeleteCount, Diff
     };
     MessageRateForQueue("%s")
-    | where Diff > max_of(PutCount/100, ${each.value.threshold})
+    | where Diff > max_of(PutCount*(${each.value.dynamic_threshold}/100.0), ${each.value.threshold})
     QUERY
     , "/${module.ecommerce_storage_transient.name}/${local.project}-${each.value.queue_key}"
   )