cisco_duo.auth: Provide option to ignore ingesting API Errors. (#12870)

Cisco Duo auth API rate limit is very low i.e., once per minute. Sometimes 429s are received even when requests are made less than once per minute. This leads to users ingesting 429 API errors very often. Provide an option to users to ignore ingesting API Errors. Currently only 429s are ignored when this option is enabled. Also add system tests for CEL input in auth data-stream.
elastic · Feb 28, 2025 · bf201c2 · bf201c2
1 parent 106d90a
commit bf201c2
Show file tree

Hide file tree

Showing 6 changed files with 54 additions and 15 deletions.
diff --git a/packages/cisco_duo/changelog.yml b/packages/cisco_duo/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.4.0"
+  changes:
+    - description: Provide option to ignore ingesting API Errors.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/12870
 - version: "2.3.2"
   changes:
     - description: Avoid obsolete cursor data in activity, telephony_v2.

diff --git a/packages/cisco_duo/data_stream/auth/_dev/test/system/test-cel-config.yml b/packages/cisco_duo/data_stream/auth/_dev/test/system/test-cel-config.yml
@@ -0,0 +1,12 @@
+input: cel
+service: cisco_duo
+vars:
+  hostname: http://{{Hostname}}:{{Port}}
+  secret_key: 40_characters_long_secret_key
+  integration_key: temp_integration_key
+  enable_request_tracer: true
+data_stream:
+  vars:
+    preserve_original_event: true
+assert:
+  hit_count: 5
diff --git a/.../_dev/test/system/test-default-config.yml → ..._dev/test/system/test-httpjson-config.yml b/.../_dev/test/system/test-default-config.yml → ..._dev/test/system/test-httpjson-config.yml
diff --git a/packages/cisco_duo/data_stream/auth/agent/stream/cel.yml.hbs b/packages/cisco_duo/data_stream/auth/agent/stream/cel.yml.hbs
@@ -26,6 +26,7 @@ state:
   limit: {{limit}}
   initial_interval: {{initial_interval}}
   want_more: false
+  ignore_api_errors: {{ignore_api_errors}}
 redact:
   fields:
     - integration_key
@@ -107,23 +108,36 @@ program: |
 
                 )
             :
-                bytes(resp.Body).decode_json().as(body,
+                resp.StatusCode == 429 && bool(state.ignore_api_errors) ?
+                    // Cisco Duo auth API rate limit is very low i.e., once per minute. 
+                    // Sometimes 429s are received even when requests are made 
+                    // less than once per minute. This leads to users ingesting 429 
+                    // API errors very often. 
+                    // If users choose not to ingest these errors, ignore them.
                     {
-                        "events": {
-                            "error": {
-                                "code": has(body.code) ? string(body.code) : string(resp.StatusCode),
-                                "id": string(resp.Status),
-                                "message": "GET:"+(
-                                    size(resp.Body) != 0 ?
-                                        string(resp.Body)
-                                    :
-                                        string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
-                                ),
-                            },
-                        },
+                        "events": [],
+                        // Log the rate limit excession at DEBUG level.
+                        "rate_limited": debug("rate_limit_exceeded", bytes(resp.Body).decode_json().?message.orValue("missing message")),
                         "want_more": false,
                     }
-                )
+                : 
+                    bytes(resp.Body).decode_json().as(body,
+                        {
+                            "events": {
+                                "error": {
+                                    "code": has(body.code) ? string(body.code) : string(resp.StatusCode),
+                                    "id": string(resp.Status),
+                                    "message": "GET:"+(
+                                        size(resp.Body) != 0 ?
+                                            string(resp.Body)
+                                        :
+                                            string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
+                                    ),
+                                },
+                            },
+                            "want_more": false,
+                        }
+                    )
             )
         )
     ))

diff --git a/packages/cisco_duo/data_stream/auth/manifest.yml b/packages/cisco_duo/data_stream/auth/manifest.yml
@@ -56,6 +56,14 @@ streams:
         type: bool
         multi: false
         default: false
+      - name: ignore_api_errors
+        required: true
+        show_user: true
+        title: Ignore Ingesting API Errors.
+        description: Events containing API errors are ignored and thus not ingested. Currently, only `429 Too Many Requests` API error is ignored when this flag is enabled. By default all errors from API are ingested.
+        type: bool
+        multi: false
+        default: false
       - name: processors
         type: yaml
         title: Processors

diff --git a/packages/cisco_duo/manifest.yml b/packages/cisco_duo/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: cisco_duo
 title: Cisco Duo
-version: "2.3.2"
+version: "2.4.0"
 description: Collect logs from Cisco Duo with Elastic Agent.
 type: integration
 categories: