Addition of filter on common stages

Apache/apache/ingest/parser.yml

@@ -33,9 +33,11 @@ pipeline:
           MODSECAPACHEERROR: "%{MODSECPREFIX} %{MODSECRULEFILE} %{MODSECRULELINE} (?:%{MODSECMATCHOFFSET} )?(?:%{MODSECRULEID} )?(?:%{MODSECRULEREV} )?(?:%{MODSECRULEMSG} )?(?:%{MODSECRULEDATA} )?(?:%{MODSECRULESEVERITY} )?%{MODSECRULETAGS}.*%{MODSECHOSTNAME} %{MODSECURI} %{MODSECUID}.*"
 
   - name: set_apache_fields
+    filter: "{{grok.event != null}}"
   - name: set_action_properties
   - name: translate_action_outcome
   - name: finalizer
+    filter: "{{grok.event != null}}"
 
 stages:
   set_action_properties:

Broadcom/edge-secure-web-gateway/ingest/parser.yml

@@ -28,6 +28,7 @@ pipeline:
         output_field: datetime
 
   - name: set_common_fields
+    filter: "{{parsed_event.message != None}}"
 
 stages:
   set_common_fields:

Broadcom/symantec-endpoint-protection/ingest/parser.yml

@@ -14,6 +14,7 @@ pipeline:
           THREAT_DETECTION: "%{HOSTNAME:host_hostname},Event Description: \\[SID(\\s|\u00a0)?: %{NUMBER:sig_id}\\] %{DATA:reason},Event Type: %{DATA:event_type}?,Local Host IP: %{IP:host_ip},Local Host MAC: %{DATA:host_mac}?,Remote Host Name: %{HOSTNAME:remote_name}?,Remote Host IP: %{IP:remote_ip},Remote Host MAC: %{DATA:remote_mac}?,%{DATA:direction},%{DATA:transport},,Begin: %{TIMESTAMP_ISO8601:start_date},End Time: %{TIMESTAMP_ISO8601:end_date},Occurrences: %{NUMBER:sightings},Application: %{DATA:application_name},Location: %{DATA},User Name: %{DATA:username},Domain Name: %{DATA:domain_name}?,Local Port: %{NUMBER:source_port},Remote Port: %{NUMBER:remote_port},CIDS Signature ID: %{NUMBER:signature_id},CIDS Signature string: %{DATA:signature_label},CIDS Signature SubID: %{NUMBER:signature_subid},Intrusion URL: %{DATA:intrusion_url},Intrusion Payload URL: %{DATA:intrusion_payload_url}?,SHA-256: %{DATA:intrusion_payload_SHA256},MD-5: %{DATA:intrusion_payload_MD5}?,Intensive Protection Level: %{DATA:protection_level},URL Risk: %{DATA:url_risk},URL Category: %{DATA:url_category}"
   - name: set_timestamp_field
   - name: set_ecs_fields
+    filter: "{{parsed_event.message != None}}"
   - name: set_broadcom_fields
 
 stages:

CEF/cef/tests/pan_ngfw_hip_match_cef.json

@@ -11,7 +11,6 @@
     },
     "@timestamp": "2021-03-01T21:20:13Z",
     "cef": {
-      "Name": "",
       "c6a1": "xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:xxxx",
       "c6a1Label": "Device IPv6 Address",
       "cat": "match_name1",

CEF/cef/tests/pan_ngfw_sctp_cef.json

@@ -12,7 +12,6 @@
     },
     "@timestamp": "2021-03-01T21:22:02Z",
     "cef": {
-      "Name": "",
       "cnt": 1,
       "cs1": "allow-business-apps",
       "cs1Label": "Rule",

Cisco/cisco-ios/ingest/parser.yml

@@ -49,7 +49,9 @@ pipeline:
         output_field: date
         timezone: "{{parsed_event.message.timezone}}"
   - name: set_ecs_fields
+    filter: '{{parsed_description.get("message") != None }}'
   - name: set_cisco_fields
+    filter: '{{parsed_description.get("message") != None }}'
 
 stages:
   set_ecs_fields:

Cisco/cisco-ise/ingest/parser.yml

@@ -73,6 +73,7 @@ pipeline:
         pattern: "%{MAC:adress}:%{GREEDYDATA:word}"
 
   - name: set_ecs_fields
+    filter: "{{pre_parsed_event.message != None}}"
   - name: set_configuration_fields
   - name: set_network_fields
   - name: set_miscellaneous_fields

Cisco/cisco-meraki/ingest/parser.yml

@@ -41,6 +41,7 @@ pipeline:
         output_field: message
         pattern: "^%{IP:dst_ip}:%{NUMBER:dst_port}$"
   - name: set_common
+    filter: "{{grok_header.message != None }}"
   - name: set_source_nat_ip_flow
     filter: '{{kv_part_message.message.get("translated_src_ip") != None}}'
   - name: set_destination_nat_ip_flow

Cisco/cisco-nx-os/ingest/parser.yml

@@ -66,7 +66,9 @@ pipeline:
         output_field: date
         timezone: "{{parsed_event.message.timezone}}"
   - name: set_ecs_fields
+    filter: '{{parsed_description.get("message") != None }}'
   - name: set_cisco_fields
+    filter: '{{parsed_description.get("message") != None }}'
 
 stages:
   set_ecs_fields:

Cisco/cisco-secure-firewall/ingest/parser.yml

@@ -167,6 +167,7 @@ pipeline:
         item_sep: ',\s'
 
   - name: set_common_fields
+    filter: "{{pre_parsing.pre_message != None}}"
   - name: set_ecs_fields
     filter: '{{pre_parsing.pre_message.message_number_grok in ["106001","110003", "106006", "106007", "106010", "106012", "106014", "106015", "106021", "106023", "106100", "110002", "111007", "113004", "199019", "302013", "302014", "302015", "302016", "302020", "302021", "304001", "305011", "313001", "313004", "313005", "313008", "305012", "402117", "402119", "419001", "419002", "500004", "602303", "602304", "609001", "609002", "710001", "710002", "710003", "710005", "710006", "716058", "713172", "716059", "722011", "722012", "722022", "722023", "722028", "722032", "722033", "722034", "722037", "725001", "733100", "725002", "725003", "725006", "725007", "737016", "852001"]}}'
   - name: set_ecs_fields_from_kv

Cisco/cisco-secure-web-appliance/_meta/smart-descriptions.json

@@ -38,5 +38,13 @@
         "field": "url.full"
       }
     ]
+  },
+  {
+    "value": "{event.reason}",
+    "conditions": [
+      {
+        "field": "event.reason"
+      }
+    ]
   }
 ]

Cisco/cisco-secure-web-appliance/ingest/parser.yml

@@ -6,10 +6,17 @@ pipeline:
       properties:
         input_field: original.message
         output_field: message
-        pattern: '%{PREFIX}%{NUMBER:timestamp}\s+%{NUMBER:elapsed}\s+%{IP:source_ip}\s+%{WORD:code}/%{NUMBER:status}\s+%{NUMBER:http_response_bytes}\s+%{WORD:method}\s+%{NOTSPACE:url}\s+%{NOTSPACE:username}\s+%{NOTSPACE:hierarchy_code}/(%{IP:peerhostip}|%{NOTSPACE:peerhost})\s+%{NOTSPACE:mime_type}\s+%{NOTSPACE:acl_decision}\s+<%{VERDICT:scanning_verdict_information}>\s+%{DATA:user_agent}.*'
+        pattern: '%{PREFIX}%{NUMBER:timestamp}\s+%{NUMBER:elapsed}\s+%{IP:source_ip}\s+%{WORD:code}/%{NUMBER:status}\s+%{NUMBER:http_response_bytes}\s+%{WORD:method}\s+%{NOTSPACE:url}\s+%{NOTSPACE:username}\s+%{NOTSPACE:hierarchy_code}/(%{IP:peerhostip}|%{NOTSPACE:peerhost})\s+%{NOTSPACE:mime_type}\s+(%{NOTSPACE}:%{NUMBER}\s+)*%{NOTSPACE:acl_decision}\s+<%{VERDICT:scanning_verdict_information}>\s+%{NOTSPACE:user_agent}.*'
         custom_patterns:
           PREFIX: '(?:Info:\s+)*'
           VERDICT: "(?:[^>]+)"
+  - name: parsed_files
+    external:
+      name: grok.match
+      properties:
+        input_field: original.message
+        output_field: message
+        pattern: 'Info: %{DATA:event_reason} (\(#counter_group: %{DATA:counter_group} #interval %{DATA:interval} #Serial number: %{GREEDYDATA:serial_number} #Time since data generated: %{NUMBER:last_gen})?(\(#files: %{DATA:file_name} %{DATA:interval} #files: %{INT:sightings} #rows: %{INT} #total rows %{INT}\) #duration\(s\): %{NUMBER:duration} #rate: %{DATA:rate})?'
   - name: parsed_timestamp
     external:
       name: date.parse
@@ -76,7 +83,11 @@ pipeline:
           - archivescan_detail
         delimiter: ","
   - name: set_ecs_fields
+    filter: "{{parsed_event.message != None}}"
   - name: set_cisco_fields
+    filter: "{{parsed_event.message != None }}"
+  - name: set_file_fields
+    filter: "{{parsed_files.message != None}}"
 stages:
   set_ecs_fields:
     actions:
@@ -150,6 +161,19 @@ stages:
           file.hash.sha256: "{{parsed_scanning_verdict.results.file_sha256}}"
         filter: '{{parsed_scanning_verdict.results.file_sha256 != "-"}}'
 
+  set_file_fields:
+    actions:
+      - set:
+          event.category:
+            - web
+            - network
+          event.reason: "{{parsed_files.message.event_reason}}"
+          file.name: "{{parsed_files.message.file_name}}"
+          observer.product: "Cisco Web Security Appliances"
+          observer.type: proxy
+          observer.vendor: Cisco
+          network.direction: egress
+
   set_cisco_fields:
     actions:
       - set:

Cisco/cisco-secure-web-appliance/tests/test_CMF_01.json

@@ -0,0 +1,82 @@
+{
+  "input": {
+    "message": "Info: 1721202449.446 100127 1.2.3.4 TCP_MISS/200 861 CONNECT http://my.site.com/ - DIRECT/my.site.com - CMF:40 DCF:200020 ERR:0 PASSTHRU_CUSTOM_0-DefaultGroup-No_authentication-DefaultGroup-NONE-NONE-DefaultGroup-NONE <\"A_BC_0\",-,-,\"-\",-,-,-,-,\"-\",-,-,-,\"-\",-,-,\"-\",\"-\",-,-,\"-\",-,\"-\",\"-\",\"-\",\"-\",\"-\",\"-\",\"-\",0.07,0,-,\"-\",\"-\",-,\"-\",-,-,\"-\",\"-\",-,-,\"-\",-,-> - 1 mysite.com 1.2.3.4",
+    "sekoiaio": {
+      "intake": {
+        "dialect": "Cisco Secure Web Appliance",
+        "dialect_uuid": "23b75d0c-2026-4d3e-b916-636c27ba4931"
+      }
+    }
+  },
+  "expected": {
+    "message": "Info: 1721202449.446 100127 1.2.3.4 TCP_MISS/200 861 CONNECT http://my.site.com/ - DIRECT/my.site.com - CMF:40 DCF:200020 ERR:0 PASSTHRU_CUSTOM_0-DefaultGroup-No_authentication-DefaultGroup-NONE-NONE-DefaultGroup-NONE <\"A_BC_0\",-,-,\"-\",-,-,-,-,\"-\",-,-,-,\"-\",-,-,\"-\",\"-\",-,-,\"-\",-,\"-\",\"-\",\"-\",\"-\",\"-\",\"-\",\"-\",0.07,0,-,\"-\",\"-\",-,\"-\",-,-,\"-\",\"-\",-,-,\"-\",-,-> - 1 mysite.com 1.2.3.4",
+    "event": {
+      "category": [
+        "network",
+        "web"
+      ],
+      "duration": 100127,
+      "start": "2024-07-17T07:47:29.446000Z"
+    },
+    "@timestamp": "2024-07-17T07:47:29.446000Z",
+    "cisco_wsa": {
+      "cache_status": "miss",
+      "hierarchy_code": "DIRECT",
+      "threat": {
+        "category": "Not Set",
+        "name": "-"
+      },
+      "url": {
+        "category": "Unclassified",
+        "category_code": "A_BC_0"
+      }
+    },
+    "destination": {
+      "address": "my.site.com",
+      "domain": "my.site.com",
+      "registered_domain": "site.com",
+      "subdomain": "my",
+      "top_level_domain": "com"
+    },
+    "http": {
+      "request": {
+        "method": "CONNECT"
+      },
+      "response": {
+        "bytes": 861,
+        "status_code": 200
+      }
+    },
+    "network": {
+      "direction": "egress",
+      "transport": "tcp"
+    },
+    "observer": {
+      "product": "Cisco Web Security Appliances",
+      "type": "proxy",
+      "vendor": "Cisco"
+    },
+    "related": {
+      "hosts": [
+        "my.site.com"
+      ],
+      "ip": [
+        "1.2.3.4"
+      ]
+    },
+    "source": {
+      "address": "1.2.3.4",
+      "ip": "1.2.3.4"
+    },
+    "url": {
+      "domain": "my.site.com",
+      "original": "http://my.site.com/",
+      "path": "/",
+      "port": 80,
+      "registered_domain": "site.com",
+      "scheme": "http",
+      "subdomain": "my",
+      "top_level_domain": "com"
+    }
+  }
+}

Cisco/cisco-secure-web-appliance/tests/test_CMF_02.json

@@ -0,0 +1,76 @@
+{
+  "input": {
+    "message": "Info: 1721202417.905 0 1.2.3.4 TCP_DENIED/407 0 CONNECT https://mysite.test.com:443/ - NONE/- - CMF:40 DCF:20 ERR:0 OTHER-NONE-Utilisateurs-NONE-NONE-NONE-NONE-NONE <\"-\",-,-,\"-\",-,-,-,-,\"-\",-,-,-,\"-\",-,-,\"-\",\"-\",-,-,\"-\",-,\"-\",\"-\",\"-\",\"-\",\"-\",\"-\",\"-\",0.00,0,-,\"-\",\"-\",-,\"-\",-,-,\"-\",\"-\",-,-,\"-\",-,-> - 1 - -",
+    "sekoiaio": {
+      "intake": {
+        "dialect": "Cisco Secure Web Appliance",
+        "dialect_uuid": "23b75d0c-2026-4d3e-b916-636c27ba4931"
+      }
+    }
+  },
+  "expected": {
+    "message": "Info: 1721202417.905 0 1.2.3.4 TCP_DENIED/407 0 CONNECT https://mysite.test.com:443/ - NONE/- - CMF:40 DCF:20 ERR:0 OTHER-NONE-Utilisateurs-NONE-NONE-NONE-NONE-NONE <\"-\",-,-,\"-\",-,-,-,-,\"-\",-,-,-,\"-\",-,-,\"-\",\"-\",-,-,\"-\",-,\"-\",\"-\",\"-\",\"-\",\"-\",\"-\",\"-\",0.00,0,-,\"-\",\"-\",-,\"-\",-,-,\"-\",\"-\",-,-,\"-\",-,-> - 1 - -",
+    "event": {
+      "category": [
+        "network",
+        "web"
+      ],
+      "duration": 0,
+      "start": "2024-07-17T07:46:57.905000Z",
+      "type": [
+        "connection",
+        "denied"
+      ]
+    },
+    "@timestamp": "2024-07-17T07:46:57.905000Z",
+    "cisco_wsa": {
+      "cache_status": "denied",
+      "hierarchy_code": "NONE",
+      "threat": {
+        "category": "Not Set",
+        "name": "-"
+      },
+      "url": {
+        "category": "Unclassified",
+        "category_code": "-"
+      }
+    },
+    "http": {
+      "request": {
+        "method": "CONNECT"
+      },
+      "response": {
+        "bytes": 0,
+        "status_code": 407
+      }
+    },
+    "network": {
+      "direction": "egress",
+      "transport": "tcp"
+    },
+    "observer": {
+      "product": "Cisco Web Security Appliances",
+      "type": "proxy",
+      "vendor": "Cisco"
+    },
+    "related": {
+      "ip": [
+        "1.2.3.4"
+      ]
+    },
+    "source": {
+      "address": "1.2.3.4",
+      "ip": "1.2.3.4"
+    },
+    "url": {
+      "domain": "mysite.test.com",
+      "original": "https://mysite.test.com:443/",
+      "path": "/",
+      "port": 443,
+      "registered_domain": "test.com",
+      "scheme": "https",
+      "subdomain": "mysite",
+      "top_level_domain": "com"
+    }
+  }
+}

Cisco/cisco-secure-web-appliance/tests/test_export_files.json

@@ -1,19 +1,18 @@
 {
   "input": {
-    "message": "Info: Completed aggregating export files (#files: DOMAINS_BY_APP_TYPE 2023-02-10-11-40 #files: 1 #rows: 2 #total rows 6698) #duration(s): 0.01 #rate: 156/s\n"
+    "message": "Info: Completed aggregating export files (#files: DOMAINS_BY_APP_TYPE 2023-02-10-11-40 #files: 1 #rows: 2 #total rows 6698) #duration(s): 0.01 #rate: 156/s"
   },
   "expected": {
-    "message": "Info: Completed aggregating export files (#files: DOMAINS_BY_APP_TYPE 2023-02-10-11-40 #files: 1 #rows: 2 #total rows 6698) #duration(s): 0.01 #rate: 156/s\n",
+    "message": "Info: Completed aggregating export files (#files: DOMAINS_BY_APP_TYPE 2023-02-10-11-40 #files: 1 #rows: 2 #total rows 6698) #duration(s): 0.01 #rate: 156/s",
     "event": {
       "category": [
         "network",
         "web"
-      ]
+      ],
+      "reason": "Completed aggregating export files"
     },
-    "cisco_wsa": {
-      "threat": {
-        "category": "Not Set"
-      }
+    "file": {
+      "name": "DOMAINS_BY_APP_TYPE"
     },
     "network": {
       "direction": "egress"
@@ -22,13 +21,6 @@
       "product": "Cisco Web Security Appliances",
       "type": "proxy",
       "vendor": "Cisco"
-    },
-    "sekoiaio": {
-      "intake": {
-        "parsing_warnings": [
-          "No fields extracted from original event"
-        ]
-      }
     }
   }
 }

Cisco/cisco-secure-web-appliance/tests/test_export_to_database.json

@@ -1,19 +1,15 @@
 {
   "input": {
-    "message": "Info: Completed writing export files to database (#counter_group: WEB_APPLICATION_TYPE_APPLICATION_NAME_DETAIL #interval 2023-02-10-11-40 #Serial number: 123456-789101112 #Time since data generated: 369\n"
+    "message": "Info: Completed writing export files to database (#counter_group: WEB_APPLICATION_TYPE_APPLICATION_NAME_DETAIL #interval 2023-02-10-11-40 #Serial number: 123456-789101112 #Time since data generated: 369"
   },
   "expected": {
-    "message": "Info: Completed writing export files to database (#counter_group: WEB_APPLICATION_TYPE_APPLICATION_NAME_DETAIL #interval 2023-02-10-11-40 #Serial number: 123456-789101112 #Time since data generated: 369\n",
+    "message": "Info: Completed writing export files to database (#counter_group: WEB_APPLICATION_TYPE_APPLICATION_NAME_DETAIL #interval 2023-02-10-11-40 #Serial number: 123456-789101112 #Time since data generated: 369",
     "event": {
       "category": [
         "network",
         "web"
-      ]
-    },
-    "cisco_wsa": {
-      "threat": {
-        "category": "Not Set"
-      }
+      ],
+      "reason": "Completed writing export files to database"
     },
     "network": {
       "direction": "egress"
@@ -22,13 +18,6 @@
       "product": "Cisco Web Security Appliances",
       "type": "proxy",
       "vendor": "Cisco"
-    },
-    "sekoiaio": {
-      "intake": {
-        "parsing_warnings": [
-          "No fields extracted from original event"
-        ]
-      }
     }
   }
 }