diff --git a/Apache/apache/ingest/parser.yml b/Apache/apache/ingest/parser.yml index 1b3bf13f2..e4ba011fe 100644 --- a/Apache/apache/ingest/parser.yml +++ b/Apache/apache/ingest/parser.yml @@ -33,9 +33,11 @@ pipeline: MODSECAPACHEERROR: "%{MODSECPREFIX} %{MODSECRULEFILE} %{MODSECRULELINE} (?:%{MODSECMATCHOFFSET} )?(?:%{MODSECRULEID} )?(?:%{MODSECRULEREV} )?(?:%{MODSECRULEMSG} )?(?:%{MODSECRULEDATA} )?(?:%{MODSECRULESEVERITY} )?%{MODSECRULETAGS}.*%{MODSECHOSTNAME} %{MODSECURI} %{MODSECUID}.*" - name: set_apache_fields + filter: "{{grok.event != null}}" - name: set_action_properties - name: translate_action_outcome - name: finalizer + filter: "{{grok.event != null}}" stages: set_action_properties: diff --git a/Broadcom/edge-secure-web-gateway/ingest/parser.yml b/Broadcom/edge-secure-web-gateway/ingest/parser.yml index 569e02b67..cabd56a7d 100644 --- a/Broadcom/edge-secure-web-gateway/ingest/parser.yml +++ b/Broadcom/edge-secure-web-gateway/ingest/parser.yml @@ -28,6 +28,7 @@ pipeline: output_field: datetime - name: set_common_fields + filter: "{{parsed_event.message != None}}" stages: set_common_fields: diff --git a/Broadcom/symantec-endpoint-protection/ingest/parser.yml b/Broadcom/symantec-endpoint-protection/ingest/parser.yml index b82c97268..2cdb22413 100644 --- a/Broadcom/symantec-endpoint-protection/ingest/parser.yml +++ b/Broadcom/symantec-endpoint-protection/ingest/parser.yml @@ -14,6 +14,7 @@ pipeline: THREAT_DETECTION: "%{HOSTNAME:host_hostname},Event Description: \\[SID(\\s|\u00a0)?: %{NUMBER:sig_id}\\] %{DATA:reason},Event Type: %{DATA:event_type}?,Local Host IP: %{IP:host_ip},Local Host MAC: %{DATA:host_mac}?,Remote Host Name: %{HOSTNAME:remote_name}?,Remote Host IP: %{IP:remote_ip},Remote Host MAC: %{DATA:remote_mac}?,%{DATA:direction},%{DATA:transport},,Begin: %{TIMESTAMP_ISO8601:start_date},End Time: %{TIMESTAMP_ISO8601:end_date},Occurrences: %{NUMBER:sightings},Application: %{DATA:application_name},Location: %{DATA},User Name: %{DATA:username},Domain Name: %{DATA:domain_name}?,Local Port: %{NUMBER:source_port},Remote Port: %{NUMBER:remote_port},CIDS Signature ID: %{NUMBER:signature_id},CIDS Signature string: %{DATA:signature_label},CIDS Signature SubID: %{NUMBER:signature_subid},Intrusion URL: %{DATA:intrusion_url},Intrusion Payload URL: %{DATA:intrusion_payload_url}?,SHA-256: %{DATA:intrusion_payload_SHA256},MD-5: %{DATA:intrusion_payload_MD5}?,Intensive Protection Level: %{DATA:protection_level},URL Risk: %{DATA:url_risk},URL Category: %{DATA:url_category}" - name: set_timestamp_field - name: set_ecs_fields + filter: "{{parsed_event.message != None}}" - name: set_broadcom_fields stages: diff --git a/CEF/cef/tests/pan_ngfw_hip_match_cef.json b/CEF/cef/tests/pan_ngfw_hip_match_cef.json index f80052d88..626aa28a4 100644 --- a/CEF/cef/tests/pan_ngfw_hip_match_cef.json +++ b/CEF/cef/tests/pan_ngfw_hip_match_cef.json @@ -11,7 +11,6 @@ }, "@timestamp": "2021-03-01T21:20:13Z", "cef": { - "Name": "", "c6a1": "xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:xxxx", "c6a1Label": "Device IPv6 Address", "cat": "match_name1", diff --git a/CEF/cef/tests/pan_ngfw_sctp_cef.json b/CEF/cef/tests/pan_ngfw_sctp_cef.json index 69bedba2e..be0654409 100644 --- a/CEF/cef/tests/pan_ngfw_sctp_cef.json +++ b/CEF/cef/tests/pan_ngfw_sctp_cef.json @@ -12,7 +12,6 @@ }, "@timestamp": "2021-03-01T21:22:02Z", "cef": { - "Name": "", "cnt": 1, "cs1": "allow-business-apps", "cs1Label": "Rule", diff --git a/Cisco/cisco-ios/ingest/parser.yml b/Cisco/cisco-ios/ingest/parser.yml index a50bfc812..a9f9dd908 100644 --- a/Cisco/cisco-ios/ingest/parser.yml +++ b/Cisco/cisco-ios/ingest/parser.yml @@ -49,7 +49,9 @@ pipeline: output_field: date timezone: "{{parsed_event.message.timezone}}" - name: set_ecs_fields + filter: '{{parsed_description.get("message") != None }}' - name: set_cisco_fields + filter: '{{parsed_description.get("message") != None }}' stages: set_ecs_fields: diff --git a/Cisco/cisco-ise/ingest/parser.yml b/Cisco/cisco-ise/ingest/parser.yml index 015a469be..69d9336d0 100644 --- a/Cisco/cisco-ise/ingest/parser.yml +++ b/Cisco/cisco-ise/ingest/parser.yml @@ -73,6 +73,7 @@ pipeline: pattern: "%{MAC:adress}:%{GREEDYDATA:word}" - name: set_ecs_fields + filter: "{{pre_parsed_event.message != None}}" - name: set_configuration_fields - name: set_network_fields - name: set_miscellaneous_fields diff --git a/Cisco/cisco-meraki/ingest/parser.yml b/Cisco/cisco-meraki/ingest/parser.yml index 85652b26a..78439f878 100644 --- a/Cisco/cisco-meraki/ingest/parser.yml +++ b/Cisco/cisco-meraki/ingest/parser.yml @@ -41,6 +41,7 @@ pipeline: output_field: message pattern: "^%{IP:dst_ip}:%{NUMBER:dst_port}$" - name: set_common + filter: "{{grok_header.message != None }}" - name: set_source_nat_ip_flow filter: '{{kv_part_message.message.get("translated_src_ip") != None}}' - name: set_destination_nat_ip_flow diff --git a/Cisco/cisco-nx-os/ingest/parser.yml b/Cisco/cisco-nx-os/ingest/parser.yml index 5b8fe2aee..c34f556ae 100644 --- a/Cisco/cisco-nx-os/ingest/parser.yml +++ b/Cisco/cisco-nx-os/ingest/parser.yml @@ -66,7 +66,9 @@ pipeline: output_field: date timezone: "{{parsed_event.message.timezone}}" - name: set_ecs_fields + filter: '{{parsed_description.get("message") != None }}' - name: set_cisco_fields + filter: '{{parsed_description.get("message") != None }}' stages: set_ecs_fields: diff --git a/Cisco/cisco-secure-firewall/ingest/parser.yml b/Cisco/cisco-secure-firewall/ingest/parser.yml index f47e3a550..7594e3bc7 100644 --- a/Cisco/cisco-secure-firewall/ingest/parser.yml +++ b/Cisco/cisco-secure-firewall/ingest/parser.yml @@ -167,6 +167,7 @@ pipeline: item_sep: ',\s' - name: set_common_fields + filter: "{{pre_parsing.pre_message != None}}" - name: set_ecs_fields filter: '{{pre_parsing.pre_message.message_number_grok in ["106001","110003", "106006", "106007", "106010", "106012", "106014", "106015", "106021", "106023", "106100", "110002", "111007", "113004", "199019", "302013", "302014", "302015", "302016", "302020", "302021", "304001", "305011", "313001", "313004", "313005", "313008", "305012", "402117", "402119", "419001", "419002", "500004", "602303", "602304", "609001", "609002", "710001", "710002", "710003", "710005", "710006", "716058", "713172", "716059", "722011", "722012", "722022", "722023", "722028", "722032", "722033", "722034", "722037", "725001", "733100", "725002", "725003", "725006", "725007", "737016", "852001"]}}' - name: set_ecs_fields_from_kv diff --git a/Cisco/cisco-secure-web-appliance/_meta/smart-descriptions.json b/Cisco/cisco-secure-web-appliance/_meta/smart-descriptions.json index e69a714cd..ffd1cf084 100644 --- a/Cisco/cisco-secure-web-appliance/_meta/smart-descriptions.json +++ b/Cisco/cisco-secure-web-appliance/_meta/smart-descriptions.json @@ -38,5 +38,13 @@ "field": "url.full" } ] + }, + { + "value": "{event.reason}", + "conditions": [ + { + "field": "event.reason" + } + ] } ] diff --git a/Cisco/cisco-secure-web-appliance/ingest/parser.yml b/Cisco/cisco-secure-web-appliance/ingest/parser.yml index 18e6dd0c1..661f86175 100644 --- a/Cisco/cisco-secure-web-appliance/ingest/parser.yml +++ b/Cisco/cisco-secure-web-appliance/ingest/parser.yml @@ -6,10 +6,17 @@ pipeline: properties: input_field: original.message output_field: message - pattern: '%{PREFIX}%{NUMBER:timestamp}\s+%{NUMBER:elapsed}\s+%{IP:source_ip}\s+%{WORD:code}/%{NUMBER:status}\s+%{NUMBER:http_response_bytes}\s+%{WORD:method}\s+%{NOTSPACE:url}\s+%{NOTSPACE:username}\s+%{NOTSPACE:hierarchy_code}/(%{IP:peerhostip}|%{NOTSPACE:peerhost})\s+%{NOTSPACE:mime_type}\s+%{NOTSPACE:acl_decision}\s+<%{VERDICT:scanning_verdict_information}>\s+%{DATA:user_agent}.*' + pattern: '%{PREFIX}%{NUMBER:timestamp}\s+%{NUMBER:elapsed}\s+%{IP:source_ip}\s+%{WORD:code}/%{NUMBER:status}\s+%{NUMBER:http_response_bytes}\s+%{WORD:method}\s+%{NOTSPACE:url}\s+%{NOTSPACE:username}\s+%{NOTSPACE:hierarchy_code}/(%{IP:peerhostip}|%{NOTSPACE:peerhost})\s+%{NOTSPACE:mime_type}\s+(%{NOTSPACE}:%{NUMBER}\s+)*%{NOTSPACE:acl_decision}\s+<%{VERDICT:scanning_verdict_information}>\s+%{NOTSPACE:user_agent}.*' custom_patterns: PREFIX: '(?:Info:\s+)*' VERDICT: "(?:[^>]+)" + - name: parsed_files + external: + name: grok.match + properties: + input_field: original.message + output_field: message + pattern: 'Info: %{DATA:event_reason} (\(#counter_group: %{DATA:counter_group} #interval %{DATA:interval} #Serial number: %{GREEDYDATA:serial_number} #Time since data generated: %{NUMBER:last_gen})?(\(#files: %{DATA:file_name} %{DATA:interval} #files: %{INT:sightings} #rows: %{INT} #total rows %{INT}\) #duration\(s\): %{NUMBER:duration} #rate: %{DATA:rate})?' - name: parsed_timestamp external: name: date.parse @@ -76,7 +83,11 @@ pipeline: - archivescan_detail delimiter: "," - name: set_ecs_fields + filter: "{{parsed_event.message != None}}" - name: set_cisco_fields + filter: "{{parsed_event.message != None }}" + - name: set_file_fields + filter: "{{parsed_files.message != None}}" stages: set_ecs_fields: actions: @@ -150,6 +161,19 @@ stages: file.hash.sha256: "{{parsed_scanning_verdict.results.file_sha256}}" filter: '{{parsed_scanning_verdict.results.file_sha256 != "-"}}' + set_file_fields: + actions: + - set: + event.category: + - web + - network + event.reason: "{{parsed_files.message.event_reason}}" + file.name: "{{parsed_files.message.file_name}}" + observer.product: "Cisco Web Security Appliances" + observer.type: proxy + observer.vendor: Cisco + network.direction: egress + set_cisco_fields: actions: - set: diff --git a/Cisco/cisco-secure-web-appliance/tests/test_CMF_01.json b/Cisco/cisco-secure-web-appliance/tests/test_CMF_01.json new file mode 100644 index 000000000..5eaa931d6 --- /dev/null +++ b/Cisco/cisco-secure-web-appliance/tests/test_CMF_01.json @@ -0,0 +1,82 @@ +{ + "input": { + "message": "Info: 1721202449.446 100127 1.2.3.4 TCP_MISS/200 861 CONNECT http://my.site.com/ - DIRECT/my.site.com - CMF:40 DCF:200020 ERR:0 PASSTHRU_CUSTOM_0-DefaultGroup-No_authentication-DefaultGroup-NONE-NONE-DefaultGroup-NONE <\"A_BC_0\",-,-,\"-\",-,-,-,-,\"-\",-,-,-,\"-\",-,-,\"-\",\"-\",-,-,\"-\",-,\"-\",\"-\",\"-\",\"-\",\"-\",\"-\",\"-\",0.07,0,-,\"-\",\"-\",-,\"-\",-,-,\"-\",\"-\",-,-,\"-\",-,-> - 1 mysite.com 1.2.3.4", + "sekoiaio": { + "intake": { + "dialect": "Cisco Secure Web Appliance", + "dialect_uuid": "23b75d0c-2026-4d3e-b916-636c27ba4931" + } + } + }, + "expected": { + "message": "Info: 1721202449.446 100127 1.2.3.4 TCP_MISS/200 861 CONNECT http://my.site.com/ - DIRECT/my.site.com - CMF:40 DCF:200020 ERR:0 PASSTHRU_CUSTOM_0-DefaultGroup-No_authentication-DefaultGroup-NONE-NONE-DefaultGroup-NONE <\"A_BC_0\",-,-,\"-\",-,-,-,-,\"-\",-,-,-,\"-\",-,-,\"-\",\"-\",-,-,\"-\",-,\"-\",\"-\",\"-\",\"-\",\"-\",\"-\",\"-\",0.07,0,-,\"-\",\"-\",-,\"-\",-,-,\"-\",\"-\",-,-,\"-\",-,-> - 1 mysite.com 1.2.3.4", + "event": { + "category": [ + "network", + "web" + ], + "duration": 100127, + "start": "2024-07-17T07:47:29.446000Z" + }, + "@timestamp": "2024-07-17T07:47:29.446000Z", + "cisco_wsa": { + "cache_status": "miss", + "hierarchy_code": "DIRECT", + "threat": { + "category": "Not Set", + "name": "-" + }, + "url": { + "category": "Unclassified", + "category_code": "A_BC_0" + } + }, + "destination": { + "address": "my.site.com", + "domain": "my.site.com", + "registered_domain": "site.com", + "subdomain": "my", + "top_level_domain": "com" + }, + "http": { + "request": { + "method": "CONNECT" + }, + "response": { + "bytes": 861, + "status_code": 200 + } + }, + "network": { + "direction": "egress", + "transport": "tcp" + }, + "observer": { + "product": "Cisco Web Security Appliances", + "type": "proxy", + "vendor": "Cisco" + }, + "related": { + "hosts": [ + "my.site.com" + ], + "ip": [ + "1.2.3.4" + ] + }, + "source": { + "address": "1.2.3.4", + "ip": "1.2.3.4" + }, + "url": { + "domain": "my.site.com", + "original": "http://my.site.com/", + "path": "/", + "port": 80, + "registered_domain": "site.com", + "scheme": "http", + "subdomain": "my", + "top_level_domain": "com" + } + } +} \ No newline at end of file diff --git a/Cisco/cisco-secure-web-appliance/tests/test_CMF_02.json b/Cisco/cisco-secure-web-appliance/tests/test_CMF_02.json new file mode 100644 index 000000000..86638eb38 --- /dev/null +++ b/Cisco/cisco-secure-web-appliance/tests/test_CMF_02.json @@ -0,0 +1,76 @@ +{ + "input": { + "message": "Info: 1721202417.905 0 1.2.3.4 TCP_DENIED/407 0 CONNECT https://mysite.test.com:443/ - NONE/- - CMF:40 DCF:20 ERR:0 OTHER-NONE-Utilisateurs-NONE-NONE-NONE-NONE-NONE <\"-\",-,-,\"-\",-,-,-,-,\"-\",-,-,-,\"-\",-,-,\"-\",\"-\",-,-,\"-\",-,\"-\",\"-\",\"-\",\"-\",\"-\",\"-\",\"-\",0.00,0,-,\"-\",\"-\",-,\"-\",-,-,\"-\",\"-\",-,-,\"-\",-,-> - 1 - -", + "sekoiaio": { + "intake": { + "dialect": "Cisco Secure Web Appliance", + "dialect_uuid": "23b75d0c-2026-4d3e-b916-636c27ba4931" + } + } + }, + "expected": { + "message": "Info: 1721202417.905 0 1.2.3.4 TCP_DENIED/407 0 CONNECT https://mysite.test.com:443/ - NONE/- - CMF:40 DCF:20 ERR:0 OTHER-NONE-Utilisateurs-NONE-NONE-NONE-NONE-NONE <\"-\",-,-,\"-\",-,-,-,-,\"-\",-,-,-,\"-\",-,-,\"-\",\"-\",-,-,\"-\",-,\"-\",\"-\",\"-\",\"-\",\"-\",\"-\",\"-\",0.00,0,-,\"-\",\"-\",-,\"-\",-,-,\"-\",\"-\",-,-,\"-\",-,-> - 1 - -", + "event": { + "category": [ + "network", + "web" + ], + "duration": 0, + "start": "2024-07-17T07:46:57.905000Z", + "type": [ + "connection", + "denied" + ] + }, + "@timestamp": "2024-07-17T07:46:57.905000Z", + "cisco_wsa": { + "cache_status": "denied", + "hierarchy_code": "NONE", + "threat": { + "category": "Not Set", + "name": "-" + }, + "url": { + "category": "Unclassified", + "category_code": "-" + } + }, + "http": { + "request": { + "method": "CONNECT" + }, + "response": { + "bytes": 0, + "status_code": 407 + } + }, + "network": { + "direction": "egress", + "transport": "tcp" + }, + "observer": { + "product": "Cisco Web Security Appliances", + "type": "proxy", + "vendor": "Cisco" + }, + "related": { + "ip": [ + "1.2.3.4" + ] + }, + "source": { + "address": "1.2.3.4", + "ip": "1.2.3.4" + }, + "url": { + "domain": "mysite.test.com", + "original": "https://mysite.test.com:443/", + "path": "/", + "port": 443, + "registered_domain": "test.com", + "scheme": "https", + "subdomain": "mysite", + "top_level_domain": "com" + } + } +} \ No newline at end of file diff --git a/Cisco/cisco-secure-web-appliance/tests/test_export_files.json b/Cisco/cisco-secure-web-appliance/tests/test_export_files.json index 1c333d934..89edd7b81 100644 --- a/Cisco/cisco-secure-web-appliance/tests/test_export_files.json +++ b/Cisco/cisco-secure-web-appliance/tests/test_export_files.json @@ -1,19 +1,18 @@ { "input": { - "message": "Info: Completed aggregating export files (#files: DOMAINS_BY_APP_TYPE 2023-02-10-11-40 #files: 1 #rows: 2 #total rows 6698) #duration(s): 0.01 #rate: 156/s\n" + "message": "Info: Completed aggregating export files (#files: DOMAINS_BY_APP_TYPE 2023-02-10-11-40 #files: 1 #rows: 2 #total rows 6698) #duration(s): 0.01 #rate: 156/s" }, "expected": { - "message": "Info: Completed aggregating export files (#files: DOMAINS_BY_APP_TYPE 2023-02-10-11-40 #files: 1 #rows: 2 #total rows 6698) #duration(s): 0.01 #rate: 156/s\n", + "message": "Info: Completed aggregating export files (#files: DOMAINS_BY_APP_TYPE 2023-02-10-11-40 #files: 1 #rows: 2 #total rows 6698) #duration(s): 0.01 #rate: 156/s", "event": { "category": [ "network", "web" - ] + ], + "reason": "Completed aggregating export files" }, - "cisco_wsa": { - "threat": { - "category": "Not Set" - } + "file": { + "name": "DOMAINS_BY_APP_TYPE" }, "network": { "direction": "egress" @@ -22,13 +21,6 @@ "product": "Cisco Web Security Appliances", "type": "proxy", "vendor": "Cisco" - }, - "sekoiaio": { - "intake": { - "parsing_warnings": [ - "No fields extracted from original event" - ] - } } } } \ No newline at end of file diff --git a/Cisco/cisco-secure-web-appliance/tests/test_export_to_database.json b/Cisco/cisco-secure-web-appliance/tests/test_export_to_database.json index c038c26b5..3a5e8f9fa 100644 --- a/Cisco/cisco-secure-web-appliance/tests/test_export_to_database.json +++ b/Cisco/cisco-secure-web-appliance/tests/test_export_to_database.json @@ -1,19 +1,15 @@ { "input": { - "message": "Info: Completed writing export files to database (#counter_group: WEB_APPLICATION_TYPE_APPLICATION_NAME_DETAIL #interval 2023-02-10-11-40 #Serial number: 123456-789101112 #Time since data generated: 369\n" + "message": "Info: Completed writing export files to database (#counter_group: WEB_APPLICATION_TYPE_APPLICATION_NAME_DETAIL #interval 2023-02-10-11-40 #Serial number: 123456-789101112 #Time since data generated: 369" }, "expected": { - "message": "Info: Completed writing export files to database (#counter_group: WEB_APPLICATION_TYPE_APPLICATION_NAME_DETAIL #interval 2023-02-10-11-40 #Serial number: 123456-789101112 #Time since data generated: 369\n", + "message": "Info: Completed writing export files to database (#counter_group: WEB_APPLICATION_TYPE_APPLICATION_NAME_DETAIL #interval 2023-02-10-11-40 #Serial number: 123456-789101112 #Time since data generated: 369", "event": { "category": [ "network", "web" - ] - }, - "cisco_wsa": { - "threat": { - "category": "Not Set" - } + ], + "reason": "Completed writing export files to database" }, "network": { "direction": "egress" @@ -22,13 +18,6 @@ "product": "Cisco Web Security Appliances", "type": "proxy", "vendor": "Cisco" - }, - "sekoiaio": { - "intake": { - "parsing_warnings": [ - "No fields extracted from original event" - ] - } } } } \ No newline at end of file diff --git a/Cloudflare/firewall_events/tests/block.json b/Cloudflare/firewall_events/tests/block.json index e27332ec4..4d7817ad1 100644 --- a/Cloudflare/firewall_events/tests/block.json +++ b/Cloudflare/firewall_events/tests/block.json @@ -18,10 +18,6 @@ "@timestamp": "2022-12-01T01:28:31.716000Z", "cloudflare": { "ClientIPClass": "noRecord", - "ClientRefererHost": "", - "ClientRefererPath": "", - "ClientRefererQuery": "", - "ClientRefererScheme": "", "EdgeColoCode": "EWR", "EdgeResponseStatus": 403, "Kind": "firewall", diff --git a/Cloudflare/http_requests/tests/http2.json b/Cloudflare/http_requests/tests/http2.json index 50af65305..4dcd50b0d 100644 --- a/Cloudflare/http_requests/tests/http2.json +++ b/Cloudflare/http_requests/tests/http2.json @@ -14,10 +14,7 @@ ] }, "cloudflare": { - "WAFMatchedVar": "", "WAFProfile": "unknown", - "WAFRuleID": "", - "WAFRuleMessage": "", "WorkerCPUTime": 0, "WorkerStatus": "unknown", "WorkerSubrequest": false, diff --git a/EfficientIP/solidserver-ddi/ingest/parser.yml b/EfficientIP/solidserver-ddi/ingest/parser.yml index 8a3d02953..f7aa298d7 100644 --- a/EfficientIP/solidserver-ddi/ingest/parser.yml +++ b/EfficientIP/solidserver-ddi/ingest/parser.yml @@ -40,6 +40,7 @@ pipeline: output_field: timestamp - name: set_ecs_fields + filter: "{{parse_event.message != None}}" - name: set_efficient_fields stages: diff --git a/F5 Networks/f5-big-ip/ingest/parser.yml b/F5 Networks/f5-big-ip/ingest/parser.yml index dcac2c0de..f5d3e3d04 100644 --- a/F5 Networks/f5-big-ip/ingest/parser.yml +++ b/F5 Networks/f5-big-ip/ingest/parser.yml @@ -1,8 +1,6 @@ name: f5-big-ip ignored_values: ["-", "N/A", ""] pipeline: - - name: set_common_fields - - name: parse_grok_event external: name: grok.match @@ -80,6 +78,8 @@ pipeline: PAYLOAD_USER_AUTH_WITH_DOMAIN: ".*authenticate with '%{DATA:user_name}@%{DATA:user_domain}'.*" PAYLOAD_NEW_SESSION: 'New session from \s*client IP %{IP:src_ip} \(%{DATA}\) at VIP %{IP:dest_ip}.*' + - name: set_common_fields + - name: set_apm_event filter: "{{ parse_apm_event.message != None }}" @@ -98,10 +98,23 @@ stages: - set: event.category: ["network"] event.type: ["info"] + os.family: "linux" + os.platform: "linux" + filter: "{{parse_cef_event.message != None}}" + + - set: + event.category: ["network"] + event.type: ["info"] + os.family: "linux" + os.platform: "linux" + filter: "{{parse_kv_event.message != None }}" - set: + event.category: ["network"] + event.type: ["info"] os.family: "linux" os.platform: "linux" + filter: "{{(parse_apm_event.message or parse_grok_event.message) != None}}" set_grokked_fields: actions: diff --git a/FreeRADIUS/freeradius/ingest/parser.yml b/FreeRADIUS/freeradius/ingest/parser.yml index 2c93113c1..a6812d62f 100644 --- a/FreeRADIUS/freeradius/ingest/parser.yml +++ b/FreeRADIUS/freeradius/ingest/parser.yml @@ -18,6 +18,8 @@ pipeline: PROTOCOL: "%{PROTOCOL_TUNNEL}|%{DATA:network_protocol}" - name: set_ecs_fields + filter: "{{parsing.message != None}}" + stages: set_ecs_fields: actions: diff --git a/HAProxy/haproxy/ingest/parser.yml b/HAProxy/haproxy/ingest/parser.yml index 98783cd3d..c2e41e4b1 100644 --- a/HAProxy/haproxy/ingest/parser.yml +++ b/HAProxy/haproxy/ingest/parser.yml @@ -28,6 +28,7 @@ pipeline: - name: grok_main_stage filter: "{{grok.message.HAPROXYHTTPBASE | length > 0}}" - name: finalizer + filter: "{{grok.message != None}}" stages: finalizer: diff --git a/IBM/ibm-aix/ingest/parser.yml b/IBM/ibm-aix/ingest/parser.yml index 008e47e36..05fb2690d 100644 --- a/IBM/ibm-aix/ingest/parser.yml +++ b/IBM/ibm-aix/ingest/parser.yml @@ -45,6 +45,7 @@ pipeline: TCP_kaccept: '(?PTCP_kaccept)\s*%{NOTSPACE:user_name}\s*%{NOTSPACE:user_group}\s*%{NOTSPACE:process_user_name}\s*%{NOTSPACE:status}\s*%{AIX_Time}\s*(No associated roles|!)\s*%{BASE16NUM} (Port\s*%{IP:source_ip}\s*%{INT:source_port})?%{DATA}' - name: set_common_fields + filter: "{{pre_parsing.pre_message != None}}" - name: set_ecs_fields - name: set_command_line_fields diff --git a/Juniper/juniper-switches/ingest/parser.yml b/Juniper/juniper-switches/ingest/parser.yml index 4a888cd85..e12d99773 100644 --- a/Juniper/juniper-switches/ingest/parser.yml +++ b/Juniper/juniper-switches/ingest/parser.yml @@ -17,6 +17,7 @@ pipeline: NETWORK_3: "FW:\\s+%{DATA:interface_name}\\s+%{DATA:filter_action}\\s+%{DATA:protocol}\\s+%{IP:source_ip}\\s+%{IP:destination_ip}\\s+%{INT:source_port}\\s+%{INT:destination_port}\\s+\\(%{INT:nb_packets} packets\\)" - name: set_ecs_fields + filter: "{{parsed_event.message != None}}" stages: set_ecs_fields: diff --git a/Log Insight Windows/log-insight-windows/ingest/parser.yml b/Log Insight Windows/log-insight-windows/ingest/parser.yml index 77fda7a69..b5ca71f22 100644 --- a/Log Insight Windows/log-insight-windows/ingest/parser.yml +++ b/Log Insight Windows/log-insight-windows/ingest/parser.yml @@ -41,8 +41,10 @@ pipeline: %{PROCESS_INFO2} - name: set_ecs_fields + filter: "{{event.message != None}}" - name: translate_action_name - name: finalizer + filter: "{{event.message != None}}" stages: finalizer: diff --git a/Microsoft/microsoft-iis/ingest/parser.yml b/Microsoft/microsoft-iis/ingest/parser.yml index aec221864..3d24722b1 100644 --- a/Microsoft/microsoft-iis/ingest/parser.yml +++ b/Microsoft/microsoft-iis/ingest/parser.yml @@ -34,6 +34,7 @@ pipeline: format: "%m/%d/%Y %H:%M:%S" - name: set_fields + filter: "{{parse_message.message != None}}" stages: set_fields: diff --git a/NGINX/nginx/ingest/parser.yml b/NGINX/nginx/ingest/parser.yml index f2fc51cc1..19c7f27aa 100644 --- a/NGINX/nginx/ingest/parser.yml +++ b/NGINX/nginx/ingest/parser.yml @@ -32,6 +32,7 @@ pipeline: output_field: datetime format: "%Y/%m/%d %H:%M:%S" - name: set_ecs_fields + filter: "{{parsed_event.message != None }}" - name: set_fields stages: diff --git a/Netskope/netskope-transaction/ingest/parser.yml b/Netskope/netskope-transaction/ingest/parser.yml index b6513f1c4..1a32a5ae2 100644 --- a/Netskope/netskope-transaction/ingest/parser.yml +++ b/Netskope/netskope-transaction/ingest/parser.yml @@ -157,6 +157,7 @@ pipeline: delimiter: " " - name: set_fields + filter: "{{parse_event.message != None}}" stages: set_fields: diff --git a/Olfeo/olfeo-secure-web-gateway/ingest/parser.yml b/Olfeo/olfeo-secure-web-gateway/ingest/parser.yml index cc61db7ad..202a1d2f4 100644 --- a/Olfeo/olfeo-secure-web-gateway/ingest/parser.yml +++ b/Olfeo/olfeo-secure-web-gateway/ingest/parser.yml @@ -16,7 +16,7 @@ pipeline: custom_patterns: {} - name: ecs_fields description: "" - filter: "" + filter: "{{parse_grok.message != None}}" external: null stages: ecs_fields: diff --git a/OpenLDAP/openldap/ingest/parser.yml b/OpenLDAP/openldap/ingest/parser.yml index 537b0c7da..a868e367f 100644 --- a/OpenLDAP/openldap/ingest/parser.yml +++ b/OpenLDAP/openldap/ingest/parser.yml @@ -63,6 +63,7 @@ pipeline: RESERVED_KEYWORDS: "(ou|o|dc)" - name: set_event_fields + filter: "{{pre_parsing.pre_message != None}}" - name: set_common_fields stages: set_event_fields: diff --git a/OpenSSH/openssh/ingest/parser.yml b/OpenSSH/openssh/ingest/parser.yml index 53d8da2b0..37946c1ac 100644 --- a/OpenSSH/openssh/ingest/parser.yml +++ b/OpenSSH/openssh/ingest/parser.yml @@ -135,6 +135,7 @@ pipeline: item_sep: '\s' - name: set_ecs_fields + filter: "{{event.message != None}}" stages: set_ecs_fields: diff --git a/OpenVPN/openvpn/ingest/parser.yml b/OpenVPN/openvpn/ingest/parser.yml index 117dde1ba..f898ffac8 100644 --- a/OpenVPN/openvpn/ingest/parser.yml +++ b/OpenVPN/openvpn/ingest/parser.yml @@ -67,6 +67,7 @@ pipeline: filter: "{{ pre_parsing.pre_message.date != None }}" - name: set_common_fields + filter: "{{parsed_event.message != None}}" stages: set_common_fields: diff --git a/Postfix/postfix/ingest/parser.yml b/Postfix/postfix/ingest/parser.yml index 5f49e1a3d..d03003622 100644 --- a/Postfix/postfix/ingest/parser.yml +++ b/Postfix/postfix/ingest/parser.yml @@ -246,6 +246,7 @@ pipeline: filter: "{{event.message.postfix_message != None}}" - name: set_basic_fields + filter: "{{event.message != None}}" - name: finalize stages: diff --git a/Stormshield/stormshield_network_security/ingest/parser.yml b/Stormshield/stormshield_network_security/ingest/parser.yml index 61236f957..427b3c4d9 100644 --- a/Stormshield/stormshield_network_security/ingest/parser.yml +++ b/Stormshield/stormshield_network_security/ingest/parser.yml @@ -34,6 +34,7 @@ pipeline: - name: set_startime - name: action_field_is_empty - name: set_fields + filter: "{{kv.result != None}}" stages: set_fields: diff --git a/Systancia/cleanroom/ingest/parser.yml b/Systancia/cleanroom/ingest/parser.yml index 0b80ff87c..4e048535a 100644 --- a/Systancia/cleanroom/ingest/parser.yml +++ b/Systancia/cleanroom/ingest/parser.yml @@ -25,6 +25,7 @@ pipeline: SESSION_SUCCESS: '%{WORD:session_event} (%{DATA})?(\s*(plugin utilisateur)|\s*(portail web))%{GREEDYDATA}' - name: set_default_fields + filter: "{{event.result.get('event_reason') != None}}" stages: set_default_fields: diff --git a/Trend Micro/trend-micro-apex-one/tests/data_loss_prevention_event.json b/Trend Micro/trend-micro-apex-one/tests/data_loss_prevention_event.json index 9e00bab13..005ac4b86 100644 --- a/Trend Micro/trend-micro-apex-one/tests/data_loss_prevention_event.json +++ b/Trend Micro/trend-micro-apex-one/tests/data_loss_prevention_event.json @@ -23,7 +23,7 @@ }, "file": { "name": "O2MDFvst.INF", - "path": "D:\\\\2. DRIVER\\\\drivers WIN7\\\\Drivers\\\\DP_CardReader_14032.7z\\\\O2Micro\\\\FORCED\\\\6x86\\\\" + "path": "D:\\\\2. DRIVER\\\\drivers WIN7\\\\Drivers\\\\DP_CardReader_14032.7z\\\\O2Micro\\\\FORCED\\\\6x86" }, "host": { "name": "shost1" diff --git a/Trend Micro/trend-micro-apex-one/tests/virus_malware_event.json b/Trend Micro/trend-micro-apex-one/tests/virus_malware_event.json index 0981a7778..0e5bb0236 100644 --- a/Trend Micro/trend-micro-apex-one/tests/virus_malware_event.json +++ b/Trend Micro/trend-micro-apex-one/tests/virus_malware_event.json @@ -19,7 +19,7 @@ "@timestamp": "2016-02-18T14:34:00Z", "file": { "name": "0348C693056617D34FC5B5BAB4643885FEE5FEDF;0xD5D56AC2", - "path": "C:\\\\Users\\\\Administrator\\\\Desktop\\\\trend_test_virus\\\\Trojans\\\\" + "path": "C:\\\\Users\\\\Administrator\\\\Desktop\\\\trend_test_virus\\\\Trojans" }, "host": { "name": "ApexOneClient01" diff --git a/Trend Micro/trend-micro-apex-one/tests/virus_malware_event2.json b/Trend Micro/trend-micro-apex-one/tests/virus_malware_event2.json index 4206ca55b..cbdc37ae1 100644 --- a/Trend Micro/trend-micro-apex-one/tests/virus_malware_event2.json +++ b/Trend Micro/trend-micro-apex-one/tests/virus_malware_event2.json @@ -19,7 +19,7 @@ "@timestamp": "2023-03-15T13:23:47Z", "file": { "name": "5f4b0aa22ce65b30fb232421673fad4c126970928207ade256d3bfee33dc3687", - "path": "C:\\\\Users\\\\adminuser\\\\Downloads\\\\5f4b0aa22ce65b30fb232421673fad4c126970928207ade256d3bfee33dc3687\\\\" + "path": "C:\\\\Users\\\\adminuser\\\\Downloads\\\\5f4b0aa22ce65b30fb232421673fad4c126970928207ade256d3bfee33dc3687" }, "host": { "name": "mymachine.local" diff --git a/Umbrella/umbrella-dns/ingest/parser.yml b/Umbrella/umbrella-dns/ingest/parser.yml index 86f36056f..6cadd3551 100644 --- a/Umbrella/umbrella-dns/ingest/parser.yml +++ b/Umbrella/umbrella-dns/ingest/parser.yml @@ -20,6 +20,7 @@ pipeline: - GREEDYDATA delimiter: "," - name: set_ecs_fields + filter: "{{event.message != None }}" stages: set_ecs_fields: diff --git a/Umbrella/umbrella-ip/ingest/parser.yml b/Umbrella/umbrella-ip/ingest/parser.yml index 5bd5ad824..0cbe43914 100644 --- a/Umbrella/umbrella-ip/ingest/parser.yml +++ b/Umbrella/umbrella-ip/ingest/parser.yml @@ -14,6 +14,7 @@ pipeline: - data delimiter: "," - name: set_ecs_fields + filter: "{{parsed_event.message != None }}" stages: set_ecs_fields: diff --git a/VMWare/vmware-vcenter/ingest/parser.yml b/VMWare/vmware-vcenter/ingest/parser.yml index 0e39e5420..45ad335db 100644 --- a/VMWare/vmware-vcenter/ingest/parser.yml +++ b/VMWare/vmware-vcenter/ingest/parser.yml @@ -34,6 +34,7 @@ pipeline: format: "%d/%b/%Y:%H:%M:%S %z" - name: set_ecs_fields + filter: "{{parsed_event.message != None}}" stages: set_ecs_fields: diff --git a/Wallix/wallix-bastion/ingest/parser.yml b/Wallix/wallix-bastion/ingest/parser.yml index f8954240a..7a5804c59 100644 --- a/Wallix/wallix-bastion/ingest/parser.yml +++ b/Wallix/wallix-bastion/ingest/parser.yml @@ -18,8 +18,11 @@ pipeline: EVENT_REASON: "session (opened|closed)" - name: set_extracted_fields + filter: "{{parsed_event.message != None}}" - name: set_ecs_fields + filter: "{{parsed_event.message != None}}" - name: set_provider + filter: "{{parsed_event.message != None}}" - name: set_auditd - name: set_ip_fields - name: set_pam_unix_fields diff --git a/WatchGuard/watchguard-firebox/ingest/parser.yml b/WatchGuard/watchguard-firebox/ingest/parser.yml index 23bdfd0d6..284c16a4a 100644 --- a/WatchGuard/watchguard-firebox/ingest/parser.yml +++ b/WatchGuard/watchguard-firebox/ingest/parser.yml @@ -36,6 +36,7 @@ pipeline: DHCPD_RELEASE: (?PRELEASE) of %{IP:dhcp_client_ip} from %{MAC:dhcp_client_mac}( \(%{DATA:dhcp_client_name}\))? %{DHCPD_VIA} \((?P(not )?found)\) DHCPD_INFORM: (?PINFORM) from %{IP:dhcp_client_ip}? %{DHCPD_VIA} - name: set_ecs_fields + filter: "{{parsed_event.message != None}}" - name: set_custom_fields stages: diff --git a/Windows/microsoft-always-on-vpn/ingest/parser.yml b/Windows/microsoft-always-on-vpn/ingest/parser.yml index bfe1b64c9..d4bed52eb 100644 --- a/Windows/microsoft-always-on-vpn/ingest/parser.yml +++ b/Windows/microsoft-always-on-vpn/ingest/parser.yml @@ -80,6 +80,7 @@ pipeline: output_field: datetime format: "%m/%d/%Y %H:%M:%S" - name: set_ecs_fields + filter: "{{parsed_event.message != None}}" - name: set_windows_fields stages: