Add creation timestamp to pre-detections (#436)

* Add creation timestamp to pre-detections
fkie-cad · Sep 25, 2023 · 80b9b41 · 80b9b41
1 parent e3e4aba
commit 80b9b41
Show file tree

Hide file tree

Showing 4 changed files with 31 additions and 11 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,10 @@
 ## next release
 ### Features
 ### Improvements
+
+* `pre_detector` processor now adds the field `creation_timestamp` to pre-detections.
+It contains the time at which a pre-detection was created by the processor.
+
 ### Bugfix
 
 ## v6.8.1

diff --git a/logprep/processor/pre_detector/processor.py b/logprep/processor/pre_detector/processor.py
@@ -38,6 +38,7 @@
 from logprep.abc.processor import Processor
 from logprep.processor.pre_detector.ip_alerter import IPAlerter
 from logprep.processor.pre_detector.rule import PreDetectorRule
+from logprep.util.time import TimeParser
 
 
 class PreDetectorError(BaseException):
@@ -125,8 +126,9 @@ def _apply_rules(self, event, rule):
             if self._logger.isEnabledFor(DEBUG):  # pragma: no cover
                 self._logger.debug(f"{self.describe()} processing matching event")
             self._get_detection_result(rule, self._extra_data)
-        if "@timestamp" in event:
-            for detection in self._extra_data:
+        for detection in self._extra_data:
+            detection["creation_timestamp"] = TimeParser.now().isoformat()
+            if "@timestamp" in event:
                 detection["@timestamp"] = event["@timestamp"]
 
     def _get_detection_result(self, rule: PreDetectorRule, detection_results: list):

diff --git a/tests/acceptance/test_pre_detection.py b/tests/acceptance/test_pre_detection.py
@@ -1,5 +1,6 @@
 # pylint: disable=missing-docstring
 # pylint: disable=line-too-long
+# pylint: disable=too-many-locals
 import json
 import re
 from logging import basicConfig, DEBUG, getLogger
@@ -39,28 +40,28 @@
         (
             {"@timestamp":"2019-07-30T14:38:16.352Z","@metadata":{"beat":"winlogbeat","type":"_doc","version":"7.2.0","topic":"wineventlog_raw"},"event":{"code":7036,"created":"2019-08-02T09:55:11.996Z","kind":"event"},"agent":{"version":"7.2.0","type":"winlogbeat","ephemeral_id":"de845cd9-5141-4c92-ad32-27a4518307e9","hostname":"CLIENT1","id":"0b755aca-0a9a-454a-9800-1979901962a0"},"ecs":{"version":"1.0.0"},"host":{"name":"CLIENT1","hostname":"CLIENT1","architecture":"x86","os":{"name":"Windows 7 Professional","kernel":"6.1.7601.18741 (win7sp1_gdr.150202-1526)","build":"7601.0","platform":"windows","version":"6.1","family":"windows"},"id":"19fc45ac-5890-4f96-81b1-50ed111c0ce4"},"log":{"level":"information"},"message":"The Software Protection service entered the stopped state.","winlog":{"provider_guid":"{555908d1-a6d7-4695-8e1e-26931d2012f4}","event_id":1234,"task":"","api":"wineventlog","event_data":{"Binary":"7300700070007300760063002F0031000000","param1":"Software Protection","param2":"stopped"},"keywords":["Classic"],"provider_name":"Service Control Manager","record_id":11580,"channel":"System","computer_name":"abcdefg1234","process":{"thread":{"id":2808},"pid":436}}},
             {"@timestamp":"2019-07-30T14:38:16.352Z","@metadata":{"beat":"winlogbeat","type":"_doc","version":"7.2.0","topic":"wineventlog_raw"},"event":{"code":7036,"created":"2019-08-02T09:55:11.996Z","kind":"event"},"agent":{"version":"7.2.0","type":"winlogbeat","ephemeral_id":"de845cd9-5141-4c92-ad32-27a4518307e9","hostname":"CLIENT1","id":"0b755aca-0a9a-454a-9800-1979901962a0"},"ecs":{"version":"1.0.0"},"host":{"name":"CLIENT1","hostname":"CLIENT1","architecture":"x86","os":{"name":"Windows 7 Professional","kernel":"6.1.7601.18741 (win7sp1_gdr.150202-1526)","build":"7601.0","platform":"windows","version":"6.1","family":"windows"},"id":"19fc45ac-5890-4f96-81b1-50ed111c0ce4"},"log":{"level":"information"},"message":"The Software Protection service entered the stopped state.","winlog":{"provider_guid":"{555908d1-a6d7-4695-8e1e-26931d2012f4}","event_id":1234,"task":"","api":"wineventlog","event_data":{"Binary":"7300700070007300760063002F0031000000","param1":"Software Protection","param2":"stopped"},"keywords":["Classic"],"provider_name":"Service Control Manager","record_id":11580,"channel":"System","computer_name":"abcdefg1234","process":{"thread":{"id":2808},"pid":436}}},
-            [{"pre_detector_topic": {"description": "", "id": "RULE_ONE_ID", "title": "RULE_ONE", "severity": "critical", "mitre": ["mitre1", "mitre2"], "case_condition": "directly", "rule_filter": 'winlog.provider_name:"Service Control Manager"', "pre_detection_id": "1cf39644-a632-4c42-a7b4-2896c4efffb5", "host": {"name": "CLIENT1"}, "@timestamp": "2019-07-30T14:38:16.352Z"}}]
+            [{"pre_detector_topic": {"description": "", "id": "RULE_ONE_ID", "title": "RULE_ONE", "severity": "critical", "mitre": ["mitre1", "mitre2"], "case_condition": "directly", "rule_filter": 'winlog.provider_name:"Service Control Manager"', "pre_detection_id": "1cf39644-a632-4c42-a7b4-2896c4efffb5", "host": {"name": "CLIENT1"}, "@timestamp": "2019-07-30T14:38:16.352Z", "creation_timestamp": "2019-07-30T14:58:16.352Z"}}]
         ),
         (
             {"@timestamp":"2019-08-02T09:46:41.906Z","@metadata":{"beat":"winlogbeat","type":"_doc","version":"7.2.0","topic":"wineventlog_raw"},"host":{"name":"CLIENT1","os":{"name":"Windows 7 Professional","kernel":"6.1.7601.18741 (win7sp1_gdr.150202-1526)","build":"7601.0","platform":"windows","version":"6.1","family":"windows"},"id":"19fc45ac-5890-4f96-81b1-50ed111c0ce4","hostname":"CLIENT1","architecture":"x86"},"agent":{"hostname":"CLIENT1","id":"0b755aca-0a9a-454a-9800-1979901962a0","version":"7.2.0","type":"winlogbeat","ephemeral_id":"de845cd9-5141-4c92-ad32-27a4518307e9"},"ecs":{"version":"1.0.0"},"winlog":{"channel":"System","provider_name":"Service Control Manager","record_id":11627,"event_id":1234,"api":"wineventlog","keywords":["Classic"],"computer_name":"abcdefg1234","process":{"pid":440,"thread":{"id":524}},"provider_guid":"{555908d1-a6d7-4695-8e1e-26931d2012f4}","event_data":{"param1":"Wazuh","param2":"running","Binary":"4F0073007300650063005300760063002F0034000000"},"task":""},"event":{"kind":"event","code":7036,"created":"2019-08-02T09:55:11.998Z"},"log":{"level":"information"},"message":"The Wazuh service entered the running state."},
             {"@timestamp":"2019-08-02T09:46:41.906Z","@metadata":{"beat":"winlogbeat","type":"_doc","version":"7.2.0","topic":"wineventlog_raw"},"host":{"name":"CLIENT1","os":{"name":"Windows 7 Professional","kernel":"6.1.7601.18741 (win7sp1_gdr.150202-1526)","build":"7601.0","platform":"windows","version":"6.1","family":"windows"},"id":"19fc45ac-5890-4f96-81b1-50ed111c0ce4","hostname":"CLIENT1","architecture":"x86"},"agent":{"hostname":"CLIENT1","id":"0b755aca-0a9a-454a-9800-1979901962a0","version":"7.2.0","type":"winlogbeat","ephemeral_id":"de845cd9-5141-4c92-ad32-27a4518307e9"},"ecs":{"version":"1.0.0"},"winlog":{"channel":"System","provider_name":"Service Control Manager","record_id":11627,"event_id":1234,"api":"wineventlog","keywords":["Classic"],"computer_name":"abcdefg1234","process":{"pid":440,"thread":{"id":524}},"provider_guid":"{555908d1-a6d7-4695-8e1e-26931d2012f4}","event_data":{"param1":"Wazuh","param2":"running","Binary":"4F0073007300650063005300760063002F0034000000"},"task":""},"event":{"kind":"event","code":7036,"created":"2019-08-02T09:55:11.998Z"},"log":{"level":"information"},"message":"The Wazuh service entered the running state."},
-            [{"pre_detector_topic": {"description": "", "id": "RULE_ONE_ID", "title": "RULE_ONE", "severity": "critical", "mitre": ["mitre1", "mitre2"], "case_condition": "directly", "rule_filter": 'winlog.provider_name:"Service Control Manager"', "pre_detection_id": "08d1aa6f-f508-464e-a13d-0b5da46b5bcc", "host": {"name": "CLIENT1"}, "@timestamp": "2019-08-02T09:46:41.906Z"}}]
+            [{"pre_detector_topic": {"description": "", "id": "RULE_ONE_ID", "title": "RULE_ONE", "severity": "critical", "mitre": ["mitre1", "mitre2"], "case_condition": "directly", "rule_filter": 'winlog.provider_name:"Service Control Manager"', "pre_detection_id": "08d1aa6f-f508-464e-a13d-0b5da46b5bcc", "host": {"name": "CLIENT1"}, "@timestamp": "2019-08-02T09:46:41.906Z", "creation_timestamp": "2019-07-30T14:58:16.352Z"}}]
         ),
         (
             {"@timestamp":"2019-08-02T09:46:54.583Z","@metadata":{"beat":"winlogbeat","type":"_doc","version":"7.2.0","topic":"wineventlog_raw"},"winlog":{"provider_name":"Service Control Manager","computer_name":"abcdefg1234","provider_guid":"{555908d1-a6d7-4695-8e1e-26931d2012f4}","process":{"pid":440,"thread":{"id":1792}},"event_data":{"param1":"Portable Device Enumerator Service","param2":"running","Binary":"57005000440042007500730045006E0075006D002F0034000000"},"channel":"System","record_id":11638,"task":"","api":"wineventlog","event_id":1234,"keywords":["Classic"]},"event":{"code":7036,"created":"2019-08-02T09:55:11.999Z","kind":"event"},"agent":{"ephemeral_id":"de845cd9-5141-4c92-ad32-27a4518307e9","hostname":"CLIENT1","id":"0b755aca-0a9a-454a-9800-1979901962a0","version":"7.2.0","type":"winlogbeat"},"ecs":{"version":"1.0.0"},"host":{"id":"19fc45ac-5890-4f96-81b1-50ed111c0ce4","name":"CLIENT1","hostname":"CLIENT1","architecture":"x86","os":{"platform":"windows","version":"6.1","family":"windows","name":"Windows 7 Professional","kernel":"6.1.7601.18741 (win7sp1_gdr.150202-1526)","build":"7601.0"}},"log":{"level":"information"},"message":"The Portable Device Enumerator Service service entered the running state."},
             {"@timestamp":"2019-08-02T09:46:54.583Z","@metadata":{"beat":"winlogbeat","type":"_doc","version":"7.2.0","topic":"wineventlog_raw"},"winlog":{"provider_name":"Service Control Manager","computer_name":"abcdefg1234","provider_guid":"{555908d1-a6d7-4695-8e1e-26931d2012f4}","process":{"pid":440,"thread":{"id":1792}},"event_data":{"param1":"Portable Device Enumerator Service","param2":"running","Binary":"57005000440042007500730045006E0075006D002F0034000000"},"channel":"System","record_id":11638,"task":"","api":"wineventlog","event_id":1234,"keywords":["Classic"]},"event":{"code":7036,"created":"2019-08-02T09:55:11.999Z","kind":"event"},"agent":{"ephemeral_id":"de845cd9-5141-4c92-ad32-27a4518307e9","hostname":"CLIENT1","id":"0b755aca-0a9a-454a-9800-1979901962a0","version":"7.2.0","type":"winlogbeat"},"ecs":{"version":"1.0.0"},"host":{"id":"19fc45ac-5890-4f96-81b1-50ed111c0ce4","name":"CLIENT1","hostname":"CLIENT1","architecture":"x86","os":{"platform":"windows","version":"6.1","family":"windows","name":"Windows 7 Professional","kernel":"6.1.7601.18741 (win7sp1_gdr.150202-1526)","build":"7601.0"}},"log":{"level":"information"},"message":"The Portable Device Enumerator Service service entered the running state."},
-            [{"pre_detector_topic": {"description": "", "id": "RULE_ONE_ID", "title": "RULE_ONE", "severity": "critical", "mitre": ["mitre1", "mitre2"], "case_condition": "directly", "rule_filter": 'winlog.provider_name:"Service Control Manager"', "pre_detection_id": "06d12743-01f0-4793-8a31-3815cfa31fc3", "host": {"name": "CLIENT1"}, "@timestamp": "2019-08-02T09:46:54.583Z"}}]
+            [{"pre_detector_topic": {"description": "", "id": "RULE_ONE_ID", "title": "RULE_ONE", "severity": "critical", "mitre": ["mitre1", "mitre2"], "case_condition": "directly", "rule_filter": 'winlog.provider_name:"Service Control Manager"', "pre_detection_id": "06d12743-01f0-4793-8a31-3815cfa31fc3", "host": {"name": "CLIENT1"}, "@timestamp": "2019-08-02T09:46:54.583Z", "creation_timestamp": "2019-07-30T14:58:16.352Z"}}]
         ),
         (
             {"@timestamp":"2019-08-02T09:54:57.125Z","@metadata":{"beat":"winlogbeat","type":"_doc","version":"7.2.0","topic":"wineventlog_raw"},"winlog":{"computer_name":"abcdefg1234","event_id":123,"record_id":11714,"provider_guid":"{555908d1-a6d7-4695-8e1e-26931d2012f4}","channel":"System","task":"","api":"wineventlog","event_data":{"param2":"running","Binary":"41007500640069006F007300720076002F0034000000","param1":"Windows Audio"},"provider_name":"Service Control Manager 2","keywords":["Classic"],"process":{"pid":440,"thread":{"id":528}}},"event":{"kind":"event","code":7036,"created":"2019-08-02T09:55:12.091Z"},"log":{"level":"information"},"message":"The Windows Audio service entered the running state.","ecs":{"version":"1.0.0"},"host":{"name":"CLIENT1","architecture":"x86","os":{"kernel":"6.1.7601.18741 (win7sp1_gdr.150202-1526)","build":"7601.0","platform":"windows","version":"6.1","family":"windows","name":"Windows 7 Professional"},"id":"19fc45ac-5890-4f96-81b1-50ed111c0ce4","hostname":"CLIENT1"},"agent":{"id":"0b755aca-0a9a-454a-9800-1979901962a0","version":"7.2.0","type":"winlogbeat","ephemeral_id":"de845cd9-5141-4c92-ad32-27a4518307e9","hostname":"CLIENT1"}},
             {"@timestamp":"2019-08-02T09:54:57.125Z","@metadata":{"beat":"winlogbeat","type":"_doc","version":"7.2.0","topic":"wineventlog_raw"},"winlog":{"computer_name":"abcdefg1234","event_id":123,"record_id":11714,"provider_guid":"{555908d1-a6d7-4695-8e1e-26931d2012f4}","channel":"System","task":"","api":"wineventlog","event_data":{"param2":"running","Binary":"41007500640069006F007300720076002F0034000000","param1":"Windows Audio"},"provider_name":"Service Control Manager 2","keywords":["Classic"],"process":{"pid":440,"thread":{"id":528}}},"event":{"kind":"event","code":7036,"created":"2019-08-02T09:55:12.091Z"},"log":{"level":"information"},"message":"The Windows Audio service entered the running state.","ecs":{"version":"1.0.0"},"host":{"name":"CLIENT1","architecture":"x86","os":{"kernel":"6.1.7601.18741 (win7sp1_gdr.150202-1526)","build":"7601.0","platform":"windows","version":"6.1","family":"windows","name":"Windows 7 Professional"},"id":"19fc45ac-5890-4f96-81b1-50ed111c0ce4","hostname":"CLIENT1"},"agent":{"id":"0b755aca-0a9a-454a-9800-1979901962a0","version":"7.2.0","type":"winlogbeat","ephemeral_id":"de845cd9-5141-4c92-ad32-27a4518307e9","hostname":"CLIENT1"}},
-            [{"pre_detector_topic": {"description": "", "id": "RULE_TWO_ID", "title": "RULE_TWO", "severity": "critical", "mitre": ["mitre2", "mitre3"], "case_condition": "directly", "rule_filter": 'winlog.event_id:"123"', "pre_detection_id": "638cc0b3-b912-4220-8551-defea8ea139d", "host": {"name": "CLIENT1"}, "@timestamp": "2019-08-02T09:54:57.125Z"}}]
+            [{"pre_detector_topic": {"description": "", "id": "RULE_TWO_ID", "title": "RULE_TWO", "severity": "critical", "mitre": ["mitre2", "mitre3"], "case_condition": "directly", "rule_filter": 'winlog.event_id:"123"', "pre_detection_id": "638cc0b3-b912-4220-8551-defea8ea139d", "host": {"name": "CLIENT1"}, "@timestamp": "2019-08-02T09:54:57.125Z", "creation_timestamp": "2019-07-30T14:58:16.352Z"}}]
         ),
         (
             {"@timestamp":"2019-08-02T09:54:57.125Z","@metadata":{"beat":"winlogbeat","type":"_doc","version":"7.2.0","topic":"wineventlog_raw"},"winlog":{"computer_name":"abcdefg1234","event_id":123,"record_id":11714,"provider_guid":"{555908d1-a6d7-4695-8e1e-26931d2012f4}","channel":"System","task":"","api":"wineventlog","event_data":{"param2":"running","Binary":"41007500640069006F007300720076002F0034000000","param1":"Windows Audio"},"provider_name":"Service Control Manager","keywords":["Classic"],"process":{"pid":440,"thread":{"id":528}}},"event":{"kind":"event","code":7036,"created":"2019-08-02T09:55:12.091Z"},"log":{"level":"information"},"message":"The Windows Audio service entered the running state.","ecs":{"version":"1.0.0"},"host":{"name":"CLIENT1","architecture":"x86","os":{"kernel":"6.1.7601.18741 (win7sp1_gdr.150202-1526)","build":"7601.0","platform":"windows","version":"6.1","family":"windows","name":"Windows 7 Professional"},"id":"19fc45ac-5890-4f96-81b1-50ed111c0ce4","hostname":"CLIENT1"},"agent":{"id":"0b755aca-0a9a-454a-9800-1979901962a0","version":"7.2.0","type":"winlogbeat","ephemeral_id":"de845cd9-5141-4c92-ad32-27a4518307e9","hostname":"CLIENT1"}},
             {"@timestamp":"2019-08-02T09:54:57.125Z","@metadata":{"beat":"winlogbeat","type":"_doc","version":"7.2.0","topic":"wineventlog_raw"},"winlog":{"computer_name":"abcdefg1234","event_id":123,"record_id":11714,"provider_guid":"{555908d1-a6d7-4695-8e1e-26931d2012f4}","channel":"System","task":"","api":"wineventlog","event_data":{"param2":"running","Binary":"41007500640069006F007300720076002F0034000000","param1":"Windows Audio"},"provider_name":"Service Control Manager","keywords":["Classic"],"process":{"pid":440,"thread":{"id":528}}},"event":{"kind":"event","code":7036,"created":"2019-08-02T09:55:12.091Z"},"log":{"level":"information"},"message":"The Windows Audio service entered the running state.","ecs":{"version":"1.0.0"},"host":{"name":"CLIENT1","architecture":"x86","os":{"kernel":"6.1.7601.18741 (win7sp1_gdr.150202-1526)","build":"7601.0","platform":"windows","version":"6.1","family":"windows","name":"Windows 7 Professional"},"id":"19fc45ac-5890-4f96-81b1-50ed111c0ce4","hostname":"CLIENT1"},"agent":{"id":"0b755aca-0a9a-454a-9800-1979901962a0","version":"7.2.0","type":"winlogbeat","ephemeral_id":"de845cd9-5141-4c92-ad32-27a4518307e9","hostname":"CLIENT1"}},
             [{"pre_detector_topic": {"description": "", "id": "RULE_ONE_ID", "title": "RULE_ONE", "severity": "critical", "mitre": ["mitre1", "mitre2"], "case_condition": "directly", "rule_filter": 'winlog.provider_name:"Service Control Manager"', "pre_detection_id": "638cc0b3-b912-4220-8551-defea8ea139d", "host": {"name": "CLIENT1"}, "@timestamp": "2019-08-02T09:54:57.125Z"}},
-             {"pre_detector_topic": {"description": "", "id": "RULE_TWO_ID", "title": "RULE_TWO", "severity": "critical", "mitre": ["mitre2", "mitre3"], "case_condition": "directly", "rule_filter": 'winlog.event_id:"123"', "pre_detection_id": "638cc0b3-b912-4220-8551-defea8ea139d", "host": {"name": "CLIENT1"}, "@timestamp": "2019-08-02T09:54:57.125Z"}}]
+             {"pre_detector_topic": {"description": "", "id": "RULE_TWO_ID", "title": "RULE_TWO", "severity": "critical", "mitre": ["mitre2", "mitre3"], "case_condition": "directly", "rule_filter": 'winlog.event_id:"123"', "pre_detection_id": "638cc0b3-b912-4220-8551-defea8ea139d", "host": {"name": "CLIENT1"}, "@timestamp": "2019-08-02T09:54:57.125Z", "creation_timestamp": "2019-07-30T14:58:16.352Z"}}]
         ),
     ],
 )
@@ -77,19 +78,29 @@ def test_events_pre_detected_correctly(
     logprep_output, logprep_extra_output, logprep_error_output = get_test_output(config_path)
     assert not logprep_error_output
     diff = DeepDiff(
-        expected_output_event, logprep_output[0], exclude_paths="root['pre_detection_id']"
+        expected_output_event,
+        logprep_output[0],  # pylint: disable=unsubscriptable-object
+        exclude_paths="root['pre_detection_id']",
     )
     assert not diff, f"The expected output event and the logprep output differ: {diff}"
     if expected_extra_output is not None:
         # compare every expected extra output with every logprep extra output and search for match
         for expected_extra_out in expected_extra_output:
             has_matching_output = False
-            for logprep_extra_out in logprep_extra_output:
-                exclude_regex_path = re.compile(
+            for logprep_extra_out in logprep_extra_output:  # pylint: disable=not-an-iterable
+                exclude_pre_detection_id_regex_path = re.compile(
                     r"root\['pre_detector_topic'\]\['pre_detection_id'\]"
                 )
+                exclude_creation_timestamp_regex_path = re.compile(
+                    r"root\['pre_detector_topic'\]\['creation_timestamp'\]"
+                )
                 diff = DeepDiff(
-                    expected_extra_out, logprep_extra_out, exclude_regex_paths=exclude_regex_path
+                    expected_extra_out,
+                    logprep_extra_out,
+                    exclude_regex_paths=[
+                        exclude_pre_detection_id_regex_path,
+                        exclude_creation_timestamp_regex_path,
+                    ],
                 )
                 if not diff:
                     has_matching_output = True

diff --git a/tests/unit/processor/pre_detector/test_pre_detector.py b/tests/unit/processor/pre_detector/test_pre_detector.py
@@ -314,6 +314,9 @@ def test_ignores_case_list(self):
     def _assert_equality_of_results(
         self, document, expected, detection_results, expected_detection_results
     ):
+        for detection_result in detection_results[0]:
+            assert detection_result.pop("creation_timestamp")
+
         pre_detection_id = document.pop("pre_detection_id", None)
 
         assert pre_detection_id is not None