first try of keeping the ordering of the source fields list

fkie-cad · Jun 5, 2024 · f6800e6 · f6800e6
1 parent 0ff8dae
commit f6800e6
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 5 deletions.
diff --git a/logprep/processor/field_manager/processor.py b/logprep/processor/field_manager/processor.py
@@ -169,7 +169,8 @@ def _write_to_single_target(self, args, extend_target_list, overwrite_target, ru
     def _overwrite_from_source_values(self, source_fields_values):
         lists, not_lists = self._separate_lists_form_other_types(source_fields_values)
         flattened_source_fields = self._get_flatten_list(lists, not_lists)
-        return flattened_source_fields
+        ordered_list = self._get_ordered_flatten_list(flattened_source_fields, source_fields_values)
+        return ordered_list
 
     def _handle_missing_fields(self, event, rule, source_fields, field_values):
         if rule.ignore_missing_fields:
@@ -210,6 +211,24 @@ def _get_flatten_list(lists: List[List], not_lists: List[Any]) -> List:
                 flatten_list.append(field_value)
         return flatten_list
 
+    @staticmethod
+    def _get_ordered_flatten_list(flatten_list, source_field_values):
+        ordered_flatten_list = []
+        flat_source_fields = []
+        duplicates = []
+        for item in source_field_values:
+            if isinstance(item, list):
+                flat_source_fields.extend(item)
+            else:
+                flat_source_fields.append(item)
+
+        for field_value in flat_source_fields:
+            if field_value not in duplicates:
+                duplicates.append(field_value)
+                if field_value in flatten_list:
+                    ordered_flatten_list.append(field_value)
+        return ordered_flatten_list
+
     @staticmethod
     def _filter_missing_fields(source_field_values, targets):
         if None in source_field_values:

diff --git a/tests/unit/processor/field_manager/test_field_manager.py b/tests/unit/processor/field_manager/test_field_manager.py
@@ -187,7 +187,7 @@
             "field3": ["value5", "value6"],
             "new_field": ["i exist"],
         },
-        {"new_field": ["i exist", "value1", "value2", "value3", "value5", "value6", "value4"]},
+        {"new_field": ["i exist", "value1", "value2", "value3", "value4", "value5", "value6"]},
     ),
     (
         (
@@ -209,7 +209,7 @@
             "field3": ["value5", "value6", "value4"],
             "new_field": ["i exist"],
         },
-        {"new_field": ["i exist", "value1", "value2", "value3", "value5", "value6", "value4"]},
+        {"new_field": ["i exist", "value1", "value2", "value3", "value5", "value4", "value6"]},
     ),
     (
         (
@@ -232,7 +232,7 @@
             "field3": ["value5", "value6", "value4"],
             "new_field": ["i exist"],
         },
-        {"new_field": ["value1", "value2", "value3", "value5", "value6", "value4"]},
+        {"new_field": ["value1", "value2", "value3", "value5", "value4", "value6"]},
     ),
     (
         "real world example from documentation",
@@ -273,8 +273,8 @@
                     "127.0.0.1",
                     "fe89::",
                     "192.168.5.1",
-                    "180.22.66.3",
                     "8.8.8.8",
+                    "180.22.66.3",
                     "10.10.2.33",
                     "180.22.66.1",
                     "223.2.3.2",