Update security service integrations packages mappings

packages/box_events/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.11.1"
+  changes:
+    - description: Add missing ECS mappings.
+      type: bugfix
+      link: http://github.com/elastic/integrations/pull/12624
 - version: "2.11.0"
   changes:
     - description: Add "preserve_original_event" tag to documents with `event.kind` set to "pipeline_error".

packages/box_events/data_stream/events/_dev/test/pipeline/test-malicious-content.log-expected.json

@@ -116,7 +116,7 @@
                         "number": 35908
                     },
                     "description": "BadMalware, MalwareBot4000, malware.exe Detected by Box Shield from IP 67.43.156.0. This is a really bad file see https://some.link/xyz",
-                    "first_seen": "2022-10-19T11:37:05-08:10",
+                    "first_seen": "2022-10-19T19:47:05.000Z",
                     "geo": {
                         "continent_name": "Asia",
                         "country_iso_code": "BT",
@@ -127,7 +127,7 @@
                         }
                     },
                     "ip": "67.43.156.0",
-                    "last_seen": "2022-10-20T11:37:05-08:10",
+                    "last_seen": "2022-10-20T19:47:05.000Z",
                     "provider": "Service name",
                     "reference": "https://some.link/xyz",
                     "type": "software"
@@ -241,8 +241,8 @@
             "threat": {
                 "indicator": {
                     "description": "BadMalware, MalwareBot4000, malware.exe Detected by Box Shield from IP Unknown IP. This is a really bad file see https://some.link/xyz",
-                    "first_seen": "2022-10-19T11:37:05-08:10",
-                    "last_seen": "2022-10-20T11:37:05-08:10",
+                    "first_seen": "2022-10-19T19:47:05.000Z",
+                    "last_seen": "2022-10-20T19:47:05.000Z",
                     "provider": "Service name",
                     "reference": "https://some.link/xyz",
                     "type": "software"
@@ -257,4 +257,4 @@
             }
         }
     ]
-}
+}

packages/box_events/data_stream/events/elasticsearch/ingest_pipeline/default.yml

@@ -889,6 +889,32 @@ processors:
         }
         ctx.threat.indicator.type = "software";
         ctx.related.indicator_type.add(ctx.threat.indicator.type);
+  - date:
+      field: threat.indicator.first_seen
+      tag: date_threat_indicator_first_seen
+      target_field: threat.indicator.first_seen
+      formats:
+        - ISO8601
+      if: ctx.threat?.indicator?.first_seen != null && ctx.threat.indicator.first_seen != ''
+      on_failure:
+        - remove:
+            field: threat.indicator.first_seen
+        - append:
+            field: error.message
+            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
+  - date:
+      field: threat.indicator.last_seen
+      tag: date_threat_indicator_last_seen
+      target_field: threat.indicator.last_seen
+      formats:
+        - ISO8601
+      if: ctx.threat?.indicator?.last_seen != null && ctx.threat.indicator.last_seen != ''
+      on_failure:
+        - remove:
+            field: threat.indicator.last_seen
+        - append:
+            field: error.message
+            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
 
 # Remove or mark invalid IPs.
   - foreach:

packages/box_events/data_stream/events/fields/ecs.yml

@@ -0,0 +1,7 @@
+# This definition could be removed once Kibana constraint is updated
+# to 8.15.2 or higher. "ecs@mappings" component template would define
+# the correct dynamic template for it.
+- external: ecs
+  name: threat.enrichments.indicator.first_seen
+- external: ecs
+  name: threat.enrichments.indicator.last_seen

packages/box_events/data_stream/events/sample_event.json

@@ -1,11 +1,11 @@
 {
     "@timestamp": "2019-12-08T08:00:00.000Z",
     "agent": {
-        "ephemeral_id": "c9ccc0f9-8d0e-4bfa-b365-1fbb4bc530c6",
-        "id": "a4d1a8b2-b45c-4d97-a37a-bd371f13111b",
-        "name": "docker-fleet-agent",
+        "ephemeral_id": "026dd623-8d74-4379-a43f-13cb90ac51ba",
+        "id": "83c9c411-7b8e-4819-9156-80e202799644",
+        "name": "elastic-agent-17757",
         "type": "filebeat",
-        "version": "8.8.1"
+        "version": "8.13.0"
     },
     "box": {
         "additional_details": {
@@ -53,16 +53,16 @@
     },
     "data_stream": {
         "dataset": "box_events.events",
-        "namespace": "ep",
+        "namespace": "76828",
         "type": "logs"
     },
     "ecs": {
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "a4d1a8b2-b45c-4d97-a37a-bd371f13111b",
+        "id": "83c9c411-7b8e-4819-9156-80e202799644",
         "snapshot": false,
-        "version": "8.8.1"
+        "version": "8.13.0"
     },
     "event": {
         "action": "SHIELD_ALERT",
@@ -71,10 +71,10 @@
             "threat",
             "file"
         ],
-        "created": "2023-09-21T01:54:30.945Z",
+        "created": "2025-02-12T10:21:05.215Z",
         "dataset": "box_events.events",
         "id": "97f1b31f-f143-4777-81f8-1b557b39ca33",
-        "ingested": "2023-09-21T01:54:31Z",
+        "ingested": "2025-02-12T10:21:08Z",
         "kind": "alert",
         "risk_score": 77,
         "type": [
@@ -85,19 +85,21 @@
     "host": {
         "architecture": "x86_64",
         "containerized": false,
-        "hostname": "docker-fleet-agent",
-        "id": "1de1e3b6561d4ccb9731539ce2f3baf3",
+        "hostname": "elastic-agent-17757",
+        "id": "8259e024976a406e8a54cdbffeb84fec",
         "ip": [
-            "172.19.0.7"
+            "172.19.0.2",
+            "172.18.0.6"
         ],
         "mac": [
-            "02-42-AC-13-00-07"
+            "02-42-AC-12-00-06",
+            "02-42-AC-13-00-02"
         ],
-        "name": "docker-fleet-agent",
+        "name": "elastic-agent-17757",
         "os": {
             "codename": "focal",
             "family": "debian",
-            "kernel": "5.10.104-linuxkit",
+            "kernel": "6.8.0-52-generic",
             "name": "Ubuntu",
             "platform": "ubuntu",
             "type": "linux",
@@ -186,4 +188,4 @@
             "name": "Some user"
         }
     }
-}
+}

packages/box_events/docs/README.md

@@ -270,4 +270,6 @@ Preserves a raw copy of the original event, added to the field `event.original`.
 | related.description | Array of `description` derived from `threat[.enrichments].indicator.description` | keyword |
 | related.indicator_type | Array of `indicator_type` derived from `threat[.enrichments].indicator.type` | keyword |
 | related.location | Array of `location` derived from `related.ip` | geo_point |
+| threat.enrichments.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date |
+| threat.enrichments.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date |
 

packages/box_events/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.3"
 name: box_events
 title: Box Events
-version: "2.11.0"
+version: "2.11.1"
 description: "Collect logs from Box with Elastic Agent"
 type: integration
 categories:

packages/claroty_ctd/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.4.3"
+  changes:
+    - description: Add missing ECS mappings.
+      type: bugfix
+      link: http://github.com/elastic/integrations/pull/12624
 - version: "0.4.2"
   changes:
     - description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.

packages/claroty_ctd/data_stream/event/fields/ecs.yml

@@ -0,0 +1,6 @@
+# This definition could be removed once Kibana constraint is updated
+# to 8.15.2 or higher. "ecs@mappings" component template would define
+# the correct dynamic template for it.
+- external: ecs
+  name: threat.indicator.modified_at
+

packages/claroty_ctd/docs/README.md

@@ -717,6 +717,7 @@ An example event for `event` looks as following:
 | log.offset | Log offset. | long |
 | log.source.address | Source address from which the log event read/sent. | keyword |
 | tags | User defined tags. | keyword |
+| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date |
 
 
 ### Assets

packages/claroty_ctd/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 3.1.4
 name: claroty_ctd
 title: Claroty CTD
-version: 0.4.2
+version: 0.4.3
 description: Collect logs from Claroty CTD using Elastic Agent.
 type: integration
 categories:

packages/crowdstrike/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.51.2"
+  changes:
+    - description: Avoid using dynamic template for flattened fields.
+      type: bugfix
+      link: http://github.com/elastic/integrations/pull/12624
 - version: "1.51.1"
   changes:
     - description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.

packages/crowdstrike/data_stream/fdr/fields/fields.yml

@@ -27,7 +27,7 @@
       type: long
     - name: AsepWrittenCount
       type: long
-    - name: assessments.*
+    - name: assessments
       type: flattened
     - name: AssociatedFile
       type: keyword

packages/crowdstrike/docs/README.md

@@ -1662,7 +1662,7 @@ and/or `session_token`.
 | crowdstrike.__mv_aip |  | keyword |
 | crowdstrike.__mv_discoverer_aid |  | keyword |
 | crowdstrike.aipCount |  | integer |
-| crowdstrike.assessments.\* |  | flattened |
+| crowdstrike.assessments |  | flattened |
 | crowdstrike.cid |  | keyword |
 | crowdstrike.discovererCount |  | integer |
 | crowdstrike.discoverer_aid |  | keyword |

packages/crowdstrike/manifest.yml

@@ -1,6 +1,6 @@
 name: crowdstrike
 title: CrowdStrike
-version: "1.51.1"
+version: "1.51.2"
 description: Collect logs from Crowdstrike with Elastic Agent.
 type: integration
 format_version: "3.0.3"

packages/mimecast/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.5.2"
+  changes:
+    - description: Add missing ECS field mappings.
+      type: bugfix
+      link: http://github.com/elastic/integrations/pull/12624
 - version: "2.5.1"
   changes:
     - description: Prevent pageToken from incorrectly reappearing in interval requests.

packages/mimecast/data_stream/threat_intel_malware_customer/fields/ecs.yml

@@ -0,0 +1,7 @@
+# This definition could be removed once Kibana constraint is updated
+# to 8.15.2 or higher. "ecs@mappings" component template would define
+# the correct dynamic template for it.
+- external: ecs
+  name: threat.indicator.modified_at
+- external: ecs
+  name: threat.indicator.first_seen

packages/mimecast/data_stream/threat_intel_malware_grid/fields/ecs.yml

@@ -0,0 +1,7 @@
+# This definition could be removed once Kibana constraint is updated
+# to 8.15.2 or higher. "ecs@mappings" component template would define
+# the correct dynamic template for it.
+- external: ecs
+  name: threat.indicator.modified_at
+- external: ecs
+  name: threat.indicator.first_seen

packages/mimecast/docs/README.md

@@ -1018,6 +1018,8 @@ An example event for `threat_intel_malware_customer` looks as following:
 | mimecast.type | The indicator type, can for example be "domain, email, FileHash-SHA256". | keyword |
 | mimecast.valid_from | The valid from date. | date |
 | mimecast.value | The value of the indicator. | keyword |
+| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date |
+| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date |
 
 
 ### Threat Intel Feed Malware: Grid
@@ -1134,6 +1136,8 @@ An example event for `threat_intel_malware_grid` looks as following:
 | mimecast.type | The indicator type, can for example be "domain, email, FileHash-SHA256". | keyword |
 | mimecast.valid_from | The valid from date. | date |
 | mimecast.value | The value of the indicator. | keyword |
+| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date |
+| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date |
 
 
 ### TTP Attachment Logs

packages/mimecast/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: mimecast
 title: "Mimecast"
-version: "2.5.1"
+version: "2.5.2"
 description: Collect logs from Mimecast with Elastic Agent.
 type: integration
 categories: ["security", "email_security"]

packages/sublime_security/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.5.2"
+  changes:
+    - description: Fix `sublime_security.email_message.headers.hops.fields` group mappings.
+      type: bugfix
+      link: http://github.com/elastic/integrations/pull/12624
 - version: "1.5.1"
   changes:
     - description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.

packages/sublime_security/data_stream/email_message/fields/fields.yml

@@ -642,8 +642,14 @@
                       type: keyword
                       description: The type of authentication result, derived from the field name.
                 - name: fields
-                  type: object
-                  object_type: keyword
+                  type: group
+                  fields:
+                    - name: "*"
+                      type: object
+                      object_type: keyword
+                    - name: position
+                      description: This field's position along the entire list of header fields.
+                      type: long
                 - name: index
                   type: long
                   description: Index indicates the order in which a hop occurred from sender to recipient.

packages/sublime_security/docs/README.md

@@ -1222,7 +1222,8 @@ An example event for `email_message` looks as following:
 | sublime_security.email_message.headers.hops.authentication_results.spf_details.server.valid | Whether the domain is valid. | boolean |
 | sublime_security.email_message.headers.hops.authentication_results.spf_details.verdict | Verdict of the SPF. | keyword |
 | sublime_security.email_message.headers.hops.authentication_results.type | The type of authentication result, derived from the field name. | keyword |
-| sublime_security.email_message.headers.hops.fields |  | object |
+| sublime_security.email_message.headers.hops.fields.\* |  | object |
+| sublime_security.email_message.headers.hops.fields.position | This field's position along the entire list of header fields. | long |
 | sublime_security.email_message.headers.hops.index | Index indicates the order in which a hop occurred from sender to recipient. | long |
 | sublime_security.email_message.headers.hops.received.additional.raw | The raw string for remaining additional clauses, such as transport information. | keyword |
 | sublime_security.email_message.headers.hops.received.id.raw | The raw string of 'id' section. | keyword |

packages/sublime_security/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 3.2.1
 name: sublime_security
 title: Sublime Security
-version: 1.5.1
+version: 1.5.2
 description: Collect logs from Sublime Security with Elastic Agent.
 type: integration
 categories:

packages/ti_anomali/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.25.2"
+  changes:
+    - description: Add missing ECS field in intelligence datastream.
+      type: bugfix
+      link: http://github.com/elastic/integrations/pull/12624
 - version: "1.25.1"
   changes:
     - description: Updated SSL description to be uniform and to include links to documentation.

packages/ti_anomali/data_stream/intelligence/_dev/test/pipeline/test-itype-apt-domain.json-expected.json

@@ -1,12 +1,12 @@
 {
     "expected": [
         {
-            "@timestamp": "2024-10-04T06:39:20.460979459Z",
+            "@timestamp": "2025-02-12T10:24:06.784043054Z",
             "anomali": {
                 "threatstream": {
                     "can_add_public_tags": true,
                     "confidence": 60,
-                    "deletion_scheduled_at": "2024-12-28T06:39:20.460979459Z",
+                    "deletion_scheduled_at": "2025-05-08T10:24:06.784043054Z",
                     "expiration_ts": "9999-12-31T00:00:00.000Z",
                     "feed_id": 0,
                     "id": "232020126",
@@ -58,4 +58,4 @@
             }
         }
     ]
-}
+}