elastic · mrodm · Feb 20, 2025 · Feb 4, 2025 · Feb 4, 2025 · Feb 4, 2025
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -30,6 +30,8 @@ env:
   ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "${ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI:-false}"
   # Disable checking for newer versions
   ELASTIC_PACKAGE_CHECK_UPDATE_DISABLED: "true"
+  # Select method to validate fields are documented
+  ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD: "mappings"
 
 steps:
   - label: "Get reference from target branch"

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
@@ -757,7 +757,7 @@ teardown_test_package() {
 }
 
 list_all_directories() {
-    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort
+    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort |grep -E '^(box_events|claroty_ctd|crowdstrike|mimecast|sublime_security|teleport|ti_anomali)$'
 }
 
 check_package() {

diff --git a/go.mod b/go.mod
@@ -231,3 +231,5 @@ require (
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )
+
+replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20250205174526-5b3f7cdba246
diff --git a/go.sum b/go.sum
@@ -125,8 +125,6 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/elastic/elastic-integration-corpus-generator-tool v0.10.0 h1:sx1lpZuTG5suJuvgix4FWQFCLFFbzkoOmPoHWYOPLCY=
 github.com/elastic/elastic-integration-corpus-generator-tool v0.10.0/go.mod h1:2/30n+2QRzRzus4TPVUV1T3U/j8g2ItUgvP0pcpjLGk=
-github.com/elastic/elastic-package v0.109.1 h1:ATZVgYOCI6L5Yr0NxjSX+MsuK4UvXkpu9tDkO4K2vgo=
-github.com/elastic/elastic-package v0.109.1/go.mod h1:vmVYISfxBrl0ejjKbm/AG0drjrmevysVg2ZIP7yewLo=
 github.com/elastic/go-elasticsearch/v7 v7.17.10 h1:TCQ8i4PmIJuBunvBS6bwT2ybzVFxxUhhltAs3Gyu1yo=
 github.com/elastic/go-elasticsearch/v7 v7.17.10/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
 github.com/elastic/go-licenser v0.4.2 h1:bPbGm8bUd8rxzSswFOqvQh1dAkKGkgAmrPxbUi+Y9+A=
@@ -372,6 +370,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0=
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
+github.com/mrodm/elastic-package v0.53.1-0.20250205174526-5b3f7cdba246 h1:EjWls8TjHBNk5E/caFYxZR7DkURTeDevDazWZgO1T7A=
+github.com/mrodm/elastic-package v0.53.1-0.20250205174526-5b3f7cdba246/go.mod h1:vmVYISfxBrl0ejjKbm/AG0drjrmevysVg2ZIP7yewLo=
 github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s=
 github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8=
 github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo=

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.11.1"
+  changes:
+    - description: Add missing ECS mappings.
+      type: bugfix
+      link: http://github.com/elastic/integrations/pull/12624
 - version: "2.11.0"
   changes:
     - description: Add "preserve_original_event" tag to documents with `event.kind` set to "pipeline_error".

@@ -116,7 +116,7 @@
                         "number": 35908
                     },
                     "description": "BadMalware, MalwareBot4000, malware.exe Detected by Box Shield from IP 67.43.156.0. This is a really bad file see https://some.link/xyz",
-                    "first_seen": "2022-10-19T11:37:05-08:10",
+                    "first_seen": "2022-10-19T19:47:05.000Z",
                     "geo": {
                         "continent_name": "Asia",
                         "country_iso_code": "BT",
@@ -127,7 +127,7 @@
                         }
                     },
                     "ip": "67.43.156.0",
-                    "last_seen": "2022-10-20T11:37:05-08:10",
+                    "last_seen": "2022-10-20T19:47:05.000Z",
                     "provider": "Service name",
                     "reference": "https://some.link/xyz",
                     "type": "software"
@@ -241,8 +241,8 @@
             "threat": {
                 "indicator": {
                     "description": "BadMalware, MalwareBot4000, malware.exe Detected by Box Shield from IP Unknown IP. This is a really bad file see https://some.link/xyz",
-                    "first_seen": "2022-10-19T11:37:05-08:10",
-                    "last_seen": "2022-10-20T11:37:05-08:10",
+                    "first_seen": "2022-10-19T19:47:05.000Z",
+                    "last_seen": "2022-10-20T19:47:05.000Z",
                     "provider": "Service name",
                     "reference": "https://some.link/xyz",
                     "type": "software"
@@ -257,4 +257,4 @@
             }
         }
     ]
-}
+}
@@ -889,6 +889,32 @@ processors:
         }
         ctx.threat.indicator.type = "software";
         ctx.related.indicator_type.add(ctx.threat.indicator.type);
+  - date:
+      field: threat.indicator.first_seen
+      tag: date_threat_indicator_first_seen
+      target_field: threat.indicator.first_seen
+      formats:
+        - ISO8601
+      if: ctx.threat?.indicator?.first_seen != null && ctx.threat.indicator.first_seen != ''
+      on_failure:
+        - remove:
+            field: threat.indicator.first_seen
+        - append:
+            field: error.message
+            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
+  - date:
+      field: threat.indicator.last_seen
+      tag: date_threat_indicator_last_seen
+      target_field: threat.indicator.last_seen
+      formats:
+        - ISO8601
+      if: ctx.threat?.indicator?.last_seen != null && ctx.threat.indicator.last_seen != ''
+      on_failure:
+        - remove:
+            field: threat.indicator.last_seen
+        - append:
+            field: error.message
+            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
 
 # Remove or mark invalid IPs.
   - foreach:

@@ -0,0 +1,7 @@
+# This definition could be removed once Kibana constraint is updated
+# to 8.15.2 or higher. "ecs@mappings" component template would define
+# the correct dynamic template for it.
+- external: ecs
+  name: threat.enrichments.indicator.first_seen
+- external: ecs
+  name: threat.enrichments.indicator.last_seen
@@ -1,11 +1,11 @@
 {
     "@timestamp": "2019-12-08T08:00:00.000Z",
     "agent": {
-        "ephemeral_id": "c9ccc0f9-8d0e-4bfa-b365-1fbb4bc530c6",
-        "id": "a4d1a8b2-b45c-4d97-a37a-bd371f13111b",
-        "name": "docker-fleet-agent",
+        "ephemeral_id": "026dd623-8d74-4379-a43f-13cb90ac51ba",
+        "id": "83c9c411-7b8e-4819-9156-80e202799644",
+        "name": "elastic-agent-17757",
         "type": "filebeat",
-        "version": "8.8.1"
+        "version": "8.13.0"
     },
     "box": {
         "additional_details": {
@@ -53,16 +53,16 @@
     },
     "data_stream": {
         "dataset": "box_events.events",
-        "namespace": "ep",
+        "namespace": "76828",
         "type": "logs"
     },
     "ecs": {
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "a4d1a8b2-b45c-4d97-a37a-bd371f13111b",
+        "id": "83c9c411-7b8e-4819-9156-80e202799644",
         "snapshot": false,
-        "version": "8.8.1"
+        "version": "8.13.0"
     },
     "event": {
         "action": "SHIELD_ALERT",
@@ -71,10 +71,10 @@
             "threat",
             "file"
         ],
-        "created": "2023-09-21T01:54:30.945Z",
+        "created": "2025-02-12T10:21:05.215Z",
         "dataset": "box_events.events",
         "id": "97f1b31f-f143-4777-81f8-1b557b39ca33",
-        "ingested": "2023-09-21T01:54:31Z",
+        "ingested": "2025-02-12T10:21:08Z",
         "kind": "alert",
         "risk_score": 77,
         "type": [
@@ -85,19 +85,21 @@
     "host": {
         "architecture": "x86_64",
         "containerized": false,
-        "hostname": "docker-fleet-agent",
-        "id": "1de1e3b6561d4ccb9731539ce2f3baf3",
+        "hostname": "elastic-agent-17757",
+        "id": "8259e024976a406e8a54cdbffeb84fec",
         "ip": [
-            "172.19.0.7"
+            "172.19.0.2",
+            "172.18.0.6"
         ],
         "mac": [
-            "02-42-AC-13-00-07"
+            "02-42-AC-12-00-06",
+            "02-42-AC-13-00-02"
         ],
-        "name": "docker-fleet-agent",
+        "name": "elastic-agent-17757",
         "os": {
             "codename": "focal",
             "family": "debian",
-            "kernel": "5.10.104-linuxkit",
+            "kernel": "6.8.0-52-generic",
             "name": "Ubuntu",
             "platform": "ubuntu",
             "type": "linux",
@@ -186,4 +188,4 @@
             "name": "Some user"
         }
     }
-}
+}
@@ -270,4 +270,6 @@ Preserves a raw copy of the original event, added to the field `event.original`.
 | related.description | Array of `description` derived from `threat[.enrichments].indicator.description` | keyword |
 | related.indicator_type | Array of `indicator_type` derived from `threat[.enrichments].indicator.type` | keyword |
 | related.location | Array of `location` derived from `related.ip` | geo_point |
+| threat.enrichments.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date |
+| threat.enrichments.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date |
 
@@ -1,7 +1,7 @@
 format_version: "3.0.3"
 name: box_events
 title: Box Events
-version: "2.11.0"
+version: "2.11.1"
 description: "Collect logs from Box with Elastic Agent"
 type: integration
 categories:

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.4.3"
+  changes:
+    - description: Add missing ECS mappings.
+      type: bugfix
+      link: http://github.com/elastic/integrations/pull/12624
 - version: "0.4.2"
   changes:
     - description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.

@@ -0,0 +1,6 @@
+# This definition could be removed once Kibana constraint is updated
+# to 8.15.2 or higher. "ecs@mappings" component template would define
+# the correct dynamic template for it.
+- external: ecs
+  name: threat.indicator.modified_at
+
@@ -717,6 +717,7 @@ An example event for `event` looks as following:
 | log.offset | Log offset. | long |
 | log.source.address | Source address from which the log event read/sent. | keyword |
 | tags | User defined tags. | keyword |
+| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date |
 
 
 ### Assets

@@ -1,7 +1,7 @@
 format_version: 3.1.4
 name: claroty_ctd
 title: Claroty CTD
-version: 0.4.2
+version: 0.4.3
 description: Collect logs from Claroty CTD using Elastic Agent.
 type: integration
 categories:

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.51.2"
+  changes:
+    - description: Avoid using dynamic template for flattened fields.
+      type: bugfix
+      link: http://github.com/elastic/integrations/pull/12624
 - version: "1.51.1"
   changes:
     - description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.

@@ -27,7 +27,7 @@
       type: long
     - name: AsepWrittenCount
       type: long
-    - name: assessments.*
+    - name: assessments
       type: flattened
     - name: AssociatedFile
       type: keyword

@@ -1662,7 +1662,7 @@ and/or `session_token`.
 | crowdstrike.__mv_aip |  | keyword |
 | crowdstrike.__mv_discoverer_aid |  | keyword |
 | crowdstrike.aipCount |  | integer |
-| crowdstrike.assessments.\* |  | flattened |
+| crowdstrike.assessments |  | flattened |
 | crowdstrike.cid |  | keyword |
 | crowdstrike.discovererCount |  | integer |
 | crowdstrike.discoverer_aid |  | keyword |

@@ -1,6 +1,6 @@
 name: crowdstrike
 title: CrowdStrike
-version: "1.51.1"
+version: "1.51.2"
 description: Collect logs from Crowdstrike with Elastic Agent.
 type: integration
 format_version: "3.0.3"

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.5.2"
+  changes:
+    - description: Add missing ECS field mappings.
+      type: bugfix
+      link: http://github.com/elastic/integrations/pull/12624
 - version: "2.5.1"
   changes:
     - description: Prevent pageToken from incorrectly reappearing in interval requests.

@@ -0,0 +1,7 @@
+# This definition could be removed once Kibana constraint is updated
+# to 8.15.2 or higher. "ecs@mappings" component template would define
+# the correct dynamic template for it.
+- external: ecs
+  name: threat.indicator.modified_at
+- external: ecs
+  name: threat.indicator.first_seen
@@ -0,0 +1,7 @@
+# This definition could be removed once Kibana constraint is updated
+# to 8.15.2 or higher. "ecs@mappings" component template would define
+# the correct dynamic template for it.
+- external: ecs
+  name: threat.indicator.modified_at
+- external: ecs
+  name: threat.indicator.first_seen
@@ -1018,6 +1018,8 @@ An example event for `threat_intel_malware_customer` looks as following:
 | mimecast.type | The indicator type, can for example be "domain, email, FileHash-SHA256". | keyword |
 | mimecast.valid_from | The valid from date. | date |
 | mimecast.value | The value of the indicator. | keyword |
+| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date |
+| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date |
 
 
 ### Threat Intel Feed Malware: Grid
@@ -1134,6 +1136,8 @@ An example event for `threat_intel_malware_grid` looks as following:
 | mimecast.type | The indicator type, can for example be "domain, email, FileHash-SHA256". | keyword |
 | mimecast.valid_from | The valid from date. | date |
 | mimecast.value | The value of the indicator. | keyword |
+| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date |
+| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date |
 
 
 ### TTP Attachment Logs

@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: mimecast
 title: "Mimecast"
-version: "2.5.1"
+version: "2.5.2"
 description: Collect logs from Mimecast with Elastic Agent.
 type: integration
 categories: ["security", "email_security"]

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.5.2"
+  changes:
+    - description: Fix `sublime_security.email_message.headers.hops.fields` group mappings.
+      type: bugfix
+      link: http://github.com/elastic/integrations/pull/12624
 - version: "1.5.1"
   changes:
     - description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.

@@ -642,8 +642,14 @@
                       type: keyword
                       description: The type of authentication result, derived from the field name.
                 - name: fields
-                  type: object
-                  object_type: keyword
+                  type: group
+                  fields:
+                    - name: "*"
+                      type: object
+                      object_type: keyword
+                    - name: position
+                      description: This field's position along the entire list of header fields.
+                      type: long
                 - name: index
                   type: long
                   description: Index indicates the order in which a hop occurred from sender to recipient.

@@ -1222,7 +1222,8 @@ An example event for `email_message` looks as following:
 | sublime_security.email_message.headers.hops.authentication_results.spf_details.server.valid | Whether the domain is valid. | boolean |
 | sublime_security.email_message.headers.hops.authentication_results.spf_details.verdict | Verdict of the SPF. | keyword |
 | sublime_security.email_message.headers.hops.authentication_results.type | The type of authentication result, derived from the field name. | keyword |
-| sublime_security.email_message.headers.hops.fields |  | object |
+| sublime_security.email_message.headers.hops.fields.\* |  | object |
+| sublime_security.email_message.headers.hops.fields.position | This field's position along the entire list of header fields. | long |
 | sublime_security.email_message.headers.hops.index | Index indicates the order in which a hop occurred from sender to recipient. | long |
 | sublime_security.email_message.headers.hops.received.additional.raw | The raw string for remaining additional clauses, such as transport information. | keyword |
 | sublime_security.email_message.headers.hops.received.id.raw | The raw string of 'id' section. | keyword |

@@ -1,7 +1,7 @@
 format_version: 3.2.1
 name: sublime_security
 title: Sublime Security
-version: 1.5.1
+version: 1.5.2
 description: Collect logs from Sublime Security with Elastic Agent.
 type: integration
 categories: