From 95a8c2f8b39688741897dd9b9293e1f861eb94b6 Mon Sep 17 00:00:00 2001
From: Gabriel Pop <gabriel.pop@elastic.co>
Date: Thu, 27 Feb 2025 11:35:44 +0200
Subject: [PATCH 01/12] dont set timestamp in node_statistics

---
 .../node_statistics/elasticsearch/ingest_pipeline/default.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/packages/rubrik/data_stream/node_statistics/elasticsearch/ingest_pipeline/default.yml b/packages/rubrik/data_stream/node_statistics/elasticsearch/ingest_pipeline/default.yml
index 6c2bbf6cb8..b730a4988f 100644
--- a/packages/rubrik/data_stream/node_statistics/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/rubrik/data_stream/node_statistics/elasticsearch/ingest_pipeline/default.yml
@@ -84,10 +84,6 @@ processors:
         - append:
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
-  - set:
-      field: '@timestamp'
-      copy_from: rubrik.node_statistics.time
-      ignore_empty_value: true      
   - remove:
       field: response
       ignore_missing: true

From 5e7e4916f2e7ee2a4a5d2162398953d73fe710d2 Mon Sep 17 00:00:00 2001
From: Gabriel Pop <gabriel.pop@elastic.co>
Date: Thu, 27 Feb 2025 11:36:05 +0200
Subject: [PATCH 02/12] enable tsdb for data streams

---
 packages/rubrik/data_stream/drives/manifest.yml                 | 2 ++
 packages/rubrik/data_stream/filesets/manifest.yml               | 2 ++
 .../rubrik/data_stream/global_cluster_performance/manifest.yml  | 2 ++
 packages/rubrik/data_stream/managed_volumes/manifest.yml        | 2 ++
 packages/rubrik/data_stream/monitoring_jobs/manifest.yml        | 2 ++
 packages/rubrik/data_stream/mssql_databases/manifest.yml        | 2 ++
 packages/rubrik/data_stream/node_statistics/manifest.yml        | 2 ++
 packages/rubrik/data_stream/physical_hosts/manifest.yml         | 2 ++
 packages/rubrik/data_stream/tasks/manifest.yml                  | 2 ++
 packages/rubrik/data_stream/virtual_machines/manifest.yml       | 2 ++
 10 files changed, 20 insertions(+)

diff --git a/packages/rubrik/data_stream/drives/manifest.yml b/packages/rubrik/data_stream/drives/manifest.yml
index 3f80213d87..bebb92e758 100644
--- a/packages/rubrik/data_stream/drives/manifest.yml
+++ b/packages/rubrik/data_stream/drives/manifest.yml
@@ -70,3 +70,5 @@ streams:
         required: false
         show_user: false
         description: The request tracer logs requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#_request_tracer_filename) for details.
+elasticsearch:
+  index_mode: time_series
\ No newline at end of file
diff --git a/packages/rubrik/data_stream/filesets/manifest.yml b/packages/rubrik/data_stream/filesets/manifest.yml
index e515089010..f8ad337064 100644
--- a/packages/rubrik/data_stream/filesets/manifest.yml
+++ b/packages/rubrik/data_stream/filesets/manifest.yml
@@ -63,3 +63,5 @@ streams:
         default: "# filter:\n#   - field: \"location\"\n#     texts: \n#       - \"prod-lab.local\"\n"
         description: >-
           Specify filters for refining the data. Filters must be defined as an array of Filter objects. Refer to the [Rubrik API documentation](https://rubrikinc.github.io/rubrik-api-documentation/schema/reference/filter.doc.html) for valid fields.
+elasticsearch:
+  index_mode: time_series
\ No newline at end of file
diff --git a/packages/rubrik/data_stream/global_cluster_performance/manifest.yml b/packages/rubrik/data_stream/global_cluster_performance/manifest.yml
index ea9a563316..1d6f739ee6 100644
--- a/packages/rubrik/data_stream/global_cluster_performance/manifest.yml
+++ b/packages/rubrik/data_stream/global_cluster_performance/manifest.yml
@@ -54,3 +54,5 @@ streams:
         required: false
         show_user: false
         description: The number of results per page in Rubrik GraphQL queries. Smaller pages mean more API requests but can be more efficient in some cases.
+elasticsearch:
+  index_mode: time_series
\ No newline at end of file
diff --git a/packages/rubrik/data_stream/managed_volumes/manifest.yml b/packages/rubrik/data_stream/managed_volumes/manifest.yml
index c2c1cfb873..c4ce997929 100644
--- a/packages/rubrik/data_stream/managed_volumes/manifest.yml
+++ b/packages/rubrik/data_stream/managed_volumes/manifest.yml
@@ -70,3 +70,5 @@ streams:
         required: false
         show_user: false
         description: The request tracer logs requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#_request_tracer_filename) for details.
+elasticsearch:
+  index_mode: time_series
\ No newline at end of file
diff --git a/packages/rubrik/data_stream/monitoring_jobs/manifest.yml b/packages/rubrik/data_stream/monitoring_jobs/manifest.yml
index 66a4fabe88..d1b2246d37 100644
--- a/packages/rubrik/data_stream/monitoring_jobs/manifest.yml
+++ b/packages/rubrik/data_stream/monitoring_jobs/manifest.yml
@@ -54,3 +54,5 @@ streams:
         required: false
         show_user: false
         description: The request tracer logs requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#_request_tracer_filename) for details.
+elasticsearch:
+  index_mode: time_series
\ No newline at end of file
diff --git a/packages/rubrik/data_stream/mssql_databases/manifest.yml b/packages/rubrik/data_stream/mssql_databases/manifest.yml
index 7001edd38d..e198bebff0 100644
--- a/packages/rubrik/data_stream/mssql_databases/manifest.yml
+++ b/packages/rubrik/data_stream/mssql_databases/manifest.yml
@@ -78,3 +78,5 @@ streams:
         required: false
         show_user: false
         description: The request tracer logs requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#_request_tracer_filename) for details.
+elasticsearch:
+  index_mode: time_series
\ No newline at end of file
diff --git a/packages/rubrik/data_stream/node_statistics/manifest.yml b/packages/rubrik/data_stream/node_statistics/manifest.yml
index c9f356b4d7..28e95a1811 100644
--- a/packages/rubrik/data_stream/node_statistics/manifest.yml
+++ b/packages/rubrik/data_stream/node_statistics/manifest.yml
@@ -54,3 +54,5 @@ streams:
         required: false
         show_user: false
         description: The number of results per page in Rubrik GraphQL queries. Smaller pages mean more API requests but can be more efficient in some cases.
+elasticsearch:
+  index_mode: time_series
\ No newline at end of file
diff --git a/packages/rubrik/data_stream/physical_hosts/manifest.yml b/packages/rubrik/data_stream/physical_hosts/manifest.yml
index 101e9d5836..0ee57fb6e9 100644
--- a/packages/rubrik/data_stream/physical_hosts/manifest.yml
+++ b/packages/rubrik/data_stream/physical_hosts/manifest.yml
@@ -78,3 +78,5 @@ streams:
         required: false
         show_user: false
         description: The request tracer logs requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#_request_tracer_filename) for details.
+elasticsearch:
+  index_mode: time_series
\ No newline at end of file
diff --git a/packages/rubrik/data_stream/tasks/manifest.yml b/packages/rubrik/data_stream/tasks/manifest.yml
index 4f3e5be007..6d592faf47 100644
--- a/packages/rubrik/data_stream/tasks/manifest.yml
+++ b/packages/rubrik/data_stream/tasks/manifest.yml
@@ -47,3 +47,5 @@ streams:
         type: bool
         multi: false
         default: false
+elasticsearch:
+  index_mode: time_series
\ No newline at end of file
diff --git a/packages/rubrik/data_stream/virtual_machines/manifest.yml b/packages/rubrik/data_stream/virtual_machines/manifest.yml
index 2bc2316137..5615b4465f 100644
--- a/packages/rubrik/data_stream/virtual_machines/manifest.yml
+++ b/packages/rubrik/data_stream/virtual_machines/manifest.yml
@@ -70,3 +70,5 @@ streams:
         required: false
         show_user: false
         description: The request tracer logs requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#_request_tracer_filename) for details.
+elasticsearch:
+  index_mode: time_series
\ No newline at end of file

From 4e44fc7363366cfc1edd60ed50e4eb31f1ea80cf Mon Sep 17 00:00:00 2001
From: Gabriel Pop <gabriel.pop@elastic.co>
Date: Thu, 27 Feb 2025 11:36:17 +0200
Subject: [PATCH 03/12] bump package version

---
 packages/rubrik/changelog.yml | 7 +++++++
 packages/rubrik/manifest.yml  | 2 +-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/packages/rubrik/changelog.yml b/packages/rubrik/changelog.yml
index c5ffc7d7da..31d4294467 100644
--- a/packages/rubrik/changelog.yml
+++ b/packages/rubrik/changelog.yml
@@ -1,4 +1,11 @@
 # newer versions go on top
+- version: "0.5.0"
+  changes:
+    - description: |
+        Enable TSDB.
+        Don't set `@timestamp` in the `node_statistics` data stream since it can fall outside the allowable range of the indices, as this may cause documents to be dropped.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/999
 - version: "0.4.0"
   changes:
     - description: Add support for Kibana `9.0.0`.
diff --git a/packages/rubrik/manifest.yml b/packages/rubrik/manifest.yml
index 540c7680d5..30d7f545d4 100644
--- a/packages/rubrik/manifest.yml
+++ b/packages/rubrik/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 3.0.2
 name: rubrik
 title: "Rubrik RSC Metrics"
-version: 0.4.0
+version: 0.5.0
 source:
   license: "Elastic-2.0"
 description: "Collect Metrics from Rubrik RSC with Elastic Agent."

From c1bcaaa52ef7d9b5075d229591481b2b3aa157f5 Mon Sep 17 00:00:00 2001
From: Gabriel Pop <gabriel.pop@elastic.co>
Date: Thu, 27 Feb 2025 12:21:24 +0200
Subject: [PATCH 04/12] fix pipeline

---
 .../_dev/test/pipeline/test-node-statistics.log-expected.json   | 2 --
 1 file changed, 2 deletions(-)

diff --git a/packages/rubrik/data_stream/node_statistics/_dev/test/pipeline/test-node-statistics.log-expected.json b/packages/rubrik/data_stream/node_statistics/_dev/test/pipeline/test-node-statistics.log-expected.json
index 96d5953f94..66b57ce767 100644
--- a/packages/rubrik/data_stream/node_statistics/_dev/test/pipeline/test-node-statistics.log-expected.json
+++ b/packages/rubrik/data_stream/node_statistics/_dev/test/pipeline/test-node-statistics.log-expected.json
@@ -1,7 +1,6 @@
 {
     "expected": [
         {
-            "@timestamp": "2025-02-04T00:29:00.000Z",
             "ecs": {
                 "version": "8.16.0"
             },
@@ -64,7 +63,6 @@
             }
         },
         {
-            "@timestamp": "2025-02-04T08:20:00.000Z",
             "ecs": {
                 "version": "8.16.0"
             },

From a8f3ee4fd274e0544a153974c3c8f31fa61bfebc Mon Sep 17 00:00:00 2001
From: Gabriel Pop <94497545+gpop63@users.noreply.github.com>
Date: Thu, 27 Feb 2025 12:39:09 +0200
Subject: [PATCH 05/12] Update packages/rubrik/changelog.yml

Co-authored-by: Richa Talwar <102972658+ritalwar@users.noreply.github.com>
---
 packages/rubrik/changelog.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/rubrik/changelog.yml b/packages/rubrik/changelog.yml
index 31d4294467..03ce06afd6 100644
--- a/packages/rubrik/changelog.yml
+++ b/packages/rubrik/changelog.yml
@@ -5,7 +5,7 @@
         Enable TSDB.
         Don't set `@timestamp` in the `node_statistics` data stream since it can fall outside the allowable range of the indices, as this may cause documents to be dropped.
       type: enhancement
-      link: https://github.com/elastic/integrations/pull/999
+      link:https://github.com/elastic/integrations/pull/12917
 - version: "0.4.0"
   changes:
     - description: Add support for Kibana `9.0.0`.

From d0ca96042d31ab6b14f08501e9fb88ced5d1cc2d Mon Sep 17 00:00:00 2001
From: Gabriel Pop <94497545+gpop63@users.noreply.github.com>
Date: Thu, 27 Feb 2025 12:39:18 +0200
Subject: [PATCH 06/12] Update packages/rubrik/changelog.yml

Co-authored-by: Richa Talwar <102972658+ritalwar@users.noreply.github.com>
---
 packages/rubrik/changelog.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/rubrik/changelog.yml b/packages/rubrik/changelog.yml
index 03ce06afd6..0565b21afb 100644
--- a/packages/rubrik/changelog.yml
+++ b/packages/rubrik/changelog.yml
@@ -3,7 +3,7 @@
   changes:
     - description: |
         Enable TSDB.
-        Don't set `@timestamp` in the `node_statistics` data stream since it can fall outside the allowable range of the indices, as this may cause documents to be dropped.
+        Remove @timestamp from the node_statistics data stream to prevent document drops due to range issues.
       type: enhancement
       link:https://github.com/elastic/integrations/pull/12917
 - version: "0.4.0"

From 5e0c569dfccf6b934aecc92ede56c4cc54b5816c Mon Sep 17 00:00:00 2001
From: Gabriel Pop <gabriel.pop@elastic.co>
Date: Thu, 27 Feb 2025 12:58:35 +0200
Subject: [PATCH 07/12] fix changelog

---
 packages/rubrik/changelog.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/rubrik/changelog.yml b/packages/rubrik/changelog.yml
index 0565b21afb..3d90885079 100644
--- a/packages/rubrik/changelog.yml
+++ b/packages/rubrik/changelog.yml
@@ -5,7 +5,7 @@
         Enable TSDB.
         Remove @timestamp from the node_statistics data stream to prevent document drops due to range issues.
       type: enhancement
-      link:https://github.com/elastic/integrations/pull/12917
+      link: https://github.com/elastic/integrations/pull/12917
 - version: "0.4.0"
   changes:
     - description: Add support for Kibana `9.0.0`.

From 647aa686ff0725c6aeb0b2a1581e11deaeecceab Mon Sep 17 00:00:00 2001
From: Gabriel Pop <gabriel.pop@elastic.co>
Date: Fri, 28 Feb 2025 12:48:59 +0200
Subject: [PATCH 08/12] Revert "dont set timestamp in node_statistics"

This reverts commit 95a8c2f8b39688741897dd9b9293e1f861eb94b6.
---
 .../node_statistics/elasticsearch/ingest_pipeline/default.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/packages/rubrik/data_stream/node_statistics/elasticsearch/ingest_pipeline/default.yml b/packages/rubrik/data_stream/node_statistics/elasticsearch/ingest_pipeline/default.yml
index b730a4988f..6c2bbf6cb8 100644
--- a/packages/rubrik/data_stream/node_statistics/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/rubrik/data_stream/node_statistics/elasticsearch/ingest_pipeline/default.yml
@@ -84,6 +84,10 @@ processors:
         - append:
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
+  - set:
+      field: '@timestamp'
+      copy_from: rubrik.node_statistics.time
+      ignore_empty_value: true      
   - remove:
       field: response
       ignore_missing: true

From a3d15fbf062c1d108969421289659bc55dd31eac Mon Sep 17 00:00:00 2001
From: Gabriel Pop <gabriel.pop@elastic.co>
Date: Fri, 28 Feb 2025 12:49:15 +0200
Subject: [PATCH 09/12] Revert "fix pipeline"

This reverts commit c1bcaaa52ef7d9b5075d229591481b2b3aa157f5.
---
 .../_dev/test/pipeline/test-node-statistics.log-expected.json   | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/rubrik/data_stream/node_statistics/_dev/test/pipeline/test-node-statistics.log-expected.json b/packages/rubrik/data_stream/node_statistics/_dev/test/pipeline/test-node-statistics.log-expected.json
index 66b57ce767..96d5953f94 100644
--- a/packages/rubrik/data_stream/node_statistics/_dev/test/pipeline/test-node-statistics.log-expected.json
+++ b/packages/rubrik/data_stream/node_statistics/_dev/test/pipeline/test-node-statistics.log-expected.json
@@ -1,6 +1,7 @@
 {
     "expected": [
         {
+            "@timestamp": "2025-02-04T00:29:00.000Z",
             "ecs": {
                 "version": "8.16.0"
             },
@@ -63,6 +64,7 @@
             }
         },
         {
+            "@timestamp": "2025-02-04T08:20:00.000Z",
             "ecs": {
                 "version": "8.16.0"
             },

From bb0e2f98b8571f0ba802ffc734662848261e1858 Mon Sep 17 00:00:00 2001
From: Gabriel Pop <gabriel.pop@elastic.co>
Date: Fri, 28 Feb 2025 12:49:28 +0200
Subject: [PATCH 10/12] change interval to 1h

---
 packages/rubrik/data_stream/node_statistics/manifest.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/rubrik/data_stream/node_statistics/manifest.yml b/packages/rubrik/data_stream/node_statistics/manifest.yml
index 28e95a1811..66e6b6e89a 100644
--- a/packages/rubrik/data_stream/node_statistics/manifest.yml
+++ b/packages/rubrik/data_stream/node_statistics/manifest.yml
@@ -21,7 +21,7 @@ streams:
         description: The interval at which the API is polled, supported in seconds, minutes, and hours.
         show_user: true
         required: true
-        default: 24h
+        default: 1h
       - name: processors
         type: yaml
         title: Processors

From 8f9a1199a41c81c0e9e7be59a4aac5dcd0a1d0d2 Mon Sep 17 00:00:00 2001
From: Gabriel Pop <gabriel.pop@elastic.co>
Date: Fri, 28 Feb 2025 12:49:44 +0200
Subject: [PATCH 11/12] add time range filter in query

---
 .../node_statistics/agent/stream/cel.yml.hbs          | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/packages/rubrik/data_stream/node_statistics/agent/stream/cel.yml.hbs b/packages/rubrik/data_stream/node_statistics/agent/stream/cel.yml.hbs
index f107aabb4e..1e56a65942 100644
--- a/packages/rubrik/data_stream/node_statistics/agent/stream/cel.yml.hbs
+++ b/packages/rubrik/data_stream/node_statistics/agent/stream/cel.yml.hbs
@@ -35,17 +35,18 @@ processors:
 
 state:
     index: 0
+    period: {{interval}}
     pageSize: {{pageSize}}
     queries:
       - |
-        query ClusterNodeStatistics($first: Int, $after: String) {
+        query ClusterNodeStatistics($first: Int, $after: String, $range: TimeRangeInput) {
           allClusterConnection(first: $first, after: $after) {
             count
             nodes {
               name
               type
               id
-              clusterNodeStats {
+              clusterNodeStats(timeRange: $range) {
                 nodeId
                 clusterPhysicalDataIngest
                 networkBytesReceived
@@ -81,6 +82,10 @@ program: |-
           "variables": {
             "first": has(state.pageSize) ? state.pageSize : null,
             "after": has(state.cursor) ? state.cursor.after : null,
+            "range": {
+              "start": now - duration(state.period),
+              "end": now.format(time_layout.RFC3339),
+            }
           }
         }.encode_json()
       }
@@ -132,7 +137,7 @@ program: |-
           "index": body.data.allClusterConnection.pageInfo.hasNextPage ? int(state.index) : (int(state.index) + 1) % size(state.queries),
           "cursor": {
             "after": body.data.allClusterConnection.pageInfo.hasNextPage ? body.data.allClusterConnection.pageInfo.endCursor : null
-          }
+          },
         }
       )
       :

From 495e6add5f2e3a9187500680a1e552e238d34eea Mon Sep 17 00:00:00 2001
From: Gabriel Pop <gabriel.pop@elastic.co>
Date: Mon, 3 Mar 2025 17:44:28 +0200
Subject: [PATCH 12/12] improve docs

---
 packages/rubrik/_dev/build/docs/README.md | 2 ++
 packages/rubrik/docs/README.md            | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/packages/rubrik/_dev/build/docs/README.md b/packages/rubrik/_dev/build/docs/README.md
index 802d5868eb..bef1e50abc 100644
--- a/packages/rubrik/_dev/build/docs/README.md
+++ b/packages/rubrik/_dev/build/docs/README.md
@@ -145,6 +145,8 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur
 
 The `node_statistics` dataset provides metrics related to the performance of the Rubrik cluster nodes.
 
+**IMPORTANT: Setting `interval` to more than `1h` may cause documents to be dropped if node statistics metrics fall outside the index time range.**
+
 **ECS Field Reference**
 
 Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields.
diff --git a/packages/rubrik/docs/README.md b/packages/rubrik/docs/README.md
index de632e157a..06f5f618dc 100644
--- a/packages/rubrik/docs/README.md
+++ b/packages/rubrik/docs/README.md
@@ -1058,6 +1058,8 @@ An example event for `global_cluster_performance` looks as following:
 
 The `node_statistics` dataset provides metrics related to the performance of the Rubrik cluster nodes.
 
+**IMPORTANT: Setting `interval` to more than `1h` may cause documents to be dropped if node statistics metrics fall outside the index time range.**
+
 **ECS Field Reference**
 
 Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields.