✨ Source Datadog: add monitors and service_level_objectives (SLOs…

…) streams (airbytehq#30999) Co-authored-by: Marcos Marx <[email protected]> Co-authored-by: marcosmarxm <[email protected]>
dlouseiro · Dec 4, 2023 · 7eb5f0e · 7eb5f0e
1 parent c11041a
commit 7eb5f0e
Show file tree

Hide file tree

Showing 10 changed files with 561 additions and 12 deletions.
diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md
@@ -108,6 +108,7 @@
 * [carlonuccio](https://github.com/carlonuccio)
 * [catpineapple](https://github.com/catpineapple)
 * [cgardens](https://github.com/cgardens)
+* [chadthman](https://github.com/chadthman)
 * [chandrasekharan98](https://github.com/chandrasekharan98)
 * [ChristoGrab](https://github.com/ChristoGrab)
 * [ChristopheDuong](https://github.com/ChristopheDuong)

diff --git a/airbyte-integrations/connectors/source-datadog/Dockerfile b/airbyte-integrations/connectors/source-datadog/Dockerfile
@@ -34,5 +34,5 @@ COPY source_datadog ./source_datadog
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.3.0
+LABEL io.airbyte.version=0.4.0
 LABEL io.airbyte.name=airbyte/source-datadog
diff --git a/airbyte-integrations/connectors/source-datadog/acceptance-test-config.yml b/airbyte-integrations/connectors/source-datadog/acceptance-test-config.yml
@@ -23,6 +23,8 @@ acceptance_tests:
             bypass_reason: Sandbox account can't seed this stream
           - name: incident_teams
             bypass_reason: Sandbox account can't seed this stream
+          - name: service_level_objectives
+            bypass_reason: Sandbox account can't seed this stream
   # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file
   #        expect_records:
   #          path: "integration_tests/expected_records.jsonl"
@@ -41,3 +43,9 @@ acceptance_tests:
     tests:
       - config_path: "secrets/config.json"
         configured_catalog_path: "integration_tests/configured_catalog.json"
+        ignored_fields:
+          logs:
+            - name: "attributes.timestamp"
+              bypass_reason: "Change everytime"
+            - name: "id"
+              bypass_reason: "Change everytime"
diff --git a/airbyte-integrations/connectors/source-datadog/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-datadog/integration_tests/configured_catalog.json
@@ -50,6 +50,16 @@
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite"
     },
+    {
+      "stream": {
+        "name": "monitors",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
     {
       "stream": {
         "name": "synthetic_tests",

diff --git a/airbyte-integrations/connectors/source-datadog/metadata.yaml b/airbyte-integrations/connectors/source-datadog/metadata.yaml
@@ -14,7 +14,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 1cfc30c7-82db-43f4-9fd7-ac1b42312cda
-  dockerImageTag: 0.3.0
+  dockerImageTag: 0.4.0
   dockerRepository: airbyte/source-datadog
   githubIssueLabel: source-datadog
   icon: datadog.svg

diff --git a/airbyte-integrations/connectors/source-datadog/setup.py b/airbyte-integrations/connectors/source-datadog/setup.py
@@ -6,14 +6,13 @@
 from setuptools import find_packages, setup
 
 MAIN_REQUIREMENTS = [
-    "airbyte-cdk~=0.1",
+    "airbyte-cdk",
 ]
 
 TEST_REQUIREMENTS = [
     "requests-mock~=1.9.3",
     "pytest~=6.2",
     "pytest-mock~=3.6.1",
-    "connector-acceptance-test",
 ]
 
 setup(

diff --git a/airbyte-integrations/connectors/source-datadog/source_datadog/manifest.yaml b/airbyte-integrations/connectors/source-datadog/source_datadog/manifest.yaml
@@ -83,6 +83,14 @@ definitions:
     end_datetime:
       datetime: "{{ config['end_date'] }}"
       datetime_format: "%Y-%m-%dT%H:%M:%SZ"
+    start_time_option:
+      type: RequestOption
+      field_name: filter[from]
+      inject_into: request_parameter
+    end_time_option:
+      type: RequestOption
+      field_name: filter[to]
+      inject_into: request_parameter
     step: "P1M"
 
   dashboards_stream:
@@ -121,6 +129,36 @@ definitions:
           type: DpathExtractor
           field_path: ["tests"]
 
+  monitors_stream:
+    $ref: "#/definitions/base_stream"
+    $parameters:
+      name: "monitors"
+      primary_key: "id"
+      path: "monitor"
+    retriever:
+      $ref: "#/definitions/retriever"
+      record_selector:
+        $ref: "#/definitions/selector"
+        type: RecordSelector
+        extractor:
+          type: DpathExtractor
+          field_path: []
+
+  service_level_objectives_stream:
+    $ref: "#/definitions/base_stream"
+    $parameters:
+      name: "service_level_objectives"
+      primary_key: "id"
+      path: "slo"
+    retriever:
+      $ref: "#/definitions/retriever"
+      record_selector:
+        $ref: "#/definitions/selector"
+        type: RecordSelector
+        extractor:
+          type: DpathExtractor
+          field_path: ["data"]
+
   audit_logs_stream:
     $ref: "#/definitions/base_stream"
     $parameters:
@@ -196,25 +234,29 @@ definitions:
         $ref: "#/definitions/base_paginator"
 
 streams:
+  - "#/definitions/audit_logs_stream"
   - "#/definitions/dashboards_stream"
   - "#/definitions/downtimes_stream"
-  - "#/definitions/synthetic_tests_stream"
-  - "#/definitions/audit_logs_stream"
+  - "#/definitions/incident_teams_stream"
+  - "#/definitions/incidents_stream"
   - "#/definitions/logs_stream"
   - "#/definitions/metrics_stream"
-  - "#/definitions/incidents_stream"
-  - "#/definitions/incident_teams_stream"
+  - "#/definitions/monitors_stream"
+  - "#/definitions/service_level_objectives_stream"
+  - "#/definitions/synthetic_tests_stream"
   - "#/definitions/users_stream"
 
 check:
   type: CheckStream
   stream_names:
+    - "audit_logs"
     - "dashboards"
     - "downtimes"
-    - "synthetic_tests"
-    - "audit_logs"
+    - "incident_teams"
+    - "incidents"
     - "logs"
     - "metrics"
-    - "incidents"
-    - "incident_teams"
+    - "monitors"
+    - "service_level_objectives"
+    - "synthetic_tests"
     - "users"