Merge pull request #190 from datafold/1017monitors1

bigquery demo monitors part 1
datafold · Oct 17, 2024 · f1cb89a · f1cb89a
2 parents 2f9ca0d + 546e197
commit f1cb89a
Showing 1 changed file with 320 additions and 0 deletions.
diff --git a/monitors.yaml b/monitors.yaml
@@ -960,3 +960,323 @@ monitors:
       different_rows_count: 10
       different_rows_percent: 10
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# ####################################################
+# OOO -> BIGQUERY
+# ####################################################
+
+# Redshift<>Bigquery XDB DIFF monitors
+
+  xdb_diff__redshift_bigquery__egress_10M_:
+    type: diff
+    schedule:
+      cron: 15 15 * * *
+    datadiff:
+      dataset_a:
+        connection_id: 8824
+        table: demo.diff.transactions_v1_10m
+      dataset_b:
+        connection_id: 8377
+        table: datafold-demo-429713.diff.transactions_v2_10m
+      primary_key:
+        - transaction_id
+      columns_to_compare:
+        - amount
+        - amount_base
+        - created_at
+        - discount
+        - first_name
+        - last_name
+        - number_of_attempts
+        - number_of_products
+        - score
+        - updated_at
+      egress_limit: 10000000
+    alert:
+      different_rows_count: 10
+      different_rows_percent: 10
+
+
+  xdb_diff__redshift_bigquery__egress_10M__sampling_0_001:
+    type: diff
+    schedule:
+      cron: 15 15 * * *
+    datadiff:
+      dataset_a:
+        connection_id: 8824
+        table: demo.diff.transactions_v1_10m
+      dataset_b:
+        connection_id: 8377
+        table: datafold-demo-429713.diff.transactions_v2_10m
+      primary_key:
+        - transaction_id
+      columns_to_compare:
+        - amount
+        - amount_base
+        - created_at
+        - discount
+        - first_name
+        - last_name
+        - number_of_attempts
+        - number_of_products
+        - score
+        - updated_at
+      egress_limit: 10000000
+      sampling:
+        rate: 0.001
+    alert:
+      different_rows_count: 10
+      different_rows_percent: 10
+
+
+
+# Microsoft SQL Server<>Bigquery XDB DIFF monitors
+
+  xdb_diff__mssqlserver_bigquery__egress_10M_:
+    type: diff
+    schedule:
+      cron: 15 15 * * *
+    datadiff:
+      dataset_a:
+        connection_id: 8182
+        table: demo.demo.transactions_v1_10m
+      dataset_b:
+        connection_id: 8377
+        table: datafold-demo-429713.diff.transactions_v2_10m
+      primary_key:
+        - transaction_id
+      columns_to_compare:
+        - amount
+        - amount_base
+        - created_at
+        - discount
+        - first_name
+        - last_name
+        - number_of_attempts
+        - number_of_products
+        - score
+        - updated_at
+      egress_limit: 10000000
+    alert:
+      different_rows_count: 10
+      different_rows_percent: 10
+
+
+  xdb_diff__mssqlserver_bigquery__egress_10M__sampling_0_001:
+    type: diff
+    schedule:
+      cron: 15 15 * * *
+    datadiff:
+      dataset_a:
+        connection_id: 8182
+        table: demo.demo.transactions_v1_10m
+      dataset_b:
+        connection_id: 8377
+        table: datafold-demo-429713.diff.transactions_v2_10m
+      primary_key:
+        - transaction_id
+      columns_to_compare:
+        - amount
+        - amount_base
+        - created_at
+        - discount
+        - first_name
+        - last_name
+        - number_of_attempts
+        - number_of_products
+        - score
+        - updated_at
+      egress_limit: 10000000
+      sampling:
+        rate: 0.001
+    alert:
+      different_rows_count: 10
+      different_rows_percent: 10
+
+
+
+# Databricks<>Bigquery XDB DIFF monitors
+
+  xdb_diff__databricks_bigquery__egress_10M_:
+    type: diff
+    schedule:
+      cron: 15 15 * * *
+    datadiff:
+      dataset_a:
+        connection_id: 8357
+        table: demo.diff.transactions_v1_10m
+      dataset_b:
+        connection_id: 8377
+        table: datafold-demo-429713.diff.transactions_v2_10m
+      primary_key:
+        - transaction_id
+      columns_to_compare:
+        - amount
+        - amount_base
+        - created_at
+        - discount
+        - first_name
+        - last_name
+        - number_of_attempts
+        - number_of_products
+        - score
+        - updated_at
+      egress_limit: 10000000
+    alert:
+      different_rows_count: 10
+      different_rows_percent: 10
+
+
+  xdb_diff__databricks_bigquery__egress_10M__sampling_0_001:
+    type: diff
+    schedule:
+      cron: 15 15 * * *
+    datadiff:
+      dataset_a:
+        connection_id: 8357
+        table: demo.diff.transactions_v1_10m
+      dataset_b:
+        connection_id: 8377
+        table: datafold-demo-429713.diff.transactions_v2_10m
+      primary_key:
+        - transaction_id
+      columns_to_compare:
+        - amount
+        - amount_base
+        - created_at
+        - discount
+        - first_name
+        - last_name
+        - number_of_attempts
+        - number_of_products
+        - score
+        - updated_at
+      egress_limit: 10000000
+      sampling:
+        rate: 0.001
+    alert:
+      different_rows_count: 10
+      different_rows_percent: 10
+
+
+
+# Snowflake<>Bigquery XDB DIFF monitors
+
+  xdb_diff__snowflake_bigquery__egress_10M_:
+    type: diff
+    schedule:
+      cron: 15 15 * * *
+    datadiff:
+      dataset_a:
+        connection_id: 4932
+        table: DEMO.DIFF.TRANSACTIONS_V1_10M
+      dataset_b:
+        connection_id: 8377
+        table: datafold-demo-429713.diff.transactions_v2_10m
+      primary_key:
+        - transaction_id
+      columns_to_compare:
+        - amount
+        - amount_base
+        - created_at
+        - discount
+        - first_name
+        - last_name
+        - number_of_attempts
+        - number_of_products
+        - score
+        - updated_at
+      egress_limit: 10000000
+    alert:
+      different_rows_count: 10
+      different_rows_percent: 10
+
+
+  xdb_diff__snowflake_bigquery__egress_10M__sampling_0_001:
+    type: diff
+    schedule:
+      cron: 15 15 * * *
+    datadiff:
+      dataset_a:
+        connection_id: 4932
+        table: DEMO.DIFF.TRANSACTIONS_V1_10M
+      dataset_b:
+        connection_id: 8377
+        table: datafold-demo-429713.diff.transactions_v2_10m
+      primary_key:
+        - transaction_id
+      columns_to_compare:
+        - amount
+        - amount_base
+        - created_at
+        - discount
+        - first_name
+        - last_name
+        - number_of_attempts
+        - number_of_products
+        - score
+        - updated_at
+      egress_limit: 10000000
+      sampling:
+        rate: 0.001
+    alert:
+      different_rows_count: 10
+      different_rows_percent: 10
+
+
+# Bigquery<>Bigquery XDB DIFF monitors
+
+  xdb_diff__bigquery_bigquery__egress_10M_:
+    type: diff
+    schedule:
+      cron: 15 15 * * *
+    datadiff:
+      dataset_a:
+        connection_id: 9171
+        table: datafold-demo-429713.diff.transactions_v1_10m
+      dataset_b:
+        connection_id: 8377
+        table: datafold-demo-429713.diff.transactions_v2_10m
+      primary_key:
+        - transaction_id
+      egress_limit: 10000000
+    alert:
+      different_rows_count: 10
+      different_rows_percent: 10
+
+
+  xdb_diff__bigquery_bigquery__egress_10M__sampling_0_001:
+    type: diff
+    schedule:
+      cron: 15 15 * * *
+    datadiff:
+      dataset_a:
+        connection_id: 9171
+        table: datafold-demo-429713.diff.transactions_v1_10m
+      dataset_b:
+        connection_id: 8377
+        table: datafold-demo-429713.diff.transactions_v2_10m
+      primary_key:
+        - transaction_id
+      egress_limit: 10000000
+      sampling:
+        rate: 0.001
+    alert:
+      different_rows_count: 10
+      different_rows_percent: 10
+