From 76e1e4fbdc4b45456c34a6a6e23a7c39c08a68ef Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 21 Oct 2024 16:24:59 +0200 Subject: [PATCH] dremio-dremio monitors and diffs --- monitors.yaml | 360 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 360 insertions(+) diff --git a/monitors.yaml b/monitors.yaml index d5cfcb4..d6d5f0b 100644 --- a/monitors.yaml +++ b/monitors.yaml @@ -1441,3 +1441,363 @@ monitors: different_rows_count: 10 different_rows_percent: 10 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +# #################################################### +# OOO -> DREMIO +# #################################################### + +# # Redshift<>Dremio XDB DIFF monitors + +# xdb_diff__redshift_dremio__egress_10M_: +# type: diff +# schedule: +# cron: 15 15 * * * +# datadiff: +# dataset_a: +# connection_id: 8824 +# table: demo.diff.transactions_v1_10m +# dataset_b: +# connection_id: 8377 +# table: datafold-demo-429713.diff.transactions_v2_10m +# primary_key: +# - transaction_id +# columns_to_compare: +# - amount +# - amount_base +# - created_at +# - discount +# - first_name +# - last_name +# - number_of_attempts +# - number_of_products +# - score +# - updated_at +# egress_limit: 10000000 +# alert: +# different_rows_count: 10 +# different_rows_percent: 10 + + +# xdb_diff__redshift_dremio__egress_10M__sampling_0_001: +# type: diff +# schedule: +# cron: 15 15 * * * +# datadiff: +# dataset_a: +# connection_id: 8824 +# table: demo.diff.transactions_v1_10m +# dataset_b: +# connection_id: 8377 +# table: datafold-demo-429713.diff.transactions_v2_10m +# primary_key: +# - transaction_id +# columns_to_compare: +# - amount +# - amount_base +# - created_at +# - discount +# - first_name +# - last_name +# - number_of_attempts +# - number_of_products +# - score +# - updated_at +# egress_limit: 10000000 +# sampling: +# rate: 0.001 +# alert: +# different_rows_count: 10 +# different_rows_percent: 10 + + + +# # Microsoft SQL Server<>Dremio XDB DIFF monitors + +# xdb_diff__mssqlserver_dremio__egress_10M_: +# type: diff +# schedule: +# cron: 15 15 * * * +# datadiff: +# dataset_a: +# connection_id: 8182 +# table: demo.demo.transactions_v1_10m +# dataset_b: +# connection_id: 8377 +# table: datafold-demo-429713.diff.transactions_v2_10m +# primary_key: +# - transaction_id +# columns_to_compare: +# - amount +# - amount_base +# - created_at +# - discount +# - first_name +# - last_name +# - number_of_attempts +# - number_of_products +# - score +# - updated_at +# egress_limit: 10000000 +# alert: +# different_rows_count: 10 +# different_rows_percent: 10 + + +# xdb_diff__mssqlserver_dremio__egress_10M__sampling_0_001: +# type: diff +# schedule: +# cron: 15 15 * * * +# datadiff: +# dataset_a: +# connection_id: 8182 +# table: demo.demo.transactions_v1_10m +# dataset_b: +# connection_id: 8377 +# table: datafold-demo-429713.diff.transactions_v2_10m +# primary_key: +# - transaction_id +# columns_to_compare: +# - amount +# - amount_base +# - created_at +# - discount +# - first_name +# - last_name +# - number_of_attempts +# - number_of_products +# - score +# - updated_at +# egress_limit: 10000000 +# sampling: +# rate: 0.001 +# alert: +# different_rows_count: 10 +# different_rows_percent: 10 + + + +# # Databricks<>Dremio XDB DIFF monitors + +# xdb_diff__databricks_dremio__egress_10M_: +# type: diff +# schedule: +# cron: 15 15 * * * +# datadiff: +# dataset_a: +# connection_id: 8357 +# table: demo.diff.transactions_v1_10m +# dataset_b: +# connection_id: 8377 +# table: datafold-demo-429713.diff.transactions_v2_10m +# primary_key: +# - transaction_id +# columns_to_compare: +# - amount +# - amount_base +# - created_at +# - discount +# - first_name +# - last_name +# - number_of_attempts +# - number_of_products +# - score +# - updated_at +# egress_limit: 10000000 +# alert: +# different_rows_count: 10 +# different_rows_percent: 10 + + +# xdb_diff__databricks_dremio__egress_10M__sampling_0_001: +# type: diff +# schedule: +# cron: 15 15 * * * +# datadiff: +# dataset_a: +# connection_id: 8357 +# table: demo.diff.transactions_v1_10m +# dataset_b: +# connection_id: 8377 +# table: datafold-demo-429713.diff.transactions_v2_10m +# primary_key: +# - transaction_id +# columns_to_compare: +# - amount +# - amount_base +# - created_at +# - discount +# - first_name +# - last_name +# - number_of_attempts +# - number_of_products +# - score +# - updated_at +# egress_limit: 10000000 +# sampling: +# rate: 0.001 +# alert: +# different_rows_count: 10 +# different_rows_percent: 10 + + + +# # Snowflake<>Dremio XDB DIFF monitors + +# xdb_diff__snowflake_dremio__egress_10M_: +# type: diff +# schedule: +# cron: 15 15 * * * +# datadiff: +# dataset_a: +# connection_id: 4932 +# table: DEMO.DIFF.TRANSACTIONS_V1_10M +# dataset_b: +# connection_id: 8377 +# table: datafold-demo-429713.diff.transactions_v2_10m +# primary_key: +# - transaction_id +# columns_to_compare: +# - amount +# - amount_base +# - created_at +# - discount +# - first_name +# - last_name +# - number_of_attempts +# - number_of_products +# - score +# - updated_at +# egress_limit: 10000000 +# alert: +# different_rows_count: 10 +# different_rows_percent: 10 + + +# xdb_diff__snowflake_dremio__egress_10M__sampling_0_001: +# type: diff +# schedule: +# cron: 15 15 * * * +# datadiff: +# dataset_a: +# connection_id: 4932 +# table: DEMO.DIFF.TRANSACTIONS_V1_10M +# dataset_b: +# connection_id: 8377 +# table: datafold-demo-429713.diff.transactions_v2_10m +# primary_key: +# - transaction_id +# columns_to_compare: +# - amount +# - amount_base +# - created_at +# - discount +# - first_name +# - last_name +# - number_of_attempts +# - number_of_products +# - score +# - updated_at +# egress_limit: 10000000 +# sampling: +# rate: 0.001 +# alert: +# different_rows_count: 10 +# different_rows_percent: 10 + + +# Dremio<>Dremio XDB DIFF monitors + + xdb_diff__dremio_dremio__egress_10M_: + type: diff + schedule: + cron: 15 15 * * * + datadiff: + dataset_a: + connection_id: 8452 + table: "Alexey S3".alexeydremiobucket.diff."transactions_v1_10m” + dataset_b: + connection_id: 8896 + table: "Alexey S3".alexeydremiobucket.diff."transactions_v2_10m” + primary_key: + - transaction_id + egress_limit: 10000000 + alert: + different_rows_count: 10 + different_rows_percent: 10 + + + xdb_diff__dremio_dremio__egress_10M__sampling_0_001: + type: diff + schedule: + cron: 15 15 * * * + datadiff: + dataset_a: + connection_id: 9171 + table: "Alexey S3".alexeydremiobucket.diff."transactions_v1_10m” + dataset_b: + connection_id: 8377 + table: "Alexey S3".alexeydremiobucket.diff."transactions_v2_10m” + primary_key: + - transaction_id + egress_limit: 10000000 + sampling: + rate: 0.001 + alert: + different_rows_count: 10 + different_rows_percent: 10 +