Usage record enable config (#13)

* only usage_record enable disable config * docs * try removing config * nvm add it back * try new schema * doc updates * Update CHANGELOG.md Co-authored-by: Avinash Kunnath <[email protected]> * update readme format * Delete package-lock.yml * Update .gitignore --------- Co-authored-by: Avinash Kunnath <[email protected]> Co-authored-by: fivetran-catfritz <[email protected]>
fivetran · Dec 2, 2024 · 577d4dc · 577d4dc
1 parent cf6d1cf
commit 577d4dc
Show file tree

Hide file tree

Showing 13 changed files with 30 additions and 15 deletions.
diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh
@@ -19,7 +19,7 @@ dbt deps
 dbt seed --target "$db" --full-refresh
 dbt run --target "$db" --full-refresh
 dbt test --target "$db"
-dbt run --vars '{using_twilio_call: false, using_twilio_messaging_service: false}' --target "$db" --full-refresh
+dbt run --vars '{using_twilio_call: false, using_twilio_messaging_service: false, using_twilio_usage_record: false}' --target "$db" --full-refresh
 dbt test --target "$db"
 
 dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"
diff --git a/.gitignore b/.gitignore
@@ -2,3 +2,4 @@
 target/
 dbt_packages/
 logs/
+package-lock.yml
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,10 @@
+# dbt_twilio_source v0.4.0
+
+[PR #13](https://github.com/fivetran/dbt_twilio_source/pull/13) includes the following updates:
+
+## Features
+- Added the ability to disable the `USAGE_RECORD` source table and its downstream fields via variable `using_twilio_usage_record` (default `true`). Refer to the [README](https://github.com/fivetran/dbt_twilio_source?tab=readme-ov-file#step-4-enablingdisabling-models) for more details.
+
 # dbt_twilio_source v0.3.0
 
 ## 🚨 Breaking Changes 🚨

diff --git a/README.md b/README.md
@@ -1,4 +1,6 @@
-<p align="center">
+# Twilio Source dbt Package ([docs](https://fivetran.github.io/dbt_twilio_source/))
+
+<p align="left">
     <a alt="License"
         href="https://github.com/fivetran/dbt_twilio_source/blob/main/LICENSE">
         <img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg" /></a>
@@ -10,7 +12,6 @@
         <img src="https://img.shields.io/badge/Contributions-welcome-blueviolet" /></a>
 </p>
 
-# Twilio Source dbt Package ([docs](https://fivetran.github.io/dbt_twilio_source/))
 ## What does this dbt package do?
 
 - Materializes [Twilio staging tables](https://fivetran.github.io/dbt_twilio_source/#!/overview/twilio_source/models/?g_v=1&g_e=seeds) which leverage data in the format described by [this ERD](https://fivetran.com/docs/applications/twilio#schemainformation). These staging tables clean, test, and prepare your Twilio data from [Fivetran's connector](https://fivetran.com/docs/applications/twilio) for analysis by doing the following:
@@ -44,7 +45,7 @@ Include the following Twilio package version in your `packages.yml` file.
 ```yaml
 packages:
   - package: fivetran/twilio_source
-    version: [">=0.3.0", "<0.4.0"]
+    version: [">=0.4.0", "<0.5.0"]
 ```
 
 ### Step 3: Define database and schema variables
@@ -66,6 +67,7 @@ By default, all variables are assumed to be `true`.
 vars:
   using_twilio_call: False # Disable this if not using call
   using_twilio_messaging_service: False # Disable this if not using messaging_service
+  using_twilio_usage_record: False # Disable this if not using usage_record
 ```
 
 ### (Optional) Step 5: Additional configurations

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,6 +1,6 @@
 
 name: 'twilio_source'
-version: '0.3.0'
+version: '0.4.0'
 config-version: 2
 
 models:

diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml
@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: twilio_source_integration_tests_5
+      schema: twilio_source_integration_tests_6
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: twilio_source_integration_tests_5
+      schema: twilio_source_integration_tests_6
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: twilio_source_integration_tests_5
+      schema: twilio_source_integration_tests_6
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: twilio_source_integration_tests_5
+      schema: twilio_source_integration_tests_6
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: twilio_source_integration_tests_5
+      schema: twilio_source_integration_tests_6
       threads: 2
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,11 +1,11 @@
 config-version: 2
 name: 'twilio_source_integration_tests'
-version: '0.3.0'
+version: '0.4.0'
 profile: 'integration_tests'
 
 vars:
   twilio_source:
-    twilio_schema: twilio_source_integration_tests_5
+    twilio_schema: twilio_source_integration_tests_6
     twilio_account_history_identifier:  "twilio_account_history_data"
     twilio_address_identifier:  "twilio_address_data"
     twilio_incoming_phone_number_identifier:  "twilio_incoming_phone_number_data"

diff --git a/models/src_twilio.yml b/models/src_twilio.yml
@@ -462,6 +462,8 @@ sources:
         freshness:
           warn_after: {count: 72, period: hour}
           error_after: {count: 168, period: hour}
+        config:
+          enabled: "{{ var('using_twilio_usage_record', true) }}"
         columns:
           - name: _fivetran_synced
             description: Timestamp of when fivetran synced a record. 

diff --git a/models/stg_twilio__usage_record.sql b/models/stg_twilio__usage_record.sql
@@ -1,3 +1,5 @@
+{{ config(enabled=var('using_twilio_usage_record', True)) }}
+
 with base as (
 
     select * 

diff --git a/models/tmp/stg_twilio__usage_record_tmp.sql b/models/tmp/stg_twilio__usage_record_tmp.sql
@@ -1,2 +1,4 @@
+{{ config(enabled=var('using_twilio_usage_record', True)) }}
+
 select {{ dbt_utils.star(source('twilio', 'usage_record')) }}
 from {{ var('usage_record') }}
-Original file line number
+Diff line change
@@ Expand Up / @@ -2,3 +2,4 @@ @@
     target/
     dbt_packages/
     logs/
+    package-lock.yml