From 67066c7d91518c2487ed3b97a3b37fbfd4987be6 Mon Sep 17 00:00:00 2001 From: Pat Nadolny Date: Thu, 16 Nov 2023 09:55:21 -0500 Subject: [PATCH] feat: de-nest properties (#57) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * denest properties * Update tap_hubspot/client.py Co-authored-by: Edgar Ramírez Mondragón <16805946+edgarrmondragon@users.noreply.github.com> --------- Co-authored-by: Edgar Ramírez Mondragón <16805946+edgarrmondragon@users.noreply.github.com> --- tap_hubspot/client.py | 20 +++ tap_hubspot/streams.py | 341 ++++++++++++++++------------------------- 2 files changed, 151 insertions(+), 210 deletions(-) diff --git a/tap_hubspot/client.py b/tap_hubspot/client.py index d6945dc..0e281cc 100644 --- a/tap_hubspot/client.py +++ b/tap_hubspot/client.py @@ -123,3 +123,23 @@ def get_url_params( params["order_by"] = self.replication_key return params + + def post_process( + self, + row: dict, + context: dict | None = None, # noqa: ARG002 + ) -> dict | None: + """Post process row. + + Args: + row: Row from response. + context: Stream sync context. + + Returns: + Transformed row. + """ + if "properties" in row: + properties = row.pop("properties") + for key, value in properties.items(): + row[key] = value + return row diff --git a/tap_hubspot/streams.py b/tap_hubspot/streams.py index cac627d..f52db1a 100644 --- a/tap_hubspot/streams.py +++ b/tap_hubspot/streams.py @@ -42,19 +42,15 @@ class ContactStream(HubspotStream): schema = PropertiesList( Property("id", StringType), - Property( - "properties", - ObjectType( - Property("company", StringType), - Property("createdate", StringType), - Property("email", StringType), - Property("firstname", StringType), - Property("lastmodifieddate", StringType), - Property("lastname", StringType), - Property("phone", StringType), - Property("website", StringType), - ), - ), + Property("company", StringType), + Property("createdate", StringType), + Property("email", StringType), + Property("firstname", StringType), + Property("lastmodifieddate", StringType), + Property("lastname", StringType), + Property("phone", StringType), + Property("website", StringType), + Property("hs_object_id", StringType), Property("createdAt", StringType), Property("updatedAt", StringType), Property("archived", BooleanType), @@ -1288,19 +1284,14 @@ class CompanyStream(HubspotStream): schema = PropertiesList( Property("id", StringType), - Property( - "properties", - ObjectType( - Property("city", StringType), - Property("createdDate", StringType), - Property("domain", StringType), - Property("hs_lastmodifieddate", StringType), - Property("industry", StringType), - Property("name", StringType), - Property("phone", StringType), - Property("state", StringType), - ), - ), + Property("city", StringType), + Property("createdDate", StringType), + Property("domain", StringType), + Property("hs_lastmodifieddate", StringType), + Property("industry", StringType), + Property("name", StringType), + Property("phone", StringType), + Property("state", StringType), Property("createdAt", StringType), Property("updatedAt", StringType), Property("archived", BooleanType), @@ -1337,19 +1328,14 @@ class DealStream(HubspotStream): schema = PropertiesList( Property("id", StringType), - Property( - "properties", - ObjectType( - Property("amount", StringType), - Property("createdDate", StringType), - Property("closedDate", StringType), - Property("dealname", StringType), - Property("dealstage", StringType), - Property("hs_lastmodifieddate", StringType), - Property("hubspot_owner_id", StringType), - Property("pipeline", StringType), - ), - ), + Property("amount", StringType), + Property("createdDate", StringType), + Property("closedDate", StringType), + Property("dealname", StringType), + Property("dealstage", StringType), + Property("hs_lastmodifieddate", StringType), + Property("hubspot_owner_id", StringType), + Property("pipeline", StringType), Property("createdAt", StringType), Property("updatedAt", StringType), Property("archived", BooleanType), @@ -1386,20 +1372,15 @@ class FeedbackSubmissionsStream(HubspotStream): schema = PropertiesList( Property("id", StringType), - Property( - "properties", - ObjectType( - Property("hs_content", StringType), - Property("hs_ingestion_id", StringType), - Property("hs_response_group", StringType), - Property("hs_submission_name", StringType), - Property("hs_survey_channel", StringType), - Property("hs_survey_id", StringType), - Property("hs_survey_name", StringType), - Property("hs_survey_type", StringType), - Property("hs_value", StringType), - ), - ), + Property("hs_content", StringType), + Property("hs_ingestion_id", StringType), + Property("hs_response_group", StringType), + Property("hs_submission_name", StringType), + Property("hs_survey_channel", StringType), + Property("hs_survey_id", StringType), + Property("hs_survey_name", StringType), + Property("hs_survey_type", StringType), + Property("hs_value", StringType), Property("createdAt", StringType), Property("updatedAt", StringType), Property("archived", BooleanType), @@ -1436,19 +1417,14 @@ class LineItemStream(HubspotStream): schema = PropertiesList( Property("id", StringType), - Property( - "properties", - ObjectType( - Property("createdate", StringType), - Property("hs_lastmodifieddate", StringType), - Property("hs_product_id", StringType), - Property("hs_recurring_billing_period", StringType), - Property("name", StringType), - Property("price", StringType), - Property("quantity", StringType), - Property("recurringbillingfrequency", StringType), - ), - ), + Property("createdate", StringType), + Property("hs_lastmodifieddate", StringType), + Property("hs_product_id", StringType), + Property("hs_recurring_billing_period", StringType), + Property("name", StringType), + Property("price", StringType), + Property("quantity", StringType), + Property("recurringbillingfrequency", StringType), Property("createdAt", StringType), Property("updatedAt", StringType), Property("archived", BooleanType), @@ -1485,19 +1461,14 @@ class ProductStream(HubspotStream): schema = PropertiesList( Property("id", StringType), - Property( - "properties", - ObjectType( - Property("createdate", StringType), - Property("description", StringType), - Property("hs_cost_of_goods_sold", StringType), - Property("hs_lastmodifieddate", StringType), - Property("hs_recurring_billing_period", StringType), - Property("hs_sku", StringType), - Property("name", StringType), - Property("price", StringType), - ), - ), + Property("createdate", StringType), + Property("description", StringType), + Property("hs_cost_of_goods_sold", StringType), + Property("hs_lastmodifieddate", StringType), + Property("hs_recurring_billing_period", StringType), + Property("hs_sku", StringType), + Property("name", StringType), + Property("price", StringType), Property("createdAt", StringType), Property("updatedAt", StringType), Property("archived", BooleanType), @@ -1534,18 +1505,13 @@ class TicketStream(HubspotStream): schema = PropertiesList( Property("id", StringType), - Property( - "properties", - ObjectType( - Property("createdate", StringType), - Property("hs_lastmodifieddate", StringType), - Property("hs_pipeline", StringType), - Property("hs_pipeline_stage", StringType), - Property("hs_ticket_priority", StringType), - Property("hubspot_owner_id", StringType), - Property("subject", StringType), - ), - ), + Property("createdate", StringType), + Property("hs_lastmodifieddate", StringType), + Property("hs_pipeline", StringType), + Property("hs_pipeline_stage", StringType), + Property("hs_ticket_priority", StringType), + Property("hubspot_owner_id", StringType), + Property("subject", StringType), Property("createdAt", StringType), Property("updatedAt", StringType), Property("archived", BooleanType), @@ -1582,19 +1548,14 @@ class QuoteStream(HubspotStream): schema = PropertiesList( Property("id", StringType), - Property( - "properties", - ObjectType( - Property("hs_createdate", StringType), - Property("hs_expiration_date", StringType), - Property("hs_quote_amount", StringType), - Property("hs_quote_number", StringType), - Property("hs_status", StringType), - Property("hs_terms", StringType), - Property("hs_title", StringType), - Property("hubspot_owner_id", StringType), - ), - ), + Property("hs_createdate", StringType), + Property("hs_expiration_date", StringType), + Property("hs_quote_amount", StringType), + Property("hs_quote_number", StringType), + Property("hs_status", StringType), + Property("hs_terms", StringType), + Property("hs_title", StringType), + Property("hubspot_owner_id", StringType), Property("createdAt", StringType), Property("updatedAt", StringType), Property("archived", BooleanType), @@ -1631,18 +1592,13 @@ class GoalStream(HubspotStream): schema = PropertiesList( Property("id", StringType), - Property( - "properties", - ObjectType( - Property("createdate", StringType), - Property("hs_created_by_user_id", StringType), - Property("hs_end_datetime", StringType), - Property("hs_goal_name", StringType), - Property("hs_lastmodifieddate", StringType), - Property("hs_start_datetime", StringType), - Property("hs_target_amount", StringType), - ), - ), + Property("createdate", StringType), + Property("hs_created_by_user_id", StringType), + Property("hs_end_datetime", StringType), + Property("hs_goal_name", StringType), + Property("hs_lastmodifieddate", StringType), + Property("hs_start_datetime", StringType), + Property("hs_target_amount", StringType), Property("createdAt", StringType), Property("updatedAt", StringType), Property("archived", BooleanType), @@ -1679,22 +1635,17 @@ class CallStream(HubspotStream): schema = PropertiesList( Property("id", StringType), - Property( - "properties", - ObjectType( - Property("createdate", StringType), - Property("hs_call_body", StringType), - Property("hs_call_duration", StringType), - Property("hs_call_from_number", StringType), - Property("hs_call_recording_url", StringType), - Property("hs_call_status", StringType), - Property("hs_call_title", StringType), - Property("hs_call_to_number", StringType), - Property("hs_lastmodifieddate", StringType), - Property("hs_timestamp", StringType), - Property("hubspot_owner_id", StringType), - ), - ), + Property("createdate", StringType), + Property("hs_call_body", StringType), + Property("hs_call_duration", StringType), + Property("hs_call_from_number", StringType), + Property("hs_call_recording_url", StringType), + Property("hs_call_status", StringType), + Property("hs_call_title", StringType), + Property("hs_call_to_number", StringType), + Property("hs_lastmodifieddate", StringType), + Property("hs_timestamp", StringType), + Property("hubspot_owner_id", StringType), Property("createdAt", StringType), Property("updatedAt", StringType), Property("archived", BooleanType), @@ -1723,7 +1674,7 @@ class CommunicationStream(HubspotStream): """ name = "communications" - path = "/objects/Communications" + path = "/objects/communications" primary_keys = ["id"] replication_key = "updatedAt" replication_method = "INCREMENTAL" @@ -1731,16 +1682,11 @@ class CommunicationStream(HubspotStream): schema = PropertiesList( Property("id", StringType), - Property( - "properties", - ObjectType( - Property("createdate", StringType), - Property("hs_communication_body", StringType), - Property("hs_communication_channel_type", StringType), - Property("hs_communication_logged_from", StringType), - Property("hs_lastmodifieddate", StringType), - ), - ), + Property("createdate", StringType), + Property("hs_communication_body", StringType), + Property("hs_communication_channel_type", StringType), + Property("hs_communication_logged_from", StringType), + Property("hs_lastmodifieddate", StringType), Property("createdAt", StringType), Property("updatedAt", StringType), Property("archived", BooleanType), @@ -1777,25 +1723,20 @@ class EmailStream(HubspotStream): schema = PropertiesList( Property("id", StringType), - Property( - "properties", - ObjectType( - Property("createdate", StringType), - Property("hs_email_direction", StringType), - Property("hs_email_sender_email", StringType), - Property("hs_email_sender_firstname", StringType), - Property("hs_email_sender_lastname", StringType), - Property("hs_email_status", StringType), - Property("hs_email_subject", StringType), - Property("hs_email_text", StringType), - Property("hs_email_to_email", StringType), - Property("hs_email_to_firstname", StringType), - Property("hs_email_to_lastname", StringType), - Property("hs_lastmodifieddate", StringType), - Property("hs_timestamp", StringType), - Property("hubspot_owner_id", StringType), - ), - ), + Property("createdate", StringType), + Property("hs_email_direction", StringType), + Property("hs_email_sender_email", StringType), + Property("hs_email_sender_firstname", StringType), + Property("hs_email_sender_lastname", StringType), + Property("hs_email_status", StringType), + Property("hs_email_subject", StringType), + Property("hs_email_text", StringType), + Property("hs_email_to_email", StringType), + Property("hs_email_to_firstname", StringType), + Property("hs_email_to_lastname", StringType), + Property("hs_lastmodifieddate", StringType), + Property("hs_timestamp", StringType), + Property("hubspot_owner_id", StringType), Property("createdAt", StringType), Property("updatedAt", StringType), Property("archived", BooleanType), @@ -1832,23 +1773,18 @@ class MeetingStream(HubspotStream): schema = PropertiesList( Property("id", StringType), - Property( - "properties", - ObjectType( - Property("createdate", StringType), - Property("hs_internal_meeting_notes", StringType), - Property("hs_lastmodifieddate", StringType), - Property("hs_meeting_body", StringType), - Property("hs_meeting_end_time", StringType), - Property("hs_meeting_external_url", StringType), - Property("hs_meeting_location", StringType), - Property("hs_meeting_outcome", StringType), - Property("hs_meeting_start_time", StringType), - Property("hs_meeting_title", StringType), - Property("hs_timestamp", StringType), - Property("hubspot_owner_id", StringType), - ), - ), + Property("createdate", StringType), + Property("hs_internal_meeting_notes", StringType), + Property("hs_lastmodifieddate", StringType), + Property("hs_meeting_body", StringType), + Property("hs_meeting_end_time", StringType), + Property("hs_meeting_external_url", StringType), + Property("hs_meeting_location", StringType), + Property("hs_meeting_outcome", StringType), + Property("hs_meeting_start_time", StringType), + Property("hs_meeting_title", StringType), + Property("hs_timestamp", StringType), + Property("hubspot_owner_id", StringType), Property("createdAt", StringType), Property("updatedAt", StringType), Property("archived", BooleanType), @@ -1885,16 +1821,11 @@ class NoteStream(HubspotStream): schema = PropertiesList( Property("id", StringType), - Property( - "properties", - ObjectType( - Property("createdate", StringType), - Property("hs_lastmodifieddate", StringType), - Property("hs_note_body", StringType), - Property("hs_timestamp", StringType), - Property("hubspot_owner_id", StringType), - ), - ), + Property("createdate", StringType), + Property("hs_lastmodifieddate", StringType), + Property("hs_note_body", StringType), + Property("hs_timestamp", StringType), + Property("hubspot_owner_id", StringType), Property("createdAt", StringType), Property("updatedAt", StringType), Property("archived", BooleanType), @@ -1931,14 +1862,9 @@ class PostalMailStream(HubspotStream): schema = PropertiesList( Property("id", StringType), - Property( - "properties", - ObjectType( - Property("createdate", StringType), - Property("hs_lastmodifieddate", StringType), - Property("hs_postal_mail_body", StringType), - ), - ), + Property("createdate", StringType), + Property("hs_lastmodifieddate", StringType), + Property("hs_postal_mail_body", StringType), Property("createdAt", StringType), Property("updatedAt", StringType), Property("archived", BooleanType), @@ -1975,19 +1901,14 @@ class TaskStream(HubspotStream): schema = PropertiesList( Property("id", StringType), - Property( - "properties", - ObjectType( - Property("createdate", StringType), - Property("hs_lastmodifieddate", StringType), - Property("hs_task_body", StringType), - Property("hs_task_priority", StringType), - Property("hs_task_status", StringType), - Property("hs_task_subject", StringType), - Property("hs_timestamp", StringType), - Property("hubspot_owner_id", StringType), - ), - ), + Property("createdate", StringType), + Property("hs_lastmodifieddate", StringType), + Property("hs_task_body", StringType), + Property("hs_task_priority", StringType), + Property("hs_task_status", StringType), + Property("hs_task_subject", StringType), + Property("hs_timestamp", StringType), + Property("hubspot_owner_id", StringType), Property("createdAt", StringType), Property("updatedAt", StringType), Property("archived", BooleanType),