refactor: Update tap to use the LinkedIn 202305 API

.env.template

@@ -1,4 +1,6 @@
 TAP_LINKEDIN_ADS_ACCESS_TOKEN=''
 TAP_LINKEDIN_ADS_ACCOUNTS=''
 TAP_LINKEDIN_ADS_CAMPAIGN=''
+TAP_LINKEDIN_ADS_CAMPAIGN_GROUP=''
+TAP_LINKEDIN_ADS_CREATIVE=''
 TAP_LINKEDIN_ADS_OWNER=''

.github/workflows/ci_workflow.yml

@@ -42,5 +42,7 @@ jobs:
          TAP_LINKEDIN_ADS_ACCOUNTS: ${{ secrets.accounts }}
          TAP_LINKEDIN_ADS_OWNER: ${{ secrets.owner }}
          TAP_LINKEDIN_ADS_CAMPAIGN: ${{ secrets.campaign }}
+         TAP_LINKEDIN_ADS_CAMPAIGN_GROUP: ${{ secrets.campaign_group }}
+         TAP_LINKEDIN_ADS_CREATIVE: ${{ secrets.creative }}
       run: |
         poetry run pytest --capture=no

README.md

@@ -15,20 +15,22 @@ Built with the [Meltano Singer SDK](https://sdk.meltano.com).
 
 ## Settings
 
-| Setting             | Required | Default | Description |
-|:--------------------|:--------:|:-------:|:------------|
-| access_token        | True     | None    | The token to authenticate against the API service |
-| start_date          | True     | None    | The earliest record date to sync |
-| end_date            | False    | 2023-05-09 02:04:18.151589 | The latest record date to sync |
-| user_agent          | False    | tap-linkedin-ads <api_user_email@your_company.com> | API ID      |
-| api_version         | False    | 202211  | LinkedInAds API Version |
-| accounts            | True     | None    | LinkedInAds Account ID |
-| campaign            | True     | None    | LinkedInAds Campaign ID |
-| owner               | True     | None    | LinkedInAds Owner ID |
-| stream_maps         | False    | None    | Config object for stream maps capability. For more information check out [Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html). |
-| stream_map_config   | False    | None    | User-defined config values to be used within map expressions. |
-| flattening_enabled  | False    | None    | 'True' to enable schema flattening and automatically expand nested properties. |
-| flattening_max_depth| False    | None    | The max depth to flatten schemas. |
+| Setting              | Required |                      Default                       | Description                                                                                                                                 |
+|:---------------------|:--------:|:--------------------------------------------------:|:--------------------------------------------------------------------------------------------------------------------------------------------|
+| access_token         | True     |                        None                        | The token to authenticate against the API service                                                                                           |
+| start_date           | True     |                        None                        | The earliest record date to sync                                                                                                            |
+| end_date             | False    |             2023-05-09 02:04:18.151589             | The latest record date to sync                                                                                                              |
+| user_agent           | False    | tap-linkedin-ads <api_user_email@your_company.com> | API ID                                                                                                                                      |
+| api_version          | False    |                       202305                       | LinkedInAds API Version                                                                                                                     |
+| accounts             | True     |                        None                        | LinkedInAds Account ID                                                                                                                      |
+| campaign             | True     |                        None                        | LinkedInAds Campaign ID                                                                                                                     |
+| creative             | True     |                        None                        | LinkedInAds Creative ID                                                                                                                     |
+| campaign_group       | True     |                        None                        | LinkedInAds Campaign Group ID                                                                                                               |
+| owner                | True     |                        None                        | LinkedInAds Owner ID                                                                                                                        |
+| stream_maps          | False    |                        None                        | Config object for stream maps capability. For more information check out [Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html). |
+| stream_map_config    | False    |                        None                        | User-defined config values to be used within map expressions.                                                                               |
+| flattening_enabled   | False    |                        None                        | 'True' to enable schema flattening and automatically expand nested properties.                                                              |
+| flattening_max_depth | False    |                        None                        | The max depth to flatten schemas.                                                                                                           |
 
 A full list of supported settings and capabilities is available by running: `tap-linkedin-ads --about`
 

meltano.yml

@@ -18,9 +18,11 @@ plugins:
     - name: user_agent
       value: 'meltano'
     - name: linkedin_version
-      value: '202211'
+      value: '202305'
     - name: campaign
     - name: owner
+    - name: campaign_group
+    - name: creative
     - name: start_date
       value: '2023-01-01T00:00:00Z'
     - name: end_date

tap_linkedin_ads/client.py

@@ -21,7 +21,7 @@ class LinkedInAdsStream(RESTStream):
 
     url_base = "https://api.linkedin.com/rest/"
 
-    records_jsonpath = "$.elements[*]"  # Or override `parse_response`.
+    records_jsonpath = "$[*]"  # Or override `parse_response`.
     next_page_token_jsonpath = (
         "$.paging.start"  # Or override `get_next_page_token`.  # noqa: S105
     )
@@ -63,16 +63,17 @@ def get_next_page_token(
         # If pagination is required, return a token which can be used to get the
         #       next page. If this is the final page, return "None" to end the
         #       pagination loop.
-
         resp_json = response.json()
         if previous_token is None:
             previous_token = 0
-
-        elements = resp_json.get("elements")
-
-        if len(elements) == 0 or len(elements) == previous_token + 1:
-            return None
-
+        try:
+            elements = resp_json.get("elements")
+            if len(elements) == 0 or len(elements) == previous_token + 1:
+                return None
+        except:
+            page = resp_json
+            if len(page) == 0 or len(page) == previous_token + 1:
+                return None
         return previous_token + 1
 
     def get_url_params(
@@ -111,13 +112,14 @@ def parse_response(  # noqa: PLR0912
             Each record from the source.
         """
         resp_json = response.json()
-
-        if isinstance(resp_json, list):
-            results = resp_json
-        elif resp_json.get("elements") is not None:
+        if resp_json.get("elements") is not None:
             results = resp_json["elements"]
             try:
                 columns = results[0]
+            except:  # noqa: E722
+                columns = results
+                pass
+            try:
                 created_time = (
                     columns.get("changeAuditStamps").get("created").get("time")
                 )
@@ -132,37 +134,54 @@ def parse_response(  # noqa: PLR0912
                     int(last_modified_time) / 1000,
                     tz=UTC,
                 ).isoformat()
-                try:
-                    account_column = columns.get("account")
-                    account_id = int(account_column.split(":")[3])
-                    columns["account_id"] = account_id
-                except:  # noqa: E722, S110
-                    pass
-                try:
-                    campaign_column = columns.get("campaignGroup")
-                    campaign = int(campaign_column.split(":")[3])
-                    columns["campaign_group_id"] = campaign
-                except:  # noqa: E722, S110
-                    pass
-                try:
-                    user_column = columns.get("user")
-                    user = user_column.split(":")[3]
-                    columns["user_person_id"] = user
-                except:  # noqa: E722, S110
-                    pass
-                try:
-                    schedule_column = columns.get("runSchedule").get("start")
-                    columns[
-                        "run_schedule_start"
-                    ] = datetime.fromtimestamp(  # noqa: DTZ006
-                        int(schedule_column) / 1000,
-                    ).isoformat()
-                except:  # noqa: E722, S110
-                    pass
-                results = [columns]
             except:  # noqa: E722, S110
                 pass
         else:
             results = resp_json
+            try:
+                columns = results
+                created_time = (
+                    columns.get("changeAuditStamps").get("created").get("time")
+                )
+                last_modified_time = (
+                    columns.get("changeAuditStamps").get("lastModified").get("time")
+                )
+                columns["created_time"] = datetime.fromtimestamp(
+                    int(created_time) / 1000,
+                    tz=UTC,
+                ).isoformat()
+                columns["last_modified_time"] = datetime.fromtimestamp(
+                    int(last_modified_time) / 1000,
+                    tz=UTC,
+                ).isoformat()
+            except:  # noqa: E722
+                columns = results
+                pass
+
+        try:
+            account_column = columns.get("account")
+            account_id = int(account_column.split(":")[3])
+            columns["account_id"] = account_id
+        except:  # noqa: E722, S110
+            pass
+        try:
+            campaign_column = columns.get("campaignGroup")
+            campaign = int(campaign_column.split(":")[3])
+            columns["campaign_group_id"] = campaign
+        except:  # noqa: E722, S110
+            pass
+        try:
+            schedule_column = columns.get("runSchedule").get("start")
+            columns["run_schedule_start"] = datetime.fromtimestamp(  # noqa: DTZ006
+                int(schedule_column) / 1000,
+            ).isoformat()
+        except:  # noqa: E722, S110
+            pass
+
+        results = (
+            resp_json["elements"]
+            if resp_json.get("elements") is not None
+            else [columns]
+        )
 
         yield from results