Merge pull request #57 from sebastianswms/api-v14

feat: v14 API and holistic cleanup

.github/workflows/ci.yml

@@ -3,15 +3,15 @@ on:
   push:
     branches:
       - main
-  pull_request:
+  pull_request_target:
+    types: [opened, synchronize, reopened]
 jobs:
   build:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9, "3.10"]
+        python-version: [3.8, 3.9, "3.10"]
         os: [ubuntu-latest, windows-latest, macos-latest]
-
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}
@@ -24,7 +24,13 @@ jobs:
           pip install poetry==1.1.12
       - name: Build tap on Python ${{ matrix.python-version }} 
         run: poetry build
+  authorize:
+    environment: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository && 'external' || 'internal' }}
+    runs-on: ubuntu-latest
+    steps:
+      - run: true
   test:
+    needs: authorize
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2

README.md

@@ -26,6 +26,8 @@ THIS IS NOT READY FOR PRODUCTION. Bearer tokens sometimes slip out to logs. Use
 | start_date       | True     | 2022-03-24T00:00:00Z (Today-7d) | Date to start our search from, applies to Streams where there is a filter date. Note that Google responds to Data in buckets of 1 Day increments |
 | end_date         | True     | 2022-03-31T00:00:00Z (Today) | Date to end our search on, applies to Streams where there is a filter date. Note that the query is BETWEEN start_date AND end_date |
 
+Note that although customer IDs are often displayed in the Google Ads UI in the format 123-456-7890, they should be provided to the tap in the format 1234567890, with no dashes.
+
 ### Get refresh token
 1. GET https://accounts.google.com/o/oauth2/v2/auth?response_type=code&client_id=client_id&redirect_uri=http://127.0.0.1&scope=https://www.googleapis.com/auth/adwords&state=autoidm&access_type=offline&prompt=select_account&include_granted_scopes=true
 1. POST https://www.googleapis.com/oauth2/v4/token?code={code}&client_id={client_id}&client_secret={client_secret}&redirect_uri=http://127.0.0.1&grant_type=authorization_code

pyproject.toml

@@ -10,15 +10,15 @@ keywords = [
 license = "Apache 2.0"
 
 [tool.poetry.dependencies]
-python = "<3.11,>=3.6.2"
+python = "<3.11,>=3.8"
 requests = "^2.25.1"
-singer-sdk = "0.3.17"
+singer-sdk = "0.29.0"
 
 [tool.poetry.dev-dependencies]
 pytest = "^6.2.5"
 tox = "^3.24.4"
 flake8 = "^3.9.2"
-black = "^21.9b0"
+black = "^23.7.0"
 pydocstyle = "^6.1.1"
 mypy = "^0.910"
 types-requests = "^2.25.8"

tap_googleads/client.py

@@ -1,6 +1,6 @@
 """REST client handling, including GoogleAdsStream base class."""
 
-import requests
+from urllib.parse import urlencode, urljoin
 from pathlib import Path
 from typing import Any, Dict, Optional, Union, List, Iterable
 
@@ -9,6 +9,7 @@
 from singer_sdk.helpers.jsonpath import extract_jsonpath
 from singer_sdk.streams import RESTStream
 from singer_sdk.exceptions import FatalAPIError, RetriableAPIError
+from singer_sdk.pagination import JSONPathPaginator
 
 from tap_googleads.auth import GoogleAdsAuthenticator
 from tap_googleads.utils import replicate_pk_at_root
@@ -20,10 +21,10 @@
 class GoogleAdsStream(RESTStream):
     """GoogleAds stream class."""
 
-    url_base = "https://googleads.googleapis.com/v12"
+    url_base = "https://googleads.googleapis.com/v14"
 
     records_jsonpath = "$[*]"  # Or override `parse_response`.
-    next_page_token_jsonpath = "$.nextPageToken"  # Or override `get_next_page_token`.
+    next_page_token_jsonpath = "$.nextPageToken"
     primary_keys_jsonpaths = None
     _LOG_REQUEST_METRIC_URLS: bool = True
 
@@ -32,11 +33,13 @@ class GoogleAdsStream(RESTStream):
     def authenticator(self) -> GoogleAdsAuthenticator:
         """Return a new authenticator object."""
         base_auth_url = "https://www.googleapis.com/oauth2/v4/token"
-        # Silly way to do parameters but it works
-        auth_url = base_auth_url + f"?refresh_token={self.config['refresh_token']}"
-        auth_url = auth_url + f"&client_id={self.config['client_id']}"
-        auth_url = auth_url + f"&client_secret={self.config['client_secret']}"
-        auth_url = auth_url + f"&grant_type=refresh_token"
+        auth_params = {
+            "refresh_token": self.config["refresh_token"],
+            "client_id": self.config["client_id"],
+            "client_secret": self.config["client_secret"],
+            "grant_type": "refresh_token",
+        }
+        auth_url = urljoin(base_auth_url, "?" + urlencode(auth_params))
         return GoogleAdsAuthenticator(stream=self, auth_endpoint=auth_url)
 
     @property
@@ -49,23 +52,8 @@ def http_headers(self) -> dict:
         headers["login-customer-id"] = self.config["login_customer_id"]
         return headers
 
-    def get_next_page_token(
-        self, response: requests.Response, previous_token: Optional[Any]
-    ) -> Optional[Any]:
-        """Return a token for identifying next page or None if no more pages."""
-        # TODO: If pagination is required, return a token which can be used to get the
-        #       next page. If this is the final page, return "None" to end the
-        #       pagination loop.
-        if self.next_page_token_jsonpath:
-            all_matches = extract_jsonpath(
-                self.next_page_token_jsonpath, response.json()
-            )
-            first_match = next(iter(all_matches), None)
-            next_page_token = first_match
-        else:
-            next_page_token = None
-
-        return next_page_token
+    def get_new_paginator(self) -> JSONPathPaginator:
+        return JSONPathPaginator(self.next_page_token_jsonpath)
 
     def get_url_params(
         self, context: Optional[dict], next_page_token: Optional[Any]
@@ -145,21 +133,6 @@ def get_records(self, context: Optional[dict]) -> Iterable[Dict[str, Any]]:
                 f"disabled after the API that lists customers is called.  {e=}"
             )
 
-    def prepare_request_payload(
-        self, context: Optional[dict], next_page_token: Optional[Any]
-    ) -> Optional[dict]:
-        """Prepare the data payload for the REST API request.
-
-        By default, no payload will be sent (return None).
-        """
-        # TODO: Delete this method if no payload is required. (Most REST APIs.)
-        return None
-
-    def parse_response(self, response: requests.Response) -> Iterable[dict]:
-        """Parse the response and return an iterator of result rows."""
-        # TODO: Parse response body and return a set of records.
-        yield from extract_jsonpath(self.records_jsonpath, input=response.json())
-
     def post_process(self, row: dict, context: Optional[dict] = None) -> Optional[dict]:
         """As needed, append or transform raw data to match expected structure."""
         return replicate_pk_at_root(row, self.primary_keys_jsonpaths)

tap_googleads/schemas/campaign_performance.json

@@ -15,6 +15,18 @@
                 },
                 "name": {
                     "type": "string"
+                },
+                "startDate": {
+                    "type": "string"
+                },
+                "endDate": {
+                    "type": "string"
+                },
+                "advertisingChannelType": {
+                    "type": "string"
+                },
+                "AdvertisingChannelSubType": {
+                    "type": "string"
                 }
             }
         },

tap_googleads/schemas/customer_hierarchy.json

@@ -0,0 +1,67 @@
+{
+    "type": "object",
+    "properties": {
+        "customerClient": {
+            "type": [
+                "object",
+                "null"
+            ],
+            "properties": {
+                "resourceName": {
+                    "type": [
+                        "string",
+                        "null"
+                    ]
+                },
+                "clientCustomer": {
+                    "type": [
+                        "string",
+                        "null"
+                    ]
+                },
+                "level": {
+                    "type": [
+                        "string",
+                        "null"
+                    ]
+                },
+                "timeZone": {
+                    "type": [
+                        "string",
+                        "null"
+                    ]
+                },
+                "manager": {
+                    "type": [
+                        "boolean",
+                        "null"
+                    ]
+                },
+                "descriptiveName": {
+                    "type": [
+                        "string",
+                        "null"
+                    ]
+                },
+                "currencyCode": {
+                    "type": [
+                        "string",
+                        "null"
+                    ]
+                },
+                "id": {
+                    "type": [
+                        "string",
+                        "null"
+                    ]
+                }
+            }
+        },
+        "_sdc_primary_key": {
+            "type": [
+                "string",
+                "null"
+            ]
+        }
+    }
+}