add date range option to readers for issue #96

nck/readers/dcm_reader.py

@@ -27,6 +27,7 @@
 from nck.streams.normalized_json_stream import NormalizedJSONStream
 from nck.clients.dcm_client import DCMClient
 from nck.helpers.dcm_helper import REPORT_TYPES
+from nck.utils.date_handler import DEFAULT_DATE_RANGE_FUNCTIONS, build_date_range
 
 DATEFORMAT = "%Y-%m-%d"
 ENCODING = "utf-8"
@@ -52,8 +53,8 @@
     multiple=True,
     help="https://developers.google.com/doubleclick-advertisers/v3.3/dimensions/#standard-dimensions",
 )
-@click.option("--dcm-start-date", type=click.DateTime(), required=True)
-@click.option("--dcm-end-date", type=click.DateTime(), required=True)
+@click.option("--dcm-start-date", type=click.DateTime(), help="Start date of the report")
+@click.option("--dcm-end-date", type=click.DateTime(), help="End date of the report")
 @click.option(
     "--dcm-filter",
     "dcm_filters",
@@ -62,6 +63,11 @@
     help="A filter is a tuple following this pattern: (dimensionName, dimensionValue). "
     "https://developers.google.com/doubleclick-advertisers/v3.3/dimensions/#standard-filters",
 )
+@click.option(
+    "--dcm-date-range",
+    type=click.Choice(DEFAULT_DATE_RANGE_FUNCTIONS.keys()),
+    help=f"One of the available NCK default date ranges: {DEFAULT_DATE_RANGE_FUNCTIONS.keys()}",
+)
 @processor("dcm_access_token", "dcm_refresh_token", "dcm_client_secret")
 def dcm(**kwargs):
     return DcmReader(**extract_args("dcm_", kwargs))
@@ -82,15 +88,15 @@ def __init__(
         start_date,
         end_date,
         filters,
+        date_range,
     ):
         self.dcm_client = DCMClient(access_token, client_id, client_secret, refresh_token)
         self.profile_ids = list(profile_ids)
         self.report_name = report_name
         self.report_type = report_type
         self.metrics = list(metrics)
         self.dimensions = list(dimensions)
-        self.start_date = start_date
-        self.end_date = end_date
+        self.start_date, self.end_date = build_date_range(start_date, end_date, date_range)
         self.filters = list(filters)
 
     def format_response(self, report_generator):

nck/readers/sa360_reader.py

@@ -23,6 +23,7 @@
 from nck.clients.sa360_client import SA360Client
 from nck.helpers.sa360_helper import REPORT_TYPES
 from nck.utils.args import extract_args
+from nck.utils.date_handler import DEFAULT_DATE_RANGE_FUNCTIONS, build_date_range
 from nck.utils.text import get_report_generator_from_flat_file
 
 DATEFORMAT = "%Y-%m-%d"
@@ -42,23 +43,23 @@
     help="If empty, all advertisers from agency will be requested",
 )
 @click.option("--sa360-report-name", default="SA360 Report")
+@click.option("--sa360-report-type", type=click.Choice(REPORT_TYPES), default=REPORT_TYPES[0])
 @click.option(
-    "--sa360-report-type", type=click.Choice(REPORT_TYPES), default=REPORT_TYPES[0]
-)
-@click.option(
-    "--sa360-column",
-    "sa360_columns",
-    multiple=True,
-    help="https://developers.google.com/search-ads/v2/report-types",
+    "--sa360-column", "sa360_columns", multiple=True, help="https://developers.google.com/search-ads/v2/report-types",
 )
 @click.option(
     "--sa360-saved-column",
     "sa360_saved_columns",
     multiple=True,
     help="https://developers.google.com/search-ads/v2/how-tos/reporting/saved-columns",
 )
-@click.option("--sa360-start-date", type=click.DateTime(), required=True)
-@click.option("--sa360-end-date", type=click.DateTime(), required=True)
+@click.option("--sa360-start-date", type=click.DateTime(), help="Start date of the report")
+@click.option("--sa360-end-date", type=click.DateTime(), help="End date of the report")
+@click.option(
+    "--sa360-date-range",
+    type=click.Choice(DEFAULT_DATE_RANGE_FUNCTIONS.keys()),
+    help=f"One of the available NCK default date ranges: {DEFAULT_DATE_RANGE_FUNCTIONS.keys()}",
+)
 @processor("sa360_access_token", "sa360_refresh_token", "sa360_client_secret")
 def sa360_reader(**kwargs):
     return SA360Reader(**extract_args("sa360_", kwargs))
@@ -79,19 +80,17 @@ def __init__(
         saved_columns,
         start_date,
         end_date,
+        date_range,
     ):
-        self.sa360_client = SA360Client(
-            access_token, client_id, client_secret, refresh_token
-        )
+        self.sa360_client = SA360Client(access_token, client_id, client_secret, refresh_token)
         self.agency_id = agency_id
         self.advertiser_ids = list(advertiser_ids)
         self.report_name = report_name
         self.report_type = report_type
         self.columns = list(columns)
         self.saved_columns = list(saved_columns)
         self.all_columns = self.columns + self.saved_columns
-        self.start_date = start_date
-        self.end_date = end_date
+        self.start_date, self.end_date = build_date_range(start_date, end_date, date_range)
 
     def result_generator(self):
         for advertiser_id in self.advertiser_ids:
@@ -109,17 +108,11 @@ def result_generator(self):
 
             report_data = self.sa360_client.assert_report_file_ready(report_id)
 
-            for line_iterator in self.sa360_client.download_report_files(
-                report_data, report_id
-            ):
+            for line_iterator in self.sa360_client.download_report_files(report_data, report_id):
                 yield from get_report_generator_from_flat_file(line_iterator)
 
     def read(self):
         if not self.advertiser_ids:
-            self.advertiser_ids = self.sa360_client.get_all_advertisers_of_agency(
-                self.agency_id
-            )
+            self.advertiser_ids = self.sa360_client.get_all_advertisers_of_agency(self.agency_id)
 
-        yield NormalizedJSONStream(
-            "results" + "_".join(self.advertiser_ids), self.result_generator()
-        )
+        yield NormalizedJSONStream("results" + "_".join(self.advertiser_ids), self.result_generator())

nck/readers/search_console_reader.py

@@ -28,6 +28,7 @@
 from nck.readers.reader import Reader
 from nck.streams.normalized_json_stream import NormalizedJSONStream
 from nck.utils.args import extract_args
+from nck.utils.date_handler import DEFAULT_DATE_RANGE_FUNCTIONS, build_date_range
 from nck.utils.retry import retry
 
 
@@ -42,6 +43,11 @@
 @click.option("--search-console-end-date", type=click.DateTime(), default=None)
 @click.option("--search-console-date-column", type=click.BOOL, default=False)
 @click.option("--search-console-row-limit", type=click.INT, default=25000)
+@click.option(
+    "--search-console-date-range",
+    type=click.Choice(DEFAULT_DATE_RANGE_FUNCTIONS.keys()),
+    help=f"One of the available NCK default date ranges: {DEFAULT_DATE_RANGE_FUNCTIONS.keys()}",
+)
 @processor()
 def search_console(**params):
     return SearchConsoleReader(**extract_args("search_console_", params))
@@ -65,15 +71,17 @@ def __init__(
         end_date,
         date_column,
         row_limit,
+        date_range,
     ):
         self.client_id = client_id
         self.client_secret = client_secret
         self.access_token = access_token
         self.refresh_token = refresh_token
         self.dimensions = list(dimensions)
         self.site_url = site_url
-        self.start_date = datetime.strftime(start_date, DATEFORMAT)
-        self.end_date = datetime.strftime(self.check_end_date(end_date), DATEFORMAT)
+        self.start_date, self.end_date = build_date_range(start_date, end_date, date_range)
+        self.start_date = datetime.strftime(self.start_date, DATEFORMAT)
+        self.end_date = datetime.strftime(self.check_end_date(self.end_date), DATEFORMAT)
         self.with_date_column = date_column
         self.row_limit = row_limit
 
@@ -101,7 +109,13 @@ def initialize_analyticsreporting(self):
 
     @staticmethod
     def check_end_date(end_date):
-        if end_date > MAX_END_DATE:
+        # adapt the class to be able to compare end_date with MAX_END_DATE (datetime)
+        if isinstance(end_date, datetime):
+            max_end_date = MAX_END_DATE
+        else:
+            max_end_date = MAX_END_DATE.date()
+
+        if end_date > max_end_date:
             logger.warning(f"The most recent date you can request is {datetime.strftime(MAX_END_DATE, DATEFORMAT)}")
         return end_date
 

nck/readers/ttd_reader.py

@@ -31,6 +31,7 @@
     ReportScheduleNotReadyError,
     format_date,
 )
+from nck.utils.date_handler import DEFAULT_DATE_RANGE_FUNCTIONS, build_date_range
 from nck.utils.text import get_report_generator_from_flat_file
 
 
@@ -50,10 +51,10 @@
     "--ttd-report-schedule-name", required=True, help="Name of the Report Schedule to create.",
 )
 @click.option(
-    "--ttd-start-date", required=True, type=click.DateTime(), help="Start date of the period to request (format: YYYY-MM-DD)",
+    "--ttd-start-date", type=click.DateTime(), help="Start date of the period to request (format: YYYY-MM-DD)",
 )
 @click.option(
-    "--ttd-end-date", required=True, type=click.DateTime(), help="End date of the period to request (format: YYYY-MM-DD)",
+    "--ttd-end-date", type=click.DateTime(), help="End date of the period to request (format: YYYY-MM-DD)",
 )
 @click.option(
     "--ttd-normalize-stream",
@@ -63,6 +64,11 @@
     "characters replaced by '_' in field names, which is useful for BigQuery). "
     "Else, yields a standard JSONStream.",
 )
+@click.option(
+    "--ttd-date-range",
+    type=click.Choice(DEFAULT_DATE_RANGE_FUNCTIONS.keys()),
+    help=f"One of the available NCK default date ranges: {DEFAULT_DATE_RANGE_FUNCTIONS.keys()}",
+)
 @processor("ttd_login", "ttd_password")
 def the_trade_desk(**kwargs):
     return TheTradeDeskReader(**extract_args("ttd_", kwargs))
@@ -79,16 +85,17 @@ def __init__(
         start_date,
         end_date,
         normalize_stream,
+        date_range,
     ):
         self.login = login
         self.password = password
         self._build_headers()
         self.advertiser_ids = list(advertiser_id)
         self.report_template_name = report_template_name
         self.report_schedule_name = report_schedule_name
-        self.start_date = start_date
+        self.start_date, self.end_date = build_date_range(start_date, end_date, date_range)
         # Report end date is exclusive: to become inclusive, it should be incremented by 1 day
-        self.end_date = end_date + timedelta(days=1)
+        self.end_date = self.end_date + timedelta(days=1)
         self.normalize_stream = normalize_stream
 
         self._validate_dates()

nck/readers/twitter_reader.py

@@ -40,6 +40,8 @@
 # from twitter_ads.creative import TweetPreview
 from twitter_ads.creative import CardsFetch
 
+from nck.utils.date_handler import DEFAULT_DATE_RANGE_FUNCTIONS, build_date_range
+
 API_DATEFORMAT = "%Y-%m-%dT%H:%M:%SZ"
 REP_DATEFORMAT = "%Y-%m-%d"
 MAX_WAITING_SEC = 3600
@@ -136,6 +138,11 @@
     default=False,
     help="If set to 'True', the date on which the request is made will appear on each report record.",
 )
+@click.option(
+    "--twitter-date-range",
+    type=click.Choice(DEFAULT_DATE_RANGE_FUNCTIONS.keys()),
+    help=f"One of the available NCK default date ranges: {DEFAULT_DATE_RANGE_FUNCTIONS.keys()}",
+)
 @processor(
     "twitter_consumer_key", "twitter_consumer_secret", "twitter_access_token", "twitter_access_token_secret",
 )
@@ -163,6 +170,7 @@ def __init__(
         start_date,
         end_date,
         add_request_date_to_report,
+        date_range,
     ):
         # Authentication inputs
         self.client = Client(consumer_key, consumer_secret, access_token, access_token_secret)
@@ -171,8 +179,8 @@ def __init__(
         # General inputs
         self.report_type = report_type
         self.entity = entity
-        self.start_date = start_date
-        self.end_date = end_date + timedelta(days=1)
+        self.start_date, self.end_date = build_date_range(start_date, end_date, date_range)
+        self.end_date = self.end_date + timedelta(days=1)
         self.add_request_date_to_report = add_request_date_to_report
 
         # Report inputs: ENTITY