Add linting to examples, fix type hints for empty dicts and list args.

.ci/lint.sh

@@ -15,7 +15,7 @@ lint() {
     return $failed
 }
 
-lint rdw_ingestion_tools/
+lint rdw_ingestion_tools/ examples/
 result=$?
 
 if [ $result = 0 ]; then

examples/mqr/mqr.py

@@ -1,5 +1,3 @@
-from datetime import datetime
-
 from api.survey import pySurvey
 
 start = "2023-01-01T00:00:00"

examples/s3_utils/unify.py

@@ -1,10 +1,10 @@
 """
-Usage example: pr python3 examples/s3_utils/unify.py --bucket 'rdw-za' --prefix 'contact-ndoh/momconnect-flow-results/flows/questions/'
+Usage example: pr python3 examples/s3_utils/unify.py --bucket 'rdw-za' --prefix
+'contact-ndoh/momconnect-flow-results/flows/questions/'
 
 """
 
 import argparse
-import os
 
 from pandas import concat
 from s3 import pyS3

examples/turn_bq/flow_results_data_packages.py

@@ -8,7 +8,7 @@
 
 print(flow_results_data_packages.head(5))
 
-flow_results_data_packages = pyTurnBQ.flow_results_data_packages.get_flow_results_data_packages_by_updated_at(
+flow_results_data_packages = pyTurnBQ.flow_results_data_packages.get_flow_results_data_packages_by_updated_at(  # noqa: E501
     from_timestamp="2023-10-01T00:00:00", to_timestamp="2023-10-10T00:00:00"
 )
 

pyproject.toml

@@ -17,6 +17,7 @@ pandas = "^2.2.2"
 awswrangler = "^3.7.3"
 attrs = "^23.2.0"
 httpx = "^0.27.0"
+types-tqdm = "^4.66.0.20240417"
 
 [tool.poetry.group.types.dependencies]
 pandas-stubs = "^2.2.1.240316"

rdw_ingestion_tools/api/__init__.py

@@ -3,6 +3,9 @@
 
 from pandas import DataFrame, concat, json_normalize
 
+# https://github.com/astral-sh/ruff/issues/3388
+from typing_extensions import Never  # noqa: UP035
+
 
 class MissingConfig(Exception):
     """Raised if a required config environment variable is not set."""
@@ -18,7 +21,9 @@ def config_from_env(key: str) -> str:
     return value
 
 
-def concatenate(objs: list[dict] | dict | list | Iterator) -> DataFrame:
+def concatenate(
+    objs: list[dict] | dict[Never, Never] | list[Never] | Iterator,
+) -> DataFrame:
     """
     Extend pandas concat to not only support dicts or lists of dicts, but
     also empty lists (which is often returned by the APIs).