Restructure models for better readability

dbtmetabase/__init__.py

@@ -5,7 +5,10 @@
 
 logger = logging.getLogger(__name__)
 
-__all__ = ["DbtReader", "MetabaseClient"]
+__all__ = [
+    "DbtReader",
+    "MetabaseClient",
+]
 
 try:
     from ._version import __version__ as version  # type: ignore

dbtmetabase/__main__.py

@@ -1,86 +1,15 @@
 import functools
 import logging
-from logging.handlers import RotatingFileHandler
 from pathlib import Path
-from typing import Callable, Iterable, List, Optional, Union
+from typing import Callable, Iterable, Optional
 
 import click
 import yaml
-from rich.logging import RichHandler
 from typing_extensions import cast
 
+from ._format import click_list_option_kwargs, setup_logging
 from .dbt import DbtReader
-from .metabase import MetabaseClient
-
-LOG_PATH = Path.home().absolute() / ".dbt-metabase" / "logs" / "dbtmetabase.log"
-
-logger = logging.getLogger(__name__)
-
-
-def _setup_logger(level: int = logging.INFO):
-    """Basic logger configuration for the CLI.
-
-    Args:
-        level (int, optional): Logging level. Defaults to logging.INFO.
-    """
-
-    LOG_PATH.parent.mkdir(parents=True, exist_ok=True)
-    file_handler = RotatingFileHandler(
-        filename=LOG_PATH,
-        maxBytes=int(1e6),
-        backupCount=3,
-    )
-    file_handler.setFormatter(
-        logging.Formatter("%(asctime)s — %(name)s — %(levelname)s — %(message)s")
-    )
-    file_handler.setLevel(logging.WARNING)
-
-    rich_handler = RichHandler(
-        level=level,
-        rich_tracebacks=True,
-        markup=True,
-        show_time=False,
-    )
-
-    logging.basicConfig(
-        level=level,
-        format="%(asctime)s — %(message)s",
-        datefmt="%Y-%m-%d %H:%M:%S %z",
-        handlers=[file_handler, rich_handler],
-        force=True,
-    )
-
-
-def _comma_separated_list_callback(
-    ctx: click.Context,
-    param: click.Option,
-    value: Union[str, List[str]],
-) -> Optional[List[str]]:
-    """Click callback for handling comma-separated lists."""
-
-    if value is None:
-        return None
-
-    assert (
-        param.type == click.UNPROCESSED or param.type.name == "list"
-    ), "comma-separated list options must be of type UNPROCESSED or list"
-
-    if ctx.get_parameter_source(str(param.name)) in (
-        click.core.ParameterSource.DEFAULT,
-        click.core.ParameterSource.DEFAULT_MAP,
-    ) and isinstance(value, list):
-        # Lists in defaults (config or option) should be lists
-        return value
-
-    elif isinstance(value, str):
-        str_value = value
-    if isinstance(value, list):
-        # When type=list, string value will be a list of chars
-        str_value = "".join(value)
-    else:
-        raise click.BadParameter("must be comma-separated list")
-
-    return str_value.split(",")
+from .metabase import MetabaseClient, MetabaseExposuresClient, MetabaseModelsClient
 
 
 @click.group()
@@ -137,26 +66,23 @@ def _add_setup(func: Callable) -> Callable:
         metavar="SCHEMAS",
         envvar="DBT_SCHEMA_EXCLUDES",
         show_envvar=True,
-        type=click.UNPROCESSED,
-        callback=_comma_separated_list_callback,
+        **click_list_option_kwargs(),
         help="Target dbt schemas to exclude. Ignored in project parser.",
     )
     @click.option(
         "--dbt-includes",
         metavar="MODELS",
         envvar="DBT_INCLUDES",
         show_envvar=True,
-        type=click.UNPROCESSED,
-        callback=_comma_separated_list_callback,
+        **click_list_option_kwargs(),
         help="Include specific dbt models names.",
     )
     @click.option(
         "--dbt-excludes",
         metavar="MODELS",
         envvar="DBT_EXCLUDES",
         show_envvar=True,
-        type=click.UNPROCESSED,
-        callback=_comma_separated_list_callback,
+        **click_list_option_kwargs(),
         help="Exclude specific dbt model names.",
     )
     @click.option(
@@ -197,7 +123,7 @@ def _add_setup(func: Callable) -> Callable:
         "metabase_verify",
         envvar="METABASE_VERIFY",
         show_envvar=True,
-        default=True,
+        default=MetabaseClient.DEFAULT_VERIFY,
         help="Verify the TLS certificate at the Metabase end.",
     )
     @click.option(
@@ -214,7 +140,7 @@ def _add_setup(func: Callable) -> Callable:
         envvar="METABASE_TIMEOUT",
         show_envvar=True,
         type=click.INT,
-        default=15,
+        default=MetabaseClient.DEFAULT_HTTP_TIMEOUT,
         show_default=True,
         help="Metabase API HTTP timeout in seconds.",
     )
@@ -242,7 +168,10 @@ def wrapper(
         verbose: bool,
         **kwargs,
     ):
-        _setup_logger(level=logging.DEBUG if verbose else logging.INFO)
+        setup_logging(
+            level=logging.DEBUG if verbose else logging.INFO,
+            path=Path.home().absolute() / ".dbt-metabase" / "logs" / "dbtmetabase.log",
+        )
 
         return func(
             dbt_reader=DbtReader(
@@ -299,14 +228,15 @@ def wrapper(
     metavar="SECS",
     envvar="METABASE_SYNC_TIMEOUT",
     show_envvar=True,
-    default=30,
+    default=MetabaseModelsClient.DEFAULT_SYNC_TIMEOUT,
     type=click.INT,
     help="Synchronization timeout in secs. When set, command fails on failed synchronization. Otherwise, command proceeds regardless. Only valid if sync is enabled.",
 )
 @click.option(
     "--metabase-exclude-sources",
     envvar="METABASE_EXCLUDE_SOURCES",
     show_envvar=True,
+    default=MetabaseModelsClient.DEFAULT_EXCLUDE_SOURCES,
     is_flag=True,
     help="Skip exporting sources to Metabase.",
 )
@@ -338,7 +268,7 @@ def models(
     envvar="OUTPUT_PATH",
     show_envvar=True,
     type=click.Path(exists=True, file_okay=False),
-    default=".",
+    default=MetabaseExposuresClient.DEFAULT_OUTPUT_PATH,
     show_default=True,
     help="Output path for generated exposure YAML files.",
 )
@@ -354,24 +284,23 @@ def models(
     envvar="METABASE_INCLUDE_PERSONAL_COLLECTIONS",
     show_envvar=True,
     is_flag=True,
+    default=MetabaseExposuresClient.DEFAULT_INCLUDE_PERSONAL_COLLECTIONS,
     help="Include personal collections when parsing exposures.",
 )
 @click.option(
     "--metabase-collection-includes",
     metavar="COLLECTIONS",
     envvar="METABASE_COLLECTION_INCLUDES",
     show_envvar=True,
-    type=click.UNPROCESSED,
-    callback=_comma_separated_list_callback,
+    **click_list_option_kwargs(),
     help="Metabase collection names to includes.",
 )
 @click.option(
     "--metabase-collection-excludes",
     metavar="COLLECTIONS",
     envvar="METABASE_COLLECTION_EXCLUDES",
     show_envvar=True,
-    type=click.UNPROCESSED,
-    callback=_comma_separated_list_callback,
+    **click_list_option_kwargs(),
     help="Metabase collection names to exclude.",
 )
 def exposures(

dbtmetabase/_format.py

@@ -1,5 +1,98 @@
+import logging
 import re
-from typing import Optional
+from logging.handlers import RotatingFileHandler
+from pathlib import Path
+from typing import Any, Iterable, List, Mapping, Optional, Union
+
+import click
+from rich.logging import RichHandler
+
+_logger = logging.getLogger(__name__)
+
+
+class _NullValue(str):
+    """Explicitly null field value."""
+
+    def __eq__(self, other: object) -> bool:
+        return other is None
+
+
+NullValue = _NullValue()
+
+
+def setup_logging(level: int, path: Path):
+    """Basic logger configuration for the CLI.
+
+    Args:
+        level (int): Logging level. Defaults to logging.INFO.
+        path (Path): Path to file logs.
+    """
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+    file_handler = RotatingFileHandler(
+        filename=path,
+        maxBytes=int(1e6),
+        backupCount=3,
+    )
+    file_handler.setFormatter(
+        logging.Formatter("%(asctime)s — %(name)s — %(levelname)s — %(message)s")
+    )
+    file_handler.setLevel(logging.WARNING)
+
+    rich_handler = RichHandler(
+        level=level,
+        rich_tracebacks=True,
+        markup=True,
+        show_time=False,
+    )
+
+    logging.basicConfig(
+        level=level,
+        format="%(asctime)s — %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S %z",
+        handlers=[file_handler, rich_handler],
+        force=True,
+    )
+
+
+def click_list_option_kwargs() -> Mapping[str, Any]:
+    """Click option that accepts comma-separated values.
+
+    Built-in list only allows repeated flags, which is ugly for larger lists.
+
+    Returns:
+        Mapping[str, Any]: Mapping of kwargs (to be unpacked with **).
+    """
+
+    def callback(
+        ctx: click.Context,
+        param: click.Option,
+        value: Union[str, List[str]],
+    ) -> Optional[List[str]]:
+        if value is None:
+            return None
+
+        if ctx.get_parameter_source(str(param.name)) in (
+            click.core.ParameterSource.DEFAULT,
+            click.core.ParameterSource.DEFAULT_MAP,
+        ) and isinstance(value, list):
+            # Lists in defaults (config or option) should be lists
+            return value
+
+        elif isinstance(value, str):
+            str_value = value
+        if isinstance(value, list):
+            # When type=list, string value will be a list of chars
+            str_value = "".join(value)
+        else:
+            raise click.BadParameter("must be comma-separated list")
+
+        return str_value.split(",")
+
+    return {
+        "type": click.UNPROCESSED,
+        "callback": callback,
+    }
 
 
 def safe_name(text: Optional[str]) -> str:
@@ -26,3 +119,23 @@ def safe_description(text: Optional[str]) -> str:
         str: Sanitized string with escaped Jinja syntax.
     """
     return re.sub(r"{{(.*)}}", r"\1", text or "")
+
+
+def scan_fields(t: Mapping, fields: Iterable[str], ns: str) -> Mapping:
+    """Reads meta fields from a schem object.
+
+    Args:
+        t (Mapping): Target to scan for fields.
+        fields (List): List of fields to accept.
+        ns (str): Field namespace (separated by .).
+
+    Returns:
+        Mapping: Field values.
+    """
+
+    vals = {}
+    for field in fields:
+        if f"{ns}.{field}" in t:
+            value = t[f"{ns}.{field}"]
+            vals[field] = value if value is not None else NullValue
+    return vals

dbtmetabase/dbt/__init__.py

@@ -0,0 +1,10 @@
+from .models import MetabaseColumn, MetabaseModel, ResourceType
+from .reader import DEFAULT_SCHEMA, DbtReader
+
+__all__ = [
+    "DEFAULT_SCHEMA",
+    "DbtReader",
+    "MetabaseModel",
+    "MetabaseColumn",
+    "ResourceType",
+]