diff --git a/.gitignore b/.gitignore index c3401424..e69bd306 100644 --- a/.gitignore +++ b/.gitignore @@ -26,7 +26,6 @@ share/python-wheels/ .installed.cfg *.egg MANIFEST -*/_version.py # PyInstaller # Usually these files are written by a python script from a template diff --git a/Makefile b/Makefile index f033f2d6..82c6e196 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ build: clean - python3 setup.py sdist bdist_wheel + python3 -m build .PHONY: build clean: @@ -39,7 +39,7 @@ check-type: mypy dbtmetabase .PHONY: check-type -check: check-fmt check-lint-python check-type +check: check-fmt check-imports check-lint-python check-type .PHONY: check test: diff --git a/dbtmetabase/__init__.py b/dbtmetabase/__init__.py index 16c1376a..70b3980e 100644 --- a/dbtmetabase/__init__.py +++ b/dbtmetabase/__init__.py @@ -1,785 +1,12 @@ -import functools +import importlib.metadata import logging -import os -from pathlib import Path -from typing import Any, Callable, Dict, Iterable, Optional -import click -import yaml - -from .logger import logging as package_logger from .models.interface import DbtInterface, MetabaseInterface -from .utils import get_version, load_config __all__ = ["MetabaseInterface", "DbtInterface"] -__version__ = get_version() - -CONFIG = load_config() -ENV_VARS = [ - "DBT_DATABASE", - "DBT_PATH", - "DBT_MANIFEST_PATH", - "MB_USER", - "MB_PASSWORD", - "MB_HOST", - "MB_DATABASE", - "MB_SESSION_TOKEN", - "MB_HTTP_TIMEOUT", -] - - -class MultiArg(click.Option): - """This class lets us pass multiple arguments after an options, equivalent to nargs=*""" - - def __init__(self, *args, **kwargs): - nargs = kwargs.pop("nargs", -1) - assert nargs == -1, "nargs, if set, must be -1 not {}".format(nargs) - super(MultiArg, self).__init__(*args, **kwargs) - self._previous_parser_process = None - self._eat_all_parser = None - - def add_to_parser(self, parser, ctx): - def parser_process(value, state): - # Method to hook to the parser.process - done = False - value = [value] - # Grab everything up to the next option - while state.rargs and not done: - for prefix in self._eat_all_parser.prefixes: - if state.rargs[0].startswith(prefix): - done = True - if not done: - value.append(state.rargs.pop(0)) - value = tuple(value) - - # Call the actual process - self._previous_parser_process(value, state) - - super().add_to_parser(parser, ctx) - for name in self.opts: - # pylint: disable=protected-access - our_parser = parser._long_opt.get(name) or parser._short_opt.get(name) - if our_parser: - self._eat_all_parser = our_parser - self._previous_parser_process = our_parser.process - our_parser.process = parser_process - break - - return - - -class ListParam(click.Tuple): - def __init__(self) -> None: - self.type = click.STRING - super().__init__([]) - - def convert( - self, value: Any, param: Optional[click.Parameter], ctx: Optional[click.Context] - ) -> Any: - len_value = len(value) - types = [self.type] * len_value - - return list(ty(x, param, ctx) for ty, x in zip(types, value)) - - -class OptionAcceptableFromConfig(click.Option): - """This class override should be used on arguments that are marked `required=True` in order to give them - more resilence to raising an error when the option exists in the users config. - - This also overrides default values for boolean CLI flags (e.g. --use_metabase_http/--use_metabase_https) in options when - no CLI flag is passed, but a value is provided in the config file (e.g. metabase_use_http: True). - """ - - def process_value(self, ctx: click.Context, value: Any) -> Any: - if value is not None: - value = self.type_cast_value(ctx, value) - - assert self.name, "none config option" - - if ( - isinstance(self.type, click.types.BoolParamType) - and ctx.get_parameter_source(self.name) - == click.core.ParameterSource.DEFAULT - and self.name in CONFIG - ): - value = CONFIG[self.name] - - if self.required and self.value_is_missing(value): - if self.name not in CONFIG: - raise click.core.MissingParameter(ctx=ctx, param=self) - value = CONFIG[self.name] - - if self.callback is not None: - value = self.callback(ctx, self, value) - - return value - - -class CommandController(click.Command): - """This class inherets from click.Command and supplies custom help text renderer to - render our docstrings a little prettier as well as a hook in the invoke to load from a config file if it exists. - """ - - def invoke(self, ctx: click.Context): - if CONFIG: - for param, value in ctx.params.items(): - if value is None and param in CONFIG: - ctx.params[param] = CONFIG[param] - - return super().invoke(ctx) - - def get_help(self, ctx: click.Context): - orig_wrap_test = click.formatting.wrap_text - - def wrap_text( - text: str, - width: int = 78, - initial_indent: str = "", - subsequent_indent: str = "", - preserve_paragraphs: bool = False, - ): - del preserve_paragraphs - return orig_wrap_test( - text.replace("\n", "\n\n"), - width, - initial_indent=initial_indent, - subsequent_indent=subsequent_indent, - preserve_paragraphs=True, - ).replace("\n\n", "\n") - - click.formatting.wrap_text = wrap_text - return super().get_help(ctx) - - -def shared_opts(func: Callable) -> Callable: - """Here we define the options shared across subcommands - - Args: - func (Callable): Wraps a subcommand - - Returns: - Callable: Subcommand with added options - """ - - @click.option( - "--dbt_database", - envvar="DBT_DATABASE", - show_envvar=True, - required=True, - cls=OptionAcceptableFromConfig, - help="Target database name as specified in dbt models to be actioned", - type=click.STRING, - ) - @click.option( - "--dbt_path", - envvar="DBT_PATH", - show_envvar=True, - help="Path to dbt project. If specified with --dbt_manifest_path, then the manifest is prioritized", - type=click.Path(exists=True, file_okay=False, dir_okay=True), - ) - @click.option( - "--dbt_manifest_path", - envvar="DBT_MANIFEST_PATH", - show_envvar=True, - help="Path to dbt manifest.json file (typically located in the /target/ directory of the dbt project)", - type=click.Path(exists=True, file_okay=True, dir_okay=False), - ) - @click.option( - "--dbt_schema", - help="Target schema. Should be passed if using folder parser", - type=click.STRING, - ) - @click.option( - "--dbt_schema_excludes", - metavar="SCHEMAS", - cls=MultiArg, - type=list, - help="Target schemas to exclude. Ignored in folder parser. Accepts multiple arguments after the flag", - ) - @click.option( - "--dbt_includes", - metavar="MODELS", - cls=MultiArg, - type=list, - help="Model names to limit processing to", - ) - @click.option( - "--dbt_excludes", - metavar="MODELS", - cls=MultiArg, - type=list, - help="Model names to exclude", - ) - @click.option( - "--metabase_database", - envvar="MB_DATABASE", - show_envvar=True, - required=True, - cls=OptionAcceptableFromConfig, - type=click.STRING, - help="Target database name as set in Metabase (typically aliased)", - ) - @click.option( - "--metabase_host", - metavar="HOST", - envvar="MB_HOST", - show_envvar=True, - required=True, - cls=OptionAcceptableFromConfig, - type=click.STRING, - help="Metabase hostname", - ) - @click.option( - "--metabase_user", - metavar="USER", - envvar="MB_USER", - show_envvar=True, - cls=OptionAcceptableFromConfig, - type=click.STRING, - help="Metabase username", - ) - @click.option( - "--metabase_password", - metavar="PASS", - envvar="MB_PASSWORD", - show_envvar=True, - cls=OptionAcceptableFromConfig, - type=click.STRING, - help="Metabase password", - ) - @click.option( - "--metabase_session_id", - metavar="TOKEN", - envvar="MB_SESSION_ID", - show_envvar=True, - default=None, - cls=OptionAcceptableFromConfig, - type=click.STRING, - help="Metabase session ID", - ) - @click.option( - "--metabase_http/--metabase_https", - "metabase_use_http", - default=False, - cls=OptionAcceptableFromConfig, - help="use HTTP or HTTPS to connect to Metabase. Default HTTPS", - ) - @click.option( - "--metabase_verify", - metavar="CERT", - type=click.Path(exists=True, file_okay=True, dir_okay=False), - help="Path to certificate bundle used by Metabase client", - ) - @click.option( - "--metabase_sync/--metabase_sync_skip", - "metabase_sync", - cls=OptionAcceptableFromConfig, - default=True, - help="Attempt to synchronize Metabase schema with local models. Default sync", - ) - @click.option( - "--metabase_sync_timeout", - metavar="SECS", - type=int, - help="Synchronization timeout (in secs). If set, we will fail hard on synchronization failure; if not set, we will proceed after attempting sync regardless of success. Only valid if sync is enabled", - ) - @click.option( - "--http_extra_headers", - cls=OptionAcceptableFromConfig, - type=(str, str), - multiple=True, - help="Additional HTTP request header to be sent to Metabase.", - ) - @click.option( - "--metabase_http_timeout", - cls=OptionAcceptableFromConfig, - type=int, - default=15, - envvar="MB_HTTP_TIMEOUT", - show_envvar=True, - help="Set the value for single requests timeout", - ) - @functools.wraps(func) - def wrapper(*args, **kwargs): - return func(*args, **kwargs) - - return wrapper - - -@click.group() -@click.version_option(__version__) -def cli(): - """Model synchronization from dbt to Metabase.""" - - -@click.command(cls=CommandController) -def check_config(): - package_logger.logger().info( - "Looking for configuration file in ~/.dbt-metabase :magnifying_glass_tilted_right:" - ) - package_logger.logger().info( - "...bootstrapping environmental variables :racing_car:" - ) - any_found = False - for env in ENV_VARS: - if env in os.environ: - package_logger.logger().info("Injecting valid env var: %s", env) - param = env.lower().replace("mb_", "metabase_") - CONFIG[param] = os.environ[env] - any_found = True - if not any_found: - package_logger.logger().info("NO valid env vars found") - - if not CONFIG: - package_logger.logger().info( - "No configuration file or env vars found, run the `config` command to interactively generate one.", - ) - else: - package_logger.logger().info("Config rendered!") - package_logger.logger().info( - {k: (v if "pass" not in k else "****") for k, v in CONFIG.items()} - ) - - -@click.command(cls=CommandController) -def check_env(): - package_logger.logger().info("All valid env vars: %s", ENV_VARS) - any_found = False - for env in ENV_VARS: - if env in os.environ: - val = os.environ[env] if "pass" not in env.lower() else "****" - package_logger.logger().info("Found value for %s --> %s", env, val) - any_found = True - if not any_found: - package_logger.logger().info("None of the env vars found in environment") - - -@cli.command(cls=CommandController) -@click.option( - "--inspect", - is_flag=True, - help="Introspect your dbt-metabase config.", -) -@click.option( - "--resolve", - is_flag=True, - help="Introspect your dbt-metabase config automatically injecting env vars into the configuration overwriting config.yml defaults. Use this flag if you are using env vars and want to see the resolved runtime output.", -) -@click.option( - "--env", - is_flag=True, - help="List all valid env vars for dbt-metabase. Env vars are evaluated before the config.yml and thus take precendence if used.", -) -@click.pass_context -def config(ctx, inspect: bool = False, resolve: bool = False, env: bool = False): - """Interactively generate a config or validate an existing config.yml - - A config allows you to omit arguments which will be substituted with config defaults. This simplifies - the execution of dbt-metabase to simply calling a command in most cases. Ex `dbt-metabase models` - - Execute the `config` command with no flags to enter an interactive session to create or update a config.yml. - - The config.yml should be located in ~/.dbt-metabase/ - Valid keys include any parameter seen in a dbt-metabase --help function - Example: `dbt-metabase models --help` - """ - if inspect: - package_logger.logger().info( - {k: (v if "pass" not in k else "****") for k, v in CONFIG.items()} - ) - if resolve: - ctx.invoke(check_config) - if env: - ctx.invoke(check_env) - if inspect or resolve or env: - ctx.exit() - click.confirm( - "Confirming you want to build or modify a dbt-metabase config file?", abort=True - ) - package_logger.logger().info( - "Preparing interactive configuration :rocket: (note defaults denoted by [...] are pulled from your existing config if found)" - ) - config_path = Path.home() / ".dbt-metabase" - config_path.mkdir(parents=True, exist_ok=True) - config_file = {} - conf_name = None - if (config_path / "config.yml").exists(): - with open(config_path / "config.yml", "r", encoding="utf-8") as f: - config_file = yaml.safe_load(f).get("config", {}) - conf_name = "config.yml" - elif (config_path / "config.yaml").exists(): - with open(config_path / "config.yaml", "r", encoding="utf-8") as f: - config_file = yaml.safe_load(f).get("config", {}) - conf_name = "config.yaml" - else: - # Default config name - conf_name = "config.yml" - if not config_file: - package_logger.logger().info("Building config file! :hammer:") - else: - package_logger.logger().info("Modifying config file! :wrench:") - config_file["dbt_database"] = click.prompt( - "Please enter the name of your dbt Database", - default=config_file.get("dbt_database"), - show_default=True, - type=click.STRING, - ) - config_file["dbt_manifest_path"] = click.prompt( - "Please enter the path to your dbt manifest.json \ntypically located in the /target directory of the dbt project", - default=config_file.get("dbt_manifest_path"), - show_default=True, - type=click.Path(exists=True, file_okay=True, dir_okay=False, resolve_path=True), - ) - if click.confirm( - "Would you like to set some default schemas to exclude when no flags are provided?" - ): - config_file["dbt_schema_excludes"] = click.prompt( - "Target schemas to exclude separated by commas", - default=config_file.get("dbt_schema_excludes"), - show_default=True, - value_proc=lambda s: list(map(str.strip, s.split(","))), - type=click.UNPROCESSED, - ) - else: - config_file.pop("dbt_schema_excludes", None) - if click.confirm( - "Would you like to set some default dbt models to exclude when no flags are provided?" - ): - config_file["dbt_excludes"] = click.prompt( - "dbt model names to exclude separated by commas", - default=config_file.get("dbt_excludes"), - show_default=True, - value_proc=lambda s: list(map(str.strip, s.split(","))), - type=ListParam(), - ) - else: - config_file.pop("dbt_excludes", None) - config_file["metabase_database"] = click.prompt( - "Target database name as set in Metabase (typically aliased)", - default=config_file.get("metabase_database"), - show_default=True, - type=click.STRING, - ) - config_file["metabase_host"] = click.prompt( - "Metabase hostname, this is the URL without the protocol (HTTP/S)", - default=config_file.get("metabase_host"), - show_default=True, - type=click.STRING, - ) - config_file["metabase_user"] = click.prompt( - "Metabase username", - default=config_file.get("metabase_user"), - show_default=True, - type=click.STRING, - ) - config_file["metabase_password"] = click.prompt( - "Metabase password [hidden]", - default=config_file.get("metabase_password"), - hide_input=True, - show_default=False, - type=click.STRING, - ) - config_file["metabase_use_http"] = click.confirm( - "Use HTTP instead of HTTPS to connect to Metabase, unless testing locally we should be saying no here", - default=config_file.get("metabase_use_http", False), - show_default=True, - ) - if click.confirm("Would you like to set a custom certificate bundle to use?"): - config_file["metabase_verify"] = click.prompt( - "Path to certificate bundle used by Metabase client", - default=config_file.get("metabase_verify"), - show_default=True, - type=click.Path( - exists=True, file_okay=True, dir_okay=False, resolve_path=True - ), - ) - else: - config_file.pop("metabase_verify", None) - config_file["metabase_sync"] = click.confirm( - "Would you like to allow Metabase schema syncs by default? Best to say yes as there is little downside", - default=config_file.get("metabase_sync", True), - show_default=True, - ) - if config_file["metabase_sync"]: - config_file["metabase_sync_timeout"] = click.prompt( - "Synchronization timeout in seconds. If set, we will fail hard on synchronization failure; \nIf set to 0, we will proceed after attempting sync regardless of success", - default=config_file.get("metabase_sync_timeout", 0), - show_default=True, - value_proc=lambda i: None if int(i) <= 0 else int(i), - type=click.INT, - ) - else: - config_file.pop("metabase_sync_timeout", None) - output_config = {"config": config_file} - package_logger.logger().info( - "Config constructed -- writing config to ~/.dbt-metabase" - ) - package_logger.logger().info( - {k: (v if "pass" not in k else "****") for k, v in config_file.items()} - ) - with open(config_path / conf_name, "w", encoding="utf-8") as outfile: - yaml.dump( - output_config, - outfile, - default_flow_style=False, - allow_unicode=True, - sort_keys=False, - ) - - -@cli.command(cls=CommandController) -@shared_opts -@click.option( - "--dbt_docs_url", - metavar="URL", - type=click.STRING, - help="Pass in URL to dbt docs site. Appends dbt docs URL for each model to Metabase table description (default None)", -) -@click.option( - "--dbt_include_tags", - is_flag=True, - help="Flag to append tags to table descriptions in Metabase (default False)", -) -@click.option( - "--metabase_exclude_sources", - is_flag=True, - help="Flag to skip exporting sources to Metabase (default False)", -) -@click.option( - "-v", - "--verbose", - is_flag=True, - help="Flag which signals verbose output", -) -def models( - metabase_host: str, - metabase_user: str, - metabase_password: str, - metabase_database: str, - dbt_database: str, - dbt_path: Optional[str] = None, - dbt_manifest_path: Optional[str] = None, - dbt_schema: Optional[str] = None, - dbt_schema_excludes: Optional[Iterable] = None, - dbt_includes: Optional[Iterable] = None, - dbt_excludes: Optional[Iterable] = None, - metabase_session_id: Optional[str] = None, - metabase_use_http: bool = False, - metabase_verify: Optional[str] = None, - metabase_sync: bool = True, - metabase_sync_timeout: Optional[int] = None, - metabase_exclude_sources: bool = False, - metabase_http_timeout: int = 15, - dbt_include_tags: bool = True, - dbt_docs_url: Optional[str] = None, - verbose: bool = False, - http_extra_headers: Optional[Dict[Any, Any]] = None, -): - """Exports model documentation and semantic types from dbt to Metabase. - - Args: - metabase_host (str): Metabase hostname. - metabase_user (str): Metabase username. - metabase_password (str): Metabase password. - metabase_database (str): Target database name as set in Metabase (typically aliased). - metabase_session_id (Optional[str], optional): Session ID. Defaults to None. - dbt_database (str): Target database name as specified in dbt models to be actioned. - dbt_path (Optional[str], optional): Path to dbt project. If specified with dbt_manifest_path, then the manifest is prioritized. Defaults to None. - dbt_manifest_path (Optional[str], optional): Path to dbt manifest.json file (typically located in the /target/ directory of the dbt project). Defaults to None. - dbt_schema (Optional[str], optional): Target schema. Should be passed if using folder parser. Defaults to None. - dbt_schema_excludes (Optional[Iterable], optional): Target schemas to exclude. Ignored in folder parser. Defaults to None. - dbt_includes (Optional[Iterable], optional): Model names to limit processing to. Defaults to None. - dbt_excludes (Optional[Iterable], optional): Model names to exclude. Defaults to None. - metabase_session_id (Optional[str], optional): Metabase session id. Defaults to none. - metabase_use_http (bool, optional): Use HTTP to connect to Metabase. Defaults to False. - metabase_verify (Optional[str], optional): Path to custom certificate bundle to be used by Metabase client. Defaults to None. - metabase_sync (bool, optional): Attempt to synchronize Metabase schema with local models. Defaults to True. - metabase_sync_timeout (Optional[int], optional): Synchronization timeout (in secs). If set, we will fail hard on synchronization failure; if not set, we will proceed after attempting sync regardless of success. Only valid if sync is enabled. Defaults to None. - metabase_exclude_sources (bool, optional): Flag to skip exporting sources to Metabase. Defaults to False. - metabase_http_timeout (int, optional): Set the timeout for the single Metabase requests. Defaults to 15. - dbt_include_tags (bool, optional): Flag to append tags to table descriptions in Metabase. Defaults to True. - dbt_docs_url (Optional[str], optional): Pass in URL to dbt docs site. Appends dbt docs URL for each model to Metabase table description. Defaults to None. - http_extra_headers (Optional[str], optional): Additional HTTP request headers to be sent to Metabase. Defaults to None. - verbose (bool, optional): Flag which signals verbose output. Defaults to False. - """ - - # Set global logging level if verbose - if verbose: - package_logger.LOGGING_LEVEL = logging.DEBUG - - # Instantiate dbt interface - dbt = DbtInterface( - path=dbt_path, - manifest_path=dbt_manifest_path, - database=dbt_database, - schema=dbt_schema, - schema_excludes=dbt_schema_excludes, - includes=dbt_includes, - excludes=dbt_excludes, - ) - - # Load models - dbt_models, aliases = dbt.read_models( - include_tags=dbt_include_tags, - docs_url=dbt_docs_url, - ) - - # Instantiate Metabase interface - metabase = MetabaseInterface( - host=metabase_host, - user=metabase_user, - password=metabase_password, - session_id=metabase_session_id, - use_http=metabase_use_http, - verify=metabase_verify, - database=metabase_database, - sync=metabase_sync, - sync_timeout=metabase_sync_timeout, - exclude_sources=metabase_exclude_sources, - http_extra_headers=http_extra_headers, - http_timeout=metabase_http_timeout, - ) - - # Load client - metabase.prepare_metabase_client(dbt_models) - - # Execute model export - metabase.client.export_models( - database=metabase.database, - models=dbt_models, - aliases=aliases, - ) - - -@cli.command(cls=CommandController) -@shared_opts -@click.option( - "--output_path", - type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), - help="Output path for generated exposure yaml. Defaults to local dir.", - default=".", -) -@click.option( - "--output_name", - type=click.STRING, - help="Output name for generated exposure yaml. Defaults to metabase_exposures.yml", -) -@click.option( - "--include_personal_collections", - is_flag=True, - help="Flag to include Personal Collections during exposure parsing", -) -@click.option( - "--collection_excludes", - cls=MultiArg, - type=list, - help="Metabase collection names to exclude", -) -@click.option( - "-v", - "--verbose", - is_flag=True, - help="Flag which signals verbose output", -) -def exposures( - metabase_host: str, - metabase_user: str, - metabase_password: str, - metabase_database: str, - dbt_database: str, - dbt_path: Optional[str] = None, - dbt_manifest_path: Optional[str] = None, - dbt_schema: Optional[str] = None, - dbt_schema_excludes: Optional[Iterable] = None, - dbt_includes: Optional[Iterable] = None, - dbt_excludes: Optional[Iterable] = None, - metabase_session_id: Optional[str] = None, - metabase_use_http: bool = False, - metabase_verify: Optional[str] = None, - metabase_sync: bool = True, - metabase_sync_timeout: Optional[int] = None, - metabase_http_timeout: int = 15, - output_path: str = ".", - output_name: str = "metabase_exposures.yml", - include_personal_collections: bool = False, - collection_excludes: Optional[Iterable] = None, - http_extra_headers: Optional[Dict[Any, Any]] = None, - verbose: bool = False, -) -> None: - """Extracts and imports exposures from Metabase to dbt. - - Args: - metabase_host (str): Metabase hostname. - metabase_user (str): Metabase username. - metabase_password (str): Metabase password. - metabase_database (str): Target database name as set in Metabase (typically aliased). - dbt_database (str): Target database name as specified in dbt models to be actioned. - dbt_path (Optional[str], optional): Path to dbt project. If specified with dbt_manifest_path, then the manifest is prioritized. Defaults to None. - dbt_manifest_path (Optional[str], optional): Path to dbt manifest.json file (typically located in the /target/ directory of the dbt project). Defaults to None. - dbt_schema (Optional[str], optional): Target schema. Should be passed if using folder parser. Defaults to None. - dbt_schema_excludes (Optional[Iterable], optional): Target schemas to exclude. Ignored in folder parser. Defaults to None. - dbt_includes (Optional[Iterable], optional): Model names to limit processing to. Defaults to None. - dbt_excludes (Optional[Iterable], optional): Model names to exclude. Defaults to None. - metabase_session_id (Optional[str], optional): Metabase session id. Defaults to none. - metabase_use_http (bool, optional): Use HTTP to connect to Metabase. Defaults to False. - metabase_verify (Optional[str], optional): Path to custom certificate bundle to be used by Metabase client. Defaults to None. - metabase_sync (bool, optional): Attempt to synchronize Metabase schema with local models. Defaults to True. - metabase_sync_timeout (Optional[int], optional): Synchronization timeout (in secs). If set, we will fail hard on synchronization failure; if not set, we will proceed after attempting sync regardless of success. Only valid if sync is enabled. Defaults to None. - metabase_http_timeout (int, optional): Set the timeout for the single Metabase requests. Default 15 - output_path (str): Output path for generated exposure yaml. Defaults to "." local dir. - output_name (str): Output name for generated exposure yaml. Defaults to metabase_exposures.yml. - include_personal_collections (bool, optional): Flag to include Personal Collections during exposure parsing. Defaults to False. - collection_excludes (Iterable, optional): Collection names to exclude. Defaults to None. - http_extra_headers (Optional[str], optional): Additional HTTP request headers to be sent to Metabase. Defaults to None. - verbose (bool, optional): Flag which signals verbose output. Defaults to False. - """ - - # Set global logging level if verbose - if verbose: - package_logger.LOGGING_LEVEL = logging.DEBUG - - # Instantiate dbt interface - dbt = DbtInterface( - path=dbt_path, - manifest_path=dbt_manifest_path, - database=dbt_database, - schema=dbt_schema, - schema_excludes=dbt_schema_excludes, - includes=dbt_includes, - excludes=dbt_excludes, - ) - - # Load models - dbt_models, _ = dbt.read_models() - - # Instantiate Metabase interface - metabase = MetabaseInterface( - host=metabase_host, - user=metabase_user, - password=metabase_password, - session_id=metabase_session_id, - use_http=metabase_use_http, - verify=metabase_verify, - database=metabase_database, - sync=metabase_sync, - sync_timeout=metabase_sync_timeout, - http_extra_headers=http_extra_headers, - http_timeout=metabase_http_timeout, - ) - - # Load client - metabase.prepare_metabase_client(dbt_models) - - # Execute exposure extraction - metabase.client.extract_exposures( - models=dbt_models, - output_path=output_path, - output_name=output_name, - include_personal_collections=include_personal_collections, - collection_excludes=collection_excludes, - ) - -def main(): - # Valid kwarg - cli(max_content_width=600) # pylint: disable=unexpected-keyword-arg +try: + __version__ = importlib.metadata.version("dbt-metabase") +except importlib.metadata.PackageNotFoundError: + __version__ = "0.0.0-UNKONWN" + logging.warning("No version found in metadata") diff --git a/dbtmetabase/__main__.py b/dbtmetabase/__main__.py new file mode 100644 index 00000000..b745cef1 --- /dev/null +++ b/dbtmetabase/__main__.py @@ -0,0 +1,3 @@ +from .cli import cli + +cli() # pylint: disable=no-value-for-parameter diff --git a/dbtmetabase/bin/dbt-metabase b/dbtmetabase/bin/dbt-metabase deleted file mode 100755 index 24a863da..00000000 --- a/dbtmetabase/bin/dbt-metabase +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python -import dbtmetabase - -if __name__ == "__main__": - dbtmetabase.main() diff --git a/dbtmetabase/cli.py b/dbtmetabase/cli.py new file mode 100644 index 00000000..9b4a4e41 --- /dev/null +++ b/dbtmetabase/cli.py @@ -0,0 +1,417 @@ +import functools +import logging +from pathlib import Path +from typing import Callable, Iterable, List, Optional, Union + +import click +import yaml +from typing_extensions import cast + +from .logger import logging as package_logger +from .models.interface import DbtInterface, MetabaseInterface + + +def _comma_separated_list_callback( + ctx: click.Context, param: click.Option, value: Union[str, List[str]] +) -> List[str]: + """Click callback for handling comma-separated lists.""" + + assert ( + param.type == click.UNPROCESSED or param.type.name == "list" + ), "comma-separated list options must be of type UNPROCESSED or list" + + if ctx.get_parameter_source(str(param.name)) in ( + click.core.ParameterSource.DEFAULT, + click.core.ParameterSource.DEFAULT_MAP, + ) and isinstance(value, list): + # lists in defaults (config or option) should be lists + return value + + elif isinstance(value, str): + str_value = value + if isinstance(value, list): + # when type=list, string value will be a list of chars + str_value = "".join(value) + else: + raise click.BadParameter("must be comma-separated list") + + return str_value.split(",") + + +@click.group() +@click.version_option(package_name="dbt-metabase") +@click.option( + "--config-path", + default="~/.dbt-metabase/config.yml", + show_default=True, + type=click.Path(), + help="Path to config.yml file with default values.", +) +@click.pass_context +def cli(ctx: click.Context, config_path: str): + group = cast(click.Group, ctx.command) + + config_path_expanded = Path(config_path).expanduser() + if config_path_expanded.exists(): + with open(config_path_expanded, "r", encoding="utf-8") as f: + config = yaml.safe_load(f).get("config", {}) + # propagate root configs to all commands + ctx.default_map = {command: config for command in group.commands} + + +def common_options(func: Callable) -> Callable: + """Common click options between commands.""" + + @click.option( + "--dbt-database", + metavar="DATABASE", + envvar="DBT_DATABASE", + show_envvar=True, + required=True, + type=click.STRING, + help="Target database name in dbt models.", + ) + @click.option( + "--dbt-manifest-path", + envvar="DBT_MANIFEST_PATH", + show_envvar=True, + type=click.Path(exists=True, dir_okay=False), + help="Path to dbt manifest.json file under /target/ in the dbt project directory. Uses dbt manifest parsing (recommended).", + ) + @click.option( + "--dbt-project-path", + envvar="DBT_PROJECT_PATH", + show_envvar=True, + type=click.Path(exists=True, file_okay=False), + help="Path to dbt project directory containing models. Uses dbt project parsing (not recommended).", + ) + @click.option( + "--dbt-schema", + metavar="SCHEMA", + envvar="DBT_SCHEMA", + show_envvar=True, + help="Target dbt schema. Must be passed if using project parser.", + type=click.STRING, + ) + @click.option( + "--dbt-schema-excludes", + metavar="SCHEMAS", + envvar="DBT_SCHEMA_EXCLUDES", + show_envvar=True, + type=click.UNPROCESSED, + callback=_comma_separated_list_callback, + help="Target dbt schemas to exclude. Ignored in project parser.", + ) + @click.option( + "--dbt-includes", + metavar="MODELS", + envvar="DBT_INCLUDES", + show_envvar=True, + type=click.UNPROCESSED, + callback=_comma_separated_list_callback, + help="Include specific dbt models names.", + ) + @click.option( + "--dbt-excludes", + metavar="MODELS", + envvar="DBT_EXCLUDES", + show_envvar=True, + type=click.UNPROCESSED, + callback=_comma_separated_list_callback, + help="Exclude specific dbt model names.", + ) + @click.option( + "--metabase-database", + metavar="DATABASE", + envvar="METABASE_DATABASE", + show_envvar=True, + required=True, + type=click.STRING, + help="Target database name in Metabase.", + ) + @click.option( + "--metabase-host", + metavar="HOST", + envvar="MB_HOST", + show_envvar=True, + required=True, + type=click.STRING, + help="Metabase hostname, excluding protocol.", + ) + @click.option( + "--metabase-user", + metavar="USER", + envvar="METABASE_USER", + show_envvar=True, + type=click.STRING, + help="Metabase username.", + ) + @click.option( + "--metabase-password", + metavar="PASSWORD", + envvar="METABASE_PASSWORD", + show_envvar=True, + type=click.STRING, + help="Metabase password.", + ) + @click.option( + "--metabase-session-id", + metavar="TOKEN", + envvar="METABASE_SESSION_ID", + show_envvar=True, + type=click.STRING, + help="Metabase session ID.", + ) + @click.option( + "--metabase-http/--metabase-https", + "metabase_use_http", + envvar="METABASE_USE_HTTP", + show_envvar=True, + default=False, + help="Force HTTP instead of HTTPS to connect to Metabase.", + ) + @click.option( + "--metabase-verify", + metavar="CERT", + envvar="METABASE_VERIFY", + show_envvar=True, + type=click.Path(exists=True, file_okay=True, dir_okay=False), + help="Path to certificate bundle used to connect to Metabase.", + ) + @click.option( + "--metabase-sync/--metabase-sync-skip", + "metabase_sync", + envvar="METABASE_SYNC", + show_envvar=True, + default=True, + show_default=True, + help="Attempt to synchronize Metabase schema with local models.", + ) + @click.option( + "--metabase-sync-timeout", + metavar="SECS", + envvar="METABASE_SYNC_TIMEOUT", + show_envvar=True, + type=click.INT, + help="Synchronization timeout in secs. When set, command fails on failed synchronization. Otherwise, command proceeds regardless. Only valid if sync is enabled.", + ) + @click.option( + "--metabase-http-timeout", + metavar="SECS", + envvar="METABASE_HTTP_TIMEOUT", + show_envvar=True, + type=click.INT, + default=15, + show_default=True, + help="Set the value for single requests timeout.", + ) + @click.option( + "-v", + "--verbose", + is_flag=True, + help="Enable verbose logging.", + ) + @functools.wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + return wrapper + + +@cli.command(help="Export dbt models to Metabase.") +@common_options +@click.option( + "--dbt-docs-url", + metavar="URL", + envvar="DBT_DOCS_URL", + show_envvar=True, + type=click.STRING, + help="URL for dbt docs to be appended to table descriptions in Metabase.", +) +@click.option( + "--dbt-include-tags", + envvar="DBT_INCLUDE_TAGS", + show_envvar=True, + is_flag=True, + help="Append tags to table descriptions in Metabase.", +) +@click.option( + "--metabase-exclude-sources", + envvar="METABASE_EXCLUDE_SOURCES", + show_envvar=True, + is_flag=True, + help="Skip exporting sources to Metabase.", +) +def models( + metabase_host: str, + metabase_user: str, + metabase_password: str, + metabase_database: str, + dbt_database: str, + dbt_path: Optional[str], + dbt_manifest_path: Optional[str], + dbt_schema: Optional[str], + dbt_schema_excludes: Optional[Iterable], + dbt_includes: Optional[Iterable], + dbt_excludes: Optional[Iterable], + metabase_session_id: Optional[str], + metabase_use_http: bool, + metabase_verify: Optional[str], + metabase_sync: bool, + metabase_sync_timeout: Optional[int], + metabase_exclude_sources: bool, + metabase_http_timeout: int, + dbt_include_tags: bool, + dbt_docs_url: Optional[str], + verbose: bool, +): + # Set global logging level if verbose + if verbose: + package_logger.LOGGING_LEVEL = logging.DEBUG + + # Instantiate dbt interface + dbt = DbtInterface( + path=dbt_path, + manifest_path=dbt_manifest_path, + database=dbt_database, + schema=dbt_schema, + schema_excludes=dbt_schema_excludes, + includes=dbt_includes, + excludes=dbt_excludes, + ) + + # Load models + dbt_models, aliases = dbt.read_models( + include_tags=dbt_include_tags, + docs_url=dbt_docs_url, + ) + + # Instantiate Metabase interface + metabase = MetabaseInterface( + host=metabase_host, + user=metabase_user, + password=metabase_password, + session_id=metabase_session_id, + use_http=metabase_use_http, + verify=metabase_verify, + database=metabase_database, + sync=metabase_sync, + sync_timeout=metabase_sync_timeout, + exclude_sources=metabase_exclude_sources, + http_timeout=metabase_http_timeout, + ) + + # Load client + metabase.prepare_metabase_client(dbt_models) + + # Execute model export + metabase.client.export_models( + database=metabase.database, + models=dbt_models, + aliases=aliases, + ) + + +@cli.command(help="Export dbt exposures to Metabase.") +@common_options +@click.option( + "--output-path", + envvar="OUTPUT_PATH", + show_envvar=True, + type=click.Path(exists=True, file_okay=False), + default=".", + show_default=True, + help="Output path for generated exposure YAML.", +) +@click.option( + "--output-name", + metavar="NAME", + envvar="OUTPUT_NAME", + show_envvar=True, + type=click.STRING, + default="metabase_exposures.yml", + show_default=True, + help="File name for generated exposure YAML.", +) +@click.option( + "--metabase-include-personal-collections", + envvar="METABASE_INCLUDE_PERSONAL_COLLECTIONS", + show_envvar=True, + is_flag=True, + help="Include personal collections when parsing exposures.", +) +@click.option( + "--metabase-collection-excludes", + metavar="COLLECTIONS", + envvar="METABASE_COLLECTION_EXCLUDES", + show_envvar=True, + type=click.UNPROCESSED, + callback=_comma_separated_list_callback, + help="Metabase collection names to exclude.", +) +def exposures( + metabase_host: str, + metabase_user: str, + metabase_password: str, + metabase_database: str, + dbt_database: str, + dbt_path: Optional[str], + dbt_manifest_path: Optional[str], + dbt_schema: Optional[str], + dbt_schema_excludes: Optional[Iterable], + dbt_includes: Optional[Iterable], + dbt_excludes: Optional[Iterable], + metabase_session_id: Optional[str], + metabase_use_http: bool, + metabase_verify: Optional[str], + metabase_sync: bool, + metabase_sync_timeout: Optional[int], + metabase_http_timeout: int, + output_path: str, + output_name: str, + metabase_include_personal_collections: bool, + metabase_collection_excludes: Optional[Iterable], + verbose: bool, +): + if verbose: + package_logger.LOGGING_LEVEL = logging.DEBUG + + # Instantiate dbt interface + dbt = DbtInterface( + path=dbt_path, + manifest_path=dbt_manifest_path, + database=dbt_database, + schema=dbt_schema, + schema_excludes=dbt_schema_excludes, + includes=dbt_includes, + excludes=dbt_excludes, + ) + + # Load models + dbt_models, _ = dbt.read_models() + + # Instantiate Metabase interface + metabase = MetabaseInterface( + host=metabase_host, + user=metabase_user, + password=metabase_password, + session_id=metabase_session_id, + use_http=metabase_use_http, + verify=metabase_verify, + database=metabase_database, + sync=metabase_sync, + sync_timeout=metabase_sync_timeout, + http_timeout=metabase_http_timeout, + ) + + # Load client + metabase.prepare_metabase_client(dbt_models) + + # Execute exposure extraction + metabase.client.extract_exposures( + models=dbt_models, + output_path=output_path, + output_name=output_name, + include_personal_collections=metabase_include_personal_collections, + collection_excludes=metabase_collection_excludes, + ) diff --git a/dbtmetabase/utils.py b/dbtmetabase/utils.py deleted file mode 100644 index 005810ea..00000000 --- a/dbtmetabase/utils.py +++ /dev/null @@ -1,44 +0,0 @@ -import logging -import sys -from pathlib import Path - -import yaml - - -def get_version() -> str: - """Checks _version.py or build metadata for package version. - - Returns: - str: Version string. - """ - - try: - from ._version import version - - return version - except ModuleNotFoundError: - logging.debug("No _version.py found") - - # importlib is only available on Python 3.8+ - if sys.version_info >= (3, 8): - # pylint: disable=no-member - import importlib.metadata - - try: - return importlib.metadata.version("dbt-metabase") - except importlib.metadata.PackageNotFoundError: - logging.warning("No version found in metadata") - - return "0.0.0-UNKONWN" - - -def load_config() -> dict: - config_data = {} - config_path = Path.home() / ".dbt-metabase" - if (config_path / "config.yml").exists(): - with open(config_path / "config.yml", "r", encoding="utf-8") as f: - config_data = yaml.safe_load(f).get("config", {}) - elif (config_path / "config.yaml").exists(): - with open(config_path / "config.yaml", "r", encoding="utf-8") as f: - config_data = yaml.safe_load(f).get("config", {}) - return config_data diff --git a/pyproject.toml b/pyproject.toml index 9756a359..6df9db02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,22 +1,21 @@ [build-system] -requires = ["setuptools>=45", "wheel", "setuptools_scm"] build-backend = "setuptools.build_meta" +requires = ["setuptools>=60", "setuptools-scm"] [tool.setuptools_scm] -write_to = "dbtmetabase/_version.py" [tool.black] -line-length = 88 -target-version = ["py36", "py37", "py38"] include = '\.pyi?$' +line-length = 88 +target-version = ["py38"] [tool.isort] profile = "black" src_paths = ["dbtmetabase", "tests", "setup.py"] [tool.mypy] -python_version = "3.8" ignore_missing_imports = true +python_version = "3.8" [tool.pylint.master] -disable = ["R", "C"] +disable = ["R", "C", "W0511"] diff --git a/requirements-test.txt b/requirements-test.txt index 9cba6a3c..078ee83c 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,9 +1,9 @@ pip>=23.3.1 -setuptools>=69.0.2 -wheel>=0.42.0 +build>=1.0.3 +twine>=4.0.2 +black>=23.11.0 +isort>=5.12.0 pylint>=3.0.2 mypy>=1.7.1 types-requests types-PyYAML -black>=23.11.0 -isort>=5.12.0 diff --git a/setup.py b/setup.py index 2a1e8de1..979cb9fb 100755 --- a/setup.py +++ b/setup.py @@ -18,21 +18,21 @@ def requires_from_file(filename: str) -> list: setup( name="dbt-metabase", - use_scm_version=True, description="Model synchronization from dbt to Metabase.", long_description=readme, long_description_content_type="text/markdown", author="Mike Gouline", url="https://github.com/gouline/dbt-metabase", license="MIT License", - scripts=["dbtmetabase/bin/dbt-metabase"], + entry_points={ + "console_scripts": ["dbt-metabase = dbtmetabase.cli:cli"], + }, packages=find_packages(exclude=["tests"]), test_suite="tests", install_requires=requires_from_file("requirements.txt"), extras_require={ "test": requires_from_file("requirements-test.txt"), }, - setup_requires=["setuptools_scm"], classifiers=[ "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", @@ -41,6 +41,8 @@ def requires_from_file(filename: str) -> list: "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Topic :: Software Development :: Libraries", "Topic :: Software Development :: Libraries :: Python Modules", ],