From 084d3e79612a99751c7b665e1fdb97b32d8f1735 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar <110245118+ankatiyar@users.noreply.github.com> Date: Thu, 29 Aug 2024 22:41:03 +0100 Subject: [PATCH] Refactor CLI with lazy subcommands and deferring imports (#1920) * try delaying import + lazy subcommands Signed-off-by: Ankita Katiyar * lazy group initial draft * fix lint * add tests and fix lint * add default command and tests * add default command and tests * address PR comments --------- Signed-off-by: Ankita Katiyar Co-authored-by: ravi-kumar-pilla --- package/kedro_viz/launchers/cli.py | 418 --------- package/kedro_viz/launchers/cli/__init__.py | 1 + package/kedro_viz/launchers/cli/build.py | 24 + package/kedro_viz/launchers/cli/deploy.py | 69 ++ .../launchers/cli/lazy_default_group.py | 76 ++ package/kedro_viz/launchers/cli/main.py | 26 + package/kedro_viz/launchers/cli/run.py | 196 +++++ package/kedro_viz/launchers/cli/utils.py | 169 ++++ package/kedro_viz/launchers/utils.py | 13 - package/pyproject.toml | 2 +- package/tests/test_launchers/test_cli.py | 797 ------------------ .../test_launchers/test_cli/test_build.py | 49 ++ .../test_launchers/test_cli/test_deploy.py | 185 ++++ .../test_cli/test_lazy_default_group.py | 82 ++ .../test_launchers/test_cli/test_main.py | 75 ++ .../tests/test_launchers/test_cli/test_run.py | 360 ++++++++ .../test_launchers/test_cli/test_utils.py | 265 ++++++ package/tests/test_launchers/test_utils.py | 22 +- 18 files changed, 1579 insertions(+), 1250 deletions(-) delete mode 100644 package/kedro_viz/launchers/cli.py create mode 100644 package/kedro_viz/launchers/cli/__init__.py create mode 100644 package/kedro_viz/launchers/cli/build.py create mode 100644 package/kedro_viz/launchers/cli/deploy.py create mode 100644 package/kedro_viz/launchers/cli/lazy_default_group.py create mode 100644 package/kedro_viz/launchers/cli/main.py create mode 100644 package/kedro_viz/launchers/cli/run.py create mode 100644 package/kedro_viz/launchers/cli/utils.py delete mode 100755 package/tests/test_launchers/test_cli.py create mode 100644 package/tests/test_launchers/test_cli/test_build.py create mode 100644 package/tests/test_launchers/test_cli/test_deploy.py create mode 100644 package/tests/test_launchers/test_cli/test_lazy_default_group.py create mode 100644 package/tests/test_launchers/test_cli/test_main.py create mode 100644 package/tests/test_launchers/test_cli/test_run.py create mode 100644 package/tests/test_launchers/test_cli/test_utils.py diff --git a/package/kedro_viz/launchers/cli.py b/package/kedro_viz/launchers/cli.py deleted file mode 100644 index 5eac79907d..0000000000 --- a/package/kedro_viz/launchers/cli.py +++ /dev/null @@ -1,418 +0,0 @@ -"""`kedro_viz.launchers.cli` launches the viz server as a CLI app.""" - -import multiprocessing -import traceback -from pathlib import Path -from typing import Dict - -import click -from click_default_group import DefaultGroup -from kedro.framework.cli.project import PARAMS_ARG_HELP -from kedro.framework.cli.utils import KedroCliError, _split_params -from kedro.framework.project import PACKAGE_NAME -from packaging.version import parse -from watchgod import RegExpWatcher, run_process - -from kedro_viz import __version__ -from kedro_viz.constants import ( - DEFAULT_HOST, - DEFAULT_PORT, - SHAREABLEVIZ_SUPPORTED_PLATFORMS, - VIZ_DEPLOY_TIME_LIMIT, -) -from kedro_viz.integrations.deployment.deployer_factory import DeployerFactory -from kedro_viz.integrations.pypi import get_latest_version, is_running_outdated_version -from kedro_viz.launchers.utils import ( - _PYPROJECT, - _check_viz_up, - _find_kedro_project, - _start_browser, - _wait_for, - viz_deploy_progress_timer, -) -from kedro_viz.server import load_and_populate_data - -try: - from azure.core.exceptions import ServiceRequestError -except ImportError: # pragma: no cover - ServiceRequestError = None # type: ignore - -_VIZ_PROCESSES: Dict[str, int] = {} - - -@click.group(name="Kedro-Viz") -def viz_cli(): # pylint: disable=missing-function-docstring - pass - - -@viz_cli.group(cls=DefaultGroup, default="run", default_if_no_args=True) -@click.pass_context -def viz(ctx): # pylint: disable=unused-argument - """Visualise a Kedro pipeline using Kedro viz.""" - - -@viz.command(context_settings={"help_option_names": ["-h", "--help"]}) -@click.option( - "--host", - default=DEFAULT_HOST, - help="Host that viz will listen to. Defaults to localhost.", -) -@click.option( - "--port", - default=DEFAULT_PORT, - type=int, - help="TCP port that viz will listen to. Defaults to 4141.", -) -@click.option( - "--browser/--no-browser", - default=True, - help="Whether to open viz interface in the default browser or not. " - "Browser will only be opened if host is localhost. Defaults to True.", -) -@click.option( - "--load-file", - default=None, - help="Path to load Kedro-Viz data from a directory", -) -@click.option( - "--save-file", - default=None, - type=click.Path(dir_okay=False, writable=True), - help="Path to save Kedro-Viz data to a directory", -) -@click.option( - "--pipeline", - "-p", - type=str, - default=None, - help="Name of the registered pipeline to visualise. " - "If not set, the default pipeline is visualised", -) -@click.option( - "--env", - "-e", - type=str, - default=None, - multiple=False, - envvar="KEDRO_ENV", - help="Kedro configuration environment. If not specified, " - "catalog config in `local` will be used", -) -@click.option( - "--autoreload", - "-a", - is_flag=True, - help="Autoreload viz server when a Python or YAML file change in the Kedro project", -) -@click.option( - "--include-hooks", - is_flag=True, - help="A flag to include all registered hooks in your Kedro Project", -) -@click.option( - "--params", - type=click.UNPROCESSED, - default="", - help=PARAMS_ARG_HELP, - callback=_split_params, -) -# pylint: disable=import-outside-toplevel, too-many-locals -def run( - host, - port, - browser, - load_file, - save_file, - pipeline, - env, - autoreload, - include_hooks, - params, -): - """Launch local Kedro Viz instance""" - from kedro_viz.server import run_server - - kedro_project_path = _find_kedro_project(Path.cwd()) - - if kedro_project_path is None: - display_cli_message( - "ERROR: Failed to start Kedro-Viz : " - "Could not find the project configuration " - f"file '{_PYPROJECT}' at '{Path.cwd()}'. ", - "red", - ) - return - - installed_version = parse(__version__) - latest_version = get_latest_version() - if is_running_outdated_version(installed_version, latest_version): - display_cli_message( - "WARNING: You are using an old version of Kedro Viz. " - f"You are using version {installed_version}; " - f"however, version {latest_version} is now available.\n" - "You should consider upgrading via the `pip install -U kedro-viz` command.\n" - "You can view the complete changelog at " - "https://github.com/kedro-org/kedro-viz/releases.", - "yellow", - ) - try: - if port in _VIZ_PROCESSES and _VIZ_PROCESSES[port].is_alive(): - _VIZ_PROCESSES[port].terminate() - - run_server_kwargs = { - "host": host, - "port": port, - "load_file": load_file, - "save_file": save_file, - "pipeline_name": pipeline, - "env": env, - "project_path": kedro_project_path, - "autoreload": autoreload, - "include_hooks": include_hooks, - "package_name": PACKAGE_NAME, - "extra_params": params, - } - if autoreload: - run_process_kwargs = { - "path": kedro_project_path, - "target": run_server, - "kwargs": run_server_kwargs, - "watcher_cls": RegExpWatcher, - "watcher_kwargs": {"re_files": r"^.*(\.yml|\.yaml|\.py|\.json)$"}, - } - viz_process = multiprocessing.Process( - target=run_process, daemon=False, kwargs={**run_process_kwargs} - ) - else: - viz_process = multiprocessing.Process( - target=run_server, daemon=False, kwargs={**run_server_kwargs} - ) - - display_cli_message("Starting Kedro Viz ...", "green") - - viz_process.start() - - _VIZ_PROCESSES[port] = viz_process - - _wait_for(func=_check_viz_up, host=host, port=port) - - display_cli_message( - "Kedro Viz started successfully. \n\n" - f"\u2728 Kedro Viz is running at \n http://{host}:{port}/", - "green", - ) - - if browser: - _start_browser(host, port) - - except Exception as ex: # pragma: no cover - traceback.print_exc() - raise KedroCliError(str(ex)) from ex - - -@viz.command(context_settings={"help_option_names": ["-h", "--help"]}) -@click.option( - "--platform", - type=str, - required=True, - help=f"Supported Cloud Platforms like {*SHAREABLEVIZ_SUPPORTED_PLATFORMS,} to host Kedro Viz", -) -@click.option( - "--endpoint", - type=str, - required=True, - help="Static Website hosted endpoint." - "(eg., For AWS - http://.s3-website..amazonaws.com/)", -) -@click.option( - "--bucket-name", - type=str, - required=True, - help="Bucket name where Kedro Viz will be hosted", -) -@click.option( - "--include-hooks", - is_flag=True, - help="A flag to include all registered hooks in your Kedro Project", -) -@click.option( - "--include-previews", - is_flag=True, - help="A flag to include preview for all the datasets", -) -def deploy(platform, endpoint, bucket_name, include_hooks, include_previews): - """Deploy and host Kedro Viz on provided platform""" - if not platform or platform.lower() not in SHAREABLEVIZ_SUPPORTED_PLATFORMS: - display_cli_message( - "ERROR: Invalid platform specified. Kedro-Viz supports \n" - f"the following platforms - {*SHAREABLEVIZ_SUPPORTED_PLATFORMS,}", - "red", - ) - return - - if not endpoint: - display_cli_message( - "ERROR: Invalid endpoint specified. If you are looking for platform \n" - "agnostic shareable viz solution, please use the `kedro viz build` command", - "red", - ) - return - - create_shareableviz_process( - platform, - include_previews, - endpoint, - bucket_name, - include_hooks, - ) - - -@viz.command(context_settings={"help_option_names": ["-h", "--help"]}) -@click.option( - "--include-hooks", - is_flag=True, - help="A flag to include all registered hooks in your Kedro Project", -) -@click.option( - "--include-previews", - is_flag=True, - help="A flag to include preview for all the datasets", -) -def build(include_hooks, include_previews): - """Create build directory of local Kedro Viz instance with Kedro project data""" - - create_shareableviz_process("local", include_previews, include_hooks=include_hooks) - - -def create_shareableviz_process( - platform, - is_all_previews_enabled, - endpoint=None, - bucket_name=None, - include_hooks=False, -): - """Creates platform specific deployer process""" - try: - process_completed = multiprocessing.Value("i", 0) - exception_queue = multiprocessing.Queue() - - viz_deploy_process = multiprocessing.Process( - target=load_and_deploy_viz, - args=( - platform, - is_all_previews_enabled, - endpoint, - bucket_name, - include_hooks, - PACKAGE_NAME, - process_completed, - exception_queue, - ), - ) - - viz_deploy_process.start() - viz_deploy_progress_timer(process_completed, VIZ_DEPLOY_TIME_LIMIT) - - if not exception_queue.empty(): # pragma: no cover - raise exception_queue.get_nowait() - - if not process_completed.value: - raise TimeoutError() - - if platform != "local": - display_cli_message( - f"\u2728 Success! Kedro Viz has been deployed on {platform.upper()}. " - "It can be accessed at :\n" - f"{endpoint}", - "green", - ) - else: - display_cli_message( - "\u2728 Success! Kedro-Viz build files have been " - "added to the `build` directory.", - "green", - ) - - except TimeoutError: # pragma: no cover - display_cli_message( - "TIMEOUT ERROR: Failed to build/deploy Kedro-Viz as the " - f"process took more than {VIZ_DEPLOY_TIME_LIMIT} seconds. " - "Please try again later.", - "red", - ) - - except KeyboardInterrupt: # pragma: no cover - display_cli_message( - "\nCreating your build/deploy Kedro-Viz process " - "is interrupted. Exiting...", - "red", - ) - - except PermissionError: # pragma: no cover - if platform != "local": - display_cli_message( - "PERMISSION ERROR: Deploying and hosting Kedro-Viz requires " - f"{platform.upper()} access keys, a valid {platform.upper()} " - "endpoint and bucket name.\n" - f"Please supply your {platform.upper()} access keys as environment variables " - f"and make sure the {platform.upper()} endpoint and bucket name are valid.\n" - "More information can be found at : " - "https://docs.kedro.org/en/stable/visualisation/share_kedro_viz.html", - "red", - ) - else: - display_cli_message( - "PERMISSION ERROR: Please make sure, " - "you have write access to the current directory", - "red", - ) - # pylint: disable=broad-exception-caught - except Exception as exc: # pragma: no cover - display_cli_message(f"ERROR: Failed to build/deploy Kedro-Viz : {exc} ", "red") - - finally: - viz_deploy_process.terminate() - - -def load_and_deploy_viz( - platform, - is_all_previews_enabled, - endpoint, - bucket_name, - include_hooks, - package_name, - process_completed, - exception_queue, -): - """Loads Kedro Project data, creates a deployer and deploys to a platform""" - try: - load_and_populate_data( - Path.cwd(), include_hooks=include_hooks, package_name=package_name - ) - - # Start the deployment - deployer = DeployerFactory.create_deployer(platform, endpoint, bucket_name) - deployer.deploy(is_all_previews_enabled) - - except ( - # pylint: disable=catching-non-exception - (FileNotFoundError, ServiceRequestError) - if ServiceRequestError is not None - else FileNotFoundError - ): # pragma: no cover - exception_queue.put(Exception("The specified bucket does not exist")) - # pylint: disable=broad-exception-caught - except Exception as exc: # pragma: no cover - exception_queue.put(exc) - finally: - process_completed.value = 1 - - -def display_cli_message(msg, msg_color=None): - """Displays message for Kedro Viz build and deploy commands""" - click.echo( - click.style( - msg, - fg=msg_color, - ) - ) diff --git a/package/kedro_viz/launchers/cli/__init__.py b/package/kedro_viz/launchers/cli/__init__.py new file mode 100644 index 0000000000..56806ffefd --- /dev/null +++ b/package/kedro_viz/launchers/cli/__init__.py @@ -0,0 +1 @@ +"""`kedro_viz.launchers.cli` launches the viz server as a CLI app.""" diff --git a/package/kedro_viz/launchers/cli/build.py b/package/kedro_viz/launchers/cli/build.py new file mode 100644 index 0000000000..d506266019 --- /dev/null +++ b/package/kedro_viz/launchers/cli/build.py @@ -0,0 +1,24 @@ +"""`kedro_viz.launchers.cli.build` provides a cli command to build +a Kedro-Viz instance""" +# pylint: disable=import-outside-toplevel +import click + +from kedro_viz.launchers.cli.main import viz + + +@viz.command(context_settings={"help_option_names": ["-h", "--help"]}) +@click.option( + "--include-hooks", + is_flag=True, + help="A flag to include all registered hooks in your Kedro Project", +) +@click.option( + "--include-previews", + is_flag=True, + help="A flag to include preview for all the datasets", +) +def build(include_hooks, include_previews): + """Create build directory of local Kedro Viz instance with Kedro project data""" + from kedro_viz.launchers.cli.utils import create_shareableviz_process + + create_shareableviz_process("local", include_previews, include_hooks=include_hooks) diff --git a/package/kedro_viz/launchers/cli/deploy.py b/package/kedro_viz/launchers/cli/deploy.py new file mode 100644 index 0000000000..10bb31870f --- /dev/null +++ b/package/kedro_viz/launchers/cli/deploy.py @@ -0,0 +1,69 @@ +"""`kedro_viz.launchers.cli.deploy` provides a cli command to deploy +a Kedro-Viz instance on cloud platforms""" +# pylint: disable=import-outside-toplevel +import click + +from kedro_viz.constants import SHAREABLEVIZ_SUPPORTED_PLATFORMS +from kedro_viz.launchers.cli.main import viz + + +@viz.command(context_settings={"help_option_names": ["-h", "--help"]}) +@click.option( + "--platform", + type=str, + required=True, + help=f"Supported Cloud Platforms like {*SHAREABLEVIZ_SUPPORTED_PLATFORMS,} to host Kedro Viz", +) +@click.option( + "--endpoint", + type=str, + required=True, + help="Static Website hosted endpoint." + "(eg., For AWS - http://.s3-website..amazonaws.com/)", +) +@click.option( + "--bucket-name", + type=str, + required=True, + help="Bucket name where Kedro Viz will be hosted", +) +@click.option( + "--include-hooks", + is_flag=True, + help="A flag to include all registered hooks in your Kedro Project", +) +@click.option( + "--include-previews", + is_flag=True, + help="A flag to include preview for all the datasets", +) +def deploy(platform, endpoint, bucket_name, include_hooks, include_previews): + """Deploy and host Kedro Viz on provided platform""" + from kedro_viz.launchers.cli.utils import ( + create_shareableviz_process, + display_cli_message, + ) + + if not platform or platform.lower() not in SHAREABLEVIZ_SUPPORTED_PLATFORMS: + display_cli_message( + "ERROR: Invalid platform specified. Kedro-Viz supports \n" + f"the following platforms - {*SHAREABLEVIZ_SUPPORTED_PLATFORMS,}", + "red", + ) + return + + if not endpoint: + display_cli_message( + "ERROR: Invalid endpoint specified. If you are looking for platform \n" + "agnostic shareable viz solution, please use the `kedro viz build` command", + "red", + ) + return + + create_shareableviz_process( + platform, + include_previews, + endpoint, + bucket_name, + include_hooks, + ) diff --git a/package/kedro_viz/launchers/cli/lazy_default_group.py b/package/kedro_viz/launchers/cli/lazy_default_group.py new file mode 100644 index 0000000000..861d023221 --- /dev/null +++ b/package/kedro_viz/launchers/cli/lazy_default_group.py @@ -0,0 +1,76 @@ +"""`kedro_viz.launchers.cli.lazy_default_group` provides a custom mutli-command +subclass for a lazy subcommand loader""" + +# pylint: disable=import-outside-toplevel +from typing import Any, Union + +import click + + +class LazyDefaultGroup(click.Group): + """A click Group that supports lazy loading of subcommands and a default command""" + + def __init__( + self, + *args: Any, + **kwargs: Any, + ): + if not kwargs.get("ignore_unknown_options", True): + raise ValueError("Default group accepts unknown options") + self.ignore_unknown_options = True + + # lazy_subcommands is a map of the form: + # + # {command-name} -> {module-name}.{command-object-name} + # + self.lazy_subcommands = kwargs.pop("lazy_subcommands", {}) + + self.default_cmd_name = kwargs.pop("default", None) + self.default_if_no_args = kwargs.pop("default_if_no_args", False) + + super().__init__(*args, **kwargs) + + def list_commands(self, ctx: click.Context) -> list[str]: + return sorted(self.lazy_subcommands.keys()) + + def get_command( # type: ignore[override] + self, ctx: click.Context, cmd_name: str + ) -> Union[click.BaseCommand, click.Command, None]: + if cmd_name in self.lazy_subcommands: + return self._lazy_load(cmd_name) + return super().get_command(ctx, cmd_name) + + def _lazy_load(self, cmd_name: str) -> click.BaseCommand: + from importlib import import_module + + # lazily loading a command, first get the module name and attribute name + import_path = self.lazy_subcommands[cmd_name] + modname, cmd_object_name = import_path.rsplit(".", 1) + + # do the import + mod = import_module(modname) + + # get the Command object from that module + cmd_object = getattr(mod, cmd_object_name) + + return cmd_object + + def parse_args(self, ctx, args): + # If no args are provided and default_command_name is specified, + # use the default command + if not args and self.default_if_no_args: + args.insert(0, self.default_cmd_name) + return super().parse_args(ctx, args) + + def resolve_command(self, ctx: click.Context, args): + # Attempt to resolve the command using the parent class method + try: + cmd_name, cmd, args = super().resolve_command(ctx, args) + return cmd_name, cmd, args + except click.UsageError as exc: + if self.default_cmd_name and not ctx.invoked_subcommand: + # No command found, use the default command + default_cmd = self.get_command(ctx, self.default_cmd_name) + if default_cmd: + return default_cmd.name, default_cmd, args + raise exc diff --git a/package/kedro_viz/launchers/cli/main.py b/package/kedro_viz/launchers/cli/main.py new file mode 100644 index 0000000000..0ccb1515e1 --- /dev/null +++ b/package/kedro_viz/launchers/cli/main.py @@ -0,0 +1,26 @@ +"""`kedro_viz.launchers.cli.main` is an entry point for Kedro-Viz cli commands.""" + +import click + +from kedro_viz.launchers.cli.lazy_default_group import LazyDefaultGroup + + +@click.group(name="Kedro-Viz") +def viz_cli(): # pylint: disable=missing-function-docstring + pass + + +@viz_cli.group( + name="viz", + cls=LazyDefaultGroup, + lazy_subcommands={ + "run": "kedro_viz.launchers.cli.run.run", + "deploy": "kedro_viz.launchers.cli.deploy.deploy", + "build": "kedro_viz.launchers.cli.build.build", + }, + default="run", + default_if_no_args=True, +) +@click.pass_context +def viz(ctx): # pylint: disable=unused-argument + """Visualise a Kedro pipeline using Kedro viz.""" diff --git a/package/kedro_viz/launchers/cli/run.py b/package/kedro_viz/launchers/cli/run.py new file mode 100644 index 0000000000..54b179b4b8 --- /dev/null +++ b/package/kedro_viz/launchers/cli/run.py @@ -0,0 +1,196 @@ +"""`kedro_viz.launchers.cli.run` provides a cli command to run +a Kedro-Viz instance""" + +from typing import Dict + +import click +from kedro.framework.cli.project import PARAMS_ARG_HELP +from kedro.framework.cli.utils import _split_params + +from kedro_viz.constants import DEFAULT_HOST, DEFAULT_PORT +from kedro_viz.launchers.cli.main import viz + +_VIZ_PROCESSES: Dict[str, int] = {} + + +@viz.command(context_settings={"help_option_names": ["-h", "--help"]}) +@click.option( + "--host", + default=DEFAULT_HOST, + help="Host that viz will listen to. Defaults to localhost.", +) +@click.option( + "--port", + default=DEFAULT_PORT, + type=int, + help="TCP port that viz will listen to. Defaults to 4141.", +) +@click.option( + "--browser/--no-browser", + default=True, + help="Whether to open viz interface in the default browser or not. " + "Browser will only be opened if host is localhost. Defaults to True.", +) +@click.option( + "--load-file", + default=None, + help="Path to load Kedro-Viz data from a directory", +) +@click.option( + "--save-file", + default=None, + type=click.Path(dir_okay=False, writable=True), + help="Path to save Kedro-Viz data to a directory", +) +@click.option( + "--pipeline", + "-p", + type=str, + default=None, + help="Name of the registered pipeline to visualise. " + "If not set, the default pipeline is visualised", +) +@click.option( + "--env", + "-e", + type=str, + default=None, + multiple=False, + envvar="KEDRO_ENV", + help="Kedro configuration environment. If not specified, " + "catalog config in `local` will be used", +) +@click.option( + "--autoreload", + "-a", + is_flag=True, + help="Autoreload viz server when a Python or YAML file change in the Kedro project", +) +@click.option( + "--include-hooks", + is_flag=True, + help="A flag to include all registered hooks in your Kedro Project", +) +@click.option( + "--params", + type=click.UNPROCESSED, + default="", + help=PARAMS_ARG_HELP, + callback=_split_params, +) +# pylint: disable=import-outside-toplevel, too-many-locals +def run( + host, + port, + browser, + load_file, + save_file, + pipeline, + env, + autoreload, + include_hooks, + params, +): + """Launch local Kedro Viz instance""" + # Deferring Imports + import multiprocessing + import traceback + from pathlib import Path + + from kedro.framework.cli.utils import KedroCliError + from kedro.framework.project import PACKAGE_NAME + from packaging.version import parse + + from kedro_viz import __version__ + from kedro_viz.integrations.pypi import ( + get_latest_version, + is_running_outdated_version, + ) + from kedro_viz.launchers.cli.utils import display_cli_message + from kedro_viz.launchers.utils import ( + _PYPROJECT, + _check_viz_up, + _find_kedro_project, + _start_browser, + _wait_for, + ) + from kedro_viz.server import run_server + + kedro_project_path = _find_kedro_project(Path.cwd()) + + if kedro_project_path is None: + display_cli_message( + "ERROR: Failed to start Kedro-Viz : " + "Could not find the project configuration " + f"file '{_PYPROJECT}' at '{Path.cwd()}'. ", + "red", + ) + return + + installed_version = parse(__version__) + latest_version = get_latest_version() + if is_running_outdated_version(installed_version, latest_version): + display_cli_message( + "WARNING: You are using an old version of Kedro Viz. " + f"You are using version {installed_version}; " + f"however, version {latest_version} is now available.\n" + "You should consider upgrading via the `pip install -U kedro-viz` command.\n" + "You can view the complete changelog at " + "https://github.com/kedro-org/kedro-viz/releases.", + "yellow", + ) + try: + if port in _VIZ_PROCESSES and _VIZ_PROCESSES[port].is_alive(): + _VIZ_PROCESSES[port].terminate() + + run_server_kwargs = { + "host": host, + "port": port, + "load_file": load_file, + "save_file": save_file, + "pipeline_name": pipeline, + "env": env, + "project_path": kedro_project_path, + "autoreload": autoreload, + "include_hooks": include_hooks, + "package_name": PACKAGE_NAME, + "extra_params": params, + } + if autoreload: + from watchgod import RegExpWatcher, run_process + + run_process_kwargs = { + "path": kedro_project_path, + "target": run_server, + "kwargs": run_server_kwargs, + "watcher_cls": RegExpWatcher, + "watcher_kwargs": {"re_files": r"^.*(\.yml|\.yaml|\.py|\.json)$"}, + } + viz_process = multiprocessing.Process( + target=run_process, daemon=False, kwargs={**run_process_kwargs} + ) + else: + viz_process = multiprocessing.Process( + target=run_server, daemon=False, kwargs={**run_server_kwargs} + ) + + display_cli_message("Starting Kedro Viz ...", "green") + + viz_process.start() + + _VIZ_PROCESSES[port] = viz_process + + _wait_for(func=_check_viz_up, host=host, port=port) + + display_cli_message( + "Kedro Viz started successfully. \n\n" + f"\u2728 Kedro Viz is running at \n http://{host}:{port}/", + "green", + ) + + if browser: + _start_browser(host, port) + + except Exception as ex: # pragma: no cover + traceback.print_exc() + raise KedroCliError(str(ex)) from ex diff --git a/package/kedro_viz/launchers/cli/utils.py b/package/kedro_viz/launchers/cli/utils.py new file mode 100644 index 0000000000..eb4efdfbc9 --- /dev/null +++ b/package/kedro_viz/launchers/cli/utils.py @@ -0,0 +1,169 @@ +"""`kedro_viz.launchers.cli.utils` provides utility functions for cli commands.""" +# pylint: disable=import-outside-toplevel +from pathlib import Path +from time import sleep +from typing import Union + +import click + +from kedro_viz.constants import VIZ_DEPLOY_TIME_LIMIT + + +def create_shareableviz_process( + platform: str, + is_all_previews_enabled: bool, + endpoint: Union[str, None] = None, + bucket_name: Union[str, None] = None, + include_hooks: bool = False, +): + """Creates platform specific deployer process""" + + import multiprocessing + + from kedro.framework.project import PACKAGE_NAME + + try: + process_completed = multiprocessing.Value("i", 0) + exception_queue = multiprocessing.Queue() # type: ignore[var-annotated] + + viz_deploy_process = multiprocessing.Process( + target=_load_and_deploy_viz, + args=( + platform, + is_all_previews_enabled, + endpoint, + bucket_name, + include_hooks, + PACKAGE_NAME, + process_completed, + exception_queue, + ), + ) + + viz_deploy_process.start() + _viz_deploy_progress_timer(process_completed, VIZ_DEPLOY_TIME_LIMIT) + + if not exception_queue.empty(): # pragma: no cover + raise exception_queue.get_nowait() + + if not process_completed.value: + raise TimeoutError() + + if platform != "local": + display_cli_message( + f"\u2728 Success! Kedro Viz has been deployed on {platform.upper()}. " + "It can be accessed at :\n" + f"{endpoint}", + "green", + ) + else: + display_cli_message( + "\u2728 Success! Kedro-Viz build files have been " + "added to the `build` directory.", + "green", + ) + + except TimeoutError: # pragma: no cover + display_cli_message( + "TIMEOUT ERROR: Failed to build/deploy Kedro-Viz as the " + f"process took more than {VIZ_DEPLOY_TIME_LIMIT} seconds. " + "Please try again later.", + "red", + ) + + except KeyboardInterrupt: # pragma: no cover + display_cli_message( + "\nCreating your build/deploy Kedro-Viz process " + "is interrupted. Exiting...", + "red", + ) + + except PermissionError: # pragma: no cover + if platform != "local": + display_cli_message( + "PERMISSION ERROR: Deploying and hosting Kedro-Viz requires " + f"{platform.upper()} access keys, a valid {platform.upper()} " + "endpoint and bucket name.\n" + f"Please supply your {platform.upper()} access keys as environment variables " + f"and make sure the {platform.upper()} endpoint and bucket name are valid.\n" + "More information can be found at : " + "https://docs.kedro.org/en/stable/visualisation/share_kedro_viz.html", + "red", + ) + else: + display_cli_message( + "PERMISSION ERROR: Please make sure, " + "you have write access to the current directory", + "red", + ) + # pylint: disable=broad-exception-caught + except Exception as exc: # pragma: no cover + display_cli_message(f"ERROR: Failed to build/deploy Kedro-Viz : {exc} ", "red") + + finally: + viz_deploy_process.terminate() + + +def display_cli_message(msg, msg_color=None): + """Displays message for Kedro Viz build and deploy commands""" + click.echo( + click.style( + msg, + fg=msg_color, + ) + ) + + +def _load_and_deploy_viz( + platform, + is_all_previews_enabled, + endpoint, + bucket_name, + include_hooks, + package_name, + process_completed, + exception_queue, +): + """Loads Kedro Project data, creates a deployer and deploys to a platform""" + try: + from kedro_viz.integrations.deployment.deployer_factory import DeployerFactory + from kedro_viz.server import load_and_populate_data + + try: + from azure.core.exceptions import ServiceRequestError + except ImportError: # pragma: no cover + ServiceRequestError = None + + load_and_populate_data( + Path.cwd(), include_hooks=include_hooks, package_name=package_name + ) + + # Start the deployment + deployer = DeployerFactory.create_deployer(platform, endpoint, bucket_name) + deployer.deploy(is_all_previews_enabled) + + except ( + # pylint: disable=catching-non-exception + (FileNotFoundError, ServiceRequestError) + if ServiceRequestError is not None + else FileNotFoundError + ): # pragma: no cover + exception_queue.put(Exception("The specified bucket does not exist")) + # pylint: disable=broad-exception-caught + except Exception as exc: # pragma: no cover + exception_queue.put(exc) + finally: + process_completed.value = 1 + + +def _viz_deploy_progress_timer(process_completed, timeout): + """Shows progress timer and message for kedro viz deploy""" + elapsed_time = 0 + while elapsed_time <= timeout and not process_completed.value: + print( + f"...Creating your build/deploy Kedro-Viz ({elapsed_time}s)", + end="\r", + flush=True, + ) + sleep(1) + elapsed_time += 1 diff --git a/package/kedro_viz/launchers/utils.py b/package/kedro_viz/launchers/utils.py index bf79602349..c4b0076677 100644 --- a/package/kedro_viz/launchers/utils.py +++ b/package/kedro_viz/launchers/utils.py @@ -96,19 +96,6 @@ def _start_browser(host: str, port: int): webbrowser.open_new(f"http://{host}:{port}/") -def viz_deploy_progress_timer(process_completed, timeout): - """Shows progress timer and message for kedro viz deploy""" - elapsed_time = 0 - while elapsed_time <= timeout and not process_completed.value: - print( - f"...Creating your build/deploy Kedro-Viz ({elapsed_time}s)", - end="\r", - flush=True, - ) - sleep(1) - elapsed_time += 1 - - def _is_project(project_path: Union[str, Path]) -> bool: metadata_file = Path(project_path).expanduser().resolve() / _PYPROJECT if not metadata_file.is_file(): diff --git a/package/pyproject.toml b/package/pyproject.toml index 1c4e4ca1f0..7c39412920 100644 --- a/package/pyproject.toml +++ b/package/pyproject.toml @@ -36,7 +36,7 @@ azure = ["adlfs>=2021.4"] gcp = ["gcsfs>=2021.4"] [project.entry-points."kedro.global_commands"] -kedro-viz = "kedro_viz.launchers.cli:viz_cli" +kedro-viz = "kedro_viz.launchers.cli.main:viz_cli" [project.entry-points."kedro.line_magic"] line_magic = "kedro_viz.launchers.jupyter:run_viz" diff --git a/package/tests/test_launchers/test_cli.py b/package/tests/test_launchers/test_cli.py deleted file mode 100755 index 82403cd5e7..0000000000 --- a/package/tests/test_launchers/test_cli.py +++ /dev/null @@ -1,797 +0,0 @@ -from unittest.mock import Mock, call - -import pytest -import requests -from click.testing import CliRunner -from packaging.version import parse -from watchgod import RegExpWatcher, run_process - -from kedro_viz import __version__ -from kedro_viz.constants import SHAREABLEVIZ_SUPPORTED_PLATFORMS, VIZ_DEPLOY_TIME_LIMIT -from kedro_viz.launchers import cli -from kedro_viz.launchers.utils import _PYPROJECT -from kedro_viz.server import run_server - - -@pytest.fixture -def patched_check_viz_up(mocker): - mocker.patch("kedro_viz.launchers.cli._check_viz_up", return_value=True) - - -@pytest.fixture -def patched_start_browser(mocker): - mocker.patch("kedro_viz.launchers.cli._start_browser") - - -@pytest.fixture -def mock_viz_deploy_process(mocker): - return mocker.patch("kedro_viz.launchers.cli.multiprocessing.Process") - - -@pytest.fixture -def mock_process_completed(mocker): - return mocker.patch( - "kedro_viz.launchers.cli.multiprocessing.Value", return_value=Mock() - ) - - -@pytest.fixture -def mock_exception_queue(mocker): - return mocker.patch( - "kedro_viz.launchers.cli.multiprocessing.Queue", return_value=Mock() - ) - - -@pytest.fixture -def mock_viz_load_and_deploy(mocker): - return mocker.patch("kedro_viz.launchers.cli.load_and_deploy_viz") - - -@pytest.fixture -def mock_viz_deploy_progress_timer(mocker): - return mocker.patch("kedro_viz.launchers.cli.viz_deploy_progress_timer") - - -@pytest.fixture -def mock_DeployerFactory(mocker): - return mocker.patch("kedro_viz.launchers.cli.DeployerFactory") - - -@pytest.fixture -def mock_load_and_populate_data(mocker): - return mocker.patch("kedro_viz.launchers.cli.load_and_populate_data") - - -@pytest.fixture -def mock_click_echo(mocker): - return mocker.patch("click.echo") - - -@pytest.fixture -def mock_project_path(mocker): - mock_path = "/tmp/project_path" - mocker.patch("pathlib.Path.cwd", return_value=mock_path) - return mock_path - - -@pytest.mark.parametrize( - "command_options,run_server_args", - [ - ( - ["viz"], - { - "host": "127.0.0.1", - "port": 4141, - "load_file": None, - "save_file": None, - "pipeline_name": None, - "env": None, - "project_path": "testPath", - "autoreload": False, - "include_hooks": False, - "package_name": None, - "extra_params": {}, - }, - ), - ( - ["viz", "run"], - { - "host": "127.0.0.1", - "port": 4141, - "load_file": None, - "save_file": None, - "pipeline_name": None, - "env": None, - "project_path": "testPath", - "autoreload": False, - "include_hooks": False, - "package_name": None, - "extra_params": {}, - }, - ), - ( - [ - "viz", - "run", - "--host", - "localhost", - ], - { - "host": "localhost", - "port": 4141, - "load_file": None, - "save_file": None, - "pipeline_name": None, - "env": None, - "project_path": "testPath", - "autoreload": False, - "include_hooks": False, - "package_name": None, - "extra_params": {}, - }, - ), - ( - [ - "viz", - "run", - "--host", - "8.8.8.8", - "--port", - "4142", - "--no-browser", - "--save-file", - "save_dir", - "--pipeline", - "data_science", - "--env", - "local", - "--params", - "extra_param=param", - ], - { - "host": "8.8.8.8", - "port": 4142, - "load_file": None, - "save_file": "save_dir", - "pipeline_name": "data_science", - "env": "local", - "project_path": "testPath", - "autoreload": False, - "include_hooks": False, - "package_name": None, - "extra_params": {"extra_param": "param"}, - }, - ), - ( - [ - "viz", - "run", - "--host", - "8.8.8.8", - "--port", - "4142", - "--no-browser", - "--save-file", - "save_dir", - "-p", - "data_science", - "-e", - "local", - "--params", - "extra_param=param", - ], - { - "host": "8.8.8.8", - "port": 4142, - "load_file": None, - "save_file": "save_dir", - "pipeline_name": "data_science", - "env": "local", - "project_path": "testPath", - "autoreload": False, - "include_hooks": False, - "package_name": None, - "extra_params": {"extra_param": "param"}, - }, - ), - ( - ["viz", "run", "--include-hooks"], - { - "host": "127.0.0.1", - "port": 4141, - "load_file": None, - "save_file": None, - "pipeline_name": None, - "env": None, - "project_path": "testPath", - "autoreload": False, - "include_hooks": True, - "package_name": None, - "extra_params": {}, - }, - ), - ], -) -def test_kedro_viz_command_run_server( - command_options, - run_server_args, - mocker, - patched_check_viz_up, - patched_start_browser, -): - process_init = mocker.patch("multiprocessing.Process") - runner = CliRunner() - # Reduce the timeout argument from 600 to 1 to make test run faster. - mocker.patch("kedro_viz.launchers.cli._wait_for.__defaults__", (True, 1, True, 1)) - # Mock finding kedro project - mocker.patch( - "kedro_viz.launchers.cli._find_kedro_project", - return_value=run_server_args["project_path"], - ) - - with runner.isolated_filesystem(): - runner.invoke(cli.viz_cli, command_options) - - process_init.assert_called_once_with( - target=run_server, daemon=False, kwargs={**run_server_args} - ) - assert run_server_args["port"] in cli._VIZ_PROCESSES - - -def test_kedro_viz_command_should_log_project_not_found( - mocker, mock_project_path, mock_click_echo -): - # Reduce the timeout argument from 600 to 1 to make test run faster. - mocker.patch("kedro_viz.launchers.cli._wait_for.__defaults__", (True, 1, True, 1)) - # Mock finding kedro project - mocker.patch("kedro_viz.launchers.cli._find_kedro_project", return_value=None) - runner = CliRunner() - with runner.isolated_filesystem(): - runner.invoke(cli.viz_cli, ["viz", "run"]) - - mock_click_echo_calls = [ - call( - "\x1b[31mERROR: Failed to start Kedro-Viz : " - "Could not find the project configuration " - f"file '{_PYPROJECT}' at '{mock_project_path}'. \x1b[0m" - ) - ] - - mock_click_echo.assert_has_calls(mock_click_echo_calls) - - -def test_kedro_viz_command_should_log_outdated_version( - mocker, mock_http_response, mock_click_echo, mock_project_path -): - installed_version = parse(__version__) - mock_version = f"{installed_version.major + 1}.0.0" - requests_get = mocker.patch("requests.get") - requests_get.return_value = mock_http_response( - data={"info": {"version": mock_version}} - ) - - mocker.patch("kedro_viz.server.run_server") - - # Reduce the timeout argument from 600 to 1 to make test run faster. - mocker.patch("kedro_viz.launchers.cli._wait_for.__defaults__", (True, 1, True, 1)) - # Mock finding kedro project - mocker.patch( - "kedro_viz.launchers.cli._find_kedro_project", return_value=mock_project_path - ) - runner = CliRunner() - with runner.isolated_filesystem(): - runner.invoke(cli.viz_cli, ["viz", "run"]) - - mock_click_echo_calls = [ - call( - "\x1b[33mWARNING: You are using an old version of Kedro Viz. " - f"You are using version {installed_version}; " - f"however, version {mock_version} is now available.\n" - "You should consider upgrading via the `pip install -U kedro-viz` command.\n" - "You can view the complete changelog at " - "https://github.com/kedro-org/kedro-viz/releases.\x1b[0m" - ) - ] - - mock_click_echo.assert_has_calls(mock_click_echo_calls) - - -def test_kedro_viz_command_should_not_log_latest_version( - mocker, mock_http_response, mock_click_echo, mock_project_path -): - requests_get = mocker.patch("requests.get") - requests_get.return_value = mock_http_response( - data={"info": {"version": str(parse(__version__))}} - ) - - mocker.patch("kedro_viz.server.run_server") - # Reduce the timeout argument from 600 to 1 to make test run faster. - mocker.patch("kedro_viz.launchers.cli._wait_for.__defaults__", (True, 1, True, 1)) - # Mock finding kedro project - mocker.patch( - "kedro_viz.launchers.cli._find_kedro_project", return_value=mock_project_path - ) - runner = CliRunner() - with runner.isolated_filesystem(): - runner.invoke(cli.viz_cli, ["viz", "run"]) - - mock_click_echo_calls = [call("\x1b[32mStarting Kedro Viz ...\x1b[0m")] - - mock_click_echo.assert_has_calls(mock_click_echo_calls) - - -def test_kedro_viz_command_should_not_log_if_pypi_is_down( - mocker, mock_http_response, mock_click_echo, mock_project_path -): - requests_get = mocker.patch("requests.get") - requests_get.side_effect = requests.exceptions.RequestException("PyPI is down") - - mocker.patch("kedro_viz.server.run_server") - # Reduce the timeout argument from 600 to 1 to make test run faster. - mocker.patch("kedro_viz.launchers.cli._wait_for.__defaults__", (True, 1, True, 1)) - # Mock finding kedro project - mocker.patch( - "kedro_viz.launchers.cli._find_kedro_project", return_value=mock_project_path - ) - runner = CliRunner() - with runner.isolated_filesystem(): - runner.invoke(cli.viz_cli, ["viz", "run"]) - - mock_click_echo_calls = [call("\x1b[32mStarting Kedro Viz ...\x1b[0m")] - - mock_click_echo.assert_has_calls(mock_click_echo_calls) - - -def test_kedro_viz_command_with_autoreload( - mocker, patched_check_viz_up, patched_start_browser, mock_project_path -): - process_init = mocker.patch("multiprocessing.Process") - - # Reduce the timeout argument from 600 to 1 to make test run faster. - mocker.patch("kedro_viz.launchers.cli._wait_for.__defaults__", (True, 1, True, 1)) - # Mock finding kedro project - mocker.patch( - "kedro_viz.launchers.cli._find_kedro_project", return_value=mock_project_path - ) - runner = CliRunner() - with runner.isolated_filesystem(): - runner.invoke(cli.viz_cli, ["viz", "run", "--autoreload"]) - - run_process_kwargs = { - "path": mock_project_path, - "target": run_server, - "kwargs": { - "host": "127.0.0.1", - "port": 4141, - "load_file": None, - "save_file": None, - "pipeline_name": None, - "env": None, - "autoreload": True, - "project_path": mock_project_path, - "include_hooks": False, - "package_name": None, - "extra_params": {}, - }, - "watcher_cls": RegExpWatcher, - "watcher_kwargs": {"re_files": "^.*(\\.yml|\\.yaml|\\.py|\\.json)$"}, - } - - process_init.assert_called_once_with( - target=run_process, daemon=False, kwargs={**run_process_kwargs} - ) - assert run_process_kwargs["kwargs"]["port"] in cli._VIZ_PROCESSES - - -def test_viz_command_group(mocker, mock_click_echo): - runner = CliRunner() - - with runner.isolated_filesystem(): - result = runner.invoke(cli.viz_cli, ["viz", "--help"]) - - assert result.output == ( - "Usage: Kedro-Viz viz [OPTIONS] COMMAND [ARGS]...\n" - "\n" - " Visualise a Kedro pipeline using Kedro viz.\n" - "\n" - "Options:\n" - " --help Show this message and exit.\n" - "\n" - "Commands:\n" - " run* Launch local Kedro Viz instance\n" - " build Create build directory of local Kedro Viz instance with Kedro...\n" - " deploy Deploy and host Kedro Viz on provided platform\n" - ) - - -@pytest.mark.parametrize( - "command_options, deployer_args", - [ - ( - [ - "viz", - "deploy", - "--platform", - "azure", - "--endpoint", - "https://example-bucket.web.core.windows.net", - "--bucket-name", - "example-bucket", - ], - { - "platform": "azure", - "endpoint": "https://example-bucket.web.core.windows.net", - "bucket_name": "example-bucket", - }, - ), - ( - [ - "viz", - "deploy", - "--platform", - "aws", - "--endpoint", - "http://example-bucket.s3-website.us-east-2.amazonaws.com/", - "--bucket-name", - "example-bucket", - ], - { - "platform": "aws", - "endpoint": "http://example-bucket.s3-website.us-east-2.amazonaws.com/", - "bucket_name": "example-bucket", - }, - ), - ( - [ - "viz", - "deploy", - "--platform", - "gcp", - "--endpoint", - "http://34.120.87.227/", - "--bucket-name", - "example-bucket", - ], - { - "platform": "gcp", - "endpoint": "http://34.120.87.227/", - "bucket_name": "example-bucket", - }, - ), - ( - [ - "viz", - "deploy", - "--platform", - "gcp", - "--endpoint", - "http://34.120.87.227/", - "--bucket-name", - "example-bucket", - "--include-hooks", - ], - { - "platform": "gcp", - "endpoint": "http://34.120.87.227/", - "bucket_name": "example-bucket", - "include_hooks": True, - }, - ), - ( - [ - "viz", - "deploy", - "--platform", - "aws", - "--endpoint", - "http://example-bucket.s3-website.us-east-2.amazonaws.com/", - "--bucket-name", - "example-bucket", - "--include-previews", - ], - { - "platform": "aws", - "endpoint": "http://example-bucket.s3-website.us-east-2.amazonaws.com/", - "bucket_name": "example-bucket", - "preview": True, - }, - ), - ], -) -def test_viz_deploy_valid_endpoint_and_bucket(command_options, deployer_args, mocker): - runner = CliRunner() - mocker.patch("fsspec.filesystem") - create_shareableviz_process_mock = mocker.patch( - "kedro_viz.launchers.cli.create_shareableviz_process" - ) - - with runner.isolated_filesystem(): - result = runner.invoke(cli.viz_cli, command_options) - - assert result.exit_code == 0 - - create_shareableviz_process_mock.assert_called_once_with( - deployer_args.get("platform"), - deployer_args.get("preview", False), - deployer_args.get("endpoint"), - deployer_args.get("bucket_name"), - deployer_args.get("include_hooks", False), - ) - - -def test_viz_deploy_invalid_platform(mocker, mock_click_echo): - runner = CliRunner() - with runner.isolated_filesystem(): - result = runner.invoke( - cli.viz_cli, - [ - "viz", - "deploy", - "--platform", - "random", - "--endpoint", - "", - "--bucket-name", - "example-bucket", - ], - ) - - assert result.exit_code == 0 - mock_click_echo_calls = [ - call( - "\x1b[31mERROR: Invalid platform specified. Kedro-Viz supports \n" - f"the following platforms - {*SHAREABLEVIZ_SUPPORTED_PLATFORMS,}\x1b[0m" - ) - ] - - mock_click_echo.assert_has_calls(mock_click_echo_calls) - - -def test_viz_deploy_invalid_endpoint(mocker, mock_click_echo): - runner = CliRunner() - with runner.isolated_filesystem(): - result = runner.invoke( - cli.viz_cli, - [ - "viz", - "deploy", - "--platform", - "aws", - "--endpoint", - "", - "--bucket-name", - "example-bucket", - ], - ) - - assert result.exit_code == 0 - mock_click_echo_calls = [ - call( - "\x1b[31mERROR: Invalid endpoint specified. If you are looking for platform \n" - "agnostic shareable viz solution, please use the `kedro viz build` command\x1b[0m" - ) - ] - - mock_click_echo.assert_has_calls(mock_click_echo_calls) - - -@pytest.mark.parametrize( - "command_options, build_args", - [ - ( - [ - "viz", - "build", - ], - { - "platform": "local", - }, - ), - ( - ["viz", "build", "--include-hooks"], - {"platform": "local", "include_hooks": True}, - ), - ( - ["viz", "build", "--include-previews"], - {"platform": "local", "preview": True}, - ), - ], -) -def test_successful_build_with_existing_static_files( - command_options, build_args, mocker -): - runner = CliRunner() - mocker.patch("fsspec.filesystem") - create_shareableviz_process_mock = mocker.patch( - "kedro_viz.launchers.cli.create_shareableviz_process" - ) - - with runner.isolated_filesystem(): - result = runner.invoke(cli.viz_cli, command_options) - - assert result.exit_code == 0 - - create_shareableviz_process_mock.assert_called_once_with( - build_args.get("platform"), - build_args.get("preview", False), - include_hooks=build_args.get("include_hooks", False), - ) - - -@pytest.mark.parametrize( - "platform, is_all_previews_enabled, endpoint, bucket_name," - "include_hooks, process_completed_value", - [ - ( - "azure", - True, - "https://example-bucket.web.core.windows.net", - "example-bucket", - True, - 1, - ), - ( - "aws", - True, - "http://example-bucket.s3-website.us-east-2.amazonaws.com/", - "example-bucket", - True, - 1, - ), - ( - "gcp", - False, - "http://34.120.87.227/", - "example-bucket", - False, - 1, - ), - ("local", False, None, None, False, 1), - ( - "azure", - True, - "https://example-bucket.web.core.windows.net", - "example-bucket", - False, - 0, - ), - ( - "aws", - False, - "http://example-bucket.s3-website.us-east-2.amazonaws.com/", - "example-bucket", - False, - 0, - ), - ( - "gcp", - True, - "http://34.120.87.227/", - "example-bucket", - True, - 0, - ), - ("local", True, None, None, True, 0), - ], -) -def test_create_shareableviz_process( - platform, - is_all_previews_enabled, - endpoint, - bucket_name, - include_hooks, - process_completed_value, - mock_viz_deploy_process, - mock_process_completed, - mock_exception_queue, - mock_viz_load_and_deploy, - mock_viz_deploy_progress_timer, - mock_click_echo, -): - mock_process_completed.return_value.value = process_completed_value - cli.create_shareableviz_process( - platform, is_all_previews_enabled, endpoint, bucket_name, include_hooks - ) - - # Assert the mocks were called as expected - mock_viz_deploy_process.assert_called_once_with( - target=mock_viz_load_and_deploy, - args=( - platform, - is_all_previews_enabled, - endpoint, - bucket_name, - include_hooks, - None, - mock_process_completed.return_value, - mock_exception_queue.return_value, - ), - ) - mock_viz_deploy_process.return_value.start.assert_called_once() - mock_viz_deploy_progress_timer.assert_called_once_with( - mock_process_completed.return_value, VIZ_DEPLOY_TIME_LIMIT - ) - mock_viz_deploy_process.return_value.terminate.assert_called_once() - - if process_completed_value: - if platform != "local": - msg = ( - "\x1b[32m\u2728 Success! Kedro Viz has been deployed on " - f"{platform.upper()}. " - "It can be accessed at :\n" - f"{endpoint}\x1b[0m" - ) - else: - msg = ( - "\x1b[32m✨ Success! Kedro-Viz build files have been " - "added to the `build` directory.\x1b[0m" - ) - else: - msg = ( - "\x1b[31mTIMEOUT ERROR: Failed to build/deploy Kedro-Viz " - f"as the process took more than {VIZ_DEPLOY_TIME_LIMIT} seconds. " - "Please try again later.\x1b[0m" - ) - - mock_click_echo_calls = [call(msg)] - mock_click_echo.assert_has_calls(mock_click_echo_calls) - - -@pytest.mark.parametrize( - "platform, is_all_previews_enabled, endpoint, bucket_name, include_hooks, package_name", - [ - ( - "azure", - False, - "https://example-bucket.web.core.windows.net", - "example-bucket", - False, - "demo_project", - ), - ( - "aws", - True, - "http://example-bucket.s3-website.us-east-2.amazonaws.com/", - "example-bucket", - True, - "demo_project", - ), - ("gcp", True, "http://34.120.87.227/", "example-bucket", False, "demo_project"), - ("local", False, None, None, True, "demo_project"), - ], -) -def test_load_and_deploy_viz_success( - platform, - is_all_previews_enabled, - endpoint, - bucket_name, - include_hooks, - package_name, - mock_DeployerFactory, - mock_load_and_populate_data, - mock_process_completed, - mock_exception_queue, - mock_click_echo, - mock_project_path, -): - deployer_mock = mock_DeployerFactory.create_deployer.return_value - - cli.load_and_deploy_viz( - platform, - is_all_previews_enabled, - endpoint, - bucket_name, - include_hooks, - package_name, - mock_process_completed, - mock_exception_queue, - ) - - mock_load_and_populate_data.assert_called_once_with( - mock_project_path, include_hooks=include_hooks, package_name=package_name - ) - mock_DeployerFactory.create_deployer.assert_called_once_with( - platform, endpoint, bucket_name - ) - deployer_mock.deploy.assert_called_once_with(is_all_previews_enabled) - mock_click_echo.echo.assert_not_called() diff --git a/package/tests/test_launchers/test_cli/test_build.py b/package/tests/test_launchers/test_cli/test_build.py new file mode 100644 index 0000000000..8918b2a21f --- /dev/null +++ b/package/tests/test_launchers/test_cli/test_build.py @@ -0,0 +1,49 @@ +import pytest +from click.testing import CliRunner + +from kedro_viz import __version__ +from kedro_viz.launchers.cli import main + + +class TestCliBuildViz: + @pytest.mark.parametrize( + "command_options, build_args", + [ + ( + [ + "viz", + "build", + ], + { + "platform": "local", + }, + ), + ( + ["viz", "build", "--include-hooks"], + {"platform": "local", "include_hooks": True}, + ), + ( + ["viz", "build", "--include-previews"], + {"platform": "local", "preview": True}, + ), + ], + ) + def test_successful_build_with_existing_static_files( + self, command_options, build_args, mocker + ): + runner = CliRunner() + mocker.patch("fsspec.filesystem") + create_shareableviz_process_mock = mocker.patch( + "kedro_viz.launchers.cli.utils.create_shareableviz_process" + ) + + with runner.isolated_filesystem(): + result = runner.invoke(main.viz_cli, command_options) + + assert result.exit_code == 0 + + create_shareableviz_process_mock.assert_called_once_with( + build_args.get("platform"), + build_args.get("preview", False), + include_hooks=build_args.get("include_hooks", False), + ) diff --git a/package/tests/test_launchers/test_cli/test_deploy.py b/package/tests/test_launchers/test_cli/test_deploy.py new file mode 100644 index 0000000000..180fe96717 --- /dev/null +++ b/package/tests/test_launchers/test_cli/test_deploy.py @@ -0,0 +1,185 @@ +from unittest.mock import call + +import pytest +from click.testing import CliRunner + +from kedro_viz import __version__ +from kedro_viz.constants import SHAREABLEVIZ_SUPPORTED_PLATFORMS +from kedro_viz.launchers.cli import main + + +@pytest.fixture +def mock_click_echo(mocker): + return mocker.patch("click.echo") + + +class TestCliDeployViz: + @pytest.mark.parametrize( + "command_options, deployer_args", + [ + ( + [ + "viz", + "deploy", + "--platform", + "azure", + "--endpoint", + "https://example-bucket.web.core.windows.net", + "--bucket-name", + "example-bucket", + ], + { + "platform": "azure", + "endpoint": "https://example-bucket.web.core.windows.net", + "bucket_name": "example-bucket", + }, + ), + ( + [ + "viz", + "deploy", + "--platform", + "aws", + "--endpoint", + "http://example-bucket.s3-website.us-east-2.amazonaws.com/", + "--bucket-name", + "example-bucket", + ], + { + "platform": "aws", + "endpoint": "http://example-bucket.s3-website.us-east-2.amazonaws.com/", + "bucket_name": "example-bucket", + }, + ), + ( + [ + "viz", + "deploy", + "--platform", + "gcp", + "--endpoint", + "http://34.120.87.227/", + "--bucket-name", + "example-bucket", + ], + { + "platform": "gcp", + "endpoint": "http://34.120.87.227/", + "bucket_name": "example-bucket", + }, + ), + ( + [ + "viz", + "deploy", + "--platform", + "gcp", + "--endpoint", + "http://34.120.87.227/", + "--bucket-name", + "example-bucket", + "--include-hooks", + ], + { + "platform": "gcp", + "endpoint": "http://34.120.87.227/", + "bucket_name": "example-bucket", + "include_hooks": True, + }, + ), + ( + [ + "viz", + "deploy", + "--platform", + "aws", + "--endpoint", + "http://example-bucket.s3-website.us-east-2.amazonaws.com/", + "--bucket-name", + "example-bucket", + "--include-previews", + ], + { + "platform": "aws", + "endpoint": "http://example-bucket.s3-website.us-east-2.amazonaws.com/", + "bucket_name": "example-bucket", + "preview": True, + }, + ), + ], + ) + def test_viz_deploy_valid_endpoint_and_bucket( + self, command_options, deployer_args, mocker + ): + runner = CliRunner() + mocker.patch("fsspec.filesystem") + create_shareableviz_process_mock = mocker.patch( + "kedro_viz.launchers.cli.utils.create_shareableviz_process" + ) + + with runner.isolated_filesystem(): + result = runner.invoke(main.viz_cli, command_options) + + assert result.exit_code == 0 + + create_shareableviz_process_mock.assert_called_once_with( + deployer_args.get("platform"), + deployer_args.get("preview", False), + deployer_args.get("endpoint"), + deployer_args.get("bucket_name"), + deployer_args.get("include_hooks", False), + ) + + def test_viz_deploy_invalid_platform(self, mock_click_echo): + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke( + main.viz_cli, + [ + "viz", + "deploy", + "--platform", + "random", + "--endpoint", + "", + "--bucket-name", + "example-bucket", + ], + ) + + assert result.exit_code == 0 + mock_click_echo_calls = [ + call( + "\x1b[31mERROR: Invalid platform specified. Kedro-Viz supports \n" + f"the following platforms - {*SHAREABLEVIZ_SUPPORTED_PLATFORMS,}\x1b[0m" + ) + ] + + mock_click_echo.assert_has_calls(mock_click_echo_calls) + + def test_viz_deploy_invalid_endpoint(self, mock_click_echo): + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke( + main.viz_cli, + [ + "viz", + "deploy", + "--platform", + "aws", + "--endpoint", + "", + "--bucket-name", + "example-bucket", + ], + ) + + assert result.exit_code == 0 + mock_click_echo_calls = [ + call( + "\x1b[31mERROR: Invalid endpoint specified. If you are looking for platform \n" + "agnostic shareable viz solution, please use the `kedro viz build` command\x1b[0m" + ) + ] + + mock_click_echo.assert_has_calls(mock_click_echo_calls) diff --git a/package/tests/test_launchers/test_cli/test_lazy_default_group.py b/package/tests/test_launchers/test_cli/test_lazy_default_group.py new file mode 100644 index 0000000000..5148d4d76d --- /dev/null +++ b/package/tests/test_launchers/test_cli/test_lazy_default_group.py @@ -0,0 +1,82 @@ +from unittest.mock import MagicMock, patch + +import pytest +from click import Context, UsageError + +from kedro_viz.launchers.cli.lazy_default_group import LazyDefaultGroup + + +@pytest.fixture +def lazy_default_group(): + """Fixture for LazyDefaultGroup.""" + return LazyDefaultGroup( + name="viz_cli_group", + lazy_subcommands={ + "run": "kedro_viz.launchers.cli.run.run", + "build": "kedro_viz.launchers.cli.build.build", + }, + default="build", + default_if_no_args=True, + ) + + +def test_lazy_loading(lazy_default_group): + """Test that lazy loading of a command works.""" + with patch("importlib.import_module") as mock_import_module: + mock_command = MagicMock() + mock_import_module.return_value.run = mock_command + + cmd = lazy_default_group.get_command(Context(lazy_default_group), "run") + + assert cmd == mock_command + mock_import_module.assert_called_once_with("kedro_viz.launchers.cli.run") + + +def test_list_commands(lazy_default_group): + """Test that the list of commands is correctly returned.""" + commands = lazy_default_group.list_commands(Context(lazy_default_group)) + assert commands == ["build", "run"] + + +def test_default_command_if_no_args(lazy_default_group): + """Test that the default command is invoked when no args are passed.""" + ctx = Context(lazy_default_group) + args = [] + + lazy_default_group.parse_args(ctx, args) + + # Assert that the default command is used + assert args == ["build"] + + +def test_resolve_command_with_valid_command(lazy_default_group): + """Test resolving a valid command.""" + ctx = Context(lazy_default_group) + cmd_name, cmd, args = lazy_default_group.resolve_command(ctx, ["run"]) + assert cmd_name == "run" + assert cmd is not None + + +def test_resolve_command_with_invalid_command(lazy_default_group): + """Test resolving an invalid command falls back to default.""" + ctx = Context(lazy_default_group) + + # When an invalid command is given, the default command should be used + cmd_name, cmd, args = lazy_default_group.resolve_command(ctx, ["invalid"]) + assert cmd_name == "build" + assert cmd is not None + + +def test_resolve_command_raises_usage_error_when_no_default(lazy_default_group): + """Test that UsageError is raised when an invalid command is given and no default is set.""" + lazy_default_group.default_cmd_name = None # Remove the default command + + ctx = Context(lazy_default_group) + with pytest.raises(UsageError): + lazy_default_group.resolve_command(ctx, ["invalid"]) + + +def test_init_raises_value_error_on_ignore_unknown_options(): + """Test that ValueError is raised when ignore_unknown_options is False.""" + with pytest.raises(ValueError): + LazyDefaultGroup(ignore_unknown_options=False) diff --git a/package/tests/test_launchers/test_cli/test_main.py b/package/tests/test_launchers/test_cli/test_main.py new file mode 100644 index 0000000000..a1546faa6e --- /dev/null +++ b/package/tests/test_launchers/test_cli/test_main.py @@ -0,0 +1,75 @@ +import pytest +from click.testing import CliRunner + +from kedro_viz.launchers.cli import main +from kedro_viz.launchers.cli.lazy_default_group import LazyDefaultGroup + + +@pytest.fixture(scope="class") +def runner(): + return CliRunner() + + +class TestCLIMain: + def test_viz_cli_group(self): + assert len(main.viz_cli.list_commands(None)) == 1 + assert len(main.viz.list_commands(None)) == 3 + + assert main.viz_cli.list_commands(None) == ["viz"] + assert main.viz.list_commands(None) == ["build", "deploy", "run"] + + assert main.viz_cli.get_command(None, "random") is None + assert main.viz_cli.get_command(None, "viz") is not None + assert main.viz.get_command(None, "run") is not None + + assert isinstance(main.viz_cli.get_command(None, "viz"), LazyDefaultGroup) + + def test_viz_help(self, runner): + with runner.isolated_filesystem(): + result = runner.invoke(main.viz_cli, ["viz", "--help"]) + + assert result.output == ( + "Usage: Kedro-Viz viz [OPTIONS] COMMAND [ARGS]...\n" + "\n" + " Visualise a Kedro pipeline using Kedro viz.\n" + "\n" + "Options:\n" + " --help Show this message and exit.\n" + "\n" + "Commands:\n" + " build Create build directory of local Kedro Viz instance with Kedro...\n" + " deploy Deploy and host Kedro Viz on provided platform\n" + " run Launch local Kedro Viz instance\n" + ) + + def test_viz_run_help(self, runner): + with runner.isolated_filesystem(): + result = runner.invoke(main.viz_cli, ["viz", "run", "--help"]) + + assert result.exit_code == 0 + assert "Launch local Kedro Viz instance" in result.output + assert "invalid-option" not in result.output + assert "--host" in result.output + + def test_viz_build_help(self, runner): + with runner.isolated_filesystem(): + result = runner.invoke(main.viz_cli, ["viz", "build", "--help"]) + + assert result.exit_code == 0 + assert ( + "Create build directory of local Kedro Viz instance with Kedro project data" + in result.output + ) + assert "invalid-option" not in result.output + assert "--include-hooks" in result.output + assert "--include-previews" in result.output + + def test_viz_deploy_help(self, runner): + with runner.isolated_filesystem(): + result = runner.invoke(main.viz_cli, ["viz", "deploy", "--help"]) + + assert result.exit_code == 0 + assert "Deploy and host Kedro Viz on provided platform" in result.output + assert "invalid-option" not in result.output + assert "--platform" in result.output + assert "--bucket-name" in result.output diff --git a/package/tests/test_launchers/test_cli/test_run.py b/package/tests/test_launchers/test_cli/test_run.py new file mode 100644 index 0000000000..bd5db7bfcf --- /dev/null +++ b/package/tests/test_launchers/test_cli/test_run.py @@ -0,0 +1,360 @@ +from unittest.mock import call + +import pytest +import requests +from click.testing import CliRunner +from packaging.version import parse +from watchgod import RegExpWatcher, run_process + +from kedro_viz import __version__ +from kedro_viz.launchers.cli import main +from kedro_viz.launchers.cli.run import _VIZ_PROCESSES +from kedro_viz.launchers.utils import _PYPROJECT +from kedro_viz.server import run_server + + +@pytest.fixture +def patched_check_viz_up(mocker): + mocker.patch("kedro_viz.launchers.utils._check_viz_up", return_value=True) + + +@pytest.fixture +def patched_start_browser(mocker): + mocker.patch("kedro_viz.launchers.utils._start_browser") + + +@pytest.fixture +def mock_click_echo(mocker): + return mocker.patch("click.echo") + + +@pytest.fixture +def mock_project_path(mocker): + mock_path = "/tmp/project_path" + mocker.patch("pathlib.Path.cwd", return_value=mock_path) + return mock_path + + +class TestCliRunViz: + @pytest.mark.parametrize( + "command_options,run_server_args", + [ + ( + ["viz"], + { + "host": "127.0.0.1", + "port": 4141, + "load_file": None, + "save_file": None, + "pipeline_name": None, + "env": None, + "project_path": "testPath", + "autoreload": False, + "include_hooks": False, + "package_name": None, + "extra_params": {}, + }, + ), + ( + ["viz", "run"], + { + "host": "127.0.0.1", + "port": 4141, + "load_file": None, + "save_file": None, + "pipeline_name": None, + "env": None, + "project_path": "testPath", + "autoreload": False, + "include_hooks": False, + "package_name": None, + "extra_params": {}, + }, + ), + ( + [ + "viz", + "run", + "--host", + "localhost", + ], + { + "host": "localhost", + "port": 4141, + "load_file": None, + "save_file": None, + "pipeline_name": None, + "env": None, + "project_path": "testPath", + "autoreload": False, + "include_hooks": False, + "package_name": None, + "extra_params": {}, + }, + ), + ( + [ + "viz", + "run", + "--host", + "8.8.8.8", + "--port", + "4142", + "--no-browser", + "--save-file", + "save_dir", + "--pipeline", + "data_science", + "--env", + "local", + "--params", + "extra_param=param", + ], + { + "host": "8.8.8.8", + "port": 4142, + "load_file": None, + "save_file": "save_dir", + "pipeline_name": "data_science", + "env": "local", + "project_path": "testPath", + "autoreload": False, + "include_hooks": False, + "package_name": None, + "extra_params": {"extra_param": "param"}, + }, + ), + ( + [ + "viz", + "run", + "--host", + "8.8.8.8", + "--port", + "4142", + "--no-browser", + "--save-file", + "save_dir", + "-p", + "data_science", + "-e", + "local", + "--params", + "extra_param=param", + ], + { + "host": "8.8.8.8", + "port": 4142, + "load_file": None, + "save_file": "save_dir", + "pipeline_name": "data_science", + "env": "local", + "project_path": "testPath", + "autoreload": False, + "include_hooks": False, + "package_name": None, + "extra_params": {"extra_param": "param"}, + }, + ), + ( + ["viz", "run", "--include-hooks"], + { + "host": "127.0.0.1", + "port": 4141, + "load_file": None, + "save_file": None, + "pipeline_name": None, + "env": None, + "project_path": "testPath", + "autoreload": False, + "include_hooks": True, + "package_name": None, + "extra_params": {}, + }, + ), + ], + ) + def test_kedro_viz_command_run_server( + self, + command_options, + run_server_args, + mocker, + patched_check_viz_up, + patched_start_browser, + ): + process_init = mocker.patch("multiprocessing.Process") + runner = CliRunner() + + # Reduce the timeout argument from 600 to 1 to make test run faster. + mocker.patch( + "kedro_viz.launchers.utils._wait_for.__defaults__", (True, 1, True, 1) + ) + + # Mock finding kedro project + mocker.patch( + "kedro_viz.launchers.utils._find_kedro_project", + return_value=run_server_args["project_path"], + ) + + with runner.isolated_filesystem(): + runner.invoke(main.viz_cli, command_options) + + process_init.assert_called_once_with( + target=run_server, daemon=False, kwargs={**run_server_args} + ) + + assert run_server_args["port"] in _VIZ_PROCESSES + + def test_kedro_viz_command_should_log_project_not_found( + self, mocker, mock_project_path, mock_click_echo + ): + # Reduce the timeout argument from 600 to 1 to make test run faster. + mocker.patch( + "kedro_viz.launchers.utils._wait_for.__defaults__", (True, 1, True, 1) + ) + # Mock finding kedro project + mocker.patch("kedro_viz.launchers.utils._find_kedro_project", return_value=None) + runner = CliRunner() + with runner.isolated_filesystem(): + runner.invoke(main.viz_cli, ["viz", "run"]) + + mock_click_echo_calls = [ + call( + "\x1b[31mERROR: Failed to start Kedro-Viz : " + "Could not find the project configuration " + f"file '{_PYPROJECT}' at '{mock_project_path}'. \x1b[0m" + ) + ] + + mock_click_echo.assert_has_calls(mock_click_echo_calls) + + def test_kedro_viz_command_should_log_outdated_version( + self, mocker, mock_http_response, mock_click_echo, mock_project_path + ): + installed_version = parse(__version__) + mock_version = f"{installed_version.major + 1}.0.0" + requests_get = mocker.patch("requests.get") + requests_get.return_value = mock_http_response( + data={"info": {"version": mock_version}} + ) + + mocker.patch("kedro_viz.server.run_server") + + # Reduce the timeout argument from 600 to 1 to make test run faster. + mocker.patch( + "kedro_viz.launchers.utils._wait_for.__defaults__", (True, 1, True, 1) + ) + # Mock finding kedro project + mocker.patch( + "kedro_viz.launchers.utils._find_kedro_project", + return_value=mock_project_path, + ) + runner = CliRunner() + with runner.isolated_filesystem(): + runner.invoke(main.viz_cli, ["viz", "run"]) + + mock_click_echo_calls = [ + call( + "\x1b[33mWARNING: You are using an old version of Kedro Viz. " + f"You are using version {installed_version}; " + f"however, version {mock_version} is now available.\n" + "You should consider upgrading via the `pip install -U kedro-viz` command.\n" + "You can view the complete changelog at " + "https://github.com/kedro-org/kedro-viz/releases.\x1b[0m" + ) + ] + + mock_click_echo.assert_has_calls(mock_click_echo_calls) + + def test_kedro_viz_command_should_not_log_latest_version( + self, mocker, mock_http_response, mock_click_echo, mock_project_path + ): + requests_get = mocker.patch("requests.get") + requests_get.return_value = mock_http_response( + data={"info": {"version": str(parse(__version__))}} + ) + + mocker.patch("kedro_viz.server.run_server") + # Reduce the timeout argument from 600 to 1 to make test run faster. + mocker.patch( + "kedro_viz.launchers.utils._wait_for.__defaults__", (True, 1, True, 1) + ) + # Mock finding kedro project + mocker.patch( + "kedro_viz.launchers.utils._find_kedro_project", + return_value=mock_project_path, + ) + runner = CliRunner() + with runner.isolated_filesystem(): + runner.invoke(main.viz_cli, ["viz", "run"]) + + mock_click_echo_calls = [call("\x1b[32mStarting Kedro Viz ...\x1b[0m")] + + mock_click_echo.assert_has_calls(mock_click_echo_calls) + + def test_kedro_viz_command_should_not_log_if_pypi_is_down( + self, mocker, mock_click_echo, mock_project_path + ): + requests_get = mocker.patch("requests.get") + requests_get.side_effect = requests.exceptions.RequestException("PyPI is down") + + mocker.patch("kedro_viz.server.run_server") + # Reduce the timeout argument from 600 to 1 to make test run faster. + mocker.patch( + "kedro_viz.launchers.utils._wait_for.__defaults__", (True, 1, True, 1) + ) + # Mock finding kedro project + mocker.patch( + "kedro_viz.launchers.utils._find_kedro_project", + return_value=mock_project_path, + ) + runner = CliRunner() + with runner.isolated_filesystem(): + runner.invoke(main.viz_cli, ["viz", "run"]) + + mock_click_echo_calls = [call("\x1b[32mStarting Kedro Viz ...\x1b[0m")] + + mock_click_echo.assert_has_calls(mock_click_echo_calls) + + def test_kedro_viz_command_with_autoreload( + self, mocker, mock_project_path, patched_check_viz_up, patched_start_browser + ): + process_init = mocker.patch("multiprocessing.Process") + + # Reduce the timeout argument from 600 to 1 to make test run faster. + mocker.patch( + "kedro_viz.launchers.utils._wait_for.__defaults__", (True, 1, True, 1) + ) + # Mock finding kedro project + mocker.patch( + "kedro_viz.launchers.utils._find_kedro_project", + return_value=mock_project_path, + ) + runner = CliRunner() + with runner.isolated_filesystem(): + runner.invoke(main.viz_cli, ["viz", "run", "--autoreload"]) + + run_process_kwargs = { + "path": mock_project_path, + "target": run_server, + "kwargs": { + "host": "127.0.0.1", + "port": 4141, + "load_file": None, + "save_file": None, + "pipeline_name": None, + "env": None, + "autoreload": True, + "project_path": mock_project_path, + "include_hooks": False, + "package_name": None, + "extra_params": {}, + }, + "watcher_cls": RegExpWatcher, + "watcher_kwargs": {"re_files": "^.*(\\.yml|\\.yaml|\\.py|\\.json)$"}, + } + + process_init.assert_called_once_with( + target=run_process, daemon=False, kwargs={**run_process_kwargs} + ) + assert run_process_kwargs["kwargs"]["port"] in _VIZ_PROCESSES diff --git a/package/tests/test_launchers/test_cli/test_utils.py b/package/tests/test_launchers/test_cli/test_utils.py new file mode 100644 index 0000000000..d8277c70ee --- /dev/null +++ b/package/tests/test_launchers/test_cli/test_utils.py @@ -0,0 +1,265 @@ +from unittest.mock import Mock, call, patch + +import pytest + +from kedro_viz import __version__ +from kedro_viz.constants import VIZ_DEPLOY_TIME_LIMIT +from kedro_viz.launchers.cli.utils import ( + _load_and_deploy_viz, + _viz_deploy_progress_timer, + create_shareableviz_process, +) + + +@pytest.fixture +def mock_viz_deploy_process(mocker): + return mocker.patch("multiprocessing.Process") + + +@pytest.fixture +def mock_process_completed(mocker): + return mocker.patch("multiprocessing.Value", return_value=Mock()) + + +@pytest.fixture +def mock_exception_queue(mocker): + return mocker.patch("multiprocessing.Queue", return_value=Mock()) + + +@pytest.fixture +def mock_viz_load_and_deploy(mocker): + return mocker.patch("kedro_viz.launchers.cli.utils._load_and_deploy_viz") + + +@pytest.fixture +def mock_viz_deploy_progress_timer(mocker): + return mocker.patch("kedro_viz.launchers.cli.utils._viz_deploy_progress_timer") + + +@pytest.fixture +def mock_DeployerFactory(mocker): + return mocker.patch( + "kedro_viz.integrations.deployment.deployer_factory.DeployerFactory" + ) + + +@pytest.fixture +def mock_load_and_populate_data(mocker): + return mocker.patch("kedro_viz.server.load_and_populate_data") + + +@pytest.fixture +def mock_click_echo(mocker): + return mocker.patch("click.echo") + + +@pytest.fixture +def mock_project_path(mocker): + mock_path = "/tmp/project_path" + mocker.patch("pathlib.Path.cwd", return_value=mock_path) + return mock_path + + +class TestCliUtils: + @pytest.mark.parametrize( + "platform, is_all_previews_enabled, endpoint, bucket_name," + "include_hooks, process_completed_value", + [ + ( + "azure", + True, + "https://example-bucket.web.core.windows.net", + "example-bucket", + True, + 1, + ), + ( + "aws", + True, + "http://example-bucket.s3-website.us-east-2.amazonaws.com/", + "example-bucket", + True, + 1, + ), + ( + "gcp", + False, + "http://34.120.87.227/", + "example-bucket", + False, + 1, + ), + ("local", False, None, None, False, 1), + ( + "azure", + True, + "https://example-bucket.web.core.windows.net", + "example-bucket", + False, + 0, + ), + ( + "aws", + False, + "http://example-bucket.s3-website.us-east-2.amazonaws.com/", + "example-bucket", + False, + 0, + ), + ( + "gcp", + True, + "http://34.120.87.227/", + "example-bucket", + True, + 0, + ), + ("local", True, None, None, True, 0), + ], + ) + def test_create_shareableviz_process( + self, + platform, + is_all_previews_enabled, + endpoint, + bucket_name, + include_hooks, + process_completed_value, + mock_viz_deploy_process, + mock_process_completed, + mock_exception_queue, + mock_viz_load_and_deploy, + mock_viz_deploy_progress_timer, + mock_click_echo, + ): + mock_process_completed.return_value.value = process_completed_value + create_shareableviz_process( + platform, is_all_previews_enabled, endpoint, bucket_name, include_hooks + ) + + # Assert the mocks were called as expected + mock_viz_deploy_process.assert_called_once_with( + target=mock_viz_load_and_deploy, + args=( + platform, + is_all_previews_enabled, + endpoint, + bucket_name, + include_hooks, + None, + mock_process_completed.return_value, + mock_exception_queue.return_value, + ), + ) + mock_viz_deploy_process.return_value.start.assert_called_once() + mock_viz_deploy_progress_timer.assert_called_once_with( + mock_process_completed.return_value, VIZ_DEPLOY_TIME_LIMIT + ) + mock_viz_deploy_process.return_value.terminate.assert_called_once() + + if process_completed_value: + if platform != "local": + msg = ( + "\x1b[32m\u2728 Success! Kedro Viz has been deployed on " + f"{platform.upper()}. " + "It can be accessed at :\n" + f"{endpoint}\x1b[0m" + ) + else: + msg = ( + "\x1b[32m✨ Success! Kedro-Viz build files have been " + "added to the `build` directory.\x1b[0m" + ) + else: + msg = ( + "\x1b[31mTIMEOUT ERROR: Failed to build/deploy Kedro-Viz " + f"as the process took more than {VIZ_DEPLOY_TIME_LIMIT} seconds. " + "Please try again later.\x1b[0m" + ) + + mock_click_echo_calls = [call(msg)] + mock_click_echo.assert_has_calls(mock_click_echo_calls) + + @pytest.mark.parametrize( + "platform, is_all_previews_enabled, endpoint, bucket_name, include_hooks, package_name", + [ + ( + "azure", + False, + "https://example-bucket.web.core.windows.net", + "example-bucket", + False, + "demo_project", + ), + ( + "aws", + True, + "http://example-bucket.s3-website.us-east-2.amazonaws.com/", + "example-bucket", + True, + "demo_project", + ), + ( + "gcp", + True, + "http://34.120.87.227/", + "example-bucket", + False, + "demo_project", + ), + ("local", False, None, None, True, "demo_project"), + ], + ) + def test_load_and_deploy_viz_success( + self, + platform, + is_all_previews_enabled, + endpoint, + bucket_name, + include_hooks, + package_name, + mock_DeployerFactory, + mock_load_and_populate_data, + mock_process_completed, + mock_exception_queue, + mock_click_echo, + mock_project_path, + ): + deployer_mock = mock_DeployerFactory.create_deployer.return_value + + _load_and_deploy_viz( + platform, + is_all_previews_enabled, + endpoint, + bucket_name, + include_hooks, + package_name, + mock_process_completed, + mock_exception_queue, + ) + + mock_load_and_populate_data.assert_called_once_with( + mock_project_path, include_hooks=include_hooks, package_name=package_name + ) + mock_DeployerFactory.create_deployer.assert_called_once_with( + platform, endpoint, bucket_name + ) + deployer_mock.deploy.assert_called_once_with(is_all_previews_enabled) + mock_click_echo.echo.assert_not_called() + + def test_viz_deploy_progress_timer(self, capsys): + mock_process_completed = Mock() + mock_process_completed.value = 0 + + with patch("kedro_viz.launchers.cli.utils.sleep") as mock_sleep: + _viz_deploy_progress_timer(mock_process_completed, VIZ_DEPLOY_TIME_LIMIT) + + assert mock_sleep.call_count == VIZ_DEPLOY_TIME_LIMIT + 1 + + expected_sleep_calls = [call(1)] * (VIZ_DEPLOY_TIME_LIMIT + 1) + mock_sleep.assert_has_calls(expected_sleep_calls) + captured = capsys.readouterr() + + for second in range(1, VIZ_DEPLOY_TIME_LIMIT + 1): + expected_output = f"...Creating your build/deploy Kedro-Viz ({second}s)" + assert expected_output in captured.out diff --git a/package/tests/test_launchers/test_utils.py b/package/tests/test_launchers/test_utils.py index 04425cfd09..83e9203bd3 100644 --- a/package/tests/test_launchers/test_utils.py +++ b/package/tests/test_launchers/test_utils.py @@ -1,17 +1,15 @@ from pathlib import Path from unittest import mock -from unittest.mock import Mock, call, patch +from unittest.mock import Mock import pytest import requests -from kedro_viz.constants import VIZ_DEPLOY_TIME_LIMIT from kedro_viz.launchers.utils import ( _check_viz_up, _find_kedro_project, _is_project, _start_browser, - viz_deploy_progress_timer, ) @@ -56,24 +54,6 @@ def test_check_viz_up(host, port, status_code, expected_result, mocker): assert result == expected_result -def test_viz_deploy_progress_timer(capsys): - mock_process_completed = Mock() - mock_process_completed.value = 0 - - with patch("kedro_viz.launchers.utils.sleep") as mock_sleep: - viz_deploy_progress_timer(mock_process_completed, VIZ_DEPLOY_TIME_LIMIT) - - assert mock_sleep.call_count == VIZ_DEPLOY_TIME_LIMIT + 1 - - expected_sleep_calls = [call(1)] * (VIZ_DEPLOY_TIME_LIMIT + 1) - mock_sleep.assert_has_calls(expected_sleep_calls) - captured = capsys.readouterr() - - for second in range(1, VIZ_DEPLOY_TIME_LIMIT + 1): - expected_output = f"...Creating your build/deploy Kedro-Viz ({second}s)" - assert expected_output in captured.out - - class TestIsProject: project_path = Path.cwd()