From 093a25458f4686c974c44df9325cbabd7a6db726 Mon Sep 17 00:00:00 2001 From: binliu Date: Wed, 14 Jun 2023 13:10:57 +0000 Subject: [PATCH 01/28] add data tracking to MLFlowHandler Signed-off-by: binliu --- monai/handlers/mlflow_handler.py | 56 ++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py index 20add9b11f..ddce9afff6 100644 --- a/monai/handlers/mlflow_handler.py +++ b/monai/handlers/mlflow_handler.py @@ -27,6 +27,7 @@ mlflow.entities, _ = optional_import( "mlflow.entities", descriptor="Please install mlflow.entities before using MLFlowHandler." ) +pandas, _ = optional_import("pandas", descriptor="Please install pandas for recording the dataset.") if TYPE_CHECKING: from ignite.engine import Engine @@ -68,10 +69,13 @@ class MLFlowHandler: epoch_log: whether to log data to MLFlow when epoch completed, default to `True`. ``epoch_log`` can be also a function and it will be interpreted as an event filter. See ``iteration_log`` argument for more details. + dataset_log: whether to log information about the dataset at the beginning. epoch_logger: customized callable logger for epoch level logging with MLFlow. Must accept parameter "engine", use default logger if None. iteration_logger: customized callable logger for iteration level logging with MLFlow. Must accept parameter "engine", use default logger if None. + dataset_logger: customized callable logger to log the dataset information with MLFlow. + Must accept parameter "engine", use the default logger if None. output_transform: a callable that is used to transform the ``ignite.engine.state.output`` into a scalar to track, or a dictionary of {key: scalar}. By default this value logging happens when every iteration completed. @@ -109,8 +113,10 @@ def __init__( tracking_uri: str | None = None, iteration_log: bool | Callable[[Engine, int], bool] = True, epoch_log: bool | Callable[[Engine, int], bool] = True, + dataset_log: bool = True, epoch_logger: Callable[[Engine], Any] | None = None, iteration_logger: Callable[[Engine], Any] | None = None, + dataset_logger: Callable[[Engine], Any] | None = None, output_transform: Callable = lambda x: x[0], global_epoch_transform: Callable = lambda x: x, state_attributes: Sequence[str] | None = None, @@ -124,8 +130,10 @@ def __init__( ) -> None: self.iteration_log = iteration_log self.epoch_log = epoch_log + self.dataset_log = dataset_log self.epoch_logger = epoch_logger self.iteration_logger = iteration_logger + self.dataset_logger = dataset_logger self.output_transform = output_transform self.global_epoch_transform = global_epoch_transform self.state_attributes = state_attributes @@ -210,6 +218,13 @@ def start(self, engine: Engine) -> None: self._delete_exist_param_in_dict(attrs) self._log_params(attrs) + if self.dataset_log: + if self.dataset_logger: + self.dataset_logger(engine) + else: + self._default_dataset_logger(engine) + + def _set_experiment(self): experiment = self.experiment if not experiment: @@ -222,6 +237,14 @@ def _set_experiment(self): raise ValueError(f"Cannot set a deleted experiment '{self.experiment_name}' as the active experiment") self.experiment = experiment + def _log_dataset(self, sample_dict:dict[str, Any]) -> None: + if not self.cur_run: + raise ValueError("Current Run is not Active to log the dataset") + sample_df = pandas.DataFrame(sample_dict) + dataset = mlflow.data.from_pandas(sample_df) + datasets = [mlflow.entities.DatasetInput(dataset._to_mlflow_entity())] + self.client.log_inputs(run_id=self.cur_run.info.run_id, datasets=datasets) + def _log_params(self, params: dict[str, Any]) -> None: if not self.cur_run: raise ValueError("Current Run is not Active to log params") @@ -352,3 +375,36 @@ def _default_iteration_log(self, engine: Engine) -> None: for i, param_group in enumerate(cur_optimizer.param_groups) } self._log_metrics(params, step=engine.state.iteration) + + def _default_dataset_logger(self, engine:Engine) -> None: + """ + Execute dataset log operation based on MONAI `engine.dataloader.dataset` data. + Abstract meta information from samples in dataset and build a Pandas DataFrame from it. + Create a PandasDataset in MLFlow using the meta information and log it. + + Args: + engine: Ignite Engine, it can be a trainer, validator or evaluator. + + """ + dataloader = getattr(engine, "data_loader", None) + dataset = getattr(dataloader, "dataset", None) if dataloader else None + if not dataset: + raise AttributeError(f"The engine dataloader is {dataloader} and the dataset of the dataloader is {dataset}.") + sample_dict = {} + sample_dict["image_name"] = [] + sample_dict["label_name"] = [] + + for sample in dataset: + if isinstance(sample, dict): + image_name = sample["image_meta_dict"]["filename_or_obj"] + label_name = sample["label_meta_dict"]["filename_or_obj"] + else: + image_name = sample[0]["image_meta_dict"]["filename_or_obj"] + label_name = sample[0]["label_meta_dict"]["filename_or_obj"] + + if not(isinstance(image_name, str) and isinstance(label_name, str)): + raise ValueError(f"Image name is type {type(image_name)} and label name is type{type(label_name)}.") + else: + sample_dict["image_name"].append(image_name) + sample_dict["label_name"].append(label_name) + self._log_dataset(sample_dict) From e22a8d04429000938e9b18f8cfa481adbbdd1f0a Mon Sep 17 00:00:00 2001 From: binliu Date: Wed, 14 Jun 2023 13:12:23 +0000 Subject: [PATCH 02/28] fix format Signed-off-by: binliu --- monai/handlers/mlflow_handler.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py index ddce9afff6..2e71836f67 100644 --- a/monai/handlers/mlflow_handler.py +++ b/monai/handlers/mlflow_handler.py @@ -224,7 +224,6 @@ def start(self, engine: Engine) -> None: else: self._default_dataset_logger(engine) - def _set_experiment(self): experiment = self.experiment if not experiment: @@ -237,7 +236,7 @@ def _set_experiment(self): raise ValueError(f"Cannot set a deleted experiment '{self.experiment_name}' as the active experiment") self.experiment = experiment - def _log_dataset(self, sample_dict:dict[str, Any]) -> None: + def _log_dataset(self, sample_dict: dict[str, Any]) -> None: if not self.cur_run: raise ValueError("Current Run is not Active to log the dataset") sample_df = pandas.DataFrame(sample_dict) @@ -376,7 +375,7 @@ def _default_iteration_log(self, engine: Engine) -> None: } self._log_metrics(params, step=engine.state.iteration) - def _default_dataset_logger(self, engine:Engine) -> None: + def _default_dataset_logger(self, engine: Engine) -> None: """ Execute dataset log operation based on MONAI `engine.dataloader.dataset` data. Abstract meta information from samples in dataset and build a Pandas DataFrame from it. @@ -389,7 +388,9 @@ def _default_dataset_logger(self, engine:Engine) -> None: dataloader = getattr(engine, "data_loader", None) dataset = getattr(dataloader, "dataset", None) if dataloader else None if not dataset: - raise AttributeError(f"The engine dataloader is {dataloader} and the dataset of the dataloader is {dataset}.") + raise AttributeError( + f"The engine dataloader is {dataloader} and the dataset of the dataloader is {dataset}." + ) sample_dict = {} sample_dict["image_name"] = [] sample_dict["label_name"] = [] @@ -402,7 +403,7 @@ def _default_dataset_logger(self, engine:Engine) -> None: image_name = sample[0]["image_meta_dict"]["filename_or_obj"] label_name = sample[0]["label_meta_dict"]["filename_or_obj"] - if not(isinstance(image_name, str) and isinstance(label_name, str)): + if not (isinstance(image_name, str) and isinstance(label_name, str)): raise ValueError(f"Image name is type {type(image_name)} and label name is type{type(label_name)}.") else: sample_dict["image_name"].append(image_name) From 8075f34a32030a82e542a88727e211dd5afff06c Mon Sep 17 00:00:00 2001 From: binliu Date: Tue, 20 Jun 2023 09:34:19 +0000 Subject: [PATCH 03/28] log the dataset to the mlflow ui Signed-off-by: binliu --- monai/handlers/mlflow_handler.py | 56 ++++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py index 2e71836f67..352d73e30d 100644 --- a/monai/handlers/mlflow_handler.py +++ b/monai/handlers/mlflow_handler.py @@ -21,6 +21,7 @@ from monai.config import IgniteInfo from monai.utils import ensure_tuple, min_version, optional_import +from monai.engines import Trainer Events, _ = optional_import("ignite.engine", IgniteInfo.OPT_IMPORT_VERSION, min_version, "Events") mlflow, _ = optional_import("mlflow", descriptor="Please install mlflow before using MLFlowHandler.") @@ -222,7 +223,7 @@ def start(self, engine: Engine) -> None: if self.dataset_logger: self.dataset_logger(engine) else: - self._default_dataset_logger(engine) + self._default_dataset_log(engine) def _set_experiment(self): experiment = self.experiment @@ -236,13 +237,32 @@ def _set_experiment(self): raise ValueError(f"Cannot set a deleted experiment '{self.experiment_name}' as the active experiment") self.experiment = experiment - def _log_dataset(self, sample_dict: dict[str, Any]) -> None: + @staticmethod + def _get_pandas_dataset_info(pandas_dataset): + dataset_name = pandas_dataset.name + return { + f"{dataset_name}_digest": pandas_dataset.digest, + f"{dataset_name}_samples": pandas_dataset.profile["num_rows"], + } + + def _log_dataset(self, sample_dict: dict[str, Any], context=None) -> None: if not self.cur_run: raise ValueError("Current Run is not Active to log the dataset") + + timestamp = str(int(time.time() * 1000))[-5:] + dataset_name = f"{context}_dataset_{timestamp}" if context else f"dataset_{timestamp}" sample_df = pandas.DataFrame(sample_dict) - dataset = mlflow.data.from_pandas(sample_df) - datasets = [mlflow.entities.DatasetInput(dataset._to_mlflow_entity())] - self.client.log_inputs(run_id=self.cur_run.info.run_id, datasets=datasets) + dataset = mlflow.data.from_pandas(sample_df, name=dataset_name) + exist_dataset_list = list( + filter(lambda x: x.dataset.digest == dataset.digest, self.cur_run.inputs.dataset_inputs) + ) + if not len(exist_dataset_list): + datasets = [mlflow.entities.DatasetInput(dataset._to_mlflow_entity())] + self.client.log_inputs(run_id=self.cur_run.info.run_id, datasets=datasets) + # Need to update the self.cur_run to sync the dataset log, otherwise the `inputs` info will be empty. + self.cur_run = self.client.get_run(self.cur_run.info.run_id) + dataset_info = MLFlowHandler._get_pandas_dataset_info(dataset) + self._log_params(dataset_info) def _log_params(self, params: dict[str, Any]) -> None: if not self.cur_run: @@ -282,6 +302,9 @@ def complete(self) -> None: """ Handler for train or validation/evaluation completed Event. """ + for input_dataset in self.cur_run.inputs.dataset_inputs: + dataset_name = "dataset_" + input_dataset.dataset.digest + if self.artifacts and self.cur_run: artifact_list = self._parse_artifacts() for artifact in artifact_list: @@ -375,7 +398,7 @@ def _default_iteration_log(self, engine: Engine) -> None: } self._log_metrics(params, step=engine.state.iteration) - def _default_dataset_logger(self, engine: Engine) -> None: + def _default_dataset_log(self, engine: Engine) -> None: """ Execute dataset log operation based on MONAI `engine.dataloader.dataset` data. Abstract meta information from samples in dataset and build a Pandas DataFrame from it. @@ -392,20 +415,19 @@ def _default_dataset_logger(self, engine: Engine) -> None: f"The engine dataloader is {dataloader} and the dataset of the dataloader is {dataset}." ) sample_dict = {} - sample_dict["image_name"] = [] - sample_dict["label_name"] = [] + sample_dict["image"] = [] for sample in dataset: if isinstance(sample, dict): - image_name = sample["image_meta_dict"]["filename_or_obj"] - label_name = sample["label_meta_dict"]["filename_or_obj"] + image = sample["image_meta_dict"]["filename_or_obj"] + elif isinstance(sample, list): + image = sample[0]["image_meta_dict"]["filename_or_obj"] else: - image_name = sample[0]["image_meta_dict"]["filename_or_obj"] - label_name = sample[0]["label_meta_dict"]["filename_or_obj"] + raise AttributeError(f"Expect samples with type list or dict, but got {type(sample)}") - if not (isinstance(image_name, str) and isinstance(label_name, str)): - raise ValueError(f"Image name is type {type(image_name)} and label name is type{type(label_name)}.") + if not isinstance(image, str): + raise ValueError(f"Expect image to be string, but got type {type(image)}.") else: - sample_dict["image_name"].append(image_name) - sample_dict["label_name"].append(label_name) - self._log_dataset(sample_dict) + sample_dict["image"].append(image) + dataset_type = "train" if isinstance(engine, Trainer) else "nontrain" + self._log_dataset(sample_dict, dataset_type) From 5f6e15833c731e51ac47c54f64f1c0ac14c3b06b Mon Sep 17 00:00:00 2001 From: binliu Date: Tue, 20 Jun 2023 09:35:23 +0000 Subject: [PATCH 04/28] fix the import order issue Signed-off-by: binliu --- monai/handlers/mlflow_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py index 352d73e30d..9242413a40 100644 --- a/monai/handlers/mlflow_handler.py +++ b/monai/handlers/mlflow_handler.py @@ -20,8 +20,8 @@ import torch from monai.config import IgniteInfo -from monai.utils import ensure_tuple, min_version, optional_import from monai.engines import Trainer +from monai.utils import ensure_tuple, min_version, optional_import Events, _ = optional_import("ignite.engine", IgniteInfo.OPT_IMPORT_VERSION, min_version, "Events") mlflow, _ = optional_import("mlflow", descriptor="Please install mlflow before using MLFlowHandler.") From 6a5e3958377793ab154f52505b8831c2e52192ca Mon Sep 17 00:00:00 2001 From: binliu Date: Tue, 20 Jun 2023 09:37:05 +0000 Subject: [PATCH 05/28] remove the dataset log from the complete method. Signed-off-by: binliu --- monai/handlers/mlflow_handler.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py index 9242413a40..0ca83551b8 100644 --- a/monai/handlers/mlflow_handler.py +++ b/monai/handlers/mlflow_handler.py @@ -302,9 +302,6 @@ def complete(self) -> None: """ Handler for train or validation/evaluation completed Event. """ - for input_dataset in self.cur_run.inputs.dataset_inputs: - dataset_name = "dataset_" + input_dataset.dataset.digest - if self.artifacts and self.cur_run: artifact_list = self._parse_artifacts() for artifact in artifact_list: From def7cc935b54165f9559ca9e01efa516e5a4f16f Mon Sep 17 00:00:00 2001 From: binliu Date: Sun, 25 Jun 2023 14:44:58 +0000 Subject: [PATCH 06/28] fix the unit test issue Signed-off-by: binliu --- monai/handlers/mlflow_handler.py | 4 ++-- tests/test_handler_mlflow.py | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py index 0ca83551b8..0c3e74a72d 100644 --- a/monai/handlers/mlflow_handler.py +++ b/monai/handlers/mlflow_handler.py @@ -411,7 +411,7 @@ def _default_dataset_log(self, engine: Engine) -> None: raise AttributeError( f"The engine dataloader is {dataloader} and the dataset of the dataloader is {dataset}." ) - sample_dict = {} + sample_dict:Dict[str, Sequence[str]] = {} sample_dict["image"] = [] for sample in dataset: @@ -420,7 +420,7 @@ def _default_dataset_log(self, engine: Engine) -> None: elif isinstance(sample, list): image = sample[0]["image_meta_dict"]["filename_or_obj"] else: - raise AttributeError(f"Expect samples with type list or dict, but got {type(sample)}") + raise AttributeError(f"Expect samples' type to be list or dict, but got {type(sample)}") if not isinstance(image, str): raise ValueError(f"Expect image to be string, but got type {type(image)}.") diff --git a/tests/test_handler_mlflow.py b/tests/test_handler_mlflow.py index f09f9b93d5..e90a945286 100644 --- a/tests/test_handler_mlflow.py +++ b/tests/test_handler_mlflow.py @@ -53,6 +53,7 @@ def _train_func(engine, batch): tracking_uri=path_to_uri(test_path), state_attributes=["test"], close_on_complete=True, + dataset_log=False, ) handler.attach(engine) engine.run(range(3), max_epochs=2) @@ -93,6 +94,7 @@ def _update_metric(engine): tracking_uri=path_to_uri(test_path), state_attributes=["test"], close_on_complete=True, + dataset_log=False, ) handler.attach(engine) engine.run(range(3), max_epochs=2) @@ -132,6 +134,7 @@ def _update_metric(engine): experiment_param=experiment_param, artifacts=[artifact_path], close_on_complete=True, + dataset_log=False, ) handler.attach(engine) engine.run(range(3), max_epochs=2) @@ -165,6 +168,7 @@ def _update_metric(engine): state_attributes=["test"], experiment_param=experiment_param, close_on_complete=True, + dataset_log=False, ) handler._default_epoch_log = MagicMock() handler.attach(engine) @@ -204,6 +208,7 @@ def _update_metric(engine): state_attributes=["test"], experiment_param=experiment_param, close_on_complete=True, + dataset_log=False, ) handler._default_iteration_log = MagicMock() handler.attach(engine) From b2a7482ddebe0a5d9a93c9231a69e46f574875e9 Mon Sep 17 00:00:00 2001 From: binliu Date: Sun, 25 Jun 2023 14:46:15 +0000 Subject: [PATCH 07/28] fix the type hint issue Signed-off-by: binliu --- monai/handlers/mlflow_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py index 0c3e74a72d..7407ae5f04 100644 --- a/monai/handlers/mlflow_handler.py +++ b/monai/handlers/mlflow_handler.py @@ -411,7 +411,7 @@ def _default_dataset_log(self, engine: Engine) -> None: raise AttributeError( f"The engine dataloader is {dataloader} and the dataset of the dataloader is {dataset}." ) - sample_dict:Dict[str, Sequence[str]] = {} + sample_dict: Dict[str, Sequence[str]] = {} sample_dict["image"] = [] for sample in dataset: From 5219a6c7359d4f6cded7389cf08078f954502c03 Mon Sep 17 00:00:00 2001 From: binliu Date: Sun, 25 Jun 2023 14:52:01 +0000 Subject: [PATCH 08/28] fix the format issue Signed-off-by: binliu --- monai/handlers/mlflow_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py index 7407ae5f04..a5d8d962b4 100644 --- a/monai/handlers/mlflow_handler.py +++ b/monai/handlers/mlflow_handler.py @@ -411,7 +411,7 @@ def _default_dataset_log(self, engine: Engine) -> None: raise AttributeError( f"The engine dataloader is {dataloader} and the dataset of the dataloader is {dataset}." ) - sample_dict: Dict[str, Sequence[str]] = {} + sample_dict: dict[str, Sequence[str]] = {} sample_dict["image"] = [] for sample in dataset: From a2d857c9bc3a775832a47fee366bb596ca67b171 Mon Sep 17 00:00:00 2001 From: binliu Date: Sun, 25 Jun 2023 15:26:16 +0000 Subject: [PATCH 09/28] add the type hint for the input parameter Signed-off-by: binliu --- monai/handlers/mlflow_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py index a5d8d962b4..9b17c2ddff 100644 --- a/monai/handlers/mlflow_handler.py +++ b/monai/handlers/mlflow_handler.py @@ -245,7 +245,7 @@ def _get_pandas_dataset_info(pandas_dataset): f"{dataset_name}_samples": pandas_dataset.profile["num_rows"], } - def _log_dataset(self, sample_dict: dict[str, Any], context=None) -> None: + def _log_dataset(self, sample_dict: dict[str, Any], context: str=None) -> None: if not self.cur_run: raise ValueError("Current Run is not Active to log the dataset") @@ -411,7 +411,7 @@ def _default_dataset_log(self, engine: Engine) -> None: raise AttributeError( f"The engine dataloader is {dataloader} and the dataset of the dataloader is {dataset}." ) - sample_dict: dict[str, Sequence[str]] = {} + sample_dict: dict[str, list[str]] = {} sample_dict["image"] = [] for sample in dataset: From a5a3801d6540df33cb84c46960af8a129b77a084 Mon Sep 17 00:00:00 2001 From: binliu Date: Sun, 25 Jun 2023 15:30:25 +0000 Subject: [PATCH 10/28] fix the format issue Signed-off-by: binliu --- monai/handlers/mlflow_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py index 9b17c2ddff..c09ebc8221 100644 --- a/monai/handlers/mlflow_handler.py +++ b/monai/handlers/mlflow_handler.py @@ -245,7 +245,7 @@ def _get_pandas_dataset_info(pandas_dataset): f"{dataset_name}_samples": pandas_dataset.profile["num_rows"], } - def _log_dataset(self, sample_dict: dict[str, Any], context: str=None) -> None: + def _log_dataset(self, sample_dict: dict[str, Any], context: str = None) -> None: if not self.cur_run: raise ValueError("Current Run is not Active to log the dataset") From 8a3186e2fd153dd8be8f0653ff6ea1d51dd926e9 Mon Sep 17 00:00:00 2001 From: binliu Date: Sun, 25 Jun 2023 15:51:08 +0000 Subject: [PATCH 11/28] fix the codeformat issue Signed-off-by: binliu --- monai/handlers/mlflow_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py index c09ebc8221..25c1830e3d 100644 --- a/monai/handlers/mlflow_handler.py +++ b/monai/handlers/mlflow_handler.py @@ -245,7 +245,7 @@ def _get_pandas_dataset_info(pandas_dataset): f"{dataset_name}_samples": pandas_dataset.profile["num_rows"], } - def _log_dataset(self, sample_dict: dict[str, Any], context: str = None) -> None: + def _log_dataset(self, sample_dict: dict[str, Any], context: str | None = None) -> None: if not self.cur_run: raise ValueError("Current Run is not Active to log the dataset") From d0ec3bbb1fbd1bf38284b80b5b830ce169bb3135 Mon Sep 17 00:00:00 2001 From: binliu Date: Mon, 26 Jun 2023 07:12:01 +0000 Subject: [PATCH 12/28] fix the fl monai algo unit test issue Signed-off-by: binliu --- monai/bundle/utils.py | 4 ++++ monai/handlers/mlflow_handler.py | 36 ++++++++++++++++++++------------ tests/test_handler_mlflow.py | 5 ----- 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py index 62d4975d94..4da63f54ad 100644 --- a/monai/bundle/utils.py +++ b/monai/bundle/utils.py @@ -114,6 +114,7 @@ "run_name": None, # may fill it at runtime "execute_config": None, + "dataset_log": True, "is_not_rank0": ( "$torch.distributed.is_available() \ and torch.distributed.is_initialized() and torch.distributed.get_rank() > 0" @@ -131,6 +132,7 @@ "tag_name": "train_loss", "output_transform": "$monai.handlers.from_engine(['loss'], first=True)", "close_on_complete": True, + "dataset_log": "@dataset_log", }, # MLFlowHandler config for the validator "validator": { @@ -140,6 +142,7 @@ "experiment_name": "@experiment_name", "run_name": "@run_name", "iteration_log": False, + "dataset_log": "@dataset_log", }, # MLFlowHandler config for the evaluator "evaluator": { @@ -151,6 +154,7 @@ "artifacts": "@execute_config", "iteration_log": False, "close_on_complete": True, + "dataset_log": "@dataset_log", }, }, } diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py index 25c1830e3d..300302a20c 100644 --- a/monai/handlers/mlflow_handler.py +++ b/monai/handlers/mlflow_handler.py @@ -13,6 +13,7 @@ import os import time +import warnings from collections.abc import Callable, Sequence from pathlib import Path from typing import TYPE_CHECKING, Any @@ -114,7 +115,7 @@ def __init__( tracking_uri: str | None = None, iteration_log: bool | Callable[[Engine, int], bool] = True, epoch_log: bool | Callable[[Engine, int], bool] = True, - dataset_log: bool = True, + dataset_log: bool = False, epoch_logger: Callable[[Engine], Any] | None = None, iteration_logger: Callable[[Engine], Any] | None = None, dataset_logger: Callable[[Engine], Any] | None = None, @@ -245,12 +246,19 @@ def _get_pandas_dataset_info(pandas_dataset): f"{dataset_name}_samples": pandas_dataset.profile["num_rows"], } - def _log_dataset(self, sample_dict: dict[str, Any], context: str | None = None) -> None: + def _log_dataset(self, sample_dict: dict[str, Any], context: str = "nontrain") -> None: if not self.cur_run: raise ValueError("Current Run is not Active to log the dataset") - timestamp = str(int(time.time() * 1000))[-5:] - dataset_name = f"{context}_dataset_{timestamp}" if context else f"dataset_{timestamp}" + # Need to update the self.cur_run to sync the dataset log, otherwise the `inputs` info will be empty. + self.cur_run = self.client.get_run(self.cur_run.info.run_id) + logged_train_set = [x for x in self.cur_run.inputs.dataset_inputs if "train" == x.dataset.name[: len("train")]] + logged_nontrain_set = [ + x for x in self.cur_run.inputs.dataset_inputs if "nontrain" == x.dataset.name[: len("nontrain")] + ] + dataset_cnt = str(len(logged_nontrain_set if context == "nontrain" else logged_train_set)) + + dataset_name = f"{context}_dataset_{dataset_cnt}" sample_df = pandas.DataFrame(sample_dict) dataset = mlflow.data.from_pandas(sample_df, name=dataset_name) exist_dataset_list = list( @@ -259,8 +267,6 @@ def _log_dataset(self, sample_dict: dict[str, Any], context: str | None = None) if not len(exist_dataset_list): datasets = [mlflow.entities.DatasetInput(dataset._to_mlflow_entity())] self.client.log_inputs(run_id=self.cur_run.info.run_id, datasets=datasets) - # Need to update the self.cur_run to sync the dataset log, otherwise the `inputs` info will be empty. - self.cur_run = self.client.get_run(self.cur_run.info.run_id) dataset_info = MLFlowHandler._get_pandas_dataset_info(dataset) self._log_params(dataset_info) @@ -412,19 +418,23 @@ def _default_dataset_log(self, engine: Engine) -> None: f"The engine dataloader is {dataloader} and the dataset of the dataloader is {dataset}." ) sample_dict: dict[str, list[str]] = {} - sample_dict["image"] = [] + sample_dict["images"] = [] for sample in dataset: if isinstance(sample, dict): - image = sample["image_meta_dict"]["filename_or_obj"] + image_name = sample["image_meta_dict"]["filename_or_obj"] if "image_meta_dict" in sample else None elif isinstance(sample, list): - image = sample[0]["image_meta_dict"]["filename_or_obj"] + # When using a transform like `RandCropByPosNegLabel`, a sample will be a list containing slices. + image_name = sample[0]["image_meta_dict"]["filename_or_obj"] if "image_meta_dict" in sample[0] else None else: - raise AttributeError(f"Expect samples' type to be list or dict, but got {type(sample)}") + image_name = None + warnings.warn(f"Don't support {type(sample)} type samples in dataset.") - if not isinstance(image, str): - raise ValueError(f"Expect image to be string, but got type {type(image)}.") + if not isinstance(image_name, str): + warnings.warn( + f"Expect the type of image name to be string, but got type {type(image_name)}. This will cause an empty dataset in MLFlow" + ) else: - sample_dict["image"].append(image) + sample_dict["images"].append(image_name) dataset_type = "train" if isinstance(engine, Trainer) else "nontrain" self._log_dataset(sample_dict, dataset_type) diff --git a/tests/test_handler_mlflow.py b/tests/test_handler_mlflow.py index e90a945286..f09f9b93d5 100644 --- a/tests/test_handler_mlflow.py +++ b/tests/test_handler_mlflow.py @@ -53,7 +53,6 @@ def _train_func(engine, batch): tracking_uri=path_to_uri(test_path), state_attributes=["test"], close_on_complete=True, - dataset_log=False, ) handler.attach(engine) engine.run(range(3), max_epochs=2) @@ -94,7 +93,6 @@ def _update_metric(engine): tracking_uri=path_to_uri(test_path), state_attributes=["test"], close_on_complete=True, - dataset_log=False, ) handler.attach(engine) engine.run(range(3), max_epochs=2) @@ -134,7 +132,6 @@ def _update_metric(engine): experiment_param=experiment_param, artifacts=[artifact_path], close_on_complete=True, - dataset_log=False, ) handler.attach(engine) engine.run(range(3), max_epochs=2) @@ -168,7 +165,6 @@ def _update_metric(engine): state_attributes=["test"], experiment_param=experiment_param, close_on_complete=True, - dataset_log=False, ) handler._default_epoch_log = MagicMock() handler.attach(engine) @@ -208,7 +204,6 @@ def _update_metric(engine): state_attributes=["test"], experiment_param=experiment_param, close_on_complete=True, - dataset_log=False, ) handler._default_iteration_log = MagicMock() handler.attach(engine) From dc0d24fea8d32f6d0c2658cca9b27770f6cc98d4 Mon Sep 17 00:00:00 2001 From: binliu Date: Mon, 26 Jun 2023 07:24:22 +0000 Subject: [PATCH 13/28] split the warning info to shorter strings Signed-off-by: binliu --- monai/handlers/mlflow_handler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py index 300302a20c..e8d90e7b21 100644 --- a/monai/handlers/mlflow_handler.py +++ b/monai/handlers/mlflow_handler.py @@ -432,7 +432,8 @@ def _default_dataset_log(self, engine: Engine) -> None: if not isinstance(image_name, str): warnings.warn( - f"Expect the type of image name to be string, but got type {type(image_name)}. This will cause an empty dataset in MLFlow" + f"Expect the type of image name to be string, but got type {type(image_name)}." + " This will cause an empty dataset in MLFlow" ) else: sample_dict["images"].append(image_name) From acd75f55bd369eabb7626b1d0070a8532d2ee5df Mon Sep 17 00:00:00 2001 From: binliu Date: Wed, 28 Jun 2023 07:03:34 +0000 Subject: [PATCH 14/28] Update the comments and warning messages in the code. Signed-off-by: binliu --- monai/handlers/mlflow_handler.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py index e8d90e7b21..493a4b57b6 100644 --- a/monai/handlers/mlflow_handler.py +++ b/monai/handlers/mlflow_handler.py @@ -77,7 +77,7 @@ class MLFlowHandler: iteration_logger: customized callable logger for iteration level logging with MLFlow. Must accept parameter "engine", use default logger if None. dataset_logger: customized callable logger to log the dataset information with MLFlow. - Must accept parameter "engine", use the default logger if None. + Must accept parameter "engine", use default logger if None. output_transform: a callable that is used to transform the ``ignite.engine.state.output`` into a scalar to track, or a dictionary of {key: scalar}. By default this value logging happens when every iteration completed. @@ -250,20 +250,21 @@ def _log_dataset(self, sample_dict: dict[str, Any], context: str = "nontrain") - if not self.cur_run: raise ValueError("Current Run is not Active to log the dataset") - # Need to update the self.cur_run to sync the dataset log, otherwise the `inputs` info will be empty. + # Need to update the self.cur_run to sync the dataset log, otherwise the `inputs` info will be out-of-date. self.cur_run = self.client.get_run(self.cur_run.info.run_id) logged_train_set = [x for x in self.cur_run.inputs.dataset_inputs if "train" == x.dataset.name[: len("train")]] logged_nontrain_set = [ x for x in self.cur_run.inputs.dataset_inputs if "nontrain" == x.dataset.name[: len("nontrain")] ] + # In case there are more datasets. dataset_cnt = str(len(logged_nontrain_set if context == "nontrain" else logged_train_set)) - dataset_name = f"{context}_dataset_{dataset_cnt}" sample_df = pandas.DataFrame(sample_dict) dataset = mlflow.data.from_pandas(sample_df, name=dataset_name) exist_dataset_list = list( filter(lambda x: x.dataset.digest == dataset.digest, self.cur_run.inputs.dataset_inputs) ) + if not len(exist_dataset_list): datasets = [mlflow.entities.DatasetInput(dataset._to_mlflow_entity())] self.client.log_inputs(run_id=self.cur_run.info.run_id, datasets=datasets) @@ -403,9 +404,12 @@ def _default_iteration_log(self, engine: Engine) -> None: def _default_dataset_log(self, engine: Engine) -> None: """ - Execute dataset log operation based on MONAI `engine.dataloader.dataset` data. - Abstract meta information from samples in dataset and build a Pandas DataFrame from it. - Create a PandasDataset in MLFlow using the meta information and log it. + Execute dataset log operation based on MONAI `Workflow.data_loader.dataset` data. + Abstract sample names in a dataset and build a Pandas DataFrame from it. To use this + function, every sample in the input dataset must have a filename, which can be fetched + from the `filename_or_obj` parameter in the `image_meta_dict` of the sample. + This function will log a PandasDataset, generated from the Pandas DataFrame, to MLFlow + inputs. Args: engine: Ignite Engine, it can be a trainer, validator or evaluator. @@ -414,26 +418,24 @@ def _default_dataset_log(self, engine: Engine) -> None: dataloader = getattr(engine, "data_loader", None) dataset = getattr(dataloader, "dataset", None) if dataloader else None if not dataset: - raise AttributeError( - f"The engine dataloader is {dataloader} and the dataset of the dataloader is {dataset}." - ) + raise AttributeError(f"The dataset of the engine is None. Cannot record it with MLFlow.") + sample_dict: dict[str, list[str]] = {} sample_dict["images"] = [] - for sample in dataset: if isinstance(sample, dict): image_name = sample["image_meta_dict"]["filename_or_obj"] if "image_meta_dict" in sample else None elif isinstance(sample, list): - # When using a transform like `RandCropByPosNegLabel`, a sample will be a list containing slices. + # When using a transform like `RandCropByPosNegLabel`, a sample will be a list containing image slices. image_name = sample[0]["image_meta_dict"]["filename_or_obj"] if "image_meta_dict" in sample[0] else None else: image_name = None - warnings.warn(f"Don't support {type(sample)} type samples in dataset.") + warnings.warn(f"Don't support {type(sample)} type samples when recording the dataset with MLFlow.") if not isinstance(image_name, str): warnings.warn( - f"Expect the type of image name to be string, but got type {type(image_name)}." - " This will cause an empty dataset in MLFlow" + f"Expected type string, got type {type(image_name)} of the image name." + "May log an empty dataset in MLFlow" ) else: sample_dict["images"].append(image_name) From f16d1ef047f628818937415c01018fcf51d60b3d Mon Sep 17 00:00:00 2001 From: binliu Date: Wed, 28 Jun 2023 08:13:04 +0000 Subject: [PATCH 15/28] Add the link to PandasDataset. Signed-off-by: binliu --- monai/handlers/mlflow_handler.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py index 493a4b57b6..1a318ed22f 100644 --- a/monai/handlers/mlflow_handler.py +++ b/monai/handlers/mlflow_handler.py @@ -409,7 +409,10 @@ def _default_dataset_log(self, engine: Engine) -> None: function, every sample in the input dataset must have a filename, which can be fetched from the `filename_or_obj` parameter in the `image_meta_dict` of the sample. This function will log a PandasDataset, generated from the Pandas DataFrame, to MLFlow - inputs. + inputs. For more details about PandasDataset, please refer to this link: + https://mlflow.org/docs/latest/python_api/mlflow.data.html#mlflow.data.pandas_dataset.PandasDataset + + Please note that it may take a while to record the dataset if it has too many samples. Args: engine: Ignite Engine, it can be a trainer, validator or evaluator. @@ -418,7 +421,7 @@ def _default_dataset_log(self, engine: Engine) -> None: dataloader = getattr(engine, "data_loader", None) dataset = getattr(dataloader, "dataset", None) if dataloader else None if not dataset: - raise AttributeError(f"The dataset of the engine is None. Cannot record it with MLFlow.") + raise AttributeError("The dataset of the engine is None. Cannot record it by MLFlow.") sample_dict: dict[str, list[str]] = {} sample_dict["images"] = [] From 93777e68b5e3aa9095570861a70d6a9bbc7735e2 Mon Sep 17 00:00:00 2001 From: binliu Date: Thu, 29 Jun 2023 14:39:48 +0000 Subject: [PATCH 16/28] add the unit test case Signed-off-by: binliu --- tests/test_handler_mlflow.py | 55 +++++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/tests/test_handler_mlflow.py b/tests/test_handler_mlflow.py index f09f9b93d5..ef44de6451 100644 --- a/tests/test_handler_mlflow.py +++ b/tests/test_handler_mlflow.py @@ -23,8 +23,13 @@ from ignite.engine import Engine, Events from parameterized import parameterized +from monai.apps import download_and_extract +from monai.bundle import ConfigWorkflow, download from monai.handlers import MLFlowHandler -from monai.utils import path_to_uri +from monai.utils import optional_import, path_to_uri +from tests.utils import skip_if_quick + +_, has_dataset_tracking = optional_import("mlflow", "2.4.0") def get_event_filter(e): @@ -230,6 +235,54 @@ def test_multi_thread(self): self.tmpdir_list.append(res) self.assertTrue(len(glob.glob(res)) > 0) + @skip_if_quick + @unittest.skipUnless(has_dataset_tracking, reason="Requires mlflow version >= 2.4.0.") + def test_dataset_tracking(self): + test_bundle_name = "endoscopic_tool_segmentation" + with tempfile.TemporaryDirectory() as tempdir: + resource = "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/endoscopic_tool_dataset.zip" + md5 = "f82da47259c0a617202fb54624798a55" + compressed_file = os.path.join(tempdir, "endoscopic_tool_segmentation.zip") + data_dir = os.path.join(tempdir, "endoscopic_tool_dataset") + if not os.path.exists(data_dir): + download_and_extract(resource, compressed_file, tempdir, md5) + + download(test_bundle_name, bundle_dir=tempdir) + + bundle_root = os.path.join(tempdir, test_bundle_name) + config_file = os.path.join(bundle_root, "configs/inference.json") + meta_file = os.path.join(bundle_root, "configs/metadata.json") + logging_file = os.path.join(bundle_root, "configs/logging.conf") + workflow = ConfigWorkflow( + workflow="infer", + config_file=config_file, + meta_file=meta_file, + logging_file=logging_file, + init_id="initialize", + run_id="run", + final_id="finalize", + ) + tracking_path = os.path.join(bundle_root, "eval") + mlflow_handler = MLFlowHandler( + iteration_log=False, + epoch_log=False, + dataset_log=True, + tracking_uri=path_to_uri(tracking_path), + ) + + workflow.bundle_root = bundle_root + workflow.dataset_dir = data_dir + workflow.initialize() + mlflow_handler.attach(workflow.evaluator) + workflow.run() + workflow.finalize() + + cur_run = mlflow_handler.client.get_run(mlflow_handler.cur_run.info.run_id) + logged_nontrain_set = [ + x for x in cur_run.inputs.dataset_inputs if "nontrain" == x.dataset.name[: len("nontrain")] + ] + self.assertEqual(len(logged_nontrain_set), 1) + mlflow_handler.close() if __name__ == "__main__": unittest.main() From 657cb0fc62fab15b215934a52c1a738e938c69a2 Mon Sep 17 00:00:00 2001 From: binliu Date: Thu, 29 Jun 2023 14:41:07 +0000 Subject: [PATCH 17/28] fix the format issue in the unit test Signed-off-by: binliu --- tests/test_handler_mlflow.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_handler_mlflow.py b/tests/test_handler_mlflow.py index ef44de6451..898bc340bc 100644 --- a/tests/test_handler_mlflow.py +++ b/tests/test_handler_mlflow.py @@ -264,10 +264,7 @@ def test_dataset_tracking(self): ) tracking_path = os.path.join(bundle_root, "eval") mlflow_handler = MLFlowHandler( - iteration_log=False, - epoch_log=False, - dataset_log=True, - tracking_uri=path_to_uri(tracking_path), + iteration_log=False, epoch_log=False, dataset_log=True, tracking_uri=path_to_uri(tracking_path) ) workflow.bundle_root = bundle_root @@ -284,5 +281,6 @@ def test_dataset_tracking(self): self.assertEqual(len(logged_nontrain_set), 1) mlflow_handler.close() + if __name__ == "__main__": unittest.main() From d514256a105b9d8010d69de2c174d12a97df7e1d Mon Sep 17 00:00:00 2001 From: binliu Date: Thu, 29 Jun 2023 14:48:04 +0000 Subject: [PATCH 18/28] add the mlflow version requirement. Signed-off-by: binliu --- monai/handlers/mlflow_handler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py index 1a318ed22f..1feec6d52d 100644 --- a/monai/handlers/mlflow_handler.py +++ b/monai/handlers/mlflow_handler.py @@ -71,7 +71,8 @@ class MLFlowHandler: epoch_log: whether to log data to MLFlow when epoch completed, default to `True`. ``epoch_log`` can be also a function and it will be interpreted as an event filter. See ``iteration_log`` argument for more details. - dataset_log: whether to log information about the dataset at the beginning. + dataset_log: whether to log information about the dataset at the beginning. This arg + is only useful when MLFlow version >= 2.4.0. epoch_logger: customized callable logger for epoch level logging with MLFlow. Must accept parameter "engine", use default logger if None. iteration_logger: customized callable logger for iteration level logging with MLFlow. From 58d975bf2c504faa52c46354e8ed94365f5f61b9 Mon Sep 17 00:00:00 2001 From: binliu Date: Wed, 5 Jul 2023 08:24:50 +0000 Subject: [PATCH 19/28] add the skip_if_downloading_fails context manager to the test case. Signed-off-by: binliu --- tests/test_handler_mlflow.py | 73 ++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 36 deletions(-) diff --git a/tests/test_handler_mlflow.py b/tests/test_handler_mlflow.py index 898bc340bc..adaf9e8252 100644 --- a/tests/test_handler_mlflow.py +++ b/tests/test_handler_mlflow.py @@ -27,7 +27,7 @@ from monai.bundle import ConfigWorkflow, download from monai.handlers import MLFlowHandler from monai.utils import optional_import, path_to_uri -from tests.utils import skip_if_quick +from tests.utils import skip_if_downloading_fails, skip_if_quick _, has_dataset_tracking = optional_import("mlflow", "2.4.0") @@ -244,42 +244,43 @@ def test_dataset_tracking(self): md5 = "f82da47259c0a617202fb54624798a55" compressed_file = os.path.join(tempdir, "endoscopic_tool_segmentation.zip") data_dir = os.path.join(tempdir, "endoscopic_tool_dataset") - if not os.path.exists(data_dir): - download_and_extract(resource, compressed_file, tempdir, md5) - - download(test_bundle_name, bundle_dir=tempdir) - - bundle_root = os.path.join(tempdir, test_bundle_name) - config_file = os.path.join(bundle_root, "configs/inference.json") - meta_file = os.path.join(bundle_root, "configs/metadata.json") - logging_file = os.path.join(bundle_root, "configs/logging.conf") - workflow = ConfigWorkflow( - workflow="infer", - config_file=config_file, - meta_file=meta_file, - logging_file=logging_file, - init_id="initialize", - run_id="run", - final_id="finalize", - ) - tracking_path = os.path.join(bundle_root, "eval") - mlflow_handler = MLFlowHandler( - iteration_log=False, epoch_log=False, dataset_log=True, tracking_uri=path_to_uri(tracking_path) - ) + with skip_if_downloading_fails(): + if not os.path.exists(data_dir): + download_and_extract(resource, compressed_file, tempdir, md5) + + download(test_bundle_name, bundle_dir=tempdir) + + bundle_root = os.path.join(tempdir, test_bundle_name) + config_file = os.path.join(bundle_root, "configs/inference.json") + meta_file = os.path.join(bundle_root, "configs/metadata.json") + logging_file = os.path.join(bundle_root, "configs/logging.conf") + workflow = ConfigWorkflow( + workflow="infer", + config_file=config_file, + meta_file=meta_file, + logging_file=logging_file, + init_id="initialize", + run_id="run", + final_id="finalize", + ) + tracking_path = os.path.join(bundle_root, "eval") + mlflow_handler = MLFlowHandler( + iteration_log=False, epoch_log=False, dataset_log=True, tracking_uri=path_to_uri(tracking_path) + ) - workflow.bundle_root = bundle_root - workflow.dataset_dir = data_dir - workflow.initialize() - mlflow_handler.attach(workflow.evaluator) - workflow.run() - workflow.finalize() - - cur_run = mlflow_handler.client.get_run(mlflow_handler.cur_run.info.run_id) - logged_nontrain_set = [ - x for x in cur_run.inputs.dataset_inputs if "nontrain" == x.dataset.name[: len("nontrain")] - ] - self.assertEqual(len(logged_nontrain_set), 1) - mlflow_handler.close() + workflow.bundle_root = bundle_root + workflow.dataset_dir = data_dir + workflow.initialize() + mlflow_handler.attach(workflow.evaluator) + workflow.run() + workflow.finalize() + + cur_run = mlflow_handler.client.get_run(mlflow_handler.cur_run.info.run_id) + logged_nontrain_set = [ + x for x in cur_run.inputs.dataset_inputs if "nontrain" == x.dataset.name[: len("nontrain")] + ] + self.assertEqual(len(logged_nontrain_set), 1) + mlflow_handler.close() if __name__ == "__main__": From 08e31c8e7090d0cf2d642e1be556c04e4fbdc8de Mon Sep 17 00:00:00 2001 From: binliu Date: Fri, 7 Jul 2023 07:43:23 +0000 Subject: [PATCH 20/28] update mlflow handler according to the reviewers opinions. Signed-off-by: binliu --- monai/bundle/utils.py | 12 +++- monai/handlers/mlflow_handler.py | 119 ++++++++++++++++++------------- tests/test_handler_mlflow.py | 12 ++-- 3 files changed, 88 insertions(+), 55 deletions(-) diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py index 4da63f54ad..b41d9c397c 100644 --- a/monai/bundle/utils.py +++ b/monai/bundle/utils.py @@ -114,7 +114,11 @@ "run_name": None, # may fill it at runtime "execute_config": None, - "dataset_log": True, + "dataset_log": False, + "tracking_dataset_train": {"train": f"@train{ID_SEP_KEY}dataset"}, + "tracking_dataset_val": {"val": f"@validate{ID_SEP_KEY}dataset"}, + "tracking_dataset_infer": {"test": f"@dataset"}, + "dataset_keys": "image", "is_not_rank0": ( "$torch.distributed.is_available() \ and torch.distributed.is_initialized() and torch.distributed.get_rank() > 0" @@ -133,6 +137,8 @@ "output_transform": "$monai.handlers.from_engine(['loss'], first=True)", "close_on_complete": True, "dataset_log": "@dataset_log", + "dataset_dict": "@tracking_dataset_train", + "dataset_keys": "@dataset_keys", }, # MLFlowHandler config for the validator "validator": { @@ -143,6 +149,8 @@ "run_name": "@run_name", "iteration_log": False, "dataset_log": "@dataset_log", + "dataset_dict": "@tracking_dataset_val", + "dataset_keys": "@dataset_keys", }, # MLFlowHandler config for the evaluator "evaluator": { @@ -155,6 +163,8 @@ "iteration_log": False, "close_on_complete": True, "dataset_log": "@dataset_log", + "dataset_dict": "@tracking_dataset_infer", + "dataset_keys": "@dataset_keys", }, }, } diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py index 1feec6d52d..248b0a5181 100644 --- a/monai/handlers/mlflow_handler.py +++ b/monai/handlers/mlflow_handler.py @@ -14,14 +14,15 @@ import os import time import warnings -from collections.abc import Callable, Sequence +from collections.abc import Callable, Mapping, Sequence from pathlib import Path from typing import TYPE_CHECKING, Any import torch +from torch.utils.data import Dataset +from tqdm import tqdm -from monai.config import IgniteInfo -from monai.engines import Trainer +from monai.config import IgniteInfo, KeysCollection from monai.utils import ensure_tuple, min_version, optional_import Events, _ = optional_import("ignite.engine", IgniteInfo.OPT_IMPORT_VERSION, min_version, "Events") @@ -72,13 +73,18 @@ class MLFlowHandler: ``epoch_log`` can be also a function and it will be interpreted as an event filter. See ``iteration_log`` argument for more details. dataset_log: whether to log information about the dataset at the beginning. This arg - is only useful when MLFlow version >= 2.4.0. + is only useful when MLFlow version >= 2.4.0. For more details, please go to the + website: https://mlflow.org/docs/latest/python_api/mlflow.data.html. epoch_logger: customized callable logger for epoch level logging with MLFlow. Must accept parameter "engine", use default logger if None. iteration_logger: customized callable logger for iteration level logging with MLFlow. Must accept parameter "engine", use default logger if None. dataset_logger: customized callable logger to log the dataset information with MLFlow. - Must accept parameter "engine", use default logger if None. + Must accept parameter "dataset_dict", use default logger if None. + dataset_dict: a dictionary in which the key is the name of the dataset and the value is a PyTorch + dataset, that needs to be recorded. + dataset_keys: a key or a collection of keys to indicate contents in the dataset that + need to be stored by MLFlow. output_transform: a callable that is used to transform the ``ignite.engine.state.output`` into a scalar to track, or a dictionary of {key: scalar}. By default this value logging happens when every iteration completed. @@ -119,7 +125,9 @@ def __init__( dataset_log: bool = False, epoch_logger: Callable[[Engine], Any] | None = None, iteration_logger: Callable[[Engine], Any] | None = None, - dataset_logger: Callable[[Engine], Any] | None = None, + dataset_logger: Callable[[Mapping[KeysCollection, Dataset]], Any] | None = None, + dataset_dict: Mapping[KeysCollection, Dataset] | None = None, + dataset_keys: KeysCollection = "image", output_transform: Callable = lambda x: x[0], global_epoch_transform: Callable = lambda x: x, state_attributes: Sequence[str] | None = None, @@ -151,6 +159,8 @@ def __init__( self.close_on_complete = close_on_complete self.experiment = None self.cur_run = None + self.dataset_dict = dataset_dict + self.dataset_keys = ensure_tuple(dataset_keys) def _delete_exist_param_in_dict(self, param_dict: dict) -> None: """ @@ -223,9 +233,9 @@ def start(self, engine: Engine) -> None: if self.dataset_log: if self.dataset_logger: - self.dataset_logger(engine) + self.dataset_logger(self.dataset_dict) else: - self._default_dataset_log(engine) + self._default_dataset_log(self.dataset_dict) def _set_experiment(self): experiment = self.experiment @@ -247,19 +257,16 @@ def _get_pandas_dataset_info(pandas_dataset): f"{dataset_name}_samples": pandas_dataset.profile["num_rows"], } - def _log_dataset(self, sample_dict: dict[str, Any], context: str = "nontrain") -> None: + def _log_dataset(self, sample_dict: dict[str, Any], context: str = "train") -> None: if not self.cur_run: raise ValueError("Current Run is not Active to log the dataset") # Need to update the self.cur_run to sync the dataset log, otherwise the `inputs` info will be out-of-date. self.cur_run = self.client.get_run(self.cur_run.info.run_id) - logged_train_set = [x for x in self.cur_run.inputs.dataset_inputs if "train" == x.dataset.name[: len("train")]] - logged_nontrain_set = [ - x for x in self.cur_run.inputs.dataset_inputs if "nontrain" == x.dataset.name[: len("nontrain")] - ] - # In case there are more datasets. - dataset_cnt = str(len(logged_nontrain_set if context == "nontrain" else logged_train_set)) - dataset_name = f"{context}_dataset_{dataset_cnt}" + logged_set = [x for x in self.cur_run.inputs.dataset_inputs if x.dataset.name.startswith(context)] + # In case there are datasets with the same name. + dataset_count = str(len(logged_set)) + dataset_name = f"{context}_dataset_{dataset_count}" sample_df = pandas.DataFrame(sample_dict) dataset = mlflow.data.from_pandas(sample_df, name=dataset_name) exist_dataset_list = list( @@ -403,45 +410,57 @@ def _default_iteration_log(self, engine: Engine) -> None: } self._log_metrics(params, step=engine.state.iteration) - def _default_dataset_log(self, engine: Engine) -> None: + def _default_dataset_log(self, dataset_dict: Mapping[KeysCollection, Dataset]) -> None: """ - Execute dataset log operation based on MONAI `Workflow.data_loader.dataset` data. - Abstract sample names in a dataset and build a Pandas DataFrame from it. To use this - function, every sample in the input dataset must have a filename, which can be fetched - from the `filename_or_obj` parameter in the `image_meta_dict` of the sample. - This function will log a PandasDataset, generated from the Pandas DataFrame, to MLFlow - inputs. For more details about PandasDataset, please refer to this link: + Execute dataset log operation based on the input dataset_dict. The dataset_dict should have a format + like: + { + "dataset_name0": dataset0, + "dataset_name1": dataset1, + ...... + } + The keys stand for names of datasets, which will be logged as prefixes of dataset names in MLFlow. + The values are PyTorch datasets from which sample names are abstracted to build a Pandas DataFrame. + + To use this function, every sample in the input datasets must contain keys specified by the `dataset_keys` + parameter. + This function will log a PandasDataset to MLFlow inputs, generated from the Pandas DataFrame. + For more details about PandasDataset, please refer to this link: https://mlflow.org/docs/latest/python_api/mlflow.data.html#mlflow.data.pandas_dataset.PandasDataset Please note that it may take a while to record the dataset if it has too many samples. Args: - engine: Ignite Engine, it can be a trainer, validator or evaluator. + dataset_dict: a dictionary in which the key is the name of the dataset and the value is a PyTorch + dataset, that needs to be recorded. """ - dataloader = getattr(engine, "data_loader", None) - dataset = getattr(dataloader, "dataset", None) if dataloader else None - if not dataset: - raise AttributeError("The dataset of the engine is None. Cannot record it by MLFlow.") - - sample_dict: dict[str, list[str]] = {} - sample_dict["images"] = [] - for sample in dataset: - if isinstance(sample, dict): - image_name = sample["image_meta_dict"]["filename_or_obj"] if "image_meta_dict" in sample else None - elif isinstance(sample, list): - # When using a transform like `RandCropByPosNegLabel`, a sample will be a list containing image slices. - image_name = sample[0]["image_meta_dict"]["filename_or_obj"] if "image_meta_dict" in sample[0] else None - else: - image_name = None - warnings.warn(f"Don't support {type(sample)} type samples when recording the dataset with MLFlow.") - - if not isinstance(image_name, str): - warnings.warn( - f"Expected type string, got type {type(image_name)} of the image name." - "May log an empty dataset in MLFlow" - ) - else: - sample_dict["images"].append(image_name) - dataset_type = "train" if isinstance(engine, Trainer) else "nontrain" - self._log_dataset(sample_dict, dataset_type) + + if len(dataset_dict) == 0: + warnings.WarningMessage("There is no dataset to log!") + + # Log datasets to MLFlow one by one. + for dataset_type, dataset in dataset_dict.items(): + if dataset is None: + raise AttributeError(f"The {dataset_type} dataset of is None. Cannot record it by MLFlow.") + + sample_dict: dict[str, list[str]] = {} + + for sample in tqdm(dataset.data, f"Recording the {dataset_type} dataset"): + for key in self.dataset_keys: + if not key in sample_dict: + sample_dict[key] = [] + + if key in sample: + value_to_log = sample[key] + else: + raise KeyError(f"Unexpect key '{key}' in the sample.") + + if not isinstance(value_to_log, str): + warnings.warn( + f"Expected type string, got type {type(value_to_log)} of the {key} name." + "May log an empty dataset in MLFlow" + ) + else: + sample_dict[key].append(value_to_log) + self._log_dataset(sample_dict, dataset_type) diff --git a/tests/test_handler_mlflow.py b/tests/test_handler_mlflow.py index adaf9e8252..8863466c21 100644 --- a/tests/test_handler_mlflow.py +++ b/tests/test_handler_mlflow.py @@ -263,9 +263,15 @@ def test_dataset_tracking(self): run_id="run", final_id="finalize", ) + workflow.initialize() + infer_dataset = workflow.dataset tracking_path = os.path.join(bundle_root, "eval") mlflow_handler = MLFlowHandler( - iteration_log=False, epoch_log=False, dataset_log=True, tracking_uri=path_to_uri(tracking_path) + iteration_log=False, + epoch_log=False, + dataset_log=True, + dataset_dict={"test": infer_dataset}, + tracking_uri=path_to_uri(tracking_path), ) workflow.bundle_root = bundle_root @@ -276,9 +282,7 @@ def test_dataset_tracking(self): workflow.finalize() cur_run = mlflow_handler.client.get_run(mlflow_handler.cur_run.info.run_id) - logged_nontrain_set = [ - x for x in cur_run.inputs.dataset_inputs if "nontrain" == x.dataset.name[: len("nontrain")] - ] + logged_nontrain_set = [x for x in cur_run.inputs.dataset_inputs if x.dataset.name.startswith("test")] self.assertEqual(len(logged_nontrain_set), 1) mlflow_handler.close() From 87edbb8af13bea309393560ed28e1a289bb684ad Mon Sep 17 00:00:00 2001 From: binliu Date: Fri, 7 Jul 2023 08:47:34 +0000 Subject: [PATCH 21/28] update the test case Signed-off-by: binliu --- tests/test_handler_mlflow.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/test_handler_mlflow.py b/tests/test_handler_mlflow.py index 8863466c21..268f3ede34 100644 --- a/tests/test_handler_mlflow.py +++ b/tests/test_handler_mlflow.py @@ -263,9 +263,12 @@ def test_dataset_tracking(self): run_id="run", final_id="finalize", ) + + tracking_path = os.path.join(bundle_root, "eval") + workflow.bundle_root = bundle_root + workflow.dataset_dir = data_dir workflow.initialize() infer_dataset = workflow.dataset - tracking_path = os.path.join(bundle_root, "eval") mlflow_handler = MLFlowHandler( iteration_log=False, epoch_log=False, @@ -273,10 +276,6 @@ def test_dataset_tracking(self): dataset_dict={"test": infer_dataset}, tracking_uri=path_to_uri(tracking_path), ) - - workflow.bundle_root = bundle_root - workflow.dataset_dir = data_dir - workflow.initialize() mlflow_handler.attach(workflow.evaluator) workflow.run() workflow.finalize() From 0ffd97c652435ea37c4e8d2a780c8d56fefc15bd Mon Sep 17 00:00:00 2001 From: binliu Date: Fri, 7 Jul 2023 08:50:30 +0000 Subject: [PATCH 22/28] fix the format issue Signed-off-by: binliu --- monai/bundle/utils.py | 2 +- monai/handlers/mlflow_handler.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py index b41d9c397c..396a33589b 100644 --- a/monai/bundle/utils.py +++ b/monai/bundle/utils.py @@ -117,7 +117,7 @@ "dataset_log": False, "tracking_dataset_train": {"train": f"@train{ID_SEP_KEY}dataset"}, "tracking_dataset_val": {"val": f"@validate{ID_SEP_KEY}dataset"}, - "tracking_dataset_infer": {"test": f"@dataset"}, + "tracking_dataset_infer": {"test": "@dataset"}, "dataset_keys": "image", "is_not_rank0": ( "$torch.distributed.is_available() \ diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py index 248b0a5181..40ebc79fd3 100644 --- a/monai/handlers/mlflow_handler.py +++ b/monai/handlers/mlflow_handler.py @@ -448,7 +448,7 @@ def _default_dataset_log(self, dataset_dict: Mapping[KeysCollection, Dataset]) - for sample in tqdm(dataset.data, f"Recording the {dataset_type} dataset"): for key in self.dataset_keys: - if not key in sample_dict: + if key not in sample_dict: sample_dict[key] = [] if key in sample: From 556618bf3e89d65e9f18d6eae9d3b9d0509f9814 Mon Sep 17 00:00:00 2001 From: binliu Date: Fri, 7 Jul 2023 09:31:46 +0000 Subject: [PATCH 23/28] make tqdm as optional import Signed-off-by: binliu --- monai/handlers/mlflow_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py index 40ebc79fd3..c768098ff6 100644 --- a/monai/handlers/mlflow_handler.py +++ b/monai/handlers/mlflow_handler.py @@ -20,7 +20,6 @@ import torch from torch.utils.data import Dataset -from tqdm import tqdm from monai.config import IgniteInfo, KeysCollection from monai.utils import ensure_tuple, min_version, optional_import @@ -31,6 +30,7 @@ "mlflow.entities", descriptor="Please install mlflow.entities before using MLFlowHandler." ) pandas, _ = optional_import("pandas", descriptor="Please install pandas for recording the dataset.") +tqdm, _ = optional_import("tqdm", "4.47.0", min_version, "tqdm") if TYPE_CHECKING: from ignite.engine import Engine From 97093030138486311861a2a7656d5244a3729959 Mon Sep 17 00:00:00 2001 From: binliu Date: Fri, 7 Jul 2023 13:58:25 +0000 Subject: [PATCH 24/28] fix the mypy issue Signed-off-by: binliu --- monai/handlers/mlflow_handler.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py index c768098ff6..3f7efe4980 100644 --- a/monai/handlers/mlflow_handler.py +++ b/monai/handlers/mlflow_handler.py @@ -257,7 +257,7 @@ def _get_pandas_dataset_info(pandas_dataset): f"{dataset_name}_samples": pandas_dataset.profile["num_rows"], } - def _log_dataset(self, sample_dict: dict[str, Any], context: str = "train") -> None: + def _log_dataset(self, sample_dict: dict[str, Any], context: str | KeysCollection = "train") -> None: if not self.cur_run: raise ValueError("Current Run is not Active to log the dataset") @@ -437,7 +437,7 @@ def _default_dataset_log(self, dataset_dict: Mapping[KeysCollection, Dataset]) - """ if len(dataset_dict) == 0: - warnings.WarningMessage("There is no dataset to log!") + warnings.warn("There is no dataset to log!") # Log datasets to MLFlow one by one. for dataset_type, dataset in dataset_dict.items(): @@ -445,8 +445,8 @@ def _default_dataset_log(self, dataset_dict: Mapping[KeysCollection, Dataset]) - raise AttributeError(f"The {dataset_type} dataset of is None. Cannot record it by MLFlow.") sample_dict: dict[str, list[str]] = {} - - for sample in tqdm(dataset.data, f"Recording the {dataset_type} dataset"): + dataset_samples = getattr(dataset, "data", []) + for sample in tqdm(dataset_samples, f"Recording the {dataset_type} dataset"): for key in self.dataset_keys: if key not in sample_dict: sample_dict[key] = [] From 856f377f8f0c1294367eddb40e8b5f3311a2e432 Mon Sep 17 00:00:00 2001 From: binliu Date: Mon, 10 Jul 2023 08:42:30 +0000 Subject: [PATCH 25/28] set the dataset name type to string Signed-off-by: binliu --- monai/handlers/mlflow_handler.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py index 3f7efe4980..5d05e20537 100644 --- a/monai/handlers/mlflow_handler.py +++ b/monai/handlers/mlflow_handler.py @@ -125,8 +125,8 @@ def __init__( dataset_log: bool = False, epoch_logger: Callable[[Engine], Any] | None = None, iteration_logger: Callable[[Engine], Any] | None = None, - dataset_logger: Callable[[Mapping[KeysCollection, Dataset]], Any] | None = None, - dataset_dict: Mapping[KeysCollection, Dataset] | None = None, + dataset_logger: Callable[[Mapping[str, Dataset]], Any] | None = None, + dataset_dict: Mapping[str, Dataset] | None = None, dataset_keys: KeysCollection = "image", output_transform: Callable = lambda x: x[0], global_epoch_transform: Callable = lambda x: x, @@ -257,7 +257,7 @@ def _get_pandas_dataset_info(pandas_dataset): f"{dataset_name}_samples": pandas_dataset.profile["num_rows"], } - def _log_dataset(self, sample_dict: dict[str, Any], context: str | KeysCollection = "train") -> None: + def _log_dataset(self, sample_dict: dict[str, Any], context: str = "train") -> None: if not self.cur_run: raise ValueError("Current Run is not Active to log the dataset") @@ -410,7 +410,7 @@ def _default_iteration_log(self, engine: Engine) -> None: } self._log_metrics(params, step=engine.state.iteration) - def _default_dataset_log(self, dataset_dict: Mapping[KeysCollection, Dataset]) -> None: + def _default_dataset_log(self, dataset_dict: Mapping[str, Dataset]) -> None: """ Execute dataset log operation based on the input dataset_dict. The dataset_dict should have a format like: From 83d8d6faabaeb62a880e6f19fefbbd1b07210f38 Mon Sep 17 00:00:00 2001 From: binliu Date: Mon, 10 Jul 2023 09:19:37 +0000 Subject: [PATCH 26/28] Remove the dataset_log parameter Signed-off-by: binliu --- monai/handlers/mlflow_handler.py | 29 ++++++++++++++--------------- tests/test_handler_mlflow.py | 1 - 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py index 5d05e20537..e7d978c70b 100644 --- a/monai/handlers/mlflow_handler.py +++ b/monai/handlers/mlflow_handler.py @@ -22,7 +22,7 @@ from torch.utils.data import Dataset from monai.config import IgniteInfo, KeysCollection -from monai.utils import ensure_tuple, min_version, optional_import +from monai.utils import CommonKeys, ensure_tuple, min_version, optional_import Events, _ = optional_import("ignite.engine", IgniteInfo.OPT_IMPORT_VERSION, min_version, "Events") mlflow, _ = optional_import("mlflow", descriptor="Please install mlflow before using MLFlowHandler.") @@ -72,9 +72,6 @@ class MLFlowHandler: epoch_log: whether to log data to MLFlow when epoch completed, default to `True`. ``epoch_log`` can be also a function and it will be interpreted as an event filter. See ``iteration_log`` argument for more details. - dataset_log: whether to log information about the dataset at the beginning. This arg - is only useful when MLFlow version >= 2.4.0. For more details, please go to the - website: https://mlflow.org/docs/latest/python_api/mlflow.data.html. epoch_logger: customized callable logger for epoch level logging with MLFlow. Must accept parameter "engine", use default logger if None. iteration_logger: customized callable logger for iteration level logging with MLFlow. @@ -82,7 +79,9 @@ class MLFlowHandler: dataset_logger: customized callable logger to log the dataset information with MLFlow. Must accept parameter "dataset_dict", use default logger if None. dataset_dict: a dictionary in which the key is the name of the dataset and the value is a PyTorch - dataset, that needs to be recorded. + dataset, that needs to be recorded. This arg is only useful when MLFlow version >= 2.4.0. + For more details about how to log data with MLFlow, please go to the website: + https://mlflow.org/docs/latest/python_api/mlflow.data.html. dataset_keys: a key or a collection of keys to indicate contents in the dataset that need to be stored by MLFlow. output_transform: a callable that is used to transform the @@ -122,12 +121,11 @@ def __init__( tracking_uri: str | None = None, iteration_log: bool | Callable[[Engine, int], bool] = True, epoch_log: bool | Callable[[Engine, int], bool] = True, - dataset_log: bool = False, epoch_logger: Callable[[Engine], Any] | None = None, iteration_logger: Callable[[Engine], Any] | None = None, dataset_logger: Callable[[Mapping[str, Dataset]], Any] | None = None, dataset_dict: Mapping[str, Dataset] | None = None, - dataset_keys: KeysCollection = "image", + dataset_keys: CommonKeys = CommonKeys.IMAGE, output_transform: Callable = lambda x: x[0], global_epoch_transform: Callable = lambda x: x, state_attributes: Sequence[str] | None = None, @@ -141,7 +139,6 @@ def __init__( ) -> None: self.iteration_log = iteration_log self.epoch_log = epoch_log - self.dataset_log = dataset_log self.epoch_logger = epoch_logger self.iteration_logger = iteration_logger self.dataset_logger = dataset_logger @@ -231,11 +228,10 @@ def start(self, engine: Engine) -> None: self._delete_exist_param_in_dict(attrs) self._log_params(attrs) - if self.dataset_log: - if self.dataset_logger: - self.dataset_logger(self.dataset_dict) - else: - self._default_dataset_log(self.dataset_dict) + if self.dataset_logger: + self.dataset_logger(self.dataset_dict) + else: + self._default_dataset_log(self.dataset_dict) def _set_experiment(self): experiment = self.experiment @@ -410,7 +406,7 @@ def _default_iteration_log(self, engine: Engine) -> None: } self._log_metrics(params, step=engine.state.iteration) - def _default_dataset_log(self, dataset_dict: Mapping[str, Dataset]) -> None: + def _default_dataset_log(self, dataset_dict: Mapping[str, Dataset] | None) -> None: """ Execute dataset log operation based on the input dataset_dict. The dataset_dict should have a format like: @@ -421,6 +417,7 @@ def _default_dataset_log(self, dataset_dict: Mapping[str, Dataset]) -> None: } The keys stand for names of datasets, which will be logged as prefixes of dataset names in MLFlow. The values are PyTorch datasets from which sample names are abstracted to build a Pandas DataFrame. + If the input dataset_dict is None, this function will directly return and do nothing. To use this function, every sample in the input datasets must contain keys specified by the `dataset_keys` parameter. @@ -436,7 +433,9 @@ def _default_dataset_log(self, dataset_dict: Mapping[str, Dataset]) -> None: """ - if len(dataset_dict) == 0: + if dataset_dict is None: + return + elif len(dataset_dict) == 0: warnings.warn("There is no dataset to log!") # Log datasets to MLFlow one by one. diff --git a/tests/test_handler_mlflow.py b/tests/test_handler_mlflow.py index 268f3ede34..d5578c01bc 100644 --- a/tests/test_handler_mlflow.py +++ b/tests/test_handler_mlflow.py @@ -272,7 +272,6 @@ def test_dataset_tracking(self): mlflow_handler = MLFlowHandler( iteration_log=False, epoch_log=False, - dataset_log=True, dataset_dict={"test": infer_dataset}, tracking_uri=path_to_uri(tracking_path), ) From cf83a7efc02d0176594c1ef62ed50c1b20903118 Mon Sep 17 00:00:00 2001 From: binliu Date: Mon, 10 Jul 2023 09:24:15 +0000 Subject: [PATCH 27/28] Remove the dataset log settings from the default MLFlow setting. Signed-off-by: binliu --- monai/bundle/utils.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py index 396a33589b..62d4975d94 100644 --- a/monai/bundle/utils.py +++ b/monai/bundle/utils.py @@ -114,11 +114,6 @@ "run_name": None, # may fill it at runtime "execute_config": None, - "dataset_log": False, - "tracking_dataset_train": {"train": f"@train{ID_SEP_KEY}dataset"}, - "tracking_dataset_val": {"val": f"@validate{ID_SEP_KEY}dataset"}, - "tracking_dataset_infer": {"test": "@dataset"}, - "dataset_keys": "image", "is_not_rank0": ( "$torch.distributed.is_available() \ and torch.distributed.is_initialized() and torch.distributed.get_rank() > 0" @@ -136,9 +131,6 @@ "tag_name": "train_loss", "output_transform": "$monai.handlers.from_engine(['loss'], first=True)", "close_on_complete": True, - "dataset_log": "@dataset_log", - "dataset_dict": "@tracking_dataset_train", - "dataset_keys": "@dataset_keys", }, # MLFlowHandler config for the validator "validator": { @@ -148,9 +140,6 @@ "experiment_name": "@experiment_name", "run_name": "@run_name", "iteration_log": False, - "dataset_log": "@dataset_log", - "dataset_dict": "@tracking_dataset_val", - "dataset_keys": "@dataset_keys", }, # MLFlowHandler config for the evaluator "evaluator": { @@ -162,9 +151,6 @@ "artifacts": "@execute_config", "iteration_log": False, "close_on_complete": True, - "dataset_log": "@dataset_log", - "dataset_dict": "@tracking_dataset_infer", - "dataset_keys": "@dataset_keys", }, }, } From 83ca9ff89ee3dbd8326126ca9c3da909ecd9fff9 Mon Sep 17 00:00:00 2001 From: binliu Date: Mon, 10 Jul 2023 14:17:56 +0000 Subject: [PATCH 28/28] fix the format issue Signed-off-by: binliu --- monai/handlers/mlflow_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py index e7d978c70b..a2bd345dc6 100644 --- a/monai/handlers/mlflow_handler.py +++ b/monai/handlers/mlflow_handler.py @@ -21,7 +21,7 @@ import torch from torch.utils.data import Dataset -from monai.config import IgniteInfo, KeysCollection +from monai.config import IgniteInfo from monai.utils import CommonKeys, ensure_tuple, min_version, optional_import Events, _ = optional_import("ignite.engine", IgniteInfo.OPT_IMPORT_VERSION, min_version, "Events") @@ -125,7 +125,7 @@ def __init__( iteration_logger: Callable[[Engine], Any] | None = None, dataset_logger: Callable[[Mapping[str, Dataset]], Any] | None = None, dataset_dict: Mapping[str, Dataset] | None = None, - dataset_keys: CommonKeys = CommonKeys.IMAGE, + dataset_keys: str = CommonKeys.IMAGE, output_transform: Callable = lambda x: x[0], global_epoch_transform: Callable = lambda x: x, state_attributes: Sequence[str] | None = None,