From 572493205f2e08421ea7f955c3b8e4b294a38b0a Mon Sep 17 00:00:00 2001 From: Hiroki Takizawa Date: Mon, 27 Mar 2023 15:14:40 +0900 Subject: [PATCH 1/8] Update studies.py --- studies.py | 55 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/studies.py b/studies.py index 87808d543..2fa858fa8 100644 --- a/studies.py +++ b/studies.py @@ -1,4 +1,5 @@ import os +import time from typing import List from typing import Optional from typing import Sequence @@ -58,7 +59,7 @@ def create_single_objective_studies() -> List[Tuple[str, StudiesType]]: def objective_single_none_categorical(trial: optuna.Trial) -> float: x = trial.suggest_float("x", -100, 100) trial.suggest_categorical("y", ["foo", None]) - return x**2 + return x ** 2 study.optimize(objective_single_none_categorical, n_trials=10) studies.append((study.study_name, study)) @@ -108,7 +109,7 @@ def create_multi_objective_studies() -> List[Tuple[str, StudiesType]]: def objective_multi(trial: optuna.Trial) -> Tuple[float, float]: x = trial.suggest_float("x", 0, 5) y = trial.suggest_float("y", 0, 3) - v0 = 4 * x**2 + 4 * y**2 + v0 = 4 * x ** 2 + 4 * y ** 2 v1 = (x - 5) ** 2 + (y - 5) ** 2 return v0, v1 @@ -132,13 +133,13 @@ def objective_multi_dynamic(trial: optuna.Trial) -> Tuple[float, float]: if category == "foo": x = trial.suggest_float("x1", 0, 5) y = trial.suggest_float("y1", 0, 3) - v0 = 4 * x**2 + 4 * y**2 + v0 = 4 * x ** 2 + 4 * y ** 2 v1 = (x - 5) ** 2 + (y - 5) ** 2 return v0, v1 else: x = trial.suggest_float("x2", 0, 5) y = trial.suggest_float("y2", 0, 3) - v0 = 2 * x**2 + 2 * y**2 + v0 = 2 * x ** 2 + 2 * y ** 2 v1 = (x - 2) ** 2 + (y - 3) ** 2 return v0, v1 @@ -170,7 +171,7 @@ def objective_single_inf_report(trial: optuna.Trial) -> float: if x > 0: raise optuna.TrialPruned() else: - return x**2 + return x ** 2 def fail_objective(_: optuna.Trial) -> float: raise ValueError @@ -316,3 +317,47 @@ def objective(trial: optuna.Trial) -> float: ) study.optimize(objective, n_trials=50, timeout=600) return study + + +def create_single_objective_studies_for_timeline() -> List[Tuple[str, StudiesType]]: + studies: List[Tuple[str, StudiesType]] = [] + storage = optuna.storages.InMemoryStorage() + + def objective_timeline(trial: optuna.Trial) -> float: + x = trial.suggest_float("x", 0, 1) + time.sleep(x * 0.1) + if x > 0.8: + raise ValueError() + if x > 0.4: + raise optuna.TrialPruned() + return x ** 2 + + # Single-objective study + study = optuna.create_study( + study_name="A single objective study consuming time", + storage=storage, + ) + + study.enqueue_trial({"x": 0.3}) + study.enqueue_trial({"x": 0.9}) + study.enqueue_trial({"x": 0.5}) + study.optimize(objective_timeline, n_trials=50, n_jobs=2, catch=(ValueError,)) + studies.append((study.study_name, study)) + + # Single-objective study + study = optuna.create_study( + study_name=( + "A single objective study consuming time and " + "the order of legends is different from the order of trials" + ), + storage=storage, + ) + study.enqueue_trial({"x": 0.9}) + study.enqueue_trial({"x": 0.5}) + study.enqueue_trial({"x": 0.3}) + study.optimize(objective_timeline, n_trials=50, n_jobs=2, catch=(ValueError,)) + studies.append((study.study_name, study)) + + # No trials single-objective study + optuna.create_study(study_name="A single objective study that has no trials", storage=storage) + return studies From aaf6ebc6e2fa5ce4059dae6133163cf74891ce74 Mon Sep 17 00:00:00 2001 From: Hiroki Takizawa Date: Mon, 27 Mar 2023 15:17:13 +0900 Subject: [PATCH 2/8] Update visual_regression_tests.py --- visual_regression_tests.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/visual_regression_tests.py b/visual_regression_tests.py index 625a8db48..14991fe9c 100644 --- a/visual_regression_tests.py +++ b/visual_regression_tests.py @@ -24,6 +24,7 @@ from studies import create_multiple_single_objective_studies from studies import create_pytorch_study from studies import create_single_objective_studies +from studies import create_single_objective_studies_for_timeline from studies import StudiesType @@ -241,6 +242,21 @@ def generate_pareto_front_plots( ) +def generate_timeline_plots( + studies: List[Tuple[str, StudiesType]], base_dir: str, plot_kwargs: Dict[str, Any] +) -> List[Tuple[str, str, str]]: + filename_prefix = "timeline" + if len(plot_kwargs) > 0: + filename_prefix = f"{filename_prefix}-{stringify_plot_kwargs(plot_kwargs)}" + return generate_plot_files( + studies, + base_dir, + wrap_plot_func(lambda s: plotly_visualization.plot_timeline(s, **plot_kwargs)), + wrap_plot_func(lambda s: matplotlib_visualization.plot_timeline(s, **plot_kwargs)), + filename_prefix=filename_prefix, + ) + + def main() -> None: if not os.path.exists(abs_output_dir): os.mkdir(abs_output_dir) @@ -256,6 +272,7 @@ def main() -> None: multi_objective_studies = create_multi_objective_studies() print("Creating studies that have intermediate values") intermediate_value_studies = create_intermediate_value_studies() + single_objective_studies_for_timeline = create_single_objective_studies_for_timeline() if args.heavy: print("Creating pytorch study") @@ -305,6 +322,7 @@ def main() -> None: generate_edf_plots, {}, ), + ("plot_timeline", single_objective_studies_for_timeline, generate_timeline_plots, {}), ]: assert isinstance(plot_kwargs, Dict) plot_files = generate(studies, abs_output_dir, plot_kwargs) From d518ff961840bdff9ba5e6752853de6f2ea66189 Mon Sep 17 00:00:00 2001 From: Hiroki Takizawa Date: Wed, 29 Mar 2023 12:10:21 +0900 Subject: [PATCH 3/8] Update studies.py --- studies.py | 1 + 1 file changed, 1 insertion(+) diff --git a/studies.py b/studies.py index 2fa858fa8..2873b0308 100644 --- a/studies.py +++ b/studies.py @@ -360,4 +360,5 @@ def objective_timeline(trial: optuna.Trial) -> float: # No trials single-objective study optuna.create_study(study_name="A single objective study that has no trials", storage=storage) + studies.append((study.study_name, study)) return studies From 4e82cf07acb89be9d66333f9750d4e4a0ad88ef9 Mon Sep 17 00:00:00 2001 From: Hiroki Takizawa Date: Wed, 29 Mar 2023 12:20:31 +0900 Subject: [PATCH 4/8] Update studies.py --- studies.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/studies.py b/studies.py index 2873b0308..d4bad79c1 100644 --- a/studies.py +++ b/studies.py @@ -59,7 +59,7 @@ def create_single_objective_studies() -> List[Tuple[str, StudiesType]]: def objective_single_none_categorical(trial: optuna.Trial) -> float: x = trial.suggest_float("x", -100, 100) trial.suggest_categorical("y", ["foo", None]) - return x ** 2 + return x**2 study.optimize(objective_single_none_categorical, n_trials=10) studies.append((study.study_name, study)) @@ -109,7 +109,7 @@ def create_multi_objective_studies() -> List[Tuple[str, StudiesType]]: def objective_multi(trial: optuna.Trial) -> Tuple[float, float]: x = trial.suggest_float("x", 0, 5) y = trial.suggest_float("y", 0, 3) - v0 = 4 * x ** 2 + 4 * y ** 2 + v0 = 4 * x**2 + 4 * y**2 v1 = (x - 5) ** 2 + (y - 5) ** 2 return v0, v1 @@ -133,13 +133,13 @@ def objective_multi_dynamic(trial: optuna.Trial) -> Tuple[float, float]: if category == "foo": x = trial.suggest_float("x1", 0, 5) y = trial.suggest_float("y1", 0, 3) - v0 = 4 * x ** 2 + 4 * y ** 2 + v0 = 4 * x**2 + 4 * y**2 v1 = (x - 5) ** 2 + (y - 5) ** 2 return v0, v1 else: x = trial.suggest_float("x2", 0, 5) y = trial.suggest_float("y2", 0, 3) - v0 = 2 * x ** 2 + 2 * y ** 2 + v0 = 2 * x**2 + 2 * y**2 v1 = (x - 2) ** 2 + (y - 3) ** 2 return v0, v1 @@ -171,7 +171,7 @@ def objective_single_inf_report(trial: optuna.Trial) -> float: if x > 0: raise optuna.TrialPruned() else: - return x ** 2 + return x**2 def fail_objective(_: optuna.Trial) -> float: raise ValueError @@ -258,7 +258,6 @@ def get_mnist() -> Tuple[torch.utils.data.DataLoader, torch.utils.data.DataLoade return train_loader, valid_loader def objective(trial: optuna.Trial) -> float: - # Generate the model. model = define_model(trial).to(DEVICE) @@ -330,7 +329,7 @@ def objective_timeline(trial: optuna.Trial) -> float: raise ValueError() if x > 0.4: raise optuna.TrialPruned() - return x ** 2 + return x**2 # Single-objective study study = optuna.create_study( From 70a9beb6556e36fe2d257a7efdb8616fe6640ef1 Mon Sep 17 00:00:00 2001 From: Hiroki Takizawa Date: Wed, 29 Mar 2023 16:35:10 +0900 Subject: [PATCH 5/8] Update studies.py --- studies.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/studies.py b/studies.py index d4bad79c1..6e435ae78 100644 --- a/studies.py +++ b/studies.py @@ -358,6 +358,9 @@ def objective_timeline(trial: optuna.Trial) -> float: studies.append((study.study_name, study)) # No trials single-objective study - optuna.create_study(study_name="A single objective study that has no trials", storage=storage) + study = optuna.create_study( + study_name="A single objective study that has no trials", + storage=storage + ) studies.append((study.study_name, study)) return studies From 929779c3dc59954e9d53c965aa6e95837d80dba8 Mon Sep 17 00:00:00 2001 From: Hiroki Takizawa Date: Wed, 29 Mar 2023 16:37:35 +0900 Subject: [PATCH 6/8] Update studies.py --- studies.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/studies.py b/studies.py index 6e435ae78..7be019569 100644 --- a/studies.py +++ b/studies.py @@ -359,8 +359,7 @@ def objective_timeline(trial: optuna.Trial) -> float: # No trials single-objective study study = optuna.create_study( - study_name="A single objective study that has no trials", - storage=storage + study_name="A single objective study that has no trials", storage=storage ) studies.append((study.study_name, study)) return studies From bf8308323668e578e8b45b1ddc1250c4a871b727 Mon Sep 17 00:00:00 2001 From: Hiroki Takizawa Date: Thu, 30 Mar 2023 18:51:46 +0900 Subject: [PATCH 7/8] Update studies.py Co-authored-by: Toshihiko Yanase --- studies.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/studies.py b/studies.py index 7be019569..069c3e084 100644 --- a/studies.py +++ b/studies.py @@ -337,9 +337,9 @@ def objective_timeline(trial: optuna.Trial) -> float: storage=storage, ) - study.enqueue_trial({"x": 0.3}) - study.enqueue_trial({"x": 0.9}) - study.enqueue_trial({"x": 0.5}) + study.enqueue_trial({"x": 0.3}) # Add a COMPLETE trial. + study.enqueue_trial({"x": 0.9}) # Add a FAIL trial. + study.enqueue_trial({"x": 0.5}) # Add a PRUNED trial. study.optimize(objective_timeline, n_trials=50, n_jobs=2, catch=(ValueError,)) studies.append((study.study_name, study)) From 2dd2ed190f56db35c324e793d07989ac5493c0cd Mon Sep 17 00:00:00 2001 From: Hiroki Takizawa Date: Thu, 30 Mar 2023 18:51:53 +0900 Subject: [PATCH 8/8] Update studies.py Co-authored-by: Toshihiko Yanase --- studies.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/studies.py b/studies.py index 069c3e084..4d233f464 100644 --- a/studies.py +++ b/studies.py @@ -351,9 +351,9 @@ def objective_timeline(trial: optuna.Trial) -> float: ), storage=storage, ) - study.enqueue_trial({"x": 0.9}) - study.enqueue_trial({"x": 0.5}) - study.enqueue_trial({"x": 0.3}) + study.enqueue_trial({"x": 0.9}) # Add a FAIL trial. + study.enqueue_trial({"x": 0.5}) # Add a PRUNED trial. + study.enqueue_trial({"x": 0.3}) # Add a COMPLETE trial. study.optimize(objective_timeline, n_trials=50, n_jobs=2, catch=(ValueError,)) studies.append((study.study_name, study))