From dd249cdb90c2614d23de7fab6553f3f04edfa60b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Sass?= Date: Wed, 26 Oct 2022 13:59:51 +0200 Subject: [PATCH] Version 2.0.0a2 (#883) ## Bugfixes - Fixed random weight (re-)generalization of multi-objective algorithms: Before the weights were generated for each call to ``build_matrix``, now we only re-generate them for every iteration. - Optimization may get stuck because of deep copying an iterator for callback: We removed the configuration call from ``on_next_configurations_end``. ## Minor - Removed example badget in README. - Added SMAC logo to README. Co-authored-by: Katharina Eggensperger Co-authored-by: Matthias Feurer Co-authored-by: dengdifan Co-authored-by: Carolin Benjamins Co-authored-by: Eric Kalosa-Kenyon --- .github/workflows/dist.yml | 2 - .github/workflows/docs.yml | 15 ++++- .github/workflows/pre-commit.yml | 2 - .github/workflows/pytest.yml | 2 - README.md | 5 +- changelog.md | 11 ++++ docs/conf.py | 5 ++ setup.py | 2 +- smac/__init__.py | 2 +- smac/callback.py | 4 +- smac/main/smbo.py | 11 ++-- .../gaussian_process/priors/tophat_prior.py | 4 +- smac/multi_objective/parego.py | 7 ++- smac/runhistory/encoder/encoder.py | 3 - smac/utils/multi_objective.py | 2 +- tests/test_callback.py | 2 +- tests/test_multi_objective/test_schaffer.py | 56 ++++++++++++++----- .../test_schaffer_upscaled.py | 28 +++++----- 18 files changed, 109 insertions(+), 54 deletions(-) diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml index 3755ba596..ffac9f005 100644 --- a/.github/workflows/dist.yml +++ b/.github/workflows/dist.yml @@ -9,14 +9,12 @@ on: branches: - main - development - - development-2.0 # Trigger on a open/push to a PR targeting one of these branches pull_request: branches: - main - development - - development-2.0 jobs: dist: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 78ed9a191..e1b16334d 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -11,14 +11,12 @@ on: branches: - main - development - - development-2.0 # Trigger on a open/push to a PR targeting one of these branches pull_request: branches: - main - development - - development-2.0 env: name: SMAC3 @@ -36,9 +34,16 @@ jobs: python-version: "3.10" - name: Install dependencies + id: install run: | pip install ".[gpytorch,dev]" + # Getting the version + SMAC_VERSION=$(python -c "import smac; print('v' + str(smac.version));") + + # Make it a global variable + echo "SMAC_VERSION=$SMAC_VERSION" >> $GITHUB_ENV + - name: Make docs run: | make clean @@ -58,6 +63,11 @@ jobs: rm -rf $branch_name cp -r ../${{ env.name }}/docs/build/html $branch_name + # we also copy the current SMAC_VERSION + rm -rf $SMAC_VERSION + cp -r ../${{ env.name }}/docs/build/html $SMAC_VERSION + + - name: Push to gh-pages if: (contains(github.ref, 'develop') || contains(github.ref, 'main')) && github.event_name == 'push' run: | @@ -65,6 +75,7 @@ jobs: cd ../gh-pages branch_name=${GITHUB_REF##*/} git add $branch_name/ + git add $SMAC_VERSION/ git config --global user.name 'Github Actions' git config --global user.email 'not@mail.com' git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index e86c722b2..5e5815016 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -9,14 +9,12 @@ on: branches: - main - development - - development-2.0 # When a push occurs on a PR that targets these branches pull_request: branches: - main - development - - development-2.0 jobs: run-all-files: diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 98addbe87..f82e49c1d 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -9,14 +9,12 @@ on: branches: - main - development - - development-2.0 # Triggers with push to a pr aimed at main pull_request: branches: - main - development - - development-2.0 schedule: # Every day at 7AM UTC diff --git a/README.md b/README.md index ba8b4e4d8..56a67e971 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,12 @@ [![Tests](https://github.com/automl/SMAC3/actions/workflows/pytest.yml/badge.svg?branch=main)](https://github.com/automl/SMAC3/actions/workflows/pytest.yml) -[![Docs](https://github.com/automl/SMAC3/actions/workflows/docs.yml/badge.svg?branch=main)](https://github.com/automl/SMAC3/actions/workflows/docs.yml) -[![Examples](https://github.com/automl/SMAC3/actions/workflows/examples.yml/badge.svg?branch=main)](https://github.com/automl/SMAC3/actions/workflows/examples.yml) +[![Documentation](https://github.com/automl/SMAC3/actions/workflows/docs.yml/badge.svg?branch=main)](https://github.com/automl/SMAC3/actions/workflows/docs.yml) [![codecov Status](https://codecov.io/gh/automl/SMAC3/branch/master/graph/badge.svg)](https://codecov.io/gh/automl/SMAC3) + + SMAC is a tool for algorithm configuration to optimize the parameters of arbitrary algorithms, including hyperparameter optimization of Machine Learning algorithms. The main core consists of Bayesian Optimization in combination with an aggressive racing mechanism to efficiently decide which of two configurations performs better. diff --git a/changelog.md b/changelog.md index 9efc44b47..2a8e1221c 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,14 @@ +# 2.0.0a2 + +## Bugfixes +- Fixed random weight (re-)generalization of multi-objective algorithms: Before the weights were generated for each call to ``build_matrix``, now we only re-generate them for every iteration. +- Optimization may get stuck because of deep copying an iterator for callback: We removed the configuration call from ``on_next_configurations_end``. + +## Minor +- Removed example badget in README. +- Added SMAC logo to README. + + # 2.0.0a1 ## Big Changes diff --git a/docs/conf.py b/docs/conf.py index 88a45a819..83041922e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,6 +10,11 @@ "copyright": copyright, "author": author, "version": version, + "versions": { + f"v{version} (unstable)": "#", + "v2.0.0a1": "https://automl.github.io/SMAC3/v2.0.0a1/", + "v1.4.0": "https://automl.github.io/SMAC3/v1.4.0/", + }, "name": name, "html_theme_options": { "github_url": "https://github.com/automl/SMAC3", diff --git a/setup.py b/setup.py index e74cb12f5..9ee50cc69 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ def read_file(filepath: str) -> str: "pytest-xdist", "pytest-timeout", # Docs - "automl-sphinx-theme>=0.1.18", + "automl-sphinx-theme>=0.2", # Others "mypy", "isort", diff --git a/smac/__init__.py b/smac/__init__.py index c39700f34..325ea5b62 100644 --- a/smac/__init__.py +++ b/smac/__init__.py @@ -19,7 +19,7 @@ Copyright {datetime.date.today().strftime('%Y')}, Marius Lindauer, Katharina Eggensperger, Matthias Feurer, André Biedenkapp, Difan Deng, Carolin Benjamins, Tim Ruhkopf, René Sass and Frank Hutter""" -version = "2.0.0a1" +version = "2.0.0a2" try: diff --git a/smac/callback.py b/smac/callback.py index 3901cf38d..925f9ce7c 100644 --- a/smac/callback.py +++ b/smac/callback.py @@ -1,7 +1,5 @@ from __future__ import annotations -from ConfigSpace import Configuration - import smac from smac.runhistory import TrialInfo, TrialInfoIntent, TrialValue @@ -37,7 +35,7 @@ def on_next_configurations_start(self, smbo: smac.main.BaseSMBO) -> None: """ pass - def on_next_configurations_end(self, smbo: smac.main.BaseSMBO, configurations: list[Configuration]) -> None: + def on_next_configurations_end(self, smbo: smac.main.BaseSMBO) -> None: """Called after the intensification asks for new configurations. Essentially, this callback is called before the surrogate model is trained and before the acquisition function is called. """ diff --git a/smac/main/smbo.py b/smac/main/smbo.py index 69a9873b0..935d4883c 100644 --- a/smac/main/smbo.py +++ b/smac/main/smbo.py @@ -2,8 +2,6 @@ from typing import Any, Iterator -import copy - import numpy as np from ConfigSpace import Configuration @@ -35,6 +33,8 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: self._considered_budgets: list[float | None] = [None] def get_next_configurations(self, n: int | None = None) -> Iterator[Configuration]: # noqa: D102 + # TODO: Let's return the initial configurations from this method too + for callback in self._callbacks: callback.on_next_configurations_start(self) @@ -53,6 +53,7 @@ def get_next_configurations(self, n: int | None = None) -> Iterator[Configuratio # the configspace. return iter([self._scenario.configspace.sample_configuration(1)]) + # TODO: Check if X/Y differs from the last run, otherwise use cached results self._model.train(X, Y) x_best_array: np.ndarray | None = None @@ -79,8 +80,7 @@ def get_next_configurations(self, n: int | None = None) -> Iterator[Configuratio ) for callback in self._callbacks: - challenger_list = list(copy.deepcopy(challengers)) - callback.on_next_configurations_end(self, challenger_list) + callback.on_next_configurations_end(self) return challengers @@ -88,6 +88,9 @@ def ask(self) -> tuple[TrialInfoIntent, TrialInfo]: # noqa: D102 for callback in self._callbacks: callback.on_ask_start(self) + if self._runhistory_encoder.multi_objective_algorithm is not None: + self._runhistory_encoder.multi_objective_algorithm.update_on_iteration_start() + intent, trial_info = self._intensifier.get_next_trial( challengers=self._initial_design_configs, incumbent=self._incumbent, diff --git a/smac/model/gaussian_process/priors/tophat_prior.py b/smac/model/gaussian_process/priors/tophat_prior.py index 8706003da..1be8ace7f 100644 --- a/smac/model/gaussian_process/priors/tophat_prior.py +++ b/smac/model/gaussian_process/priors/tophat_prior.py @@ -35,6 +35,8 @@ def __init__( self._log_min = np.log(lower_bound) self._max = upper_bound self._log_max = np.log(upper_bound) + self._prob = 1 / (self._max - self._min) + self._log_prob = np.log(self._prob) if not (self._max > self._min): raise Exception("Upper bound of Tophat prior must be greater than the lower bound.") @@ -50,7 +52,7 @@ def _get_log_probability(self, theta: float) -> float: if theta < self._min or theta > self._max: return -np.inf else: - return 0 + return self._log_prob def _sample_from_prior(self, n_samples: int) -> np.ndarray: if np.ndim(n_samples) != 0: diff --git a/smac/multi_objective/parego.py b/smac/multi_objective/parego.py index 60d02e44a..4345d1532 100644 --- a/smac/multi_objective/parego.py +++ b/smac/multi_objective/parego.py @@ -37,8 +37,8 @@ def __init__( self._rng = np.random.RandomState(seed) self._rho = rho - self._theta = self._rng.rand(self._n_objectives) - self.update_on_iteration_start() + # Will be set on starting an SMBO iteration + self._theta: np.ndarray | None = None @property def meta(self) -> dict[str, Any]: # noqa: D102 @@ -61,5 +61,8 @@ def update_on_iteration_start(self) -> None: # noqa: D102 def __call__(self, values: list[float]) -> float: # noqa: D102 # Weight the values + if self._theta is None: + raise ValueError("Iteration not yet initalized; Call `update_on_iteration_start()` first") + theta_f = self._theta * values return float(np.max(theta_f, axis=0) + self._rho * np.sum(theta_f, axis=0)) diff --git a/smac/runhistory/encoder/encoder.py b/smac/runhistory/encoder/encoder.py index a8a82c8ea..12ba8c035 100644 --- a/smac/runhistory/encoder/encoder.py +++ b/smac/runhistory/encoder/encoder.py @@ -33,9 +33,6 @@ def _build_matrix( # TODO: Extend for native multi-objective y = np.ones([n_rows, 1]) - if self._multi_objective_algorithm is not None: - self._multi_objective_algorithm.update_on_iteration_start() - # Then populate matrix for row, (key, run) in enumerate(trials.items()): # Scaling is automatically done in configSpace diff --git a/smac/utils/multi_objective.py b/smac/utils/multi_objective.py index edbc88774..1eb527473 100644 --- a/smac/utils/multi_objective.py +++ b/smac/utils/multi_objective.py @@ -38,6 +38,6 @@ def normalize_costs( cost = 1.0 else: cost = p / q - costs += [cost] + costs.append(cost) return costs diff --git a/tests/test_callback.py b/tests/test_callback.py index 7d3c84574..124520fb8 100644 --- a/tests/test_callback.py +++ b/tests/test_callback.py @@ -39,7 +39,7 @@ def on_iteration_end(self, smbo: smac.main.BaseSMBO) -> None: def on_next_configurations_start(self, smbo: smac.main.BaseSMBO) -> None: self.next_configurations_start_counter += 1 - def on_next_configurations_end(self, smbo: smac.main.BaseSMBO, configurations: list[Configuration]) -> None: + def on_next_configurations_end(self, smbo: smac.main.BaseSMBO) -> None: self.next_configurations_end_counter += 1 def on_ask_start(self, smbo: smac.main.BaseSMBO) -> None: diff --git a/tests/test_multi_objective/test_schaffer.py b/tests/test_multi_objective/test_schaffer.py index 9b712e877..c4af32b48 100644 --- a/tests/test_multi_objective/test_schaffer.py +++ b/tests/test_multi_objective/test_schaffer.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import numpy as np import pytest from ConfigSpace import ConfigurationSpace, Float @@ -5,9 +7,11 @@ from smac import ( AlgorithmConfigurationFacade, BlackBoxFacade, + Callback, HyperparameterOptimizationFacade, RandomFacade, ) +from smac.multi_objective import AbstractMultiObjectiveAlgorithm from smac.multi_objective.aggregation_strategy import MeanAggregationStrategy from smac.multi_objective.parego import ParEGO @@ -52,16 +56,33 @@ def configspace(): return cs +class WrapStrategy(AbstractMultiObjectiveAlgorithm): + def __init__(self, strategy: AbstractMultiObjectiveAlgorithm, *args, **kwargs): + self.strategy = strategy(*args, **kwargs) + self.n_calls_update_on_iteration_start = 0 + self.n_calls___call__ = 0 + + def update_on_iteration_start(self) -> None: # noqa: D102 + self.n_calls_update_on_iteration_start += 1 + return self.strategy.update_on_iteration_start() + + def __call__(self, values: list[float]) -> float: # noqa: D102 + self.n_calls___call__ += 1 + return self.strategy(values) + + @pytest.mark.parametrize( "facade", [BlackBoxFacade, HyperparameterOptimizationFacade, AlgorithmConfigurationFacade, RandomFacade] ) def test_mean_aggregation(facade, make_scenario, configspace): scenario = make_scenario(configspace, use_multi_objective=True) + multi_objective_algorithm = WrapStrategy(MeanAggregationStrategy, scenario=scenario) + smac = facade( scenario=scenario, target_function=tae, - multi_objective_algorithm=MeanAggregationStrategy(scenario=scenario), + multi_objective_algorithm=multi_objective_algorithm, overwrite=True, ) incumbent = smac.optimize() @@ -75,6 +96,10 @@ def test_mean_aggregation(facade, make_scenario, configspace): assert diff < 0.06 + assert multi_objective_algorithm.n_calls_update_on_iteration_start >= 100 + assert multi_objective_algorithm.n_calls_update_on_iteration_start <= 130 + assert multi_objective_algorithm.n_calls___call__ >= 100 + @pytest.mark.parametrize( "facade", [BlackBoxFacade, HyperparameterOptimizationFacade, AlgorithmConfigurationFacade, RandomFacade] @@ -82,25 +107,30 @@ def test_mean_aggregation(facade, make_scenario, configspace): def test_parego(facade, make_scenario, configspace): scenario = make_scenario(configspace, use_multi_objective=True) + multi_objective_algorithm = WrapStrategy(ParEGO, scenario=scenario) + smac = facade( scenario=scenario, target_function=tae, - multi_objective_algorithm=ParEGO(scenario=scenario), + multi_objective_algorithm=multi_objective_algorithm, overwrite=True, ) - # The incumbent is not ambiguously because we have a Pareto front + smac.optimize() - # We use the mean aggregation strategy to get the same weights - multi_objective_algorithm = MeanAggregationStrategy(scenario=scenario) - smac.runhistory.multi_objective_algorithm = multi_objective_algorithm + # The incumbent is not ambiguous because we have a Pareto front + confs, vals = smac.runhistory.get_pareto_front() - incumbent, _ = smac.runhistory.get_incumbent() + min_ = np.inf + for x, y in zip(confs, vals): + tr = schaffer(x["x"]) + assert np.allclose(tr, y) + if np.sum(y) < min_: + min_ = np.sum(y) - f1_inc, f2_inc = schaffer(incumbent["x"]) - f1_opt, f2_opt = get_optimum() - inc = f1_inc + f2_inc - opt = f1_opt + f2_opt - diff = abs(inc - opt) + opt = np.sum(get_optimum()) + assert abs(np.sum(min_) - opt) <= 0.06 - assert diff < 0.06 + assert multi_objective_algorithm.n_calls_update_on_iteration_start >= 100 + assert multi_objective_algorithm.n_calls_update_on_iteration_start <= 120 + assert multi_objective_algorithm.n_calls___call__ >= 100 diff --git a/tests/test_multi_objective/test_schaffer_upscaled.py b/tests/test_multi_objective/test_schaffer_upscaled.py index 541e6e81f..5bb995e72 100644 --- a/tests/test_multi_objective/test_schaffer_upscaled.py +++ b/tests/test_multi_objective/test_schaffer_upscaled.py @@ -88,18 +88,18 @@ def test_parego(facade, make_scenario, configspace): ) smac.optimize() - # We use the mean aggregation strategy to get the same weights - multi_objective_algorithm = MeanAggregationStrategy(scenario=scenario) - smac.runhistory.multi_objective_algorithm = multi_objective_algorithm - - incumbent, _ = smac.runhistory.get_incumbent() - - f1_inc, f2_inc = schaffer(incumbent["x"]) - f1_opt, f2_opt = get_optimum() + # The incumbent is not ambiguous because we have a Pareto front + confs, vals = smac.runhistory.get_pareto_front() + + min_ = [np.inf, np.inf] + for x, y in zip(confs, vals): + tr = schaffer(x["x"]) + assert np.allclose(tr, y) + if y[0] + y[1] / UPSCALING_FACTOR < min_[0] + min_[1] / UPSCALING_FACTOR: + min_ = y + + opt = np.sum(get_optimum()) + f1_inc, f2_inc = min_ f2_inc = f2_inc / UPSCALING_FACTOR - - inc = f1_inc + f2_inc - opt = f1_opt + f2_opt - diff = abs(inc - opt) - - assert diff < 0.06 + min_ = (f1_inc, f2_inc) + assert abs(np.sum(min_) - opt) <= 0.06