diff --git a/.binder/apt.txt b/.binder/apt.txt new file mode 100644 index 0000000000..f0a795a247 --- /dev/null +++ b/.binder/apt.txt @@ -0,0 +1,2 @@ +build-essential +swig \ No newline at end of file diff --git a/.binder/postBuild b/.binder/postBuild new file mode 100644 index 0000000000..311cee5918 --- /dev/null +++ b/.binder/postBuild @@ -0,0 +1,43 @@ +#!/bin/bash + +set -e + +python -m pip install -e .[dev] + +# Taken from https://github.com/scikit-learn/scikit-learn/blob/22cd233e1932457947e9994285dc7fd4e93881e4/.binder/postBuild +# under BSD3 license, copyright the scikit-learn contributors + +# This script is called in a binder context. When this script is called, we are +# inside a git checkout of the automl/SMAC3 repo. This script +# generates notebooks from the SMAC3 python examples. + +if [[ ! -f /.dockerenv ]]; then + echo "This script was written for repo2docker and is supposed to run inside a docker container." + echo "Exiting because this script can delete data if run outside of a docker container." + exit 1 +fi + +# Copy content we need from the SMAC3 repo +TMP_CONTENT_DIR=/tmp/SMAC3 +mkdir -p $TMP_CONTENT_DIR +cp -r examples .binder $TMP_CONTENT_DIR +# delete everything in current directory including dot files and dot folders +find . -delete + +# Generate notebooks and remove other files from examples folder +GENERATED_NOTEBOOKS_DIR=examples +cp -r $TMP_CONTENT_DIR/examples $GENERATED_NOTEBOOKS_DIR + +find $GENERATED_NOTEBOOKS_DIR -name 'example_*.py' -exec sphx_glr_python_to_jupyter.py '{}' + +# Keep __init__.py and custom_metrics.py +NON_NOTEBOOKS=$(find $GENERATED_NOTEBOOKS_DIR -type f | grep -v '\.ipynb' | grep -v 'init' | grep -v 'custom_metrics') +rm -f $NON_NOTEBOOKS + +# Modify path to be consistent by the path given by sphinx-gallery +mkdir notebooks +mv $GENERATED_NOTEBOOKS_DIR notebooks/ + +# Put the .binder folder back (may be useful for debugging purposes) +mv $TMP_CONTENT_DIR/.binder . +# Final clean up +rm -rf $TMP_CONTENT_DIR diff --git a/.binder/requirements.txt b/.binder/requirements.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e5e5092a20..4cad3b2867 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,5 +5,12 @@ updates: # https://docs.github.com/en/enterprise-server@3.4/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot - package-ecosystem: "github-actions" directory: "/" + target-branch: "development" schedule: - interval: "daily" + interval: "weekly" + assignees: + - "eddiebergman" + reviewers: + - "eddiebergman" + commit-message: + include: "chore: " diff --git a/CHANGELOG.md b/CHANGELOG.md index b048337a0f..807cbaf7ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,33 @@ +# 2.0.1 + +## Improvements +- Callbacks registration is now a public method of the optimizer and allows callbacks to be inserted at a specific position. +- Adapt developer install instructions to include pre-commit installation +- Add option to pass a dask client to the facade, e.g. enables running on a hpc cluster (#983). +- Added scenario.use_default_config argument/attribute=False, that adds the user's configspace default configuration + as an additional_config to the inital design if set to True. This adds one additional configuration to the number of configs + originating from the initial design. Since n_trials is still respected, this results in one fewer BO steps +- Adapt developer install instructions to include pre-commit installation. +- Add option to pass a dask client to the facade, e.g. enables running on a hpc cluster (#983). +- Add example for using a callback to log run metadata to a file (#996). +- Move base callback and metadata callback files to own callback directory. +- Add a workaround to be able to pass a dataset via dask.scatter so that serialization/deserialization in Dask becomes much quicker (#993). + +## Bugfixes +- The ISB-pair differences over the incumbent's configurations are computed correctly now (#956). +- Adjust amount of configurations in different stages of hyperband brackets to conform to the original paper. +- Fix validation in smbo to use the seed in the scenario. +- Change order of callbacks, intensifier callback for incumbent selection is now the first callback. +- intensifier.get_state() will now check if the configurations contained in the queue is stored in the runhistory (#997) + + # 2.0.0 ## Improvements - Clarify origin of configurations (#908). - Random forest with instances predicts the marginalized costs by using a C++ implementation in `pyrfr`, which is much faster (#903). -- Add version to makefile to install correct test release version +- Add version to makefile to install correct test release version. +- Add option to disable logging by setting `logging_level=False`. (#947) ## Bugfixes - Continue run when setting incumbent selection to highest budget when using Successive Halving (#907). diff --git a/CITATION.cff b/CITATION.cff index 65bcf359fa..36435d6ecf 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -9,7 +9,7 @@ date-released: "2016-08-17" url: "https://automl.github.io/SMAC3/master/index.html" repository-code: "https://github.com/automl/SMAC3" -version: "1.0.1" +version: "2.0.1" type: "software" keywords: @@ -25,84 +25,108 @@ license: "BSD-3-Clause" authors: - family-names: "Lindauer" given-names: "Marius" - affiliation: "Leibniz Universität Hannover" + affiliation: "Leibniz University Hannover, Germany" + + - family-names: "Benjamins" + given-names: "Carolin" + affiliation: "Leibniz University Hannover, Germany" + + - family-names: "Biedenkapp" + given-names: "André" + orcid: "https://orcid.org/0000-0002-8703-8559" + affiliation: "University of Freiburg, Germany" + + - family-names: "Deng" + given-names: "Difan" + affiliation: "Leibniz University Hannover, Germany" - family-names: "Eggensperger" given-names: "Katharina" orcid: "https://orcid.org/0000-0002-0309-401X" affiliation: "University of Freiburg, Germany" + - family-names: "Falkner" + given-names: "Stefan" + orcid: "https://orcid.org/0000-0002-6303-9418" + affiliation: "Bosch Center for Artificial Intelligence, Rennigen, Germany" + - family-names: "Feurer" given-names: "Matthias" orcid: "https://orcid.org/0000-0001-9611-8588" affiliation: "University of Freiburg, Germany" - - family-names: "Biedenkapp" - given-names: "André" - orcid: "https://orcid.org/0000-0002-8703-8559" - affiliation: "University of Freiburg, Germany" - - - family-names: "Deng" - given-names: "Difan" - affiliation: "Leibniz Universität Hannover" + - family-names: "Graf" + given-names: "Helena" + affiliation: "Leibniz University Hannover, Germany" - - family-names: "Benjamins" - given-names: "Carolin" - affiliation: "Leibniz Universität Hannover" + - family-names: "Ruhkopf" + given-names: "Tim" + affiliation: "Leibniz University Hannover, Germany" - family-names: "Sass" given-names: "René" - affiliation: "Leibniz Universität Hannover" + affiliation: "Leibniz University Hannover, Germany" + + - family-names: "Segel" + given-names: "Sarah" + affiliation: "Leibniz University Hannover, Germany" + + - family-names: "Tornede" + given-names: "Alexander" + affiliation: "Leibniz University Hannover, Germany" - family-names: "Hutter" given-names: "Frank" affiliation: "University of Freiburg, Germany" - - family-names: "Falkner" - given-names: "Stefan" - orcid: "https://orcid.org/0000-0002-6303-9418" - affiliation: "Bosch Center for Artificial Intelligence, Rennigen, Germany" - preferred-citation: type: "article" title: "SMAC3: A Versatile Bayesian Optimization Package for Hyperparameter Optimization" - month: "9" - year: "2021" - url: "https://arxiv.org/abs/2109.09831" + journal: "Journal of Machine Learning Research" + year: "2022" + volume: "23" + number: "54" + start: "1" + end: "9" + url: "https://www.jmlr.org/papers/volume23/21-0888/21-0888.pdf" authors: - family-names: "Lindauer" given-names: "Marius" - affiliation: "Leibniz Universität Hannover" + affiliation: "Leibniz University Hannover" - family-names: "Eggensperger" given-names: "Katharina" orcid: "https://orcid.org/0000-0002-0309-401X" - affiliation: "University of Freiburg, Germany" + affiliation: "University of Freiburg" - family-names: "Feurer" given-names: "Matthias" orcid: "https://orcid.org/0000-0001-9611-8588" - affiliation: "University of Freiburg, Germany" + affiliation: "University of Freiburg" - family-names: "Biedenkapp" given-names: "André " orcid: "https://orcid.org/0000-0002-8703-8559" - affiliation: "University of Freiburg, Germany" + affiliation: "University of Freiburg" - family-names: "Deng" given-names: "Difan" - affiliation: "Leibniz Universität Hannover" + affiliation: "Leibniz University Hannover" - family-names: "Benjamins" given-names: "Carolin" - affiliation: "Leibniz Universität Hannover" + affiliation: "Leibniz University Hannover" + + - family-names: "Ruhkopf" + given-names: "Tim" + affiliation: "Leibniz University Hannover" - family-names: "Sass" given-names: "René" - affiliation: "Leibniz Universität Hannover" + affiliation: "Leibniz University Hannover" - family-names: "Hutter" given-names: "Frank" - affiliation: "University of Freiburg, Germany" + affiliation: "University of Freiburg" ... diff --git a/Makefile b/Makefile index 522acebc41..f7727cf7f6 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ SHELL := /bin/bash NAME := SMAC3 PACKAGE_NAME := smac -VERSION := 2.0.0 +VERSION := 2.0.1 DIR := "${CURDIR}" SOURCE_DIR := ${PACKAGE_NAME} @@ -136,7 +136,7 @@ clean-data: # Will echo the commands to actually publish to be run to publish to actual PyPi # This is done to prevent accidental publishing but provide the same conveniences publish: clean build - read -p "Did you update the version number in Makefile, smac/__init__.py, benchmark/src/wrappers/v20.py? \ + read -p "Did you update the version number in Makefile, smac/__init__.py, benchmark/src/wrappers/v20.py, CITATION.cff? \ Did you add the old version to docs/conf.py? Did you add changes to CHANGELOG.md?" $(PIP) install twine @@ -152,3 +152,5 @@ publish: clean build @echo @echo "Once you have decided it works, publish to actual pypi with" @echo "--- python -m twine upload dist/*" + @echo "After publishing via pypi, please also add a new release on Github and edit the version in the SMAC link \ + on the SMAC Github page." diff --git a/README.md b/README.md index 03999059a9..e546e8356d 100644 --- a/README.md +++ b/README.md @@ -57,10 +57,10 @@ Install SMAC via PyPI: pip install smac ``` -Or alternatively, clone the environment: +If you want to contribute to SMAC, use the following steps instead: ``` git clone https://github.com/automl/SMAC3.git && cd SMAC3 -pip install -e .[dev] +make install-dev ``` diff --git a/benchmark/src/wrappers/v20.py b/benchmark/src/wrappers/v20.py index c7b9db028c..8ed0864ecb 100644 --- a/benchmark/src/wrappers/v20.py +++ b/benchmark/src/wrappers/v20.py @@ -6,7 +6,7 @@ class Version20(Wrapper): - supported_versions: list[str] = ["2.0.0"] + supported_versions: list[str] = ["2.0.1"] def __init__(self, task: Task, seed: int) -> None: super().__init__(task, seed) diff --git a/docs/6_references.rst b/docs/6_references.rst index a5dfd1987e..de2cf083b5 100644 --- a/docs/6_references.rst +++ b/docs/6_references.rst @@ -14,7 +14,7 @@ References .. [Know06] J. Knowles; ParEGO: A Hybrid Algorithm with on-Line Landscape Approximation for Expensive Multiobjective Optimization Problems; - https://ieeexplore.ieee.org/document/1583627 + https://www.cs.bham.ac.uk/~jdk/parego/ParEGO-TR3.pdf .. [SKKS10] N. Srinivas, S. M. Kakade, A. Krause, M. Seeger; diff --git a/docs/advanced_usage/9_parallelism.rst b/docs/advanced_usage/9_parallelism.rst index f978e74b31..9912c782f5 100644 --- a/docs/advanced_usage/9_parallelism.rst +++ b/docs/advanced_usage/9_parallelism.rst @@ -1,7 +1,7 @@ Parallelism =========== -SMAC supports multiple workers natively. Just specify ``n_workers`` in the scenario and you are ready to go. +SMAC supports multiple workers natively via Dask. Just specify ``n_workers`` in the scenario and you are ready to go. .. note :: @@ -19,3 +19,24 @@ SMAC supports multiple workers natively. Just specify ``n_workers`` in the scena .. warning :: When using multiple workers, SMAC is not reproducible anymore. + + +Running on a Cluster +-------------------- +You can also pass a custom dask client, e.g. to run on a slurm cluster. +See our :ref:`parallelism example`. + +.. warning :: + + On some clusters you cannot spawn new jobs when running a SLURMCluster inside a + job instead of on the login node. No obvious errors might be raised but it can hang silently. + +.. warning :: + + Sometimes you need to modify your launch command which can be done with + ``SLURMCluster.job_class.submit_command``. + +.. code-block:: python + + cluster.job_cls.submit_command = submit_command + cluster.job_cls.cancel_command = cancel_command diff --git a/docs/conf.py b/docs/conf.py index 4f6de7d9a8..d7b5598884 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,6 +12,7 @@ "version": version, "versions": { f"v{version}": "#", + "v2.0.0": "https://automl.github.io/SMAC3/v2.0.0/", "v2.0.0b1": "https://automl.github.io/SMAC3/v2.0.0b1/", "v2.0.0a2": "https://automl.github.io/SMAC3/v2.0.0a2/", "v2.0.0a1": "https://automl.github.io/SMAC3/v2.0.0a1/", @@ -27,6 +28,20 @@ "plot_gallery": True, "within_subsection_order": FileNameSortKey, "filename_pattern": "/", # We want to execute all files in `examples` + "binder": { + # Required keys + "org": "automl", + "repo": "SMAC3", + "branch": "main", + "binderhub_url": "https://mybinder.org", + "dependencies": ["../.binder/apt.txt", "../.binder/requirements.txt"], + # "filepath_prefix": "" # A prefix to prepend to any filepaths in Binder links. + # Jupyter notebooks for Binder will be copied to this directory (relative to built documentation root). + "notebooks_dir": "notebooks/", + "use_jupyter_lab": True, + # Whether Binder links should start Jupyter Lab instead of the Jupyter Notebook interface. + }, + "ignore_pattern": ".*7_parallelization_cluster.py$", }, } diff --git a/docs/index.rst b/docs/index.rst index 0c48e51dcb..736448d784 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -29,12 +29,15 @@ If you use SMAC, please cite our `JMLR paper ` as its surrogate model. +The facade works best on a numerical hyperparameter configuration space and should not +be applied to problems with large evaluation budgets (up to 1000 evaluations). +""" + +import numpy as np +from ConfigSpace import Configuration, ConfigurationSpace, Float +from dask.distributed import Client +from dask_jobqueue import SLURMCluster + +from smac import BlackBoxFacade, Scenario + +__copyright__ = "Copyright 2023, AutoML.org Freiburg-Hannover" +__license__ = "3-clause BSD" + + +class Branin(object): + @property + def configspace(self) -> ConfigurationSpace: + cs = ConfigurationSpace(seed=0) + x0 = Float("x0", (-5, 10), default=-5, log=False) + x1 = Float("x1", (0, 15), default=2, log=False) + cs.add_hyperparameters([x0, x1]) + + return cs + + def train(self, config: Configuration, seed: int = 0) -> float: + """Branin function + + Parameters + ---------- + config : Configuration + Contains two continuous hyperparameters, x0 and x1 + seed : int, optional + Not used, by default 0 + + Returns + ------- + float + Branin function value + """ + x0 = config["x0"] + x1 = config["x1"] + a = 1.0 + b = 5.1 / (4.0 * np.pi**2) + c = 5.0 / np.pi + r = 6.0 + s = 10.0 + t = 1.0 / (8.0 * np.pi) + ret = a * (x1 - b * x0**2 + c * x0 - r) ** 2 + s * (1 - t) * np.cos(x0) + s + + return ret + + +if __name__ == "__main__": + model = Branin() + + # Scenario object specifying the optimization "environment" + scenario = Scenario(model.configspace, deterministic=True, n_trials=100) + + # Create cluster + n_workers = 4 # Use 4 workers on the cluster + # Please note that the number of workers is directly set in the + # cluster / client. `scenario.n_workers` is ignored in this case. + + cluster = SLURMCluster( + # This is the partition of our slurm cluster. + queue="cpu_short", + # Your account name + # account="myaccount", + cores=1, + memory="1 GB", + # Walltime limit for each worker. Ensure that your function evaluations + # do not exceed this limit. + # More tips on this here: https://jobqueue.dask.org/en/latest/advanced-tips-and-tricks.html#how-to-handle-job-queueing-system-walltime-killing-workers + walltime="00:10:00", + processes=1, + log_directory="tmp/smac_dask_slurm", + ) + cluster.scale(jobs=n_workers) + + # Dask will create n_workers jobs on the cluster which stay open. + # Then, SMAC/Dask will schedule individual runs on the workers like on your local machine. + client = Client( + address=cluster, + ) + # Instead, you can also do + # client = cluster.get_client() + + # Now we use SMAC to find the best hyperparameters + smac = BlackBoxFacade( + scenario, + model.train, # We pass the target function here + overwrite=True, # Overrides any previous results that are found that are inconsistent with the meta-data + dask_client=client, + ) + + incumbent = smac.optimize() + + # Get cost of default configuration + default_cost = smac.validate(model.configspace.get_default_configuration()) + print(f"Default cost: {default_cost}") + + # Let's calculate the cost of the incumbent + incumbent_cost = smac.validate(incumbent) + print(f"Incumbent cost: {incumbent_cost}") diff --git a/examples/4_advanced_optimizer/3_metadata_callback.py b/examples/4_advanced_optimizer/3_metadata_callback.py new file mode 100644 index 0000000000..1f7078f941 --- /dev/null +++ b/examples/4_advanced_optimizer/3_metadata_callback.py @@ -0,0 +1,72 @@ +""" +Callback for logging run metadata +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +An example for using a callback to log run metadata to a file. Any arguments passed to the callback will be logged +to a json file at the beginning of the SMAC run (arguments must be json serializable). + +Instead of editing the Git-related information (repository, branch, commit) by hand each time they change, +this information can also be added automatically using GitPython (install via "pip install GitPython"). +There is an example for obtaining the information via GitPython below: + from git import Repo + repo = Repo(".", search_parent_directories=True) + MetadataCallback( + repository=repo.working_tree_dir.split("/")[-1], + branch=str(repo.active_branch), + commit=str(repo.head.commit), + command=" ".join([sys.argv[0][len(repo.working_tree_dir) + 1:]] + sys.argv[1:]), + ) +""" + +import sys +from ConfigSpace import Configuration, ConfigurationSpace, Float + +from smac import HyperparameterOptimizationFacade as HPOFacade +from smac import Scenario +from smac.callback.metadata_callback import MetadataCallback + +__copyright__ = "Copyright 2023, AutoML.org Freiburg-Hannover" +__license__ = "3-clause BSD" + + +class Rosenbrock2D: + @property + def configspace(self) -> ConfigurationSpace: + cs = ConfigurationSpace(seed=0) + x0 = Float("x0", (-5, 10), default=-3) + x1 = Float("x1", (-5, 10), default=-4) + cs.add_hyperparameters([x0, x1]) + + return cs + + def train(self, config: Configuration, seed: int = 0) -> float: + x1 = config["x0"] + x2 = config["x1"] + + cost = 100.0 * (x2 - x1**2.0) ** 2.0 + (1 - x1) ** 2.0 + return cost + + +if __name__ == "__main__": + model = Rosenbrock2D() + + # Scenario object specifying the optimization "environment" + scenario = Scenario(model.configspace, n_trials=200) + + # Now we use SMAC to find the best hyperparameters and add the metadata callback defined above + HPOFacade( + scenario, + model.train, + overwrite=True, + callbacks=[ + MetadataCallback( + project_name="My Project Name", + repository="My Repository Name", + branch="Name of Active Branch", + commit="Commit Hash", + command=" ".join(sys.argv), + additional_information="Some Additional Information" + ) + ], + logging_level=999999, + ).optimize() diff --git a/setup.py b/setup.py index 9838322ffa..79249cd5f9 100644 --- a/setup.py +++ b/setup.py @@ -68,6 +68,7 @@ def read_file(filepath: str) -> str: "scikit-learn>=1.1.2", "pyrfr>=0.9.0", "dask[distributed]", + "dask_jobqueue", "emcee>=3.0.0", "regex", "pyyaml", diff --git a/smac/__init__.py b/smac/__init__.py index c3446c136d..f3fe5805b9 100644 --- a/smac/__init__.py +++ b/smac/__init__.py @@ -19,15 +19,11 @@ Copyright {datetime.date.today().strftime('%Y')}, Marius Lindauer, Katharina Eggensperger, Matthias Feurer, André Biedenkapp, Difan Deng, Carolin Benjamins, Tim Ruhkopf, René Sass and Frank Hutter""" -version = "2.0.0" +version = "2.0.1" try: - from smac.utils.logging import setup_logging - - setup_logging(0) - - from smac.callback import Callback + from smac.callback.callback import Callback from smac.facade import ( AlgorithmConfigurationFacade, BlackBoxFacade, diff --git a/smac/acquisition/maximizer/helpers.py b/smac/acquisition/maximizer/helpers.py index 82afdf1e71..d7319f20fb 100644 --- a/smac/acquisition/maximizer/helpers.py +++ b/smac/acquisition/maximizer/helpers.py @@ -21,7 +21,7 @@ class ChallengerList(Iterator): configspace : ConfigurationSpace challenger_callback : Callable Callback function which returns a list of challengers (without interleaved random configurations, must a be a - closure: https://www.programiz.com/python-programming/closure) + python closure. random_design : AbstractRandomDesign | None, defaults to ModulusRandomDesign(modulus=2.0) Which random design should be used. """ diff --git a/smac/callback/__init__.py b/smac/callback/__init__.py new file mode 100644 index 0000000000..73e9dc6e47 --- /dev/null +++ b/smac/callback/__init__.py @@ -0,0 +1,7 @@ +from smac.callback.callback import Callback +from smac.callback.metadata_callback import MetadataCallback + +__all__ = [ + "Callback", + "MetadataCallback", +] diff --git a/smac/callback.py b/smac/callback/callback.py similarity index 100% rename from smac/callback.py rename to smac/callback/callback.py diff --git a/smac/callback/metadata_callback.py b/smac/callback/metadata_callback.py new file mode 100644 index 0000000000..626de5dee5 --- /dev/null +++ b/smac/callback/metadata_callback.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import json +import platform +from datetime import datetime + +import smac +from smac.callback.callback import Callback +from smac.main.smbo import SMBO + +__copyright__ = "Copyright 2023, AutoML.org Freiburg-Hannover" +__license__ = "3-clause BSD" + + +class MetadataCallback(Callback): + def __init__(self, **kwargs: str | int | float | dict | list) -> None: + # Arguments must be json serializable + self.kwargs = kwargs + + def on_start(self, smbo: SMBO) -> None: + """Called before the optimization starts.""" + path = smbo._scenario.output_directory + meta_dict = { + "utc_time": str(datetime.utcnow()), + "os": platform.platform(), + "smac_version": getattr(smac, "version"), + } + for key, value in self.kwargs.items(): + meta_dict[key] = value + + path.mkdir(parents=True, exist_ok=True) + + with open(path / "metadata.json", "w") as fp: + json.dump(meta_dict, fp, indent=2) diff --git a/smac/facade/abstract_facade.py b/smac/facade/abstract_facade.py index 728194de6c..adb1411123 100644 --- a/smac/facade/abstract_facade.py +++ b/smac/facade/abstract_facade.py @@ -7,6 +7,8 @@ import joblib from ConfigSpace import Configuration +from dask.distributed import Client +from typing_extensions import Literal import smac from smac.acquisition.function.abstract_acquisition_function import ( @@ -15,7 +17,7 @@ from smac.acquisition.maximizer.abstract_acqusition_maximizer import ( AbstractAcquisitionMaximizer, ) -from smac.callback import Callback +from smac.callback.callback import Callback from smac.initial_design.abstract_initial_design import AbstractInitialDesign from smac.intensifier.abstract_intensifier import AbstractIntensifier from smac.main.config_selector import ConfigSelector @@ -81,15 +83,21 @@ class AbstractFacade: Based on the runhistory, the surrogate model is trained. However, the data first needs to be encoded, which is done by the runhistory encoder. For example, inactive hyperparameters need to be encoded or cost values can be log transformed. - logging_level: int | Path | None + logging_level: int | Path | Literal[False] | None The level of logging (the lowest level 0 indicates the debug level). If a path is passed, a yaml file is expected with the logging configuration. If nothing is passed, the default logging.yml from SMAC is used. + If False is passed, SMAC will not do any customization of the logging setup and the responsibility is left + to the user. callbacks: list[Callback], defaults to [] Callbacks, which are incorporated into the optimization loop. overwrite: bool, defaults to False When True, overwrites the run results if a previous run is found that is inconsistent in the meta data with the current setup. If ``overwrite`` is set to False, the user is asked for the exact behaviour (overwrite completely, save old run, or use old results). + dask_client: Client | None, defaults to None + User-created dask client, which can be used to start a dask cluster and then attach SMAC to it. This will not + be closed automatically and will have to be closed manually if provided explicitly. If none is provided + (default), a local one will be created for you and closed upon completion. """ def __init__( @@ -106,9 +114,10 @@ def __init__( multi_objective_algorithm: AbstractMultiObjectiveAlgorithm | None = None, runhistory_encoder: AbstractRunHistoryEncoder | None = None, config_selector: ConfigSelector | None = None, - logging_level: int | Path | None = None, + logging_level: int | Path | Literal[False] | None = None, callbacks: list[Callback] = [], overwrite: bool = False, + dask_client: Client | None = None, ): setup_logging(logging_level) @@ -178,14 +187,19 @@ def __init__( ) # In case of multiple jobs, we need to wrap the runner again using DaskParallelRunner - if (n_workers := scenario.n_workers) > 1: - available_workers = joblib.cpu_count() - if n_workers > available_workers: - logger.info(f"Workers are reduced to {n_workers}.") - n_workers = available_workers + if (n_workers := scenario.n_workers) > 1 or dask_client is not None: + if dask_client is not None: + logger.warning( + "Provided `dask_client`. Ignore `scenario.n_workers`, directly set `n_workers` in `dask_client`." + ) + else: + available_workers = joblib.cpu_count() + if n_workers > available_workers: + logger.info(f"Workers are reduced to {n_workers}.") + n_workers = available_workers # We use a dask runner for parallelization - runner = DaskParallelRunner(single_worker=runner) + runner = DaskParallelRunner(single_worker=runner, dask_client=dask_client) # Set the runner to access it globally self._runner = runner @@ -205,11 +219,11 @@ def __init__( # Register callbacks here for callback in callbacks: - self._optimizer._register_callback(callback) + self._optimizer.register_callback(callback) # Additionally, we register the runhistory callback from the intensifier to efficiently update our incumbent # every time new information are available - self._optimizer._register_callback(self._intensifier.get_callback()) + self._optimizer.register_callback(self._intensifier.get_callback(), index=0) @property def scenario(self) -> Scenario: @@ -275,10 +289,21 @@ def tell(self, info: TrialInfo, value: TrialValue, save: bool = True) -> None: """ return self._optimizer.tell(info, value, save=save) - def optimize(self) -> Configuration | list[Configuration]: + def optimize(self, *, data_to_scatter: dict[str, Any] | None = None) -> Configuration | list[Configuration]: """ Optimizes the configuration of the algorithm. + Parameters + ---------- + data_to_scatter: dict[str, Any] | None + We first note that this argument is valid only dask_runner! + When a user scatters data from their local process to the distributed network, + this data is distributed in a round-robin fashion grouping by number of cores. + Roughly speaking, we can keep this data in memory and then we do not have to (de-)serialize the data + every time we would like to execute a target function with a big dataset. + For example, when your target function has a big dataset shared across all the target function, + this argument is very useful. + Returns ------- incumbent : Configuration @@ -286,7 +311,7 @@ def optimize(self) -> Configuration | list[Configuration]: """ incumbents = None try: - incumbents = self._optimizer.optimize() + incumbents = self._optimizer.optimize(data_to_scatter=data_to_scatter) finally: self._optimizer.save() diff --git a/smac/initial_design/abstract_initial_design.py b/smac/initial_design/abstract_initial_design.py index ddf505e30c..7e17c88c24 100644 --- a/smac/initial_design/abstract_initial_design.py +++ b/smac/initial_design/abstract_initial_design.py @@ -51,7 +51,7 @@ def __init__( n_configs: int | None = None, n_configs_per_hyperparameter: int | None = 10, max_ratio: float = 0.25, - additional_configs: list[Configuration] = [], + additional_configs: list[Configuration] = None, seed: int | None = None, ): self._configspace = scenario.configspace @@ -59,9 +59,22 @@ def __init__( if seed is None: seed = scenario.seed + self.use_default_config = scenario.use_default_config + self._seed = seed self._rng = np.random.RandomState(seed) self._n_configs_per_hyperparameter = n_configs_per_hyperparameter + + # make sure that additional configs is not a mutable default value + # this avoids issues + if additional_configs is None: + additional_configs = [] + + if self.use_default_config: + default_config = self._configspace.get_default_configuration() + default_config.origin = "Initial Design: Default configuration" + additional_configs.append(default_config) + self._additional_configs = additional_configs n_params = len(self._configspace.get_hyperparameters()) diff --git a/smac/intensifier/abstract_intensifier.py b/smac/intensifier/abstract_intensifier.py index 5d2954ac3d..b944867273 100644 --- a/smac/intensifier/abstract_intensifier.py +++ b/smac/intensifier/abstract_intensifier.py @@ -12,7 +12,7 @@ from ConfigSpace import Configuration import smac -from smac.callback import Callback +from smac.callback.callback import Callback from smac.constants import MAXINT from smac.main.config_selector import ConfigSelector from smac.runhistory import TrialInfo @@ -271,13 +271,12 @@ def get_instance_seed_keys_of_interest( i = 0 while True: - # We have two conditions to stop the loop: - # A) We found enough configs - # B) We used enough seeds - A = self._max_config_calls is not None and len(instance_seed_keys) >= self._max_config_calls - B = self._n_seeds is not None and i >= self._n_seeds + found_enough_configs = ( + self._max_config_calls is not None and len(instance_seed_keys) >= self._max_config_calls + ) + used_enough_seeds = self._n_seeds is not None and i >= self._n_seeds - if A or B: + if found_enough_configs or used_enough_seeds: break if validate: @@ -419,7 +418,10 @@ def get_incumbent_instance_seed_budget_key_differences(self, compare: bool = Fal if len(incumbent_isb_keys) <= 1: return [] - incumbent_isb_keys = list(set.difference(*map(set, incumbent_isb_keys))) # type: ignore + # Compute the actual differences + intersection_isb_keys = set.intersection(*map(set, incumbent_isb_keys)) # type: ignore + union_isb_keys = set.union(*map(set, incumbent_isb_keys)) # type: ignore + incumbent_isb_keys = list(union_isb_keys - intersection_isb_keys) # type: ignore if len(incumbent_isb_keys) == 0: return [] diff --git a/smac/intensifier/hyperband.py b/smac/intensifier/hyperband.py index fda1c805ff..69d21d63c3 100644 --- a/smac/intensifier/hyperband.py +++ b/smac/intensifier/hyperband.py @@ -2,8 +2,6 @@ from typing import Any -import numpy as np - from smac.intensifier.successive_halving import SuccessiveHalving @@ -22,36 +20,22 @@ def __post_init__(self) -> None: min_budget = self._min_budget max_budget = self._max_budget + assert min_budget is not None and max_budget is not None eta = self._eta # The only difference we have to do is change max_iterations, n_configs_in_stage, budgets_in_stage - s_max = int(np.floor(np.log(max_budget / min_budget) / np.log(eta))) # type: ignore[operator] - - max_iterations: dict[int, int] = {} - n_configs_in_stage: dict[int, list] = {} - budgets_in_stage: dict[int, list] = {} - - for i in range(s_max + 1): - max_iter = s_max - i - n_initial_challengers = int(eta**max_iter) - - # How many configs in each stage - linspace = -np.linspace(0, max_iter, max_iter + 1) - n_configs_ = n_initial_challengers * np.power(eta, linspace) - n_configs = np.array(np.round(n_configs_), dtype=int).tolist() - - # How many budgets in each stage - linspace = -np.linspace(max_iter, 0, max_iter + 1) - budgets = (max_budget * np.power(eta, linspace)).tolist() - - max_iterations[i] = max_iter + 1 - n_configs_in_stage[i] = n_configs - budgets_in_stage[i] = budgets - - self._s_max = s_max - self._max_iterations = max_iterations - self._n_configs_in_stage = n_configs_in_stage - self._budgets_in_stage = budgets_in_stage + self._s_max = self._get_max_iterations(eta, max_budget, min_budget) # type: ignore[operator] + self._max_iterations: dict[int, int] = {} + self._n_configs_in_stage: dict[int, list] = {} + self._budgets_in_stage: dict[int, list] = {} + + for i in range(self._s_max + 1): + max_iter = self._s_max - i + + self._budgets_in_stage[i], self._n_configs_in_stage[i] = self._compute_configs_and_budgets_for_stages( + eta, max_budget, max_iter, self._s_max + ) + self._max_iterations[i] = max_iter + 1 def get_state(self) -> dict[str, Any]: # noqa: D102 state = super().get_state() diff --git a/smac/intensifier/intensifier.py b/smac/intensifier/intensifier.py index 7617961032..345013f878 100644 --- a/smac/intensifier/intensifier.py +++ b/smac/intensifier/intensifier.py @@ -81,7 +81,11 @@ def uses_instances(self) -> bool: # noqa: D102 def get_state(self) -> dict[str, Any]: # noqa: D102 return { - "queue": [(self.runhistory.get_config_id(config), n) for config, n in self._queue], + "queue": [ + (self.runhistory.get_config_id(config), n) + for config, n in self._queue + if self.runhistory.has_config(config) + ], } def set_state(self, state: dict[str, Any]) -> None: # noqa: D102 diff --git a/smac/intensifier/successive_halving.py b/smac/intensifier/successive_halving.py index f4d19eac6e..e7061a190b 100644 --- a/smac/intensifier/successive_halving.py +++ b/smac/intensifier/successive_halving.py @@ -2,6 +2,7 @@ from typing import Any, Iterator +import math from collections import defaultdict import numpy as np @@ -166,17 +167,8 @@ def __post_init__(self) -> None: ) # Pre-computing Successive Halving variables - max_iter = int(np.floor(np.log(max_budget / min_budget) / np.log(eta))) - n_initial_challengers = int(eta**max_iter) - - # How many configs in each stage - linspace = -np.linspace(0, max_iter, max_iter + 1) - n_configs_ = n_initial_challengers * np.power(eta, linspace) - n_configs = np.array(np.round(n_configs_), dtype=int).tolist() - - # How many budgets in each stage - linspace = -np.linspace(max_iter, 0, max_iter + 1) - budgets = (max_budget * np.power(eta, linspace)).tolist() + max_iter = self._get_max_iterations(eta, max_budget, min_budget) + budgets, n_configs = self._compute_configs_and_budgets_for_stages(eta, max_budget, max_iter) # Global variables self._min_budget = min_budget @@ -187,6 +179,30 @@ def __post_init__(self) -> None: self._n_configs_in_stage: dict[int, list] = {0: n_configs} self._budgets_in_stage: dict[int, list] = {0: budgets} + @staticmethod + def _get_max_iterations(eta: int, max_budget: float | int, min_budget: float | int) -> int: + return int(np.floor(np.log(max_budget / min_budget) / np.log(eta))) + + @staticmethod + def _compute_configs_and_budgets_for_stages( + eta: int, max_budget: float | int, max_iter: int, s_max: int | None = None + ) -> tuple[list[int], list[int]]: + if s_max is None: + s_max = max_iter + + n_initial_challengers = math.ceil((eta**max_iter) * (s_max + 1) / (max_iter + 1)) + + # How many configs in each stage + lin_space = -np.linspace(0, max_iter, max_iter + 1) + n_configs_ = np.floor(n_initial_challengers * np.power(eta, lin_space)) + n_configs = np.array(np.round(n_configs_), dtype=int).tolist() + + # How many budgets in each stage + lin_space = -np.linspace(max_iter, 0, max_iter + 1) + budgets = (max_budget * np.power(eta, lin_space)).tolist() + + return budgets, n_configs + def get_state(self) -> dict[str, Any]: # noqa: D102 # Replace config by dict tracker: dict[str, list[tuple[int | None, list[dict]]]] = defaultdict(list) diff --git a/smac/main/config_selector.py b/smac/main/config_selector.py index 312e6d05f8..76ace17f84 100644 --- a/smac/main/config_selector.py +++ b/smac/main/config_selector.py @@ -13,7 +13,7 @@ from smac.acquisition.maximizer.abstract_acqusition_maximizer import ( AbstractAcquisitionMaximizer, ) -from smac.callback import Callback +from smac.callback.callback import Callback from smac.initial_design import AbstractInitialDesign from smac.model.abstract_model import AbstractModel from smac.random_design.abstract_random_design import AbstractRandomDesign diff --git a/smac/main/smbo.py b/smac/main/smbo.py index 9a96620d4c..30e01c2e97 100644 --- a/smac/main/smbo.py +++ b/smac/main/smbo.py @@ -8,17 +8,19 @@ import numpy as np from ConfigSpace import Configuration +from numpy import ndarray from smac.acquisition.function.abstract_acquisition_function import ( AbstractAcquisitionFunction, ) -from smac.callback import Callback +from smac.callback.callback import Callback from smac.intensifier.abstract_intensifier import AbstractIntensifier from smac.model.abstract_model import AbstractModel from smac.runhistory import StatusType, TrialInfo, TrialValue from smac.runhistory.runhistory import RunHistory from smac.runner import FirstRunCrashedException from smac.runner.abstract_runner import AbstractRunner +from smac.runner.dask_runner import DaskParallelRunner from smac.scenario import Scenario from smac.utils.data_structures import recursively_compare_dicts from smac.utils.logging import get_logger @@ -244,9 +246,19 @@ def update_acquisition_function(self, acquisition_function: AbstractAcquisitionF assert config_selector._acquisition_maximizer is not None config_selector._acquisition_maximizer.acquisition_function = acquisition_function - def optimize(self) -> Configuration | list[Configuration]: + def optimize(self, *, data_to_scatter: dict[str, Any] | None = None) -> Configuration | list[Configuration]: """Runs the Bayesian optimization loop. + Parameters + ---------- + data_to_scatter: dict[str, Any] | None + When a user scatters data from their local process to the distributed network, + this data is distributed in a round-robin fashion grouping by number of cores. + Roughly speaking, we can keep this data in memory and then we do not have to (de-)serialize the data + every time we would like to execute a target function with a big dataset. + For example, when your target function has a big dataset shared across all the target function, + this argument is very useful. + Returns ------- incumbent : Configuration @@ -269,6 +281,15 @@ def optimize(self) -> Configuration | list[Configuration]: for callback in self._callbacks: callback.on_start(self) + dask_data_to_scatter = {} + if isinstance(self._runner, DaskParallelRunner) and data_to_scatter is not None: + dask_data_to_scatter = dict(data_to_scatter=self._runner._client.scatter(data_to_scatter, broadcast=True)) + elif data_to_scatter is not None: + raise ValueError( + "data_to_scatter is valid only for DaskParallelRunner, " + f"but {dask_data_to_scatter} was provided for {self._runner.__class__.__name__}" + ) + # Main BO loop while True: for callback in self._callbacks: @@ -280,7 +301,7 @@ def optimize(self) -> Configuration | list[Configuration]: # We submit the trial to the runner # In multi-worker mode, SMAC waits till a new worker is available here - self._runner.submit_trial(trial_info=trial_info) + self._runner.submit_trial(trial_info=trial_info, **dask_data_to_scatter) except StopIteration: self._stop = True @@ -440,9 +461,19 @@ def _add_results(self) -> None: logger.info("Cost threshold was reached. Abort is requested.") self._stop = True - def _register_callback(self, callback: Callback) -> None: - """Registers a callback to be called before, in between, and after the Bayesian optimization loop.""" - self._callbacks += [callback] + def register_callback(self, callback: Callback, index: int = -1) -> None: + """ + Registers a callback to be called before, in between, and after the Bayesian optimization loop. + + + Parameters + ---------- + callback : Callback + The callback to be registered. + index : int + The index at which the callback should be registered. + """ + self._callbacks.insert(index, callback) def _initialize_state(self) -> None: """Detects whether the optimization is restored from a previous state.""" @@ -512,28 +543,27 @@ def validate( config: Configuration, *, seed: int | None = None, - ) -> float | list[float]: + ) -> float | ndarray[float]: """Validates a configuration on other seeds than the ones used in the optimization process and on the highest - budget (if budget type is real-valued). + budget (if budget type is real-valued). Does not exceed the maximum number of config calls or seeds as defined + in the scenario. Parameters ---------- config : Configuration Configuration to validate - instances : list[str] | None, defaults to None - Which instances to validate. If None, all instances specified in the scenario are used. In case that the budget type is real-valued budget, this argument is ignored. seed : int | None, defaults to None If None, the seed from the scenario is used. Returns ------- - cost : float | list[float] + cost : float | ndarray[float] The averaged cost of the configuration. In case of multi-fidelity, the cost of each objective is averaged. """ if seed is None: - seed = 0 + seed = self._scenario.seed costs = [] for trial in self._intensifier.get_trials_of_interest(config, validate=True, seed=seed): diff --git a/smac/runhistory/runhistory.py b/smac/runhistory/runhistory.py index 9098acbb52..a30e1356ee 100644 --- a/smac/runhistory/runhistory.py +++ b/smac/runhistory/runhistory.py @@ -221,6 +221,9 @@ def add( config_id = self._n_id + # Set the id attribute of the config object, so that users can access it + config.config_id = config_id + if status != StatusType.RUNNING: if self._n_objectives == -1: self._n_objectives = n_objectives @@ -593,6 +596,10 @@ def get_config_id(self, config: Configuration) -> int: """Returns the configuration id from a configuration.""" return self._config_ids[config] + def has_config(self, config: Configuration) -> bool: + """Check if the config is stored in the runhistory""" + return config in self._config_ids + def get_configs(self, sort_by: str | None = None) -> list[Configuration]: """Return all configurations in this RunHistory object. diff --git a/smac/runner/abstract_runner.py b/smac/runner/abstract_runner.py index b7ba1f26f5..2631bd966d 100644 --- a/smac/runner/abstract_runner.py +++ b/smac/runner/abstract_runner.py @@ -76,7 +76,9 @@ def __init__( assert isinstance(scenario.crash_cost, float) self._crash_cost = [scenario.crash_cost for _ in range(self._n_objectives)] - def run_wrapper(self, trial_info: TrialInfo) -> tuple[TrialInfo, TrialValue]: + def run_wrapper( + self, trial_info: TrialInfo, **dask_data_to_scatter: dict[str, Any] + ) -> tuple[TrialInfo, TrialValue]: """Wrapper around run() to execute and check the execution of a given config. This function encapsulates common handling/processing, so that run() implementation is simplified. @@ -85,6 +87,13 @@ def run_wrapper(self, trial_info: TrialInfo) -> tuple[TrialInfo, TrialValue]: ---------- trial_info : RunInfo Object that contains enough information to execute a configuration run in isolation. + dask_data_to_scatter: dict[str, Any] + When a user scatters data from their local process to the distributed network, + this data is distributed in a round-robin fashion grouping by number of cores. + Roughly speaking, we can keep this data in memory and then we do not have to (de-)serialize the data + every time we would like to execute a target function with a big dataset. + For example, when your target function has a big dataset shared across all the target function, + this argument is very useful. Returns ------- @@ -101,6 +110,7 @@ def run_wrapper(self, trial_info: TrialInfo) -> tuple[TrialInfo, TrialValue]: instance=trial_info.instance, budget=trial_info.budget, seed=trial_info.seed, + **dask_data_to_scatter, ) except Exception as e: status = StatusType.CRASHED diff --git a/smac/runner/dask_runner.py b/smac/runner/dask_runner.py index 2afad7bd30..b9aade4015 100644 --- a/smac/runner/dask_runner.py +++ b/smac/runner/dask_runner.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Iterator +from typing import Any, Iterator import time from pathlib import Path @@ -98,7 +98,7 @@ def __init__( self._client = dask_client self._close_client_at_del = False - def submit_trial(self, trial_info: TrialInfo) -> None: + def submit_trial(self, trial_info: TrialInfo, **dask_data_to_scatter: dict[str, Any]) -> None: """This function submits a configuration embedded in a ``trial_info`` object, and uses one of the workers to produce a result locally to each worker. @@ -115,6 +115,14 @@ def submit_trial(self, trial_info: TrialInfo) -> None: ---------- trial_info : TrialInfo An object containing the configuration launched. + + dask_data_to_scatter: dict[str, Any] + When a user scatters data from their local process to the distributed network, + this data is distributed in a round-robin fashion grouping by number of cores. + Roughly speaking, we can keep this data in memory and then we do not have to (de-)serialize the data + every time we would like to execute a target function with a big dataset. + For example, when your target function has a big dataset shared across all the target function, + this argument is very useful. """ # Check for resources or block till one is available if self.count_available_workers() <= 0: @@ -133,7 +141,7 @@ def submit_trial(self, trial_info: TrialInfo) -> None: ) # At this point we can submit the job - trial = self._client.submit(self._single_worker.run_wrapper, trial_info=trial_info) + trial = self._client.submit(self._single_worker.run_wrapper, trial_info=trial_info, **dask_data_to_scatter) self._pending_trials.append(trial) def iter_results(self) -> Iterator[tuple[TrialInfo, TrialValue]]: # noqa: D102 @@ -154,8 +162,11 @@ def run( instance: str | None = None, budget: float | None = None, seed: int | None = None, + **dask_data_to_scatter: dict[str, Any], ) -> tuple[StatusType, float | list[float], float, dict]: # noqa: D102 - return self._single_worker.run(config=config, instance=instance, seed=seed, budget=budget) + return self._single_worker.run( + config=config, instance=instance, seed=seed, budget=budget, **dask_data_to_scatter + ) def count_available_workers(self) -> int: """Total number of workers available. This number is dynamic as more resources diff --git a/smac/runner/target_function_runner.py b/smac/runner/target_function_runner.py index 44df75a0c8..28837caa09 100644 --- a/smac/runner/target_function_runner.py +++ b/smac/runner/target_function_runner.py @@ -72,7 +72,12 @@ def __init__( # Pynisher limitations if (memory := self._scenario.trial_memory_limit) is not None: + unit = None + if isinstance(memory, (tuple, list)): + memory, unit = memory memory = int(math.ceil(memory)) + if unit is not None: + memory = (memory, unit) if (time := self._scenario.trial_walltime_limit) is not None: time = int(math.ceil(time)) @@ -93,6 +98,7 @@ def run( instance: str | None = None, budget: float | None = None, seed: int | None = None, + **dask_data_to_scatter: dict[str, Any], ) -> tuple[StatusType, float | list[float], float, dict]: """Calls the target function with pynisher if algorithm wall time limit or memory limit is set. Otherwise, the function is called directly. @@ -107,6 +113,14 @@ def run( A positive, real-valued number representing an arbitrary limit to the target function handled by the target function internally. seed : int, defaults to None + dask_data_to_scatter: dict[str, Any] + This kwargs must be empty when we do not use dask! () + When a user scatters data from their local process to the distributed network, + this data is distributed in a round-robin fashion grouping by number of cores. + Roughly speaking, we can keep this data in memory and then we do not have to (de-)serialize the data + every time we would like to execute a target function with a big dataset. + For example, when your target function has a big dataset shared across all the target function, + this argument is very useful. Returns ------- @@ -121,6 +135,7 @@ def run( """ # The kwargs are passed to the target function. kwargs: dict[str, Any] = {} + kwargs.update(dask_data_to_scatter) if "seed" in self._required_arguments: kwargs["seed"] = seed diff --git a/smac/scenario.py b/smac/scenario.py index 2451013121..133ae57b07 100644 --- a/smac/scenario.py +++ b/smac/scenario.py @@ -57,6 +57,11 @@ class Scenario: n_trials : int, defaults to 100 The maximum number of trials (combination of configuration, seed, budget, and instance, depending on the task) to run. + use_default_config: bool, defaults to False. + If True, the configspace's default configuration is evaluated in the initial design. + For historic benchmark reasons, this is False by default. + Notice, that this will result in n_configs + 1 for the initial design. Respecting n_trials, + this will result in one fewer evaluated configuration in the optimization. instances : list[str] | None, defaults to None Names of the instances to use. If None, no instances are used. Instances could be dataset names, seeds, subsets, etc. @@ -93,6 +98,7 @@ class Scenario: trial_walltime_limit: float | None = None trial_memory_limit: int | None = None n_trials: int = 100 + use_default_config: bool = False # Algorithm Configuration instances: list[str] | None = None diff --git a/smac/utils/logging.py b/smac/utils/logging.py index 0e66be776e..64e26ac207 100644 --- a/smac/utils/logging.py +++ b/smac/utils/logging.py @@ -5,6 +5,7 @@ from pathlib import Path import yaml +from typing_extensions import Literal import smac @@ -12,14 +13,20 @@ __license__ = "3-clause BSD" -def setup_logging(level: int | Path | None = None) -> None: +def setup_logging( + level: int | Path | Literal[False] | None = False, +) -> None: """Sets up the logging configuration for all modules. Parameters ---------- - level : int | Path | None, defaults to None + level : int | Path | Literal[False] | None, defaults to None An integer representing the logging level. An custom logging configuration can be used when passing a path. + If False, no logging setup is performed. """ + if level is False: + return + if isinstance(level, Path): log_filename = level else: diff --git a/tests/fixtures/scenario.py b/tests/fixtures/scenario.py index 8cc131ed15..6211939879 100644 --- a/tests/fixtures/scenario.py +++ b/tests/fixtures/scenario.py @@ -20,6 +20,7 @@ def _make( max_budget: int = 5, n_workers: int = 1, n_trials: int = 100, + use_default_config: bool = False, ) -> Scenario: objectives = "cost" if use_multi_objective: @@ -50,6 +51,7 @@ def _make( instance_features=instance_features, min_budget=min_budget, max_budget=max_budget, + use_default_config=use_default_config ) return _make diff --git a/tests/test_callback.py b/tests/test_callback.py index 18a7d6f985..fe369f0252 100644 --- a/tests/test_callback.py +++ b/tests/test_callback.py @@ -4,7 +4,7 @@ import smac from smac import HyperparameterOptimizationFacade, Scenario -from smac.callback import Callback +from smac.callback.callback import Callback from smac.initial_design import DefaultInitialDesign from smac.intensifier.intensifier import Intensifier from smac.runhistory import TrialInfo, TrialValue diff --git a/tests/test_continue.py b/tests/test_continue.py index 85d2661d81..0264ba82cb 100644 --- a/tests/test_continue.py +++ b/tests/test_continue.py @@ -13,7 +13,7 @@ from smac import MultiFidelityFacade as MFFacade from smac import RandomFacade as RFacade from smac import Scenario -from smac.callback import Callback +from smac.callback.callback import Callback from smac.runhistory.dataclasses import TrialInfo, TrialValue FACADES = [BBFacade, HPOFacade, MFFacade, RFacade, HBFacade, ACFacade] @@ -129,7 +129,7 @@ def on_tell_end(self, smbo, info: TrialInfo, value: TrialValue) -> bool | None: # Let's see if we restored the runhistory correctly; used walltime should be roughly the same # However, since some more things happen in the background, it might be slightly different - assert pytest.approx(smac2._optimizer.used_walltime, 0.2) == smac._optimizer.used_walltime + assert pytest.approx(smac2._optimizer.used_walltime, 0.5) == smac._optimizer.used_walltime assert smac2.runhistory.finished == smac.runhistory.finished smac2.optimize() diff --git a/tests/test_initial_design/test_initial_design.py b/tests/test_initial_design/test_initial_design.py index d4b221cd4e..d12e503ff5 100644 --- a/tests/test_initial_design/test_initial_design.py +++ b/tests/test_initial_design/test_initial_design.py @@ -121,3 +121,16 @@ def test_select_configurations(make_scenario, configspace_small): # We expect empty list here with pytest.raises(NotImplementedError): dc.select_configurations() + +def test_include_default_config(make_scenario, configspace_small): + scenario = make_scenario(configspace_small, use_default_config=True) + + dc = AbstractInitialDesign( + scenario=scenario, + n_configs=15, + ) + + # if use_default_config is True, then the default config should be included in the additional_configs + default_config = scenario.configspace.get_default_configuration() + assert default_config in dc._additional_configs + diff --git a/tests/test_intensifier/test_hyperband.py b/tests/test_intensifier/test_hyperband.py index b27437c1b6..bc0b912168 100644 --- a/tests/test_intensifier/test_hyperband.py +++ b/tests/test_intensifier/test_hyperband.py @@ -30,10 +30,10 @@ def test_initialization(make_scenario, configspace_small): assert intensifier._max_iterations[4] == 1 assert intensifier._n_configs_in_stage[0] == [81, 27, 9, 3, 1] - assert intensifier._n_configs_in_stage[1] == [27, 9, 3, 1] - assert intensifier._n_configs_in_stage[2] == [9, 3, 1] - assert intensifier._n_configs_in_stage[3] == [3, 1] # in the paper it's 6 and 2 which is false - assert intensifier._n_configs_in_stage[4] == [1] # in the paper it's 5 which is false? + assert intensifier._n_configs_in_stage[1] == [34, 11, 3, 1] + assert intensifier._n_configs_in_stage[2] == [15, 5, 1] + assert intensifier._n_configs_in_stage[3] == [8, 2] + assert intensifier._n_configs_in_stage[4] == [5] assert intensifier._budgets_in_stage[0] == [1, 3, 9, 27, 81] assert intensifier._budgets_in_stage[1] == [3, 9, 27, 81] @@ -75,7 +75,7 @@ def test_state(make_scenario, configspace_small, make_config_selector): gen = iter(intensifier) # Add some configs to the tracker - for _ in range(10): + for _ in range(12): trial = next(gen) runhistory.add_running_trial(trial) # We have to mark it as running manually intensifier.update_incumbents(trial.config)