diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..e5e5092a2 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,9 @@ +version: 2 + +updates: + # This will check for updates to github actions every day + # https://docs.github.com/en/enterprise-server@3.4/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" diff --git a/.github/stale.yml b/.github/stale.yml index f1508d0f2..7d15239c7 100644 --- a/.github/stale.yml +++ b/.github/stale.yml @@ -6,11 +6,16 @@ daysUntilClose: 7 # Issues with these labels will never be considered stale exemptLabels: - - pinned - - security + - bug + - dependency + - documentation + - enhancement + - feature + - test + - example # Label to use when marking an issue as stale -staleLabel: wontfix +staleLabel: stale # Comment to post when marking an issue as stale. Set to `false` to disable markComment: > diff --git a/.github/workflows/citation.yml b/.github/workflows/citation.yml index ec841dced..9ca4ace4b 100644 --- a/.github/workflows/citation.yml +++ b/.github/workflows/citation.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out a copy of the repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Check whether the citation metadata from CITATION.cff is valid uses: citation-file-format/cffconvert-github-action@2.0.0 diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml index 798ebd608..e43d6bcfb 100644 --- a/.github/workflows/dist.yml +++ b/.github/workflows/dist.yml @@ -22,10 +22,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: "3.9" diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 22cc9fb78..bbe410603 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -26,10 +26,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: "3.9" diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index 999165bab..001e6be3e 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -26,9 +26,9 @@ jobs: fail-fast: false steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index f8968ee98..5e5815016 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -21,12 +21,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: submodules: recursive - name: Setup Python 3.9 - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.9 diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index fcc32a3d7..f40927270 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -48,10 +48,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -101,7 +101,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Conda install uses: conda-incubator/setup-miniconda@v2 @@ -137,10 +137,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} diff --git a/README.md b/README.md index d06e47747..af1a5644b 100644 --- a/README.md +++ b/README.md @@ -66,20 +66,10 @@ X_val, y_val = np.random.randint(2, size=(5, 2)), np.random.randint(2, size=5) def train_random_forest(config): - """ - Trains a random forest on the given hyperparameters, defined by config, and returns the accuracy - on the validation data. - - Input: - config (Configuration): Configuration object derived from ConfigurationSpace. - - Return: - cost (float): Performance measure on the validation data. - """ model = RandomForestClassifier(max_depth=config["depth"]) model.fit(X_train, y_train) - # define the evaluation metric as return + # Define the evaluation metric as return return 1 - model.score(X_val, y_val) @@ -125,16 +115,18 @@ Freiburg](http://www.automl.org/). If you have found a bug, please report to [issues](https://github.com/automl/SMAC3/issues). Moreover, we are appreciating any kind of help. Find our guidlines for contributing to this package [here](https://github.com/automl/SMAC3/blob/master/.github/CONTRIBUTING.md). -If you use SMAC in one of your research projects, please cite us: +If you use SMAC in one of your research projects, please cite our [JMLR paper](https://jmlr.org/papers/v23/21-0888.html): ``` -@misc{lindauer2021smac3, - title={SMAC3: A Versatile Bayesian Optimization Package for Hyperparameter Optimization}, - author={Marius Lindauer and Katharina Eggensperger and Matthias Feurer and André Biedenkapp and Difan Deng and Carolin Benjamins and Tim Ruhkopf and René Sass and Frank Hutter}, - year={2021}, - eprint={2109.09831}, - archivePrefix={arXiv}, - primaryClass={cs.LG} +@article{JMLR:v23:21-0888, + author = {Marius Lindauer and Katharina Eggensperger and Matthias Feurer and André Biedenkapp and Difan Deng and Carolin Benjamins and Tim Ruhkopf and René Sass and Frank Hutter}, + title = {SMAC3: A Versatile Bayesian Optimization Package for Hyperparameter Optimization}, + journal = {Journal of Machine Learning Research}, + year = {2022}, + volume = {23}, + number = {54}, + pages = {1--9}, + url = {http://jmlr.org/papers/v23/21-0888.html} } ``` -Copyright (C) 2016-2021 [AutoML Group](http://www.automl.org/). \ No newline at end of file +Copyright (C) 2016-2022 [AutoML Group](http://www.automl.org/). diff --git a/changelog.md b/changelog.md index 65e156b2d..07965ca22 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,10 @@ +# 1.3.4 +* Added reference to JMLR paper. +* Typos in documentations. +* Code more readable since all typings are imported at the beginning of the file. +* Updated stale bot options. + + # 1.3.3 * Hotfix: Since multi-objective implementation depends on normalized costs, it now is ensured that the cached costs are updated everytime a new entry is added. diff --git a/docs/details/callbacks.rst b/docs/details/callbacks.rst index 5f0832d0d..b885784f5 100644 --- a/docs/details/callbacks.rst +++ b/docs/details/callbacks.rst @@ -9,15 +9,15 @@ How to add a new callback ^^^^^^^^^^^^^^^^^^^^^^^^^ * Implement a callback class in ``smac/callbacks.py``. There are no restrictions on how such a -callback must look like, but it is recommended to implement the main logic inside the `__call__` -function, such as for example in ``IncorporateRunResultCallback``. + callback must look like, but it is recommended to implement the main logic inside the `__call__` + function, such as for example in ``IncorporateRunResultCallback``. * Add your callback to ``smac.smbo.optimizer.SMBO._callbacks``, using the name of your callback -as the key, and an empty list as the value. + as the key, and an empty list as the value. * Add your callback to ``smac.smbo.optimizer.SMBO._callback_to_key``, using the callback class as -the key, and the name as value (the name used in 2.). + the key, and the name as value (the name used in 2.). * Implement calling all registered callbacks at the correct place. This is as simple as -``for callback in self._callbacks['your_callback']: callback(*args, **kwargs)``, where you -obviously need to change the callback name and signature. \ No newline at end of file + ``for callback in self._callbacks['your_callback']: callback(*args, **kwargs)``, where you + obviously need to change the callback name and signature. \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 2fab5ed88..c972796eb 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -22,7 +22,7 @@ efficiently decide which of two configurations performs better. SMAC3 is written in Python3 and continuously tested with Python 3.7, 3.8 and 3.9. Its Random Forest is written in C++. In further texts, SMAC is representatively mentioned for SMAC3. -If you use SMAC, please cite our paper: +If you use SMAC, please cite our `JMLR paper `_: .. code-block:: text diff --git a/examples/python/plot_svm_eips.py b/examples/python/plot_svm_eips.py index 56a83f208..b4fc5bf70 100644 --- a/examples/python/plot_svm_eips.py +++ b/examples/python/plot_svm_eips.py @@ -13,21 +13,25 @@ logging.basicConfig(level=logging.INFO) import numpy as np +from ConfigSpace.hyperparameters import ( + CategoricalHyperparameter, + UniformFloatHyperparameter, +) from sklearn import datasets, svm from sklearn.model_selection import cross_val_score -from ConfigSpace.hyperparameters import UniformFloatHyperparameter, CategoricalHyperparameter - from smac.configspace import ConfigurationSpace +from smac.epm.uncorrelated_mo_rf_with_instances import ( + UncorrelatedMultiObjectiveRandomForestWithInstances, +) from smac.facade.smac_ac_facade import SMAC4AC -# Import SMAC-utilities -from smac.scenario.scenario import Scenario - # EIPS related from smac.optimizer.acquisition import EIPS from smac.runhistory.runhistory2epm import RunHistory2EPM4EIPS -from smac.epm.uncorrelated_mo_rf_with_instances import UncorrelatedMultiObjectiveRandomForestWithInstances + +# Import SMAC-utilities +from smac.scenario.scenario import Scenario __copyright__ = "Copyright 2021, AutoML.org Freiburg-Hannover" __license__ = "3-clause BSD" @@ -91,10 +95,10 @@ def svm_from_cfg(cfg): # It returns: Status, Cost, Runtime, Additional Infos def_value = svm_from_cfg(cs.get_default_configuration()) print("Default Value: %.2f" % def_value) - + # Optimize, using a SMAC-object print("Optimizing! Depending on your machine, this might take a few minutes.") - + # Besides the kwargs used for initializing UncorrelatedMultiObjectiveRandomForestWithInstances, # we also need kwargs for initializing the model insides UncorrelatedMultiObjectiveModel model_kwargs = {"target_names": ["loss", "time"], "model_kwargs": {"seed": 1}} @@ -105,7 +109,7 @@ def svm_from_cfg(cfg): model_kwargs=model_kwargs, tae_runner=svm_from_cfg, acquisition_function=EIPS, - runhistory2epm=RunHistory2EPM4EIPS + runhistory2epm=RunHistory2EPM4EIPS, ) incumbent = smac.optimize() diff --git a/setup.py b/setup.py index f22d02acc..2493b327a 100644 --- a/setup.py +++ b/setup.py @@ -16,20 +16,6 @@ def read_file(filepath: str) -> str: - """ - Read in a files contents - - Parameters - ---------- - filepath : str - The name of the file. - - Returns - ------- - str - The contents of the file. - """ - with open(filepath, "r", encoding="utf-8") as fh: return fh.read() diff --git a/smac/__init__.py b/smac/__init__.py index 0993ca306..a004c2746 100644 --- a/smac/__init__.py +++ b/smac/__init__.py @@ -22,7 +22,7 @@ Matthias Feurer, André Biedenkapp, Difan Deng, Carolin Benjamins, Tim Ruhkopf, René Sass and Frank Hutter """ -version = "1.3.3" +version = "1.3.4" if os.name != "posix": diff --git a/smac/epm/base_uncorrelated_mo_model.py b/smac/epm/base_uncorrelated_mo_model.py index 94815cdbe..44cec9d8e 100644 --- a/smac/epm/base_uncorrelated_mo_model.py +++ b/smac/epm/base_uncorrelated_mo_model.py @@ -135,7 +135,7 @@ def _predict(self, X: np.ndarray, cov_return_type: Optional[str] = "diagonal_cov ---------- X : np.ndarray of shape = [n_samples, n_features (config + instance features)] - cov_return_type: typing.Optional[str] + cov_return_type: Optional[str] Specifies what to return along with the mean. Refer ``predict()`` for more information. Returns diff --git a/smac/epm/gaussian_process_mcmc.py b/smac/epm/gaussian_process_mcmc.py index 87c4a862c..5f3e21de4 100644 --- a/smac/epm/gaussian_process_mcmc.py +++ b/smac/epm/gaussian_process_mcmc.py @@ -1,4 +1,4 @@ -import typing +from typing import List, Optional, Tuple, Union, cast import logging import warnings @@ -74,8 +74,8 @@ class GaussianProcessMCMC(BaseModel): def __init__( self, configspace: ConfigurationSpace, - types: typing.List[int], - bounds: typing.List[typing.Tuple[float, float]], + types: List[int], + bounds: List[Tuple[float, float]], seed: int, kernel: Kernel, n_mcmc_walkers: int = 20, @@ -84,8 +84,8 @@ def __init__( normalize_y: bool = True, mcmc_sampler: str = "emcee", average_samples: bool = False, - instance_features: typing.Optional[np.ndarray] = None, - pca_components: typing.Optional[int] = None, + instance_features: Optional[np.ndarray] = None, + pca_components: Optional[int] = None, ): super().__init__( configspace=configspace, @@ -101,7 +101,7 @@ def __init__( self.chain_length = chain_length self.burned = False self.burnin_steps = burnin_steps - self.models = [] # type: typing.List[GaussianProcess] + self.models = [] # type: List[GaussianProcess] self.normalize_y = normalize_y self.mcmc_sampler = mcmc_sampler self.average_samples = average_samples @@ -155,7 +155,7 @@ def _train(self, X: np.ndarray, y: np.ndarray, do_optimize: bool = True) -> "Gau # Initialize the walkers by sampling from the prior dim_samples = [] - prior = None # type: typing.Optional[typing.Union[typing.List[Prior], Prior]] + prior = None # type: Optional[Union[List[Prior], Prior]] for dim, prior in enumerate(self._all_priors): # Always sample from the first prior if isinstance(prior, list): @@ -163,7 +163,7 @@ def _train(self, X: np.ndarray, y: np.ndarray, do_optimize: bool = True) -> "Gau prior = None else: prior = prior[0] - prior = typing.cast(typing.Optional[Prior], prior) + prior = cast(Optional[Prior], prior) if prior is None: raise NotImplementedError() else: @@ -325,7 +325,7 @@ def _ll(self, theta: np.ndarray) -> float: else: return lml - def _ll_w_grad(self, theta: np.ndarray) -> typing.Tuple[float, np.ndarray]: + def _ll_w_grad(self, theta: np.ndarray) -> Tuple[float, np.ndarray]: """Returns the marginal log likelihood (+ the prior) for a hyperparameter configuration theta. @@ -375,8 +375,8 @@ def _ll_w_grad(self, theta: np.ndarray) -> typing.Tuple[float, np.ndarray]: return lml, grad def _predict( - self, X_test: np.ndarray, cov_return_type: typing.Optional[str] = "diagonal_cov" - ) -> typing.Tuple[np.ndarray, np.ndarray]: + self, X_test: np.ndarray, cov_return_type: Optional[str] = "diagonal_cov" + ) -> Tuple[np.ndarray, np.ndarray]: r""" Returns the predictive mean and variance of the objective function at X average over all hyperparameter samples. @@ -389,7 +389,7 @@ def _predict( ---------- X_test: np.ndarray (N, D) Input test points - cov_return_type: typing.Optional[str] + cov_return_type: Optional[str] Specifies what to return along with the mean. Refer ``predict()`` for more information. Returns diff --git a/smac/epm/random_epm.py b/smac/epm/random_epm.py index 748c4d624..77d553a77 100644 --- a/smac/epm/random_epm.py +++ b/smac/epm/random_epm.py @@ -83,7 +83,7 @@ def _predict(self, X: np.ndarray, cov_return_type: Optional[str] = "diagonal_cov Parameters ---------- X : np.ndarray of shape = [n_samples, n_features (config + instance features)] - cov_return_type: typing.Optional[str] + cov_return_type: Optional[str] Specifies what to return along with the mean. Refer ``predict()`` for more information. Returns diff --git a/smac/facade/hyperband_facade.py b/smac/facade/hyperband_facade.py index cd41fe3d6..ae0b489ba 100644 --- a/smac/facade/hyperband_facade.py +++ b/smac/facade/hyperband_facade.py @@ -1,4 +1,4 @@ -import typing +from typing import Any from smac.facade.roar_facade import ROAR from smac.initial_design.random_configuration_design import RandomConfigurations @@ -29,7 +29,7 @@ class HB4AC(ROAR): List of all incumbents """ - def __init__(self, **kwargs: typing.Any): + def __init__(self, **kwargs: Any): kwargs["initial_design"] = kwargs.get("initial_design", RandomConfigurations) # Intensification parameters diff --git a/smac/facade/smac_bb_facade.py b/smac/facade/smac_bb_facade.py index b26a3f956..b5b52f44d 100644 --- a/smac/facade/smac_bb_facade.py +++ b/smac/facade/smac_bb_facade.py @@ -1,4 +1,4 @@ -import typing +from typing import Any, Type import numpy as np @@ -52,7 +52,7 @@ class SMAC4BB(SMAC4AC): List of all incumbents """ - def __init__(self, model_type: str = "gp_mcmc", **kwargs: typing.Any): + def __init__(self, model_type: str = "gp_mcmc", **kwargs: Any): scenario = kwargs["scenario"] if len(scenario.cs.get_hyperparameters()) <= 21201: @@ -126,7 +126,7 @@ def __init__(self, model_type: str = "gp_mcmc", **kwargs: typing.Any): raise ValueError() if model_type == "gp": - model_class = GaussianProcess # type: typing.Type[BaseModel] + model_class = GaussianProcess # type: Type[BaseModel] kwargs["model"] = model_class model_kwargs["kernel"] = kernel model_kwargs["normalize_y"] = True diff --git a/smac/facade/smac_hpo_facade.py b/smac/facade/smac_hpo_facade.py index 22c893536..89c80066d 100644 --- a/smac/facade/smac_hpo_facade.py +++ b/smac/facade/smac_hpo_facade.py @@ -1,4 +1,4 @@ -import typing +from typing import Any from smac.epm.rf_with_instances import RandomForestWithInstances from smac.facade.smac_ac_facade import SMAC4AC @@ -32,7 +32,7 @@ class SMAC4HPO(SMAC4AC): List of all incumbents """ - def __init__(self, **kwargs: typing.Any): + def __init__(self, **kwargs: Any): scenario = kwargs["scenario"] kwargs["initial_design"] = kwargs.get("initial_design", SobolDesign) diff --git a/smac/facade/smac_mf_facade.py b/smac/facade/smac_mf_facade.py index 196f5285e..1d5571b3d 100644 --- a/smac/facade/smac_mf_facade.py +++ b/smac/facade/smac_mf_facade.py @@ -1,4 +1,4 @@ -import typing +from typing import Any from smac.facade.smac_hpo_facade import SMAC4HPO from smac.initial_design.random_configuration_design import RandomConfigurations @@ -32,7 +32,7 @@ class SMAC4MF(SMAC4HPO): List of all incumbents """ - def __init__(self, **kwargs: typing.Any): + def __init__(self, **kwargs: Any): scenario = kwargs["scenario"] kwargs["initial_design"] = kwargs.get("initial_design", RandomConfigurations) diff --git a/smac/initial_design/factorial_design.py b/smac/initial_design/factorial_design.py index 67fad0161..67786ecb8 100644 --- a/smac/initial_design/factorial_design.py +++ b/smac/initial_design/factorial_design.py @@ -22,7 +22,7 @@ class FactorialInitialDesign(InitialDesign): Attributes ---------- - configs : typing.List[Configuration] + configs : List[Configuration] List of configurations to be evaluated Don't pass configs to the constructor; otherwise factorial design is overwritten diff --git a/smac/initial_design/initial_design.py b/smac/initial_design/initial_design.py index da9e95df7..b27a14856 100644 --- a/smac/initial_design/initial_design.py +++ b/smac/initial_design/initial_design.py @@ -1,4 +1,4 @@ -import typing +from typing import List, Optional import logging from collections import OrderedDict @@ -34,7 +34,7 @@ class InitialDesign: design. ta_run_limit: int Number of iterations allowed for the target algorithm - configs: typing.Optional[typing.List[Configuration]] + configs: Optional[List[Configuration]] List of initial configurations. Disables the arguments ``n_configs_x_params`` if given. Either this, or ``n_configs_x_params`` or ``init_budget`` must be provided. n_configs_x_params: int @@ -51,7 +51,7 @@ class InitialDesign: Attributes ---------- cs : ConfigurationSpace - configs : typing.List[Configuration] + configs : List[Configuration] List of configurations to be evaluated """ @@ -61,10 +61,10 @@ def __init__( rng: np.random.RandomState, traj_logger: TrajLogger, ta_run_limit: int, - configs: typing.Optional[typing.List[Configuration]] = None, - n_configs_x_params: typing.Optional[int] = 10, + configs: Optional[List[Configuration]] = None, + n_configs_x_params: Optional[int] = 10, max_config_fracs: float = 0.25, - init_budget: typing.Optional[int] = None, + init_budget: Optional[int] = None, ): self.cs = cs self.rng = rng @@ -96,7 +96,7 @@ def __init__( ) self.logger.info("Running initial design for %d configurations" % self.init_budget) - def select_configurations(self) -> typing.List[Configuration]: + def select_configurations(self) -> List[Configuration]: """Selects the initial configurations.""" if self.configs is None: self.configs = self._select_configurations() @@ -113,12 +113,12 @@ def select_configurations(self) -> typing.List[Configuration]: self.configs = list(OrderedDict.fromkeys(self.configs)) return self.configs - def _select_configurations(self) -> typing.List[Configuration]: + def _select_configurations(self) -> List[Configuration]: raise NotImplementedError def _transform_continuous_designs( self, design: np.ndarray, origin: str, cs: ConfigurationSpace - ) -> typing.List[Configuration]: + ) -> List[Configuration]: params = cs.get_hyperparameters() for idx, param in enumerate(params): diff --git a/smac/initial_design/latin_hypercube_design.py b/smac/initial_design/latin_hypercube_design.py index e2b0f9988..27c79122e 100644 --- a/smac/initial_design/latin_hypercube_design.py +++ b/smac/initial_design/latin_hypercube_design.py @@ -1,4 +1,4 @@ -import typing +from typing import List from ConfigSpace.configuration_space import Configuration from ConfigSpace.hyperparameters import Constant @@ -16,13 +16,13 @@ class LHDesign(InitialDesign): Attributes ---------- - configs : typing.List[Configuration] + configs : List[Configuration] List of configurations to be evaluated Don't pass configs to the constructor; otherwise factorial design is overwritten """ - def _select_configurations(self) -> typing.List[Configuration]: + def _select_configurations(self) -> List[Configuration]: """Selects a single configuration to run. Returns diff --git a/smac/initial_design/sobol_design.py b/smac/initial_design/sobol_design.py index 64d49a89a..48ace4b84 100644 --- a/smac/initial_design/sobol_design.py +++ b/smac/initial_design/sobol_design.py @@ -1,4 +1,4 @@ -import typing +from typing import List import warnings @@ -20,13 +20,13 @@ class SobolDesign(InitialDesign): Attributes ---------- - configs : typing.List[Configuration] + configs : List[Configuration] List of configurations to be evaluated Don't pass configs to the constructor; otherwise factorial design is overwritten """ - def _select_configurations(self) -> typing.List[Configuration]: + def _select_configurations(self) -> List[Configuration]: """Selects a single configuration to run. Returns diff --git a/smac/intensification/abstract_racer.py b/smac/intensification/abstract_racer.py index 738075633..574c8722d 100644 --- a/smac/intensification/abstract_racer.py +++ b/smac/intensification/abstract_racer.py @@ -1,4 +1,4 @@ -import typing +from typing import Iterator, List, Mapping, Optional, Tuple import logging import time @@ -14,7 +14,7 @@ from smac.utils.io.traj_logging import TrajLogger from smac.utils.logging import format_array -_config_to_run_type = typing.Iterator[typing.Optional[Configuration]] +_config_to_run_type = Iterator[Optional[Configuration]] __author__ = "Ashwin Raaghav Narayanan" __copyright__ = "Copyright 2019, ML4AAD" @@ -53,9 +53,9 @@ class AbstractRacer(object): traj_logger: smac.utils.io.traj_logging.TrajLogger TrajLogger object to log all new incumbents rng : np.random.RandomState - instances : typing.List[str] + instances : List[str] list of all instance ids - instance_specifics : typing.Mapping[str, str] + instance_specifics : Mapping[str, str] mapping from instance name to instance specific string cutoff : float runtime cutoff of TA runs @@ -80,9 +80,9 @@ def __init__( stats: Stats, traj_logger: TrajLogger, rng: np.random.RandomState, - instances: typing.List[str], - instance_specifics: typing.Optional[typing.Mapping[str, str]] = None, - cutoff: typing.Optional[float] = None, + instances: List[str], + instance_specifics: Optional[Mapping[str, str]] = None, + cutoff: Optional[float] = None, deterministic: bool = False, run_obj_time: bool = True, minR: int = 1, @@ -114,7 +114,7 @@ def __init__( # removing duplicates in the user provided instances self.instances = list(OrderedDict.fromkeys(instances)) if instance_specifics is None: - self.instance_specifics = {} # type: typing.Mapping[str, str] + self.instance_specifics = {} # type: Mapping[str, str] else: self.instance_specifics = instance_specifics @@ -137,13 +137,13 @@ def __init__( def get_next_run( self, - challengers: typing.Optional[typing.List[Configuration]], + challengers: Optional[List[Configuration]], incumbent: Configuration, - chooser: typing.Optional[EPMChooser], + chooser: Optional[EPMChooser], run_history: RunHistory, repeat_configs: bool = True, num_workers: int = 1, - ) -> typing.Tuple[RunInfoIntent, RunInfo]: + ) -> Tuple[RunInfoIntent, RunInfo]: """Abstract method for choosing the next challenger, to allow for different selections across intensifiers uses ``_next_challenger()`` by default. @@ -152,7 +152,7 @@ def get_next_run( Parameters ---------- - challengers : typing.List[Configuration] + challengers : List[Configuration] promising configurations incumbent: Configuration incumbent configuration @@ -178,12 +178,12 @@ def get_next_run( def process_results( self, run_info: RunInfo, - incumbent: typing.Optional[Configuration], + incumbent: Optional[Configuration], run_history: RunHistory, time_bound: float, result: RunValue, log_traj: bool = True, - ) -> typing.Tuple[Configuration, float]: + ) -> Tuple[Configuration, float]: """The intensifier stage will be updated based on the results/status of a configuration execution. Also, a incumbent will be determined. @@ -191,7 +191,7 @@ def process_results( ---------- run_info : RunInfo A RunInfo containing the configuration that was evaluated - incumbent : typing.Optional[Configuration] + incumbent : Optional[Configuration] Best configuration seen so far run_history : RunHistory stores all runs we ran so far @@ -215,17 +215,17 @@ def process_results( def _next_challenger( self, - challengers: typing.Optional[typing.List[Configuration]], - chooser: typing.Optional[EPMChooser], + challengers: Optional[List[Configuration]], + chooser: Optional[EPMChooser], run_history: RunHistory, repeat_configs: bool = True, - ) -> typing.Optional[Configuration]: + ) -> Optional[Configuration]: """Retuns the next challenger to use in intensification If challenger is None, then optimizer will be used to generate the next challenger. Parameters ---------- - challengers : typing.List[Configuration] + challengers : List[Configuration] promising configurations to evaluate next chooser : smac.optimizer.epm_configuration_chooser.EPMChooser a sampler that generates next configurations to use for racing @@ -315,7 +315,7 @@ def _compare_configs( challenger: Configuration, run_history: RunHistory, log_traj: bool = True, - ) -> typing.Optional[Configuration]: + ) -> Optional[Configuration]: """Compare two configuration wrt the runhistory and return the one which performs better (or None if the decision is not safe) diff --git a/smac/intensification/hyperband.py b/smac/intensification/hyperband.py index bcf9f9b5c..c80fda4b8 100644 --- a/smac/intensification/hyperband.py +++ b/smac/intensification/hyperband.py @@ -1,4 +1,4 @@ -import typing +from typing import List, Mapping, Optional, Tuple import logging @@ -41,25 +41,25 @@ class with the actual implementation of the method. traj_logger: smac.utils.io.traj_logging.TrajLogger TrajLogger object to log all new incumbents rng : np.random.RandomState - instances : typing.List[str] + instances : List[str] list of all instance ids - instance_specifics : typing.Mapping[str, str] + instance_specifics : Mapping[str, str] mapping from instance name to instance specific string - cutoff : typing.Optional[int] + cutoff : Optional[int] runtime cutoff of TA runs deterministic : bool whether the TA is deterministic or not - initial_budget : typing.Optional[float] + initial_budget : Optional[float] minimum budget allowed for 1 run of successive halving - max_budget : typing.Optional[float] + max_budget : Optional[float] maximum budget allowed for 1 run of successive halving eta : float 'halving' factor after each iteration in a successive halving run. Defaults to 3 run_obj_time : bool whether the run objective is runtime or not (if true, apply adaptive capping) - n_seeds : typing.Optional[int] + n_seeds : Optional[int] Number of seeds to use, if TA is not deterministic. Defaults to None, i.e., seed is set as 0 - instance_order : typing.Optional[str] + instance_order : Optional[str] how to order instances. Can be set to: [None, shuffle_once, shuffle] * None - use as is given by the user * shuffle_once - shuffle once and use across all SH run (default) @@ -84,15 +84,15 @@ def __init__( stats: Stats, traj_logger: TrajLogger, rng: np.random.RandomState, - instances: typing.List[str], - instance_specifics: typing.Mapping[str, str] = None, - cutoff: typing.Optional[float] = None, + instances: List[str], + instance_specifics: Mapping[str, str] = None, + cutoff: Optional[float] = None, deterministic: bool = False, - initial_budget: typing.Optional[float] = None, - max_budget: typing.Optional[float] = None, + initial_budget: Optional[float] = None, + max_budget: Optional[float] = None, eta: float = 3, run_obj_time: bool = True, - n_seeds: typing.Optional[int] = None, + n_seeds: Optional[int] = None, instance_order: str = "shuffle_once", adaptive_capping_slackfactor: float = 1.2, min_chall: int = 1, @@ -133,12 +133,12 @@ def __init__( def process_results( self, run_info: RunInfo, - incumbent: typing.Optional[Configuration], + incumbent: Optional[Configuration], run_history: RunHistory, time_bound: float, result: RunValue, log_traj: bool = True, - ) -> typing.Tuple[Configuration, float]: + ) -> Tuple[Configuration, float]: """The intensifier stage will be updated based on the results/status of a configuration execution. Also, a incumbent will be determined. @@ -146,7 +146,7 @@ def process_results( ---------- run_info : RunInfo A RunInfo containing the configuration that was evaluated - incumbent : typing.Optional[Configuration] + incumbent : Optional[Configuration] Best configuration seen so far run_history : RunHistory stores all runs we ran so far @@ -185,13 +185,13 @@ def process_results( def get_next_run( self, - challengers: typing.Optional[typing.List[Configuration]], + challengers: Optional[List[Configuration]], incumbent: Configuration, - chooser: typing.Optional[EPMChooser], + chooser: Optional[EPMChooser], run_history: RunHistory, repeat_configs: bool = True, num_workers: int = 1, - ) -> typing.Tuple[RunInfoIntent, RunInfo]: + ) -> Tuple[RunInfoIntent, RunInfo]: """Selects which challenger to use based on the iteration stage and set the iteration parameters. First iteration will choose configurations from the ``chooser`` or input challengers, while the later iterations pick top configurations from the previously selected @@ -201,7 +201,7 @@ def get_next_run( Parameters ---------- - challengers : typing.List[Configuration] + challengers : List[Configuration] promising configurations incumbent: Configuration incumbent configuration @@ -331,25 +331,25 @@ class Hyperband(ParallelScheduler): traj_logger: smac.utils.io.traj_logging.TrajLogger TrajLogger object to log all new incumbents rng : np.random.RandomState - instances : typing.List[str] + instances : List[str] list of all instance ids - instance_specifics : typing.Mapping[str, str] + instance_specifics : Mapping[str, str] mapping from instance name to instance specific string - cutoff : typing.Optional[int] + cutoff : Optional[int] runtime cutoff of TA runs deterministic : bool whether the TA is deterministic or not - initial_budget : typing.Optional[float] + initial_budget : Optional[float] minimum budget allowed for 1 run of successive halving - max_budget : typing.Optional[float] + max_budget : Optional[float] maximum budget allowed for 1 run of successive halving eta : float 'halving' factor after each iteration in a successive halving run. Defaults to 3 run_obj_time : bool whether the run objective is runtime or not (if true, apply adaptive capping) - n_seeds : typing.Optional[int] + n_seeds : Optional[int] Number of seeds to use, if TA is not deterministic. Defaults to None, i.e., seed is set as 0 - instance_order : typing.Optional[str] + instance_order : Optional[str] how to order instances. Can be set to: [None, shuffle_once, shuffle] * None - use as is given by the user * shuffle_once - shuffle once and use across all SH run (default) @@ -372,15 +372,15 @@ def __init__( stats: Stats, traj_logger: TrajLogger, rng: np.random.RandomState, - instances: typing.List[str], - instance_specifics: typing.Mapping[str, str] = None, - cutoff: typing.Optional[float] = None, + instances: List[str], + instance_specifics: Mapping[str, str] = None, + cutoff: Optional[float] = None, deterministic: bool = False, - initial_budget: typing.Optional[float] = None, - max_budget: typing.Optional[float] = None, + initial_budget: Optional[float] = None, + max_budget: Optional[float] = None, eta: float = 3, run_obj_time: bool = True, - n_seeds: typing.Optional[int] = None, + n_seeds: Optional[int] = None, instance_order: str = "shuffle_once", adaptive_capping_slackfactor: float = 1.2, min_chall: int = 1, @@ -412,7 +412,7 @@ def __init__( self.max_budget = max_budget self.eta = eta - def _get_intensifier_ranking(self, intensifier: AbstractRacer) -> typing.Tuple[int, int]: + def _get_intensifier_ranking(self, intensifier: AbstractRacer) -> Tuple[int, int]: """Given a intensifier, returns how advance it is. This metric will be used to determine what priority to assign to the intensifier. diff --git a/smac/intensification/intensification.py b/smac/intensification/intensification.py index 08f76344c..119367571 100644 --- a/smac/intensification/intensification.py +++ b/smac/intensification/intensification.py @@ -1,4 +1,4 @@ -import typing +from typing import List, Mapping, Optional, Tuple, cast import logging from collections import Counter @@ -96,9 +96,9 @@ class Intensifier(AbstractRacer): traj_logger: TrajLogger TrajLogger object to log all new incumbents rng : np.random.RandomState - instances : typing.List[str] + instances : List[str] list of all instance ids - instance_specifics : typing.Mapping[str, str] + instance_specifics : Mapping[str, str] mapping from instance name to instance specific string cutoff : int runtime cutoff of TA runs @@ -132,8 +132,8 @@ def __init__( stats: Stats, traj_logger: TrajLogger, rng: np.random.RandomState, - instances: typing.List[str], - instance_specifics: typing.Mapping[str, str] = None, + instances: List[str], + instance_specifics: Mapping[str, str] = None, cutoff: int = None, deterministic: bool = False, run_obj_time: bool = True, @@ -192,19 +192,19 @@ def __init__( self.update_configs_to_run = True # racing related variables - self.to_run = [] # type: typing.List[InstSeedBudgetKey] + self.to_run = [] # type: List[InstSeedBudgetKey] self.inc_sum_cost = np.inf self.N = -1 def get_next_run( self, - challengers: typing.Optional[typing.List[Configuration]], + challengers: Optional[List[Configuration]], incumbent: Configuration, - chooser: typing.Optional[EPMChooser], + chooser: Optional[EPMChooser], run_history: RunHistory, repeat_configs: bool = True, num_workers: int = 1, - ) -> typing.Tuple[RunInfoIntent, RunInfo]: + ) -> Tuple[RunInfoIntent, RunInfo]: """This procedure is in charge of generating a RunInfo object to comply with lines 7 (in case stage is stage==RUN_INCUMBENT) or line 12 (In case of stage==RUN_CHALLENGER) @@ -221,7 +221,7 @@ def get_next_run( Parameters ---------- - challengers : typing.List[Configuration] + challengers : List[Configuration] promising configurations incumbent: Configuration incumbent configuration @@ -451,12 +451,12 @@ def get_next_run( def process_results( self, run_info: RunInfo, - incumbent: typing.Optional[Configuration], + incumbent: Optional[Configuration], run_history: RunHistory, time_bound: float, result: RunValue, log_traj: bool = True, - ) -> typing.Tuple[Configuration, float]: + ) -> Tuple[Configuration, float]: """The intensifier stage will be updated based on the results/status of a configuration execution. @@ -476,7 +476,7 @@ def process_results( ---------- run_info : RunInfo A RunInfo containing the configuration that was evaluated - incumbent : typing.Optional[Configuration] + incumbent : Optional[Configuration] best configuration so far, None in 1st run run_history : RunHistory stores all runs we ran so far @@ -567,13 +567,13 @@ def process_results( def _get_next_inc_run( self, - available_insts: typing.List[str], - ) -> typing.Tuple[str, int, typing.Optional[float]]: + available_insts: List[str], + ) -> Tuple[str, int, Optional[float]]: """Method to extract the next seed/instance in which a incumbent run most be evaluated. Parameters ---------- - available_insts : typing.List[str] + available_insts : List[str] A list of instances from which to extract the next incumbent run Returns @@ -609,7 +609,7 @@ def _get_inc_available_inst( incumbent: Configuration, run_history: RunHistory, log_traj: bool = True, - ) -> typing.List[str]: + ) -> List[str]: """Implementation of line 4 of Intensification. This method queries the inc runs in the run history @@ -692,7 +692,7 @@ def _get_next_racer( incumbent: Configuration, run_history: RunHistory, log_traj: bool = True, - ) -> typing.Tuple[Configuration, str, int, typing.Optional[float]]: + ) -> Tuple[Configuration, str, int, Optional[float]]: """Method to return the next config setting to aggressively race challenger against incumbent. @@ -775,7 +775,7 @@ def _process_racer_results( incumbent: Configuration, run_history: RunHistory, log_traj: bool = True, - ) -> typing.Optional[Configuration]: + ) -> Optional[Configuration]: """Process the result of a racing configuration against the current incumbent. Might propose a new incumbent. @@ -790,7 +790,7 @@ def _process_racer_results( Returns ------- - new_incumbent: typing.Optional[Configuration] + new_incumbent: Optional[Configuration] Either challenger or incumbent """ chal_runs = run_history.get_runs_for_config(challenger, only_max_observed_budget=True) @@ -856,7 +856,7 @@ def _get_instances_to_run( incumbent: Configuration, N: int, run_history: RunHistory, - ) -> typing.Tuple[typing.List[InstSeedBudgetKey], float]: + ) -> Tuple[List[InstSeedBudgetKey], float]: """Returns the minimum list of pairs to run the challenger on before comparing it with the incumbent. @@ -873,7 +873,7 @@ def _get_instances_to_run( Returns ------- - typing.List[InstSeedBudgetKey] + List[InstSeedBudgetKey] list of tuples to run float total (runtime) cost of running the incumbent on the instances (used for adaptive capping while racing) @@ -905,9 +905,9 @@ def _get_instances_to_run( def get_next_challenger( self, - challengers: typing.Optional[typing.List[Configuration]], - chooser: typing.Optional[EPMChooser], - ) -> typing.Tuple[typing.Optional[Configuration], bool]: + challengers: Optional[List[Configuration]], + chooser: Optional[EPMChooser], + ) -> Tuple[Optional[Configuration], bool]: """This function returns the next challenger, that should be exercised though lines 8-17. It does so by populating configs_to_run, which is a pool of configuration @@ -925,14 +925,14 @@ def get_next_challenger( Parameters ---------- - challengers : typing.List[Configuration] + challengers : List[Configuration] promising configurations chooser : smac.optimizer.epm_configuration_chooser.EPMChooser optimizer that generates next configurations to use for racing Returns ------- - typing.Optional[Configuration] + Optional[Configuration] next configuration to evaluate bool flag telling if the configuration is newly sampled or one currently being tracked @@ -944,7 +944,7 @@ def get_next_challenger( # this is a new intensification run, get the next list of configurations to run if self.update_configs_to_run: configs_to_run = self._generate_challengers(challengers=challengers, chooser=chooser) - self.configs_to_run = typing.cast(_config_to_run_type, configs_to_run) + self.configs_to_run = cast(_config_to_run_type, configs_to_run) self.update_configs_to_run = False # pick next configuration from the generator @@ -969,22 +969,22 @@ def get_next_challenger( def _generate_challengers( self, - challengers: typing.Optional[typing.List[Configuration]], - chooser: typing.Optional[EPMChooser], + challengers: Optional[List[Configuration]], + chooser: Optional[EPMChooser], ) -> _config_to_run_type: """Retuns a sequence of challengers to use in intensification If challengers are not provided, then optimizer will be used to generate the challenger list. Parameters ---------- - challengers : typing.List[Configuration] + challengers : List[Configuration] promising configurations to evaluate next chooser : smac.optimizer.epm_configuration_chooser.EPMChooser a sampler that generates next configurations to use for racing Returns ------- - typing.Optional[typing.Generator[Configuration]] + Optional[Generator[Configuration]] A generator containing the next challengers to use """ if challengers: diff --git a/smac/intensification/parallel_scheduling.py b/smac/intensification/parallel_scheduling.py index 4baa8a0e8..4c15173f3 100644 --- a/smac/intensification/parallel_scheduling.py +++ b/smac/intensification/parallel_scheduling.py @@ -1,4 +1,4 @@ -import typing +from typing import Dict, List, Mapping, Optional, Tuple import warnings @@ -29,34 +29,34 @@ class ParallelScheduler(AbstractRacer): traj_logger: smac.utils.io.traj_logging.TrajLogger TrajLogger object to log all new incumbents rng : np.random.RandomState - instances : typing.List[str] + instances : List[str] list of all instance ids - instance_specifics : typing.Mapping[str, str] + instance_specifics : Mapping[str, str] mapping from instance name to instance specific string - cutoff : typing.Optional[int] + cutoff : Optional[int] cutoff of TA runs deterministic : bool whether the TA is deterministic or not - initial_budget : typing.Optional[float] + initial_budget : Optional[float] minimum budget allowed for 1 run of successive halving - max_budget : typing.Optional[float] + max_budget : Optional[float] maximum budget allowed for 1 run of successive halving eta : float 'halving' factor after each iteration in a successive halving run. Defaults to 3 - num_initial_challengers : typing.Optional[int] + num_initial_challengers : Optional[int] number of challengers to consider for the initial budget. If None, calculated internally run_obj_time : bool whether the run objective is runtime or not (if true, apply adaptive capping) - n_seeds : typing.Optional[int] + n_seeds : Optional[int] Number of seeds to use, if TA is not deterministic. Defaults to None, i.e., seed is set as 0 - instance_order : typing.Optional[str] + instance_order : Optional[str] how to order instances. Can be set to: [None, shuffle_once, shuffle] * None - use as is given by the user * shuffle_once - shuffle once and use across all SH run (default) * shuffle - shuffle before every SH run adaptive_capping_slackfactor : float slack factor of adpative capping (factor * adaptive cutoff) - inst_seed_pairs : typing.List[typing.Tuple[str, int]], optional + inst_seed_pairs : List[Tuple[str, int]], optional Do not set this argument, it will only be used by hyperband! min_chall: int minimal number of challengers to be considered (even if time_bound is exhausted earlier). This class will @@ -74,19 +74,19 @@ def __init__( stats: Stats, traj_logger: TrajLogger, rng: np.random.RandomState, - instances: typing.List[str], - instance_specifics: typing.Mapping[str, str] = None, - cutoff: typing.Optional[float] = None, + instances: List[str], + instance_specifics: Mapping[str, str] = None, + cutoff: Optional[float] = None, deterministic: bool = False, - initial_budget: typing.Optional[float] = None, - max_budget: typing.Optional[float] = None, + initial_budget: Optional[float] = None, + max_budget: Optional[float] = None, eta: float = 3, - num_initial_challengers: typing.Optional[int] = None, + num_initial_challengers: Optional[int] = None, run_obj_time: bool = True, - n_seeds: typing.Optional[int] = None, - instance_order: typing.Optional[str] = "shuffle_once", + n_seeds: Optional[int] = None, + instance_order: Optional[str] = "shuffle_once", adaptive_capping_slackfactor: float = 1.2, - inst_seed_pairs: typing.Optional[typing.List[typing.Tuple[str, int]]] = None, + inst_seed_pairs: Optional[List[Tuple[str, int]]] = None, min_chall: int = 1, incumbent_selection: str = "highest_executed_budget", num_obj: int = 1, @@ -107,18 +107,18 @@ def __init__( ) # We have a pool of instances that yield configurations ot run - self.intensifier_instances = {} # type: typing.Dict[int, AbstractRacer] + self.intensifier_instances = {} # type: Dict[int, AbstractRacer] self.print_worker_warning = True def get_next_run( self, - challengers: typing.Optional[typing.List[Configuration]], + challengers: Optional[List[Configuration]], incumbent: Configuration, - chooser: typing.Optional[EPMChooser], + chooser: Optional[EPMChooser], run_history: RunHistory, repeat_configs: bool = False, num_workers: int = 1, - ) -> typing.Tuple[RunInfoIntent, RunInfo]: + ) -> Tuple[RunInfoIntent, RunInfo]: """This procedure decides from which instance to pick a config, in order to determine the next run. @@ -130,7 +130,7 @@ def get_next_run( Parameters ---------- - challengers : typing.List[Configuration] + challengers : List[Configuration] promising configurations incumbent: Configuration incumbent configuration @@ -210,12 +210,12 @@ def get_next_run( def process_results( self, run_info: RunInfo, - incumbent: typing.Optional[Configuration], + incumbent: Optional[Configuration], run_history: RunHistory, time_bound: float, result: RunValue, log_traj: bool = True, - ) -> typing.Tuple[Configuration, float]: + ) -> Tuple[Configuration, float]: """The intensifier stage will be updated based on the results/status of a configuration execution. @@ -231,7 +231,7 @@ def process_results( ---------- run_info : RunInfo A RunInfo containing the configuration that was evaluated - incumbent : typing.Optional[Configuration] + incumbent : Optional[Configuration] Best configuration seen so far run_history : RunHistory stores all runs we ran so far @@ -276,7 +276,7 @@ def _add_new_instance(self, num_workers: int) -> bool: """ raise NotImplementedError() - def _get_intensifier_ranking(self, intensifier: AbstractRacer) -> typing.Tuple[int, int]: + def _get_intensifier_ranking(self, intensifier: AbstractRacer) -> Tuple[int, int]: """Given a intensifier, returns how advance it is. This metric will be used to determine what priority to assign to the intensifier. @@ -298,14 +298,14 @@ def _get_intensifier_ranking(self, intensifier: AbstractRacer) -> typing.Tuple[i """ raise NotImplementedError() - def _sort_instances_by_stage(self, instances: typing.Dict[int, AbstractRacer]) -> typing.List[int]: + def _sort_instances_by_stage(self, instances: Dict[int, AbstractRacer]) -> List[int]: """This procedure dictates what SH to prioritize in launching jobs. It prioritizes resource allocation to SH instances that have higher stages. In case of tie, we prioritize the SH instance with more launched configs. Parameters ---------- - instances: typing.Dict[int, AbstractRacer] + instances: Dict[int, AbstractRacer] Dict with the instances to prioritize Returns diff --git a/smac/intensification/simple_intensifier.py b/smac/intensification/simple_intensifier.py index b35fe9d55..fe8d6d1d2 100644 --- a/smac/intensification/simple_intensifier.py +++ b/smac/intensification/simple_intensifier.py @@ -1,4 +1,4 @@ -import typing +from typing import Any, Dict, List, Mapping, Optional, Tuple import numpy as np @@ -24,11 +24,11 @@ class SimpleIntensifier(AbstractRacer): traj_logger: smac.utils.io.traj_logging.TrajLogger TrajLogger object to log all new incumbents rng : np.random.RandomState - instances : typing.List[str] + instances : List[str] list of all instance ids - instance_specifics : typing.Mapping[str, str] + instance_specifics : Mapping[str, str] mapping from instance name to instance specific string - cutoff : typing.Optional[int] + cutoff : Optional[int] cutoff of TA runs deterministic : bool whether the TA is deterministic or not @@ -41,13 +41,13 @@ def __init__( stats: Stats, traj_logger: TrajLogger, rng: np.random.RandomState, - instances: typing.List[str], - instance_specifics: typing.Mapping[str, str] = None, - cutoff: typing.Optional[float] = None, + instances: List[str], + instance_specifics: Mapping[str, str] = None, + cutoff: Optional[float] = None, deterministic: bool = False, run_obj_time: bool = True, num_obj: int = 1, - **kwargs: typing.Any, + **kwargs: Any, ) -> None: super().__init__( @@ -69,17 +69,17 @@ def __init__( # the workers. At any time, we want to make sure that if there # are just W workers, there should be at max W active runs # Below variable tracks active runs not processed - self.run_tracker = {} # type: typing.Dict[typing.Tuple[Configuration, str, int, float], bool] + self.run_tracker = {} # type: Dict[Tuple[Configuration, str, int, float], bool] def process_results( self, run_info: RunInfo, - incumbent: typing.Optional[Configuration], + incumbent: Optional[Configuration], run_history: RunHistory, time_bound: float, result: RunValue, log_traj: bool = True, - ) -> typing.Tuple[Configuration, float]: + ) -> Tuple[Configuration, float]: """The intensifier stage will be updated based on the results/status of a configuration execution. Also, a incumbent will be determined. @@ -87,7 +87,7 @@ def process_results( ---------- run_info : RunInfo A RunInfo containing the configuration that was evaluated - incumbent : typing.Optional[Configuration] + incumbent : Optional[Configuration] Best configuration seen so far run_history : RunHistory stores all runs we ran so far @@ -130,20 +130,20 @@ def process_results( def get_next_run( self, - challengers: typing.Optional[typing.List[Configuration]], + challengers: Optional[List[Configuration]], incumbent: Configuration, - chooser: typing.Optional[EPMChooser], + chooser: Optional[EPMChooser], run_history: RunHistory, repeat_configs: bool = True, num_workers: int = 1, - ) -> typing.Tuple[RunInfoIntent, RunInfo]: + ) -> Tuple[RunInfoIntent, RunInfo]: """Selects which challenger to be used. As in a traditional BO loop, we sample from the EPM, which is the next configuration based on the acquisition function. The input data is read from the runhistory. Parameters ---------- - challengers : typing.List[Configuration] + challengers : List[Configuration] promising configurations incumbent: Configuration incumbent configuration diff --git a/smac/optimizer/epm_configuration_chooser.py b/smac/optimizer/epm_configuration_chooser.py index cc0f9b075..d01e94d72 100644 --- a/smac/optimizer/epm_configuration_chooser.py +++ b/smac/optimizer/epm_configuration_chooser.py @@ -1,4 +1,4 @@ -import typing +from typing import Iterator, List, Optional, Tuple import logging @@ -64,7 +64,7 @@ def __init__( acquisition_func: AbstractAcquisitionFunction, rng: np.random.RandomState, restore_incumbent: Configuration = None, - random_configuration_chooser: typing.Union[RandomConfigurationChooser] = ChooserNoCoolDown(modulus=2.0), + random_configuration_chooser: RandomConfigurationChooser = ChooserNoCoolDown(modulus=2.0), predict_x_best: bool = True, min_samples_model: int = 1, ): @@ -87,7 +87,7 @@ def __init__( rng, ) - self.initial_design_configs = [] # type: typing.List[Configuration] + self.initial_design_configs = [] # type: List[Configuration] self.predict_x_best = predict_x_best @@ -96,7 +96,7 @@ def __init__( 0.0, ] - def _collect_data_to_train_model(self) -> typing.Tuple[np.ndarray, np.ndarray, np.ndarray]: + def _collect_data_to_train_model(self) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: # if we use a float value as a budget, we want to train the model only on the highest budget available_budgets = [] for run_key in self.runhistory.data.keys(): @@ -132,10 +132,10 @@ def _collect_data_to_train_model(self) -> typing.Tuple[np.ndarray, np.ndarray, n np.empty(shape=[0, 0]), ) - def _get_evaluated_configs(self) -> typing.List[Configuration]: + def _get_evaluated_configs(self) -> List[Configuration]: return self.runhistory.get_all_configs_per_budget(budget_subset=self.currently_considered_budgets) - def choose_next(self, incumbent_value: float = None) -> typing.Iterator[Configuration]: + def choose_next(self, incumbent_value: float = None) -> Iterator[Configuration]: """Choose next candidate solution with Bayesian optimization. The suggested configurations depend on the argument ``acq_optimizer`` to the ``SMBO`` class. @@ -161,7 +161,7 @@ def choose_next(self, incumbent_value: float = None) -> typing.Iterator[Configur if incumbent_value is not None: best_observation = incumbent_value - x_best_array = None # type: typing.Optional[np.ndarray] + x_best_array = None # type: Optional[np.ndarray] else: if self.runhistory.empty(): raise ValueError("Runhistory is empty and the cost value of " "the incumbent is unknown.") @@ -183,7 +183,7 @@ def choose_next(self, incumbent_value: float = None) -> typing.Iterator[Configur ) return challengers - def _get_x_best(self, predict: bool, X: np.ndarray) -> typing.Tuple[np.ndarray, float]: + def _get_x_best(self, predict: bool, X: np.ndarray) -> Tuple[np.ndarray, float]: """Get value, configuration, and array representation of the "best" configuration. The definition of best varies depending on the argument ``predict``. If set to ``True``, diff --git a/smac/optimizer/multi_objective/abstract_multi_objective_algorithm.py b/smac/optimizer/multi_objective/abstract_multi_objective_algorithm.py index ca2fad8f8..6ce18d141 100644 --- a/smac/optimizer/multi_objective/abstract_multi_objective_algorithm.py +++ b/smac/optimizer/multi_objective/abstract_multi_objective_algorithm.py @@ -1,5 +1,5 @@ -import typing from abc import ABC +from typing import Optional import numpy as np @@ -10,7 +10,7 @@ class AbstractMultiObjectiveAlgorithm(ABC): It can be applied to rh2epm or epmchooser. """ - def __init__(self, num_obj: int, rng: typing.Optional[np.random.RandomState] = None): + def __init__(self, num_obj: int, rng: Optional[np.random.RandomState] = None): if rng is None: rng = np.random.RandomState(0) diff --git a/smac/optimizer/smbo.py b/smac/optimizer/smbo.py index 315549ad5..32c190cd6 100644 --- a/smac/optimizer/smbo.py +++ b/smac/optimizer/smbo.py @@ -1,4 +1,4 @@ -import typing +from typing import Callable, Dict, List, Optional, Type, Union import logging import os @@ -127,7 +127,7 @@ def __init__( self._min_time = 10**-5 self.tae_runner = tae_runner - self.initial_design_configs = [] # type: typing.List[Configuration] + self.initial_design_configs = [] # type: List[Configuration] # TODO: consider if we need an additional EPMChooser for multi-objective optimization self.epm_chooser = EPMChooser( @@ -150,10 +150,10 @@ def __init__( # Callbacks. All known callbacks have a key. If something does not have a key here, there is # no callback available. - self._callbacks = {"_incorporate_run_results": list()} # type: typing.Dict[str, typing.List[typing.Callable]] + self._callbacks = {"_incorporate_run_results": list()} # type: Dict[str, List[Callable]] self._callback_to_key = { IncorporateRunResultCallback: "_incorporate_run_results", - } # type: typing.Dict[typing.Type, str] + } # type: Dict[Type, str] def start(self) -> None: """Starts the Bayesian Optimization loop. @@ -347,8 +347,8 @@ def run(self) -> Configuration: def validate( self, - config_mode: typing.Union[str, typing.List[Configuration]] = "inc", - instance_mode: typing.Union[str, typing.List[str]] = "train+test", + config_mode: Union[str, List[Configuration]] = "inc", + instance_mode: Union[str, List[str]] = "train+test", repetitions: int = 1, use_epm: bool = False, n_jobs: int = -1, @@ -385,13 +385,13 @@ def validate( traj_fn = os.path.join(self.scenario.output_dir_for_this_run, "traj_aclib2.json") trajectory = TrajLogger.read_traj_aclib_format( fn=traj_fn, cs=self.config_space - ) # type: typing.Optional[typing.List[typing.Dict[str, typing.Union[float, int, Configuration]]]] + ) # type: Optional[List[Dict[str, Union[float, int, Configuration]]]] else: trajectory = None if self.scenario.output_dir_for_this_run: new_rh_path = os.path.join( self.scenario.output_dir_for_this_run, "validated_runhistory.json" - ) # type: typing.Optional[str] # noqa E501 + ) # type: Optional[str] # noqa E501 else: new_rh_path = None diff --git a/smac/runhistory/runhistory2epm.py b/smac/runhistory/runhistory2epm.py index 768e41345..10c72cf48 100644 --- a/smac/runhistory/runhistory2epm.py +++ b/smac/runhistory/runhistory2epm.py @@ -1,5 +1,5 @@ import abc -import typing +from typing import Dict, List, Mapping, Optional, Tuple import logging @@ -78,14 +78,14 @@ def __init__( self, scenario: Scenario, num_params: int, - success_states: typing.List[StatusType], + success_states: List[StatusType], impute_censored_data: bool = False, - impute_state: typing.Optional[typing.List[StatusType]] = None, - consider_for_higher_budgets_state: typing.Optional[typing.List[StatusType]] = None, - imputor: typing.Optional[BaseImputor] = None, + impute_state: Optional[List[StatusType]] = None, + consider_for_higher_budgets_state: Optional[List[StatusType]] = None, + imputor: Optional[BaseImputor] = None, scale_perc: int = 5, - rng: typing.Optional[np.random.RandomState] = None, - multi_objective_algorithm: typing.Optional[AggregationStrategy] = None, + rng: Optional[np.random.RandomState] = None, + multi_objective_algorithm: Optional[AggregationStrategy] = None, ) -> None: self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) @@ -117,13 +117,13 @@ def __init__( if impute_state is None: # please mypy - self.impute_state = [] # type: typing.List[StatusType] + self.impute_state = [] # type: List[StatusType] else: self.impute_state = impute_state if consider_for_higher_budgets_state is None: # please mypy - self.consider_for_higher_budgets_state = [] # type: typing.List[StatusType] + self.consider_for_higher_budgets_state = [] # type: List[StatusType] else: self.consider_for_higher_budgets_state = consider_for_higher_budgets_state @@ -160,11 +160,11 @@ def __init__( @abc.abstractmethod def _build_matrix( self, - run_dict: typing.Mapping[RunKey, RunValue], + run_dict: Mapping[RunKey, RunValue], runhistory: RunHistory, return_time_as_y: bool = False, store_statistics: bool = False, - ) -> typing.Tuple[np.ndarray, np.ndarray]: + ) -> Tuple[np.ndarray, np.ndarray]: """Builds x,y matrixes from selected runs from runhistory. Parameters @@ -188,8 +188,8 @@ def _build_matrix( def _get_s_run_dict( self, runhistory: RunHistory, - budget_subset: typing.Optional[typing.List] = None, - ) -> typing.Dict[RunKey, RunValue]: + budget_subset: Optional[List] = None, + ) -> Dict[RunKey, RunValue]: # Get only successfully finished runs if budget_subset is not None: if len(budget_subset) != 1: @@ -218,8 +218,8 @@ def _get_s_run_dict( def _get_t_run_dict( self, runhistory: RunHistory, - budget_subset: typing.Optional[typing.List] = None, - ) -> typing.Dict[RunKey, RunValue]: + budget_subset: Optional[List] = None, + ) -> Dict[RunKey, RunValue]: if budget_subset is not None: t_run_dict = { run: runhistory.data[run] @@ -239,7 +239,7 @@ def _get_t_run_dict( def get_configurations( self, runhistory: RunHistory, - budget_subset: typing.Optional[typing.List] = None, + budget_subset: Optional[List] = None, ) -> np.ndarray: """Returns vector representation of only the configurations. Instance features are not appended and cost values are not taken into account. @@ -266,8 +266,8 @@ def get_configurations( def transform( self, runhistory: RunHistory, - budget_subset: typing.Optional[typing.List] = None, - ) -> typing.Tuple[np.ndarray, np.ndarray]: + budget_subset: Optional[List] = None, + ) -> Tuple[np.ndarray, np.ndarray]: """Returns vector representation of runhistory; if imputation is disabled, censored (TIMEOUT with time < cutoff) will be skipped. @@ -381,7 +381,7 @@ def transform_response_values( """ raise NotImplementedError - def get_X_y(self, runhistory: RunHistory) -> typing.Tuple[np.ndarray, np.ndarray, np.ndarray]: + def get_X_y(self, runhistory: RunHistory) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: """ Simple interface to obtain all data in runhistory in X, y format. Note: This function should not be used as it does not consider all available StatusTypes. @@ -424,11 +424,11 @@ class RunHistory2EPM4Cost(AbstractRunHistory2EPM): def _build_matrix( self, - run_dict: typing.Mapping[RunKey, RunValue], + run_dict: Mapping[RunKey, RunValue], runhistory: RunHistory, return_time_as_y: bool = False, store_statistics: bool = False, - ) -> typing.Tuple[np.ndarray, np.ndarray]: + ) -> Tuple[np.ndarray, np.ndarray]: """Builds X,y matrixes from selected runs from runhistory. Parameters @@ -664,11 +664,11 @@ class RunHistory2EPM4EIPS(AbstractRunHistory2EPM): def _build_matrix( self, - run_dict: typing.Mapping[RunKey, RunValue], + run_dict: Mapping[RunKey, RunValue], runhistory: RunHistory, return_time_as_y: bool = False, store_statistics: bool = False, - ) -> typing.Tuple[np.ndarray, np.ndarray]: + ) -> Tuple[np.ndarray, np.ndarray]: """TODO.""" if return_time_as_y: raise NotImplementedError() diff --git a/smac/smac_cli.py b/smac/smac_cli.py index bdda69f64..b11205584 100644 --- a/smac/smac_cli.py +++ b/smac/smac_cli.py @@ -1,4 +1,4 @@ -import typing +from typing import List, Optional, Tuple import logging import os @@ -40,7 +40,7 @@ class SMACCLI(object): def __init__(self) -> None: self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) - def main_cli(self, commandline_arguments: typing.Optional[typing.List[str]] = None) -> None: + def main_cli(self, commandline_arguments: Optional[List[str]] = None) -> None: """Main function of SMAC for CLI interface.""" self.logger.info("SMAC call: %s" % (" ".join(sys.argv))) @@ -181,7 +181,7 @@ def restore_state( self, scen: Scenario, restore_state: str, - ) -> typing.Tuple[RunHistory, Stats, typing.List, typing.List]: + ) -> Tuple[RunHistory, Stats, List, List]: """Read in files for state-restoration: runhistory, stats, trajectory.""" # Check for folder and files rh_path = os.path.join(restore_state, "runhistory.json") @@ -208,8 +208,8 @@ def restore_state_after_output_dir( self, scen: Scenario, stats: Stats, - traj_list_aclib: typing.List, - traj_list_old: typing.List, + traj_list_aclib: List, + traj_list_old: List, ) -> Configuration: """Finish processing files for state-restoration. diff --git a/smac/stats/stats.py b/smac/stats/stats.py index 84cb3062c..903dc0fde 100644 --- a/smac/stats/stats.py +++ b/smac/stats/stats.py @@ -1,4 +1,4 @@ -import typing +from typing import Optional import json import logging @@ -74,7 +74,7 @@ def save(self) -> None: with open(path, "w") as fh: json.dump(data, fh) - def load(self, fn: typing.Optional[str] = None) -> None: + def load(self, fn: Optional[str] = None) -> None: """Load all attributes from dictionary in file into stats-object. Parameters diff --git a/smac/tae/__init__.py b/smac/tae/__init__.py index 861483fbb..527c1ee56 100644 --- a/smac/tae/__init__.py +++ b/smac/tae/__init__.py @@ -1,4 +1,4 @@ -import typing +from typing import Any, Dict from enum import Enum @@ -30,7 +30,7 @@ class StatusType(Enum): RUNNING = 9 @staticmethod - def enum_hook(obj: typing.Dict) -> typing.Any: + def enum_hook(obj: Dict) -> Any: """Hook function passed to json-deserializer as "object_hook". EnumEncoder in runhistory/runhistory. diff --git a/smac/tae/dask_runner.py b/smac/tae/dask_runner.py index 3cecdc03b..3e7fa8d62 100644 --- a/smac/tae/dask_runner.py +++ b/smac/tae/dask_runner.py @@ -1,4 +1,4 @@ -import typing +from typing import Dict, List, Optional, Tuple import os import time @@ -82,8 +82,8 @@ def __init__( single_worker: BaseRunner, n_workers: int, patience: int = 5, - output_directory: typing.Optional[str] = None, - dask_client: typing.Optional[dask.distributed.Client] = None, + output_directory: Optional[str] = None, + dask_client: Optional[dask.distributed.Client] = None, ): super(DaskParallelRunner, self).__init__( ta=single_worker.ta, @@ -122,7 +122,7 @@ def __init__( else: self.close_client_at_del = False self.client = dask_client - self.futures = [] # type: typing.List[Future] + self.futures = [] # type: List[Future] self.scheduler_info = self.client._get_scheduler_info() @@ -168,7 +168,7 @@ def submit_run(self, run_info: RunInfo) -> None: # http://distributed.dask.org/en/stable/client.html#pure-functions-by-default self.futures.append(self.client.submit(self.single_worker.run_wrapper, run_info, pure=False)) - def get_finished_runs(self) -> typing.List[typing.Tuple[RunInfo, RunValue]]: + def get_finished_runs(self) -> List[Tuple[RunInfo, RunValue]]: """This method returns any finished configuration, and returns a list with the results of exercising the configurations. This class keeps populating results to self.results until a call to get_finished runs is done. In this case, the self.results list is emptied and all @@ -233,11 +233,11 @@ def run( self, config: Configuration, instance: str, - cutoff: typing.Optional[float] = None, + cutoff: Optional[float] = None, seed: int = 12345, - budget: typing.Optional[float] = None, + budget: Optional[float] = None, instance_specific: str = "0", - ) -> typing.Tuple[StatusType, float, float, typing.Dict]: + ) -> Tuple[StatusType, float, float, Dict]: """This method only complies with the abstract parent class. In the parallel case, we call the single worker run() method. diff --git a/smac/tae/execute_ta_run_aclib.py b/smac/tae/execute_ta_run_aclib.py index be58566cd..65180d14e 100644 --- a/smac/tae/execute_ta_run_aclib.py +++ b/smac/tae/execute_ta_run_aclib.py @@ -1,4 +1,4 @@ -import typing +from typing import Dict, List, Optional, Tuple import json from subprocess import PIPE, Popen @@ -26,11 +26,11 @@ def run( self, config: Configuration, instance: str, - cutoff: typing.Optional[float] = None, + cutoff: Optional[float] = None, seed: int = 12345, - budget: typing.Optional[float] = None, + budget: Optional[float] = None, instance_specific: str = "0", - ) -> typing.Tuple[StatusType, float, float, typing.Dict]: + ) -> Tuple[StatusType, float, float, Dict]: """Runs target algorithm with configuration on instance with instance specifics. @@ -138,11 +138,11 @@ def _call_ta( instance_specific: str, cutoff: float, seed: int, - ) -> typing.Tuple[typing.Dict, str, str]: + ) -> Tuple[Dict, str, str]: # TODO: maybe replace fixed instance specific and cutoff_length (0) to # other value - cmd = [] # type: typing.List[str] + cmd = [] # type: List[str] if not isinstance(self.ta, (list, tuple)): raise TypeError("self.ta needs to be of type list or tuple, but is %s" % type(self.ta)) cmd.extend(self.ta) diff --git a/smac/tae/execute_ta_run_hydra.py b/smac/tae/execute_ta_run_hydra.py index 78de006f0..39563fd95 100644 --- a/smac/tae/execute_ta_run_hydra.py +++ b/smac/tae/execute_ta_run_hydra.py @@ -1,4 +1,4 @@ -import typing +from typing import Any, Dict, Mapping, Optional, Tuple, Type from smac.configspace import Configuration from smac.tae import StatusType @@ -17,17 +17,17 @@ class ExecuteTARunHydra(SerialRunner): Parameters ---------- - cost_oracle: typing.Mapping[str,float] + cost_oracle: Mapping[str,float] cost of oracle per instance - tae: typing.Type[SerialRunner] + tae: Type[SerialRunner] target algorithm evaluator """ def __init__( self, - cost_oracle: typing.Mapping[str, float], - tae: typing.Type[SerialRunner] = ExecuteTARunOld, - **kwargs: typing.Any, + cost_oracle: Mapping[str, float], + tae: Type[SerialRunner] = ExecuteTARunOld, + **kwargs: Any, ) -> None: super().__init__(**kwargs) self.cost_oracle = cost_oracle @@ -46,11 +46,11 @@ def run( self, config: Configuration, instance: str, - cutoff: typing.Optional[float] = None, + cutoff: Optional[float] = None, seed: int = 12345, - budget: typing.Optional[float] = None, + budget: Optional[float] = None, instance_specific: str = "0", - ) -> typing.Tuple[StatusType, float, float, typing.Dict]: + ) -> Tuple[StatusType, float, float, Dict]: """See ~smac.tae.execute_ta_run.ExecuteTARunOld for docstring.""" if cutoff is None: raise ValueError("Cutoff of type None is not supported") diff --git a/smac/tae/execute_ta_run_old.py b/smac/tae/execute_ta_run_old.py index 581ecce9d..546606886 100644 --- a/smac/tae/execute_ta_run_old.py +++ b/smac/tae/execute_ta_run_old.py @@ -1,4 +1,4 @@ -import typing +from typing import Dict, List, Optional, Tuple from subprocess import PIPE, Popen @@ -24,12 +24,12 @@ class ExecuteTARunOld(SerialRunner): def run( self, config: Configuration, - instance: typing.Optional[str] = None, - cutoff: typing.Optional[float] = None, + instance: Optional[str] = None, + cutoff: Optional[float] = None, seed: int = 12345, - budget: typing.Optional[float] = 0.0, + budget: Optional[float] = 0.0, instance_specific: str = "0", - ) -> typing.Tuple[StatusType, float, float, typing.Dict]: + ) -> Tuple[StatusType, float, float, Dict]: """Runs target algorithm with configuration on instance with instance specifics. @@ -79,7 +79,7 @@ def run( status_string = "CRASHED" quality = 1234567890.0 runtime = 1234567890.0 - additional_info = {} # type: typing.Dict[str, str] + additional_info = {} # type: Dict[str, str] for line in stdout_.split("\n"): if ( line.startswith("Result of this algorithm run:") @@ -156,10 +156,10 @@ def _call_ta( instance_specific: str, cutoff: float, seed: int, - ) -> typing.Tuple[str, str]: + ) -> Tuple[str, str]: # TODO: maybe replace fixed instance specific and cutoff_length (0) to other value - cmd = [] # type: typing.List[str] + cmd = [] # type: List[str] if not isinstance(self.ta, (list, tuple)): raise TypeError("self.ta needs to be of type list or tuple, but is %s" % type(self.ta)) cmd.extend(self.ta) diff --git a/smac/utils/dependencies.py b/smac/utils/dependencies.py index d81f563a7..303b5d1ee 100644 --- a/smac/utils/dependencies.py +++ b/smac/utils/dependencies.py @@ -1,4 +1,4 @@ -import typing +from typing import List, Union import importlib import re @@ -14,12 +14,12 @@ RE_PATTERN = re.compile(r"^(?P[\w\-]+)%s?(,%s)?$" % (SUBPATTERN % (1, 1), SUBPATTERN % (2, 2))) -def verify_packages(packages: typing.Union[typing.List[str], str]) -> None: +def verify_packages(packages: Union[List[str], str]) -> None: """Verifies packages. Calls `_verify_packages` as subroutine. Parameters ---------- - packages : typing.Union[typing.List[str], str] + packages : Union[List[str], str] Packages to verify. Raises diff --git a/smac/utils/io/output_writer.py b/smac/utils/io/output_writer.py index ac0ac26e5..8fb7cde47 100644 --- a/smac/utils/io/output_writer.py +++ b/smac/utils/io/output_writer.py @@ -1,4 +1,4 @@ -import typing +from typing import TYPE_CHECKING, Any, Dict, Iterable, List import os import shutil @@ -7,7 +7,7 @@ from smac.configspace import ConfigurationSpace, json, pcs_new from smac.utils.logging import PickableLoggerAdapter -if typing.TYPE_CHECKING: +if TYPE_CHECKING: from smac.scenario.scenario import Scenario __copyright__ = "Copyright 2021, AutoML.org Freiburg-Hannover" @@ -72,7 +72,7 @@ def write_scenario_file(self, scenario: "Scenario") -> None: if new_value is not None: fh.write("{} = {}\n".format(options_dest2name[key], new_value)) - def _parse_argument(self, scenario: "Scenario", key: str, value: typing.Any) -> typing.Any: + def _parse_argument(self, scenario: "Scenario", key: str, value: Any) -> Any: """Some values of the scenario-file need to be changed upon writing, such as the 'ta' (target algorithm), due to it's callback. Also, the configspace, features, train_inst- and test-inst-lists are saved @@ -152,7 +152,7 @@ def _parse_argument(self, scenario: "Scenario", key: str, value: typing.Any) -> return value - def write_inst_file(self, insts: typing.List[str], fn: str) -> None: + def write_inst_file(self, insts: List[str], fn: str) -> None: """Writes instance-list to file. Parameters @@ -168,7 +168,7 @@ def write_inst_file(self, insts: typing.List[str], fn: str) -> None: def write_inst_features_file( self, n_features: int, - feat_dict: typing.Dict[str, typing.Iterable[float]], + feat_dict: Dict[str, Iterable[float]], fn: str, ) -> None: """Writes features to file. diff --git a/smac/utils/merge_foreign_data.py b/smac/utils/merge_foreign_data.py index 49088618f..22662f10b 100644 --- a/smac/utils/merge_foreign_data.py +++ b/smac/utils/merge_foreign_data.py @@ -1,4 +1,4 @@ -import typing +from typing import List, Tuple from smac.configspace import ConfigurationSpace from smac.runhistory.runhistory import DataOrigin, RunHistory @@ -11,10 +11,10 @@ def merge_foreign_data_from_file( scenario: Scenario, runhistory: RunHistory, - in_scenario_fn_list: typing.List[str], - in_runhistory_fn_list: typing.List[str], + in_scenario_fn_list: List[str], + in_runhistory_fn_list: List[str], cs: ConfigurationSpace, -) -> typing.Tuple[Scenario, RunHistory]: +) -> Tuple[Scenario, RunHistory]: """Extend and with runhistory data from another. assuming the same pcs, feature space, but different instances @@ -25,9 +25,9 @@ def merge_foreign_data_from_file( original scenario -- feature dictionary will be extended runhistory: RunHistory original runhistory -- will be extended by further data points - in_scenario_fn_list: typing.List[str] + in_scenario_fn_list: List[str] input scenario file names - in_runhistory_fn_list: typing.List[str] + in_runhistory_fn_list: List[str] list filenames of runhistory dumps cs: ConfigurationSpace parameter configuration space to read runhistory from file @@ -55,9 +55,9 @@ def merge_foreign_data_from_file( def merge_foreign_data( scenario: Scenario, runhistory: RunHistory, - in_scenario_list: typing.List[Scenario], - in_runhistory_list: typing.List[RunHistory], -) -> typing.Tuple[Scenario, RunHistory]: + in_scenario_list: List[Scenario], + in_runhistory_list: List[RunHistory], +) -> Tuple[Scenario, RunHistory]: """Extend and with runhistory data from another. assuming the same pcs, feature space, but different instances @@ -68,9 +68,9 @@ def merge_foreign_data( original scenario -- feature dictionary will be extended runhistory: RunHistory original runhistory -- will be extended by further data points - in_scenario_list: typing.List[Scenario] + in_scenario_list: List[Scenario] input scenario - in_runhistory_list: typing.List[RunHistory] + in_runhistory_list: List[RunHistory] list of runhistories wrt Returns diff --git a/smac/utils/validate.py b/smac/utils/validate.py index c60da84b7..653661d14 100644 --- a/smac/utils/validate.py +++ b/smac/utils/validate.py @@ -1,5 +1,4 @@ -import typing -from typing import Union +from typing import Any, Dict, List, Optional, Sequence, Tuple, Union, cast import logging import os @@ -29,10 +28,10 @@ def _unbound_tae_starter( tae: BaseRunner, - runhistory: typing.Optional[RunHistory], + runhistory: Optional[RunHistory], run_info: RunInfo, - *args: typing.Any, - **kwargs: typing.Any, + *args: Any, + **kwargs: Any, ) -> RunValue: """Unbound function to be used by joblibs Parallel, since directly passing the TAE results in pickling-problems. @@ -93,14 +92,14 @@ class Validator(object): def __init__( self, scenario: Scenario, - trajectory: typing.Optional[typing.List], + trajectory: Optional[List], rng: Union[np.random.RandomState, int, None] = None, ) -> None: self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) self.traj = trajectory self.scen = scenario - self.epm = None # type: typing.Optional[RandomForestWithInstances] + self.epm = None # type: Optional[RandomForestWithInstances] if isinstance(rng, np.random.RandomState): self.rng = rng @@ -114,8 +113,8 @@ def __init__( def _save_results( self, rh: RunHistory, - output_fn: typing.Optional[str], - backup_fn: typing.Optional[str] = None, + output_fn: Optional[str], + backup_fn: Optional[str] = None, ) -> None: """Helper to save results to file. @@ -150,14 +149,14 @@ def _save_results( def validate( self, - config_mode: Union[str, typing.List[Configuration]] = "def", - instance_mode: Union[str, typing.List[str]] = "test", + config_mode: Union[str, List[Configuration]] = "def", + instance_mode: Union[str, List[str]] = "test", repetitions: int = 1, n_jobs: int = 1, backend: str = "threading", - runhistory: typing.Optional[RunHistory] = None, + runhistory: Optional[RunHistory] = None, tae: BaseRunner = None, - output_fn: typing.Optional[str] = None, + output_fn: Optional[str] = None, ) -> RunHistory: """Validate configs on instances and save result in runhistory. If a runhistory is provided as input it is important that you run it on the same/comparable hardware. @@ -255,11 +254,11 @@ def validate( def _validate_parallel( self, tae: BaseRunner, - runs: typing.List[_Run], + runs: List[_Run], n_jobs: int, backend: str, - runhistory: typing.Optional[RunHistory] = None, - ) -> typing.List[RunValue]: + runhistory: Optional[RunHistory] = None, + ) -> List[RunValue]: """Validate runs with joblibs Parallel-interface. Parameters @@ -302,11 +301,11 @@ def _validate_parallel( def validate_epm( self, - config_mode: Union[str, typing.List[Configuration]] = "def", - instance_mode: Union[str, typing.List[str]] = "test", + config_mode: Union[str, List[Configuration]] = "def", + instance_mode: Union[str, List[str]] = "test", repetitions: int = 1, - runhistory: typing.Optional[RunHistory] = None, - output_fn: typing.Optional[str] = None, + runhistory: Optional[RunHistory] = None, + output_fn: Optional[str] = None, reuse_epm: bool = True, ) -> RunHistory: """Use EPM to predict costs/runtimes for unknown config/inst-pairs. @@ -388,7 +387,7 @@ def validate_epm( # Train random forest epm.train(X, y) else: - epm = typing.cast(RandomForestWithInstances, self.epm) + epm = cast(RandomForestWithInstances, self.epm) # Predict desired runs runs, rh_epm = self._get_runs(config_mode, instance_mode, repetitions, runhistory) @@ -431,11 +430,11 @@ def validate_epm( def _get_runs( self, - configs: Union[str, typing.List[Configuration]], - insts: Union[str, typing.List[str]], + configs: Union[str, List[Configuration]], + insts: Union[str, List[str]], repetitions: int = 1, runhistory: RunHistory = None, - ) -> typing.Tuple[typing.List[_Run], RunHistory]: + ) -> Tuple[List[_Run], RunHistory]: """Generate list of SMAC-TAE runs to be executed. This means combinations of configs with all instances on a certain number of seeds. @@ -469,7 +468,7 @@ def _get_runs( if isinstance(configs, str): configs = self._get_configs(configs) if isinstance(insts, str): - instances = sorted(self._get_instances(insts)) # type: typing.Sequence[typing.Union[str, None]] + instances = sorted(self._get_instances(insts)) # type: Sequence[Union[str, None]] elif insts is not None: instances = sorted(insts) else: @@ -554,10 +553,10 @@ def _get_runs( def _process_runhistory( self, - configs: typing.List[Configuration], - insts: typing.Sequence[typing.Optional[str]], - runhistory: typing.Optional[RunHistory], - ) -> typing.Dict[str, typing.List[typing.Tuple[int, typing.List[Configuration]]]]: + configs: List[Configuration], + insts: Sequence[Optional[str]], + runhistory: Optional[RunHistory], + ) -> Dict[str, List[Tuple[int, List[Configuration]]]]: """Processes runhistory from self._get_runs by extracting already evaluated (relevant) config-inst-seed tuples. @@ -582,7 +581,7 @@ def _process_runhistory( # Like this we can easily retrieve the most used instance-seed pairs to # minimize the number of runs to be evaluated if runhistory: - inst_seed_config = {} # type: typing.Dict[str, typing.Dict[int, typing.List[Configuration]]] + inst_seed_config = {} # type: Dict[str, Dict[int, List[Configuration]]] relevant = dict() for key in runhistory.data: if runhistory.ids_config[key.config_id] in configs and key.instance_id in insts: @@ -609,10 +608,10 @@ def _process_runhistory( for i in inst_seed_config } else: - rval = {} # type: typing.Dict[str, typing.List[typing.Tuple[int, typing.List[Configuration]]]] + rval = {} # type: Dict[str, List[Tuple[int, List[Configuration]]]] return rval - def _get_configs(self, mode: str) -> typing.List[str]: + def _get_configs(self, mode: str) -> List[str]: """Return desired configs. Parameters @@ -664,7 +663,7 @@ def _get_configs(self, mode: str) -> typing.List[str]: self.logger.debug("Gathered %d configurations for mode %s.", len(configs), mode) return configs - def _get_instances(self, mode: str) -> typing.List[str]: + def _get_instances(self, mode: str) -> List[str]: """Get desired instances. Parameters @@ -694,7 +693,7 @@ def _get_instances(self, mode: str) -> typing.List[str]: ) instance_mode = "train+test" - instances = [] # type: typing.List[str] + instances = [] # type: List[str] if (instance_mode == "train" or instance_mode == "train+test") and not self.scen.train_insts == [None]: instances.extend(self.scen.train_insts) if (instance_mode == "test" or instance_mode == "train+test") and not self.scen.test_insts == [None]: diff --git a/tests/test_epm/test_gp_priors.py b/tests/test_epm/test_gp_priors.py index 5bc889bc0..86cd345b3 100644 --- a/tests/test_epm/test_gp_priors.py +++ b/tests/test_epm/test_gp_priors.py @@ -145,7 +145,7 @@ def test_lnprob_and_grad_scalar(self): # Legal scalar x = -1 - self.assertEqual(prior.lnprob(x), -0.46155023498761205) + self.assertAlmostEqual(prior.lnprob(x), -0.46155023, 7) self.assertEqual(prior.gradient(x), -1.2357588823428847) def test_lnprob_and_grad_array(self): diff --git a/tests/test_facade/test_smac_facade.py b/tests/test_facade/test_smac_facade.py index 7f19695e6..39fa75f40 100644 --- a/tests/test_facade/test_smac_facade.py +++ b/tests/test_facade/test_smac_facade.py @@ -332,7 +332,7 @@ def test_init_EIPS_as_arguments(self): ) self.assertIsInstance(smbo.epm_chooser.rh2EPM, RunHistory2EPM4EIPS) - with self.assertRaisesRegex(TypeError, "the surrogate model must support multi-objective prediction!"): + with self.assertRaisesRegex(TypeError, "the surrogate model must support multi-objective prediction!"): SMAC4AC(self.scenario, acquisition_function=EIPS, runhistory2epm=RunHistory2EPM4EIPS) ####################################################################################################################