Merge pull request #322 from automl/development

Prepare Release 0.7

.travis.yml

@@ -4,11 +4,11 @@ matrix:
 
   include:
   - os: linux
-    env: PYTHON_VERSION="3.4" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh"
+    env: PYTHON_VERSION="3.4" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-4.3.21-Linux-x86_64.sh"
   - os: linux
-    env: PYTHON_VERSION="3.5" COVERAGE="true" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh"
+    env: PYTHON_VERSION="3.5" COVERAGE="true" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-4.3.21-Linux-x86_64.sh"
   - os: linux
-    env: PYTHON_VERSION="3.6" COVERAGE="true" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh"
+    env: PYTHON_VERSION="3.6" COVERAGE="true" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-4.3.21-Linux-x86_64.sh"
 
   # Disable OSX building because it takes too long and hinders progress
   # Set language to generic to not break travis-ci
@@ -18,15 +18,15 @@ matrix:
   #- os: osx
   #  sudo: required
   #  language: generic
-  #  env: PYTHON_VERSION="3.4" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh"
+  #  env: PYTHON_VERSION="3.4" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-4.3.21-MacOSX-x86_64.sh"
   #- os: osx
   #  sudo: required
   #  language: generic
-  #  env: PYTHON_VERSION="3.5" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh"
+  #  env: PYTHON_VERSION="3.5" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-4.3.21-MacOSX-x86_64.sh"
   #- os: osx
   #  sudo: required
   #  language: generic
-  #  env: PYTHON_VERSION="3.6" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh"
+  #  env: PYTHON_VERSION="3.6" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-4.3.21-MacOSX-x86_64.sh"
 
 cache:
   directories:
@@ -43,7 +43,6 @@ before_install:
   - bash miniconda.sh -b -p $HOME/miniconda
   - export PATH="$HOME/miniconda/bin:$PATH"
   - if [[ `which conda` ]]; then echo 'Conda installation successful'; else exit 1; fi
-  - conda update --yes conda
   - conda create -n testenv --yes python=$PYTHON_VERSION pip wheel nose
   - source activate testenv
   - conda install --yes gcc swig

changelog.md

@@ -1,3 +1,10 @@
+# 0.7
+
+* FIX #193, restoring the scenario now possible
+* ADD #271 validation
+* FIX #311 abort on first crash
+* FIX #318, ExecuteTARunOld now always returns a StatusType
+
 # 0.6
 
 ## Major changes

circle.yml

@@ -27,9 +27,8 @@ dependencies:
     - sudo -E apt-get -yq update
     - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install dvipng texlive-latex-base texlive-latex-extra
     # Conda installation
-    - wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh
+    - wget http://repo.continuum.io/miniconda/Miniconda3-4.3.21-Linux-x86_64.sh -O ~/miniconda.sh
     - bash ~/miniconda.sh -b -p $HOME/miniconda
-    - conda update --yes conda
     - conda create -n testenv --yes python=3.6 pip wheel nose gcc swig
 
   # The --user is needed to let sphinx see the source and the binaries

doc/basic_usage.rst

@@ -43,6 +43,7 @@ Optional:
      * *seed*: The integer that the random-generator will be based upon. **Default**: 12345
      * *verbose_level*: in [INFO, DEBUG], specifies the logging-verbosity. **Default**: INFO
      * *mode*: in [SMAC, ROAR]. SMAC will use the bayeasian optimization with an intensification process, whereas ROAR stands for Random Online Adaptive Racing. **Default**: SMAC
+     * *restore_state*: A string specifying the folder of the *SMAC*-run to be continued. **Assuming exactly the same scenario, except for budget-options.**
 
 In the scenario file, there are two mandatory parameters: The **algo**-parameter
 defines how *SMAC* will call the target algorithm. Parameters will be appended to the call
@@ -52,6 +53,18 @@ The **paramfile**-parameter defines the path to the `PCS-file <options.html#pcs>
 which describes the ranges and default values of the tunable parameters.
 Both will interpret paths *from the execution-directory*.
 
+.. _restorestate:
+
+To restore a previous *SMAC*-run, either because it was interrupted or because
+you want to extend its computation- or time-limits, use the
+``--restore_state FOLDER``-option in the commandline. If you want to increase
+computation- or time-limits, change the scenario-file specified with the
+``--scenario SCENARIOFILE``-option (not the one in the folder to be restored).
+Restarting a *SMAC*-run that quit due to budget-exhaustion will do nothing,
+because the budget is still exhausted.
+**Changing any other options than *wallclock_limit*, *runcount_limit* or
+*tuner-timeout* in the scenario-file is NOT intended and will likely lead
+to unexpected behaviour!**
 
 .. note::
 

doc/faq.rst

@@ -18,6 +18,10 @@ Refer to `commandline execution <basic_usage.html#commandline>`_ for details on
 
 You can also run SMAC with :code:`--verbose DEBUG` to see how *SMAC* tried to call your algorithm.
 
+.. rubric:: Can I restore SMAC from a previous state?
+
+Use the `restore-option <basic_usage.html#restorestate>`_.
+
 .. rubric:: I discovered a bug/have criticism or ideas on *SMAC*. Where should I report to?
 
 *SMAC* uses the `GitHub issue-tracker <https://github.com/automl/SMAC3/issues>`_ to take care of bugs and questions. If you

doc/installation.rst

@@ -13,7 +13,7 @@ please call:
 
 .. code-block:: bash
 
-    sudo apt-get install build-essentials swig
+    sudo apt-get install build-essential swig
 
 If you use Anaconda, you have to install both gcc and SWIG from Anaconda to
 prevent broken links:

doc/options.rst

@@ -20,7 +20,7 @@ Optional:
 
 Scenario
 ~~~~~~~~
-The scenario-object is used to configure *SMAC* and can be constructed either by providing an actual
+The scenario-object (:class:`smac.scenario.scenario.Scenario`) is used to configure *SMAC* and can be constructed either by providing an actual
 scenario-object (see `SVM-example <quickstart.html#using-smac-in-python-svm>`_), or by specifing the options in a
 scenario file (see `SPEAR example <quickstart.html#spear-example>`_).
 The format of the scenario file is one option per line:

doc/quickstart.rst

@@ -202,7 +202,7 @@ and evaluation of the function, SMAC needs as inputs the scenario object
 as well as the function.
 
 .. literalinclude:: ../examples/svm.py
-   :lines: 103-
+   :lines: 103-112
    :lineno-match:
 
 We start the optimization loop.
@@ -236,6 +236,13 @@ so that as final output we can see the error value of the incumbent.
 
    Optimized Value: 0.02
 
+As a bonus, we can validate our results. This is more useful when optimizing on
+instances, but we include the code so it is easily applicable for any usecase.
+
+.. literalinclude:: ../examples/svm.py
+   :lines: 115-
+   :lineno-match:
+
 .. _spear-example:
 
 Spear-QCP

doc/tae.rst

@@ -7,7 +7,7 @@ Target Algorithm Evaluator
 --------------------------
 
 *SMAC* evaluates the algorithm to be optimized by invoking it through a Target
-Algortihm Evaluator (TAE). There are different TAEs implemented in *SMAC* which
+Algorithm Evaluator (TAE). There are different TAEs implemented in *SMAC* which
 provide slightly different interfaces. You can see `SVM-example
 <quickstart.html#using-smac-in-python-svm>`_ for an example on the usage.
 

examples/svm.py

@@ -110,3 +110,10 @@ def svm_from_cfg(cfg):
 inc_value = svm_from_cfg(incumbent)
 
 print("Optimized Value: %.2f" % (inc_value))
+
+
+# We can also validate our results (though this makes a lot more sense with instances)
+smac.validate(config_mode='inc',      # We can choose which configurations to evaluate
+              #instance_mode='train+test',  # Defines what instances to validate
+              repetitions=100,        # Ignored, unless you set "deterministic" to "false" in line 95
+              n_jobs=1)               # How many cores to use in parallel for optimization

requirements.txt

@@ -10,3 +10,4 @@ typing
 pyrfr>=0.5.0
 sphinx
 sphinx_rtd_theme
+joblib

scripts/smac-validate.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+
+from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
+import logging
+import sys
+import os
+import inspect
+cmd_folder = os.path.realpath(os.path.abspath(os.path.split(inspect.getfile( inspect.currentframe() ))[0]))
+cmd_folder = os.path.realpath(os.path.join(cmd_folder, ".."))
+if cmd_folder not in sys.path:
+    sys.path.insert(0, cmd_folder)
+
+from smac.scenario.scenario import Scenario
+from smac.stats.stats import Stats
+from smac.tae.execute_ta_run_old import ExecuteTARunOld
+from smac.tae.execute_ta_run_aclib import ExecuteTARunAClib
+from smac.utils.validate import Validator
+from smac.utils.io.traj_logging import TrajLogger
+
+
+if __name__ == "__main__":
+
+
+    parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
+    req_opts = parser.add_argument_group("Required Options")
+    req_opts.add_argument("--scenario", required=True,
+                          help="path to SMAC scenario")
+    req_opts.add_argument("--trajectory", required=True,
+                          help="path to SMAC trajectory")
+    req_opts.add_argument("--output", required=True,
+                          help="path to save runhistory to")
+
+    req_opts = parser.add_argument_group("Optional Options")
+    req_opts.add_argument("--configs", default="def+inc", type=str,
+                          choices=["def", "inc", "def+inc", "time", "all"],
+                          help="what configurations to evaluate: "
+                               "def: default, inc: incumbent, "
+                               "time: configs at timesteps 2^1, 2^2, 2^3, ..., "
+                               "all: all configurations in the trajectory")
+    req_opts.add_argument("--instances", default="test", type=str,
+                          choices=["train", "test", "train+test"],
+                          help="what instances to evaluate")
+    req_opts.add_argument("--runhistory", default=None, type=str,
+                          help="path to runhistory to impute runs from")
+    req_opts.add_argument("--seed", type=int, help="random seed")
+    req_opts.add_argument("--repetitions", default=1, type=int,
+                          help="number of repetitions for nondeterministic "
+                               "algorithms")
+    req_opts.add_argument("--n_jobs", default=1, type=int,
+                          help="number of cpu-cores to use")
+    req_opts.add_argument("--tae", default="old", type=str,
+                          help="what tae to use", choices=["aclib", "old"])
+    req_opts.add_argument("--verbose_level", default=logging.INFO,
+                          choices=["INFO", "DEBUG"],
+                          help="verbose level")
+
+    args_, misc = parser.parse_known_args()
+
+    # remove leading '-' in option names
+    misc = dict((k.lstrip("-"), v.strip("'"))
+                for k, v in zip(misc[::2], misc[1::2]))
+
+    if args_.verbose_level == "INFO":
+        logging.basicConfig(level=logging.INFO)
+    else:
+        logging.basicConfig(level=logging.DEBUG)
+
+    scenario = Scenario(args_.scenario)
+    traj_logger = TrajLogger(None, Stats(scenario))
+    trajectory = traj_logger.read_traj_aclib_format(args_.trajectory, scenario.cs)
+    if args_.tae == "old":
+        tae = ExecuteTARunOld(ta=scenario.ta,
+                              run_obj=scenario.run_obj,
+                              par_factor=scenario.par_factor,
+                              cost_for_crash=scenario.cost_for_crash)
+    if args_.tae == "aclib":
+        tae = ExecuteTARunAClib(ta=scenario.ta,
+                              run_obj=scenario.run_obj,
+                              par_factor=scenario.par_factor,
+                              cost_for_crash=scenario.cost_for_crash)
+
+    validator = Validator(scenario, trajectory, args_.output,
+                          args_.seed)
+    validator.validate(config_mode=args_.configs,
+                       instance_mode=args_.instances,
+                       repetitions=args_.repetitions,
+                       n_jobs=args_.n_jobs,
+                       runhistory=args_.runhistory,
+                       tae=tae)

smac/__version__.py

@@ -1,4 +1,4 @@
 """Version information."""
 
 # The following line *must* be the last in the module, exactly as formatted:
-__version__ = "0.6.0"
+__version__ = "0.7.1"

smac/facade/smac_facade.py

@@ -65,6 +65,7 @@ def __init__(self,
                  initial_design: InitialDesign=None,
                  initial_configurations: typing.List[Configuration]=None,
                  stats: Stats=None,
+                 restore_incumbent: Configuration=None,
                  rng: np.random.RandomState=None):
         """Constructor
 
@@ -105,6 +106,8 @@ def __init__(self,
             optional stats object
         rng : np.random.RandomState
             Random number generator
+        restore_incumbent: Configuration
+            incumbent used if restoring to previous state
         """
 
         self.logger = logging.getLogger(
@@ -319,7 +322,8 @@ def __init__(self,
                            model=model,
                            acq_optimizer=local_search,
                            acquisition_func=acquisition_function,
-                           rng=rng)
+                           rng=rng,
+                           restore_incumbent=restore_incumbent)
 
     @staticmethod
     def _get_rng(rng):
@@ -364,6 +368,7 @@ def optimize(self):
         try:
             incumbent = self.solver.run()
         finally:
+            self.solver.stats.save()
             self.solver.stats.print_stats()
             self.logger.info("Final Incumbent: %s" % (self.solver.incumbent))
             self.runhistory = self.solver.runhistory
@@ -375,6 +380,35 @@ def optimize(self):
                                     "runhistory.json"))
         return incumbent
 
+    def validate(self, config_mode='inc', instance_mode='train+test',
+                 repetitions=1, n_jobs=-1, backend='threading'):
+        """Create validator-object and run validation, using
+        scenario-information, runhistory from smbo and tae_runner from intensify
+
+        Parameters
+        ----------
+        config_mode: string
+            what configurations to validate
+            from [def, inc, def+inc, time, all], time means evaluation at
+            timesteps 2^-4, 2^-3, 2^-2, 2^-1, 2^0, 2^1, ...
+        instance_mode: string
+            what instances to use for validation, from [train, test, train+test]
+        repetitions: int
+            number of repetitions in nondeterministic algorithms (in
+            deterministic will be fixed to 1)
+        n_jobs: int
+            number of parallel processes used by joblib
+        backend: string
+            what backend to be used by joblib
+
+        Returns
+        -------
+        runhistory: RunHistory
+            runhistory containing all specified runs
+        """
+        return self.solver.validate(config_mode, instance_mode, repetitions,
+                                    n_jobs, backend)
+
     def get_tae_runner(self):
         """Returns target algorithm evaluator (TAE) object which can run the
         target algorithm given a configuration

smac/initial_design/single_config_initial_design.py

@@ -1,11 +1,12 @@
 import numpy as np
 
 from smac.initial_design.initial_design import InitialDesign
-from smac.tae.execute_ta_run import ExecuteTARun
+from smac.tae.execute_ta_run import ExecuteTARun, StatusType
 from smac.stats.stats import Stats
 from smac.utils.io.traj_logging import TrajLogger
 from smac.scenario.scenario import Scenario
 from smac.utils import constants
+from smac.tae.execute_ta_run import FirstRunCrashedException
 
 __author__ = "Marius Lindauer, Katharina Eggensperger"
 __copyright__ = "Copyright 2016, ML4AAD"
@@ -73,13 +74,25 @@ def run(self):
         else:
             initial_seed = self.rng.randint(0, constants.MAXINT)
 
-        status, cost, runtime, additional_info = self.tae_runner.start(
-            initial_incumbent,
-            instance=rand_inst,
-            cutoff=self.scenario.cutoff,
-            seed=initial_seed,
-            instance_specific=self.scenario.instance_specific.get(rand_inst,
-                                                                  "0"))
+        try:
+            status, cost, runtime, additional_info = self.tae_runner.start(
+                initial_incumbent,
+                instance=rand_inst,
+                cutoff=self.scenario.cutoff,
+                seed=initial_seed,
+                instance_specific=self.scenario.instance_specific.get(rand_inst,
+                                                                      "0"))
+        except FirstRunCrashedException as err:
+            if self.scenario.abort_on_first_run_crash:
+                raise
+            else:
+                status = StatusType.CRASHED
+                if self.scenario.run_obj == "quality":
+                    cost = self.scenario.cost_for_crash
+                else:
+                    cost = self.scenario.cutoff * scenario.par_factor
+                runtime = 0
+                additional_info = {}
 
         self.stats.inc_changed += 1  # first incumbent