From 144c99926033b59481050e64c62d5e7ce0a211c3 Mon Sep 17 00:00:00 2001 From: Teytaud Date: Tue, 19 Sep 2023 11:39:48 +0200 Subject: [PATCH] NgIoh variants (#1551) * po * newplots * Dactoid with many new methods (#1546) * bobyqa_coming * black_and_static * Update recastlib.py * Add_ax * smac * scipyplus * scipyplus * fix * fix * fix * fix * fix * fix * black * Update recastlib.py * Update mypy.ini * Update recastlib.py * Update mypy.ini * fix * fix * fix * fix * wtf * fix * fix * flute * fix * fix * wtf * black * smac * fix * fix * fix * fix * so_dirty * smac2 * Add a bound option to ArtificialFunction * Use normalizer for BO optimizers * changelo * fix * Polishing * Changes in experiments.py for xp in bounded domains * black * Add normalizer to AX * Add normalizer to SMAC * smac / no-transform: fix bounds thx Jeremy * fix bounds ax * risky * fixadcas * fix * fix * ouie * fix * fix * fix * fix * fix * fix * fix * fix * po * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * po * fix * fix * fix * fix * fix * fix * fix * okarc * fix * fix * fix * fix * fix * fix * bfgs (#1532) * fix * fix * fix * fix * fix * fix * fix * fix * pou * fix * fix * fix * fix * fix * oldstuff * qopso * fix * fix * fix * fix * po * po * fix * fix * fix * noslowcircleci * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * po * po * fo --------- Co-authored-by: Elena Raponi Co-authored-by: Jeremy Rapin * fix * ZZpo * rmrfvenv * fixbudgetnone * black * po * fix * fix * po * fix * fix * fix --------- Co-authored-by: Elena Raponi Co-authored-by: Jeremy Rapin --- nevergrad/benchmark/core.py | 7 +- nevergrad/benchmark/experiments.py | 37 +- nevergrad/benchmark/plotting.py | 15 +- nevergrad/functions/ml/mlfunctionlib.py | 17 +- nevergrad/functions/rocket/rocket.py | 2 +- nevergrad/optimization/optimizerlib.py | 326 ++++++++++++++++++ .../optimization/recorded_recommendations.csv | 5 + requirements/bench.txt | 9 +- scripts/dagstuhloid.sh | 5 + scripts/latexize.sh | 17 +- 10 files changed, 390 insertions(+), 50 deletions(-) diff --git a/nevergrad/benchmark/core.py b/nevergrad/benchmark/core.py index 433ac2b4c1..cc1181bddf 100644 --- a/nevergrad/benchmark/core.py +++ b/nevergrad/benchmark/core.py @@ -38,7 +38,12 @@ def save_or_append_to_csv(df: pd.DataFrame, path: Path) -> None: """Saves a dataframe to a file in append mode""" if path.exists(): print("Appending to existing file") - predf = pd.read_csv(str(path)) + try: + predf = pd.read_csv( + str(path), on_bad_lines="warn" + ) # , error_bad_lines=False)#, error_bad_lines=False) + except: # for some versions of Panda. + predf = pd.read_csv(str(path)) # , error_bad_lines=False)#, error_bad_lines=False) df = pd.concat([predf, df], sort=False) df.to_csv(path, index=False) diff --git a/nevergrad/benchmark/experiments.py b/nevergrad/benchmark/experiments.py index 7010195f77..9090bbb7b5 100644 --- a/nevergrad/benchmark/experiments.py +++ b/nevergrad/benchmark/experiments.py @@ -51,7 +51,7 @@ def refactor_optims(x: tp.List[tp.Any]) -> tp.List[tp.Any]: # return ["RandomSearch", "OnePlusOne", "DE", "PSO"] - return ["NgIoh", "Shiwa", "NGOpt"] + algos = {} algos["aquacrop_fao"] = [ "CMA", @@ -409,7 +409,7 @@ def refactor_optims(x: tp.List[tp.Any]) -> tp.List[tp.Any]: # Below, we use the best in the records above. benchmark = str(inspect.stack()[1].function) - if benchmark in algos and "tunin" in benchmark and np.random.randint(2) > 0 and False: + if benchmark in algos and "tunin" in benchmark and np.random.randint(2) > 0: return algos[benchmark][:5] # Here, we pseudo-randomly draw one optim in the provided list, @@ -462,34 +462,11 @@ def doint(s): # Converting a string into an int. import socket host = socket.gethostname() - list_optims = ["Carola1", "Carola2", "Carola3", "NgIoh", "NgIoh", "NgIoh", "MetaModel", "Cobyla"] - list_optims = ["NgIoh", "NgOpt"] - if "tuning" in benchmark: - list_optims = [ - "NgIoh", - "NgIoh", - "NgIoh", - "NgIoh", - "NGOpt", - "NGOpt", - "NGOpt", - "NGOpt", - "HyperOpt", - "RandomSearch", - "PSO", - "DE", - "SQOPSO", - "Cobyla", - # "AX", - "LHSSearch", - "QODE", - "SODE", - ] - if "iscre" in benchmark: - list_optims = ["RecombiningPortfolioDiscreteOnePlusOne"] return [list_optims[doint(host) % len(list_optims)]] - return x # ["Zero"] #return x + + +# return x # ["Zero"] #return x # return ["MultiSQP", "MultiCobyla", "MultiBFGS"] @@ -3373,8 +3350,8 @@ def pbo_suite(seed: tp.Optional[int] = None, reduced: bool = False) -> tp.Iterat for x in ng.optimizers.registry.keys() if "iscre" in x and "ois" not in x and "ptim" not in x and "oerr" not in x ] - optims = ["NGOpt", "NGOptRW"] - optims = refactor_optims(optims) + list_optims = ["NGOpt", "NGOptRW"] + list_optims = refactor_optims(list_optims) for dim in [16, 64, 100]: for fid in range(1, 24): for iid in range(1, 5): diff --git a/nevergrad/benchmark/plotting.py b/nevergrad/benchmark/plotting.py index fcc55d3333..223b381390 100644 --- a/nevergrad/benchmark/plotting.py +++ b/nevergrad/benchmark/plotting.py @@ -367,7 +367,7 @@ def create_plots( print("\n# new case #", fixed, case) casedf = df.select(**dict(zip(fixed, case))) data_df = FightPlotter.winrates_from_selection( - casedf, fight_descriptors, num_rows=num_rows, num_cols=50 + casedf, fight_descriptors, num_rows=num_rows, num_cols=350 ) fplotter = FightPlotter(data_df) # Competence maps: we find out the best algorithm for each attribute1=valuei/attribute2=valuej. @@ -403,7 +403,7 @@ def create_plots( fplotter.save(str(output_folder / "fight_all_pure.png"), dpi=_DPI) else: fplotter.save(str(output_folder / name) + "_pure.png", dpi=_DPI) - print(f"# {len(data_df.columns[:])}") + print(f"# {len(data_df.columns[:])} {data_df.columns[:]}") if order == 2 and competencemaps and best_algo: # With order 2 we can create a competence map. print("\n# Competence map") name = "competencemap_" + ",".join("{}".format(x) for x in fixed) + ".tex" @@ -779,10 +779,15 @@ def __init__(self, winrates_df: pd.DataFrame) -> None: self.winrates = winrates_df self._fig = plt.figure() self._ax = self._fig.add_subplot(111) + max_cols = 25 self._cax = self._ax.imshow( - 100 * np.array(self.winrates), cmap=cm.seismic, interpolation="none", vmin=0, vmax=100 + 100 * np.array(self.winrates)[:, :max_cols], + cmap=cm.seismic, + interpolation="none", + vmin=0, + vmax=100, ) - x_names = self.winrates.columns + x_names = self.winrates.columns[:max_cols] # we plot only the 50 best self._ax.set_xticks(list(range(len(x_names)))) self._ax.set_xticklabels(x_names, rotation=45, ha="right", fontsize=7) y_names = self.winrates.index @@ -799,7 +804,7 @@ def winrates_from_selection( df: utils.Selector, categories: tp.List[str], num_rows: int = 5, - num_cols: int = 50, + num_cols: int = 350, complete_runs_only: bool = False, ) -> pd.DataFrame: """Creates a fight plot win rate data out of the given run dataframe, diff --git a/nevergrad/functions/ml/mlfunctionlib.py b/nevergrad/functions/ml/mlfunctionlib.py index 43fe028c03..d357d2cab6 100644 --- a/nevergrad/functions/ml/mlfunctionlib.py +++ b/nevergrad/functions/ml/mlfunctionlib.py @@ -244,9 +244,15 @@ def make_dataset(self, data_dimension: tp.Optional[int], dataset: str) -> None: assert dataset in ["diabetes", "kerasBoston", "auto-mpg", "red-wine", "white-wine"] assert data_dimension is None sets_url = { - "auto-mpg": "http://www-lisic.univ-littoral.fr/~teytaud/files/Cours/Apprentissage/data/auto-mpg.data", - "red-wine": "http://www-lisic.univ-littoral.fr/~teytaud/files/Cours/Apprentissage/data/winequality-red.csv", - "white-wine": "http://www-lisic.univ-littoral.fr/~teytaud/files/Cours/Apprentissage/data/winequality-white.csv", + # "auto-mpg": "http://www-lisic.univ-littoral.fr/~teytaud/files/Cours/Apprentissage/data/auto-mpg.data", + # "red-wine": "http://www-lisic.univ-littoral.fr/~teytaud/files/Cours/Apprentissage/data/winequality-red.csv", + # "white-wine": "http://www-lisic.univ-littoral.fr/~teytaud/files/Cours/Apprentissage/data/winequality-white.csv", + # "auto-mpg": "https://github.com/plotly/datasets/blob/master/auto-mpg.csv", + # "red-wine": "https://github.com/plotly/datasets/blob/master/winequality-red.csv", + # "white-wine": "https://github.com/stedy/Machine-Learning-with-R-datasets/blob/master/winequality-white.csv", + "auto-mpg": "https://raw.githubusercontent.com/plotly/datasets/master/auto-mpg.csv", + "red-wine": "https://raw.githubusercontent.com/plotly/datasets/master/winequality-red.csv", + "white-wine": "https://raw.githubusercontent.com/stedy/Machine-Learning-with-R-datasets/master/winequality-white.csv", } sets_tag = {"auto-mpg": "mpg", "red-wine": "quality", "white-wine": "quality"} if dataset == "kerasBoston": @@ -259,7 +265,10 @@ def make_dataset(self, data_dimension: tp.Optional[int], dataset: str) -> None: data = keras.datasets.boston_housing elif dataset in sets_tag: - data = pd.read_csv(sets_url[dataset]) + try: + data = pd.read_csv(sets_url[dataset]) + except Exception as e: + assert False, f"failing with error {e} for dataset {dataset}" else: data = {"diabetes": sklearn.datasets.load_diabetes,}[ # data = {"boston": sklearn.datasets.load_boston, "diabetes": sklearn.datasets.load_diabetes,}[ diff --git a/nevergrad/functions/rocket/rocket.py b/nevergrad/functions/rocket/rocket.py index e5b63b9219..9d088177bb 100644 --- a/nevergrad/functions/rocket/rocket.py +++ b/nevergrad/functions/rocket/rocket.py @@ -228,7 +228,7 @@ def net_force(Ex, Ey, Ez, Evx, Evy, Evz, m): percentage = row[0] / np.sum( thrust_list ) # percentage of total thrust to find percentage of mass lost - assert percentage >= 0.0 + assert percentage >= -1e-5, f"percentage is {percentage}" assert percentage <= 1.0 mass_loss = mass_reman * percentage mass_reman -= mass_loss diff --git a/nevergrad/optimization/optimizerlib.py b/nevergrad/optimization/optimizerlib.py index 75962429ea..5256fce17b 100644 --- a/nevergrad/optimization/optimizerlib.py +++ b/nevergrad/optimization/optimizerlib.py @@ -2929,6 +2929,7 @@ def _select_optimizer_cls(self) -> base.OptCls: if self.dimension > 1 else OnePlusOne ) + print(f"NGOptbase: budget={self.budget}, dim={self.dimension}, nw={self.num_workers}, {cls}") return cls def _internal_ask_candidate(self) -> p.Parameter: @@ -3586,6 +3587,331 @@ def _select_optimizer_cls(self) -> base.OptCls: return super()._select_optimizer_cls() +@registry.register +class NgIoh2(NGOpt16): + def _select_optimizer_cls(self) -> base.OptCls: + if self.fully_continuous and self.has_noise: # In particular for neuro-DPS. + DeterministicMix = ConfPortfolio(optimizers=[DiagonalCMA, PSO, GeneticDE]) + return Chaining([DeterministicMix, OptimisticNoisyOnePlusOne], ["half"]) + + cma_vars = max(1, 4 + int(3 * np.log(self.dimension))) + num36 = ( + 1 + int(np.sqrt(4.0 * (4 * self.budget) // (self.dimension * 1000))) + if self.budget is not None + else 1 + ) + num21 = 1 + (4 * self.budget) // (self.dimension * 1000) if self.budget is not None else 1 + num_dim10 = ( + 1 + int(np.sqrt(8.0 * (8 * self.budget) // (self.dimension * 1000))) + if self.budget is not None + else 1 + ) + num_dim20 = self.budget // (500 * self.dimension) if self.budget is not None else 1 + para = 1 + if self.budget is not None and self.budget > 5000 * self.dimension: + para = num36 * cma_vars + elif self.dimension < 5: + para = num21 * cma_vars + elif self.dimension < 10: + para = num_dim10 * cma_vars + elif self.dimension < 20: + para = num_dim20 * cma_vars + if ( + self.fully_continuous + and self.num_workers == 1 + and self.budget is not None + and self.budget < 1000 * self.dimension + and self.budget > 20 * self.dimension + and not self.has_noise + and self.dimension > 1 + and self.dimension < 200 + ): + return Carola2 + if ( + self.fully_continuous + and self.num_workers == 1 + and self.budget is not None + and self.budget >= 1000 * self.dimension + and not self.has_noise + and self.dimension > 1 + and self.dimension < 200 + ): + return Carola2 + # Special cases in the bounded case + if ( + self.budget is not None + and self.budget > 500 * self.dimension + and self.fully_continuous + and not self.has_noise + and self.num_objectives < 2 + and self.num_workers <= para + and p.helpers.Normalizer(self.parametrization).fully_bounded + ): + if self.dimension == 1: + return NGOpt16 + if ( + self.budget > 5000 * self.dimension + ): # Asymptotically let us trust NGOpt36 and its subtle restart. + return NGOpt36 + if self.dimension < 5: # Low dimension: let us hit the bounds. + return NGOpt21 + if self.dimension < 10: # Moderate dimension: reasonable restart + bet and run. + num = 1 + int(np.sqrt(8.0 * (8 * self.budget) // (self.dimension * 1000))) + return ConfPortfolio(optimizers=[NGOpt14] * num, warmup_ratio=0.7) + if self.dimension < 20: # Nobody knows why this seems to be so good. + num = self.budget // (500 * self.dimension) + return ConfPortfolio( + optimizers=[Rescaled(base_optimizer=NGOpt14, scale=1.3**i) for i in range(num)], + warmup_ratio=0.5, + ) + if self.num_workers == 1: + return CmaFmin2 + # We need a special case for dim < 30 ---> let's see later. + # Otherwise, let us go back to normal life: NGOpt16 which rocks in many cases, possibly Cobyla. + return NGOpt16 + elif ( # This might be specific of high-precision cases. + self.budget is not None + and self.fully_continuous + and not self.has_noise + and self.num_objectives < 2 + and self.num_workers <= cma_vars + and self.budget > 50 * self.dimension + and p.helpers.Normalizer(self.parametrization).fully_bounded + ): + if self.dimension < 3: + return NGOpt8 + if self.dimension <= 20 and self.num_workers == 1: + MetaModelFmin2 = ParametrizedMetaModel(multivariate_optimizer=CmaFmin2) + MetaModelFmin2.no_parallelization = True + return MetaModelFmin2 + return NGOpt15 + else: + return super()._select_optimizer_cls() + + +@registry.register +class NgIoh3(NGOpt16): + def _select_optimizer_cls(self) -> base.OptCls: + if self.fully_continuous and self.has_noise: # In particular for neuro-DPS. + DeterministicMix = ConfPortfolio(optimizers=[DiagonalCMA, PSO, GeneticDE]) + return Chaining([DeterministicMix, OptimisticNoisyOnePlusOne], ["half"]) + + cma_vars = max(1, 4 + int(3 * np.log(self.dimension))) + num36 = ( + 1 + int(np.sqrt(4.0 * (4 * self.budget) // (self.dimension * 1000))) + if self.budget is not None + else 1 + ) + num21 = 1 + (4 * self.budget) // (self.dimension * 1000) if self.budget is not None else 1 + num_dim10 = ( + 1 + int(np.sqrt(8.0 * (8 * self.budget) // (self.dimension * 1000))) + if self.budget is not None + else 1 + ) + num_dim20 = self.budget // (500 * self.dimension) if self.budget is not None else 1 + para = 1 + if self.budget is not None and self.budget > 5000 * self.dimension: + para = num36 * cma_vars + elif self.dimension < 5: + para = num21 * cma_vars + elif self.dimension < 10: + para = num_dim10 * cma_vars + elif self.dimension < 20: + para = num_dim20 * cma_vars + if ( + self.fully_continuous + and self.num_workers == 1 + and self.budget is not None + and self.budget < 1000 * self.dimension + and self.budget > 20 * self.dimension + and not self.has_noise + and self.dimension > 1 + and self.dimension < 100 + ): + return Carola2 + if ( + self.fully_continuous + and self.num_workers == 1 + and self.budget is not None + and self.budget >= 1000 * self.dimension + and not self.has_noise + and self.dimension > 1 + and self.dimension < 50 + ): + return Carola2 + # Special cases in the bounded case + if ( + self.budget is not None + and self.budget > 500 * self.dimension + and self.fully_continuous + and not self.has_noise + and self.num_objectives < 2 + and self.num_workers <= para + and p.helpers.Normalizer(self.parametrization).fully_bounded + ): + if self.dimension == 1: + return NGOpt16 + if ( + self.budget > 5000 * self.dimension + ): # Asymptotically let us trust NGOpt36 and its subtle restart. + return NGOpt36 + if self.dimension < 5: # Low dimension: let us hit the bounds. + return NGOpt21 + if self.dimension < 10: # Moderate dimension: reasonable restart + bet and run. + num = 1 + int(np.sqrt(8.0 * (8 * self.budget) // (self.dimension * 1000))) + return ConfPortfolio(optimizers=[NGOpt14] * num, warmup_ratio=0.7) + if self.dimension < 20: # Nobody knows why this seems to be so good. + num = self.budget // (500 * self.dimension) + return ConfPortfolio( + optimizers=[Rescaled(base_optimizer=NGOpt14, scale=1.3**i) for i in range(num)], + warmup_ratio=0.5, + ) + if self.num_workers == 1: + return CmaFmin2 + # We need a special case for dim < 30 ---> let's see later. + # Otherwise, let us go back to normal life: NGOpt16 which rocks in many cases, possibly Cobyla. + return NGOpt16 + elif ( # This might be specific of high-precision cases. + self.budget is not None + and self.fully_continuous + and not self.has_noise + and self.num_objectives < 2 + and self.num_workers <= cma_vars + and self.budget > 50 * self.dimension + and p.helpers.Normalizer(self.parametrization).fully_bounded + ): + if self.dimension < 3: + return NGOpt8 + if self.dimension <= 20 and self.num_workers == 1: + MetaModelFmin2 = ParametrizedMetaModel(multivariate_optimizer=CmaFmin2) + MetaModelFmin2.no_parallelization = True + return MetaModelFmin2 + return NGOpt15 + else: + return super()._select_optimizer_cls() + + +@registry.register +class NgIoh4(NGOptBase): + """Nevergrad optimizer by competence map. You might modify this one for designing your own competence map.""" + + def _select_optimizer_cls(self) -> base.OptCls: + optCls: base.OptCls = NGOptBase + funcinfo = self.parametrization.function + if ( + self.fully_continuous + and self.num_workers == 1 + and self.budget is not None + and self.budget < 1000 * self.dimension + and self.budget > 20 * self.dimension + and not self.has_noise + and self.dimension > 1 + and self.dimension < 100 + ): + print(f"budget={self.budget}, dim={self.dimension}, nw={self.num_workers}, Carola2") + return Carola2 + if ( + self.fully_continuous + and self.num_workers == 1 + and self.budget is not None + and self.budget >= 1000 * self.dimension + and not self.has_noise + and self.dimension > 1 + and self.dimension < 50 + ): + print(f"budget={self.budget}, dim={self.dimension}, nw={self.num_workers}, Carola2") + return Carola2 + # Special cases in the bounded case + if self.has_noise and (self.has_discrete_not_softmax or not funcinfo.metrizable): + optCls = RecombiningPortfolioOptimisticNoisyDiscreteOnePlusOne + elif self.dimension >= 60 and not funcinfo.metrizable: + optCls = CMA + print(f"budget={self.budget}, dim={self.dimension}, nw={self.num_workers}, we choose {optCls}") + return optCls + + +@registry.register +class NgIoh5(NGOptBase): + """Nevergrad optimizer by competence map. You might modify this one for designing your own competence map.""" + + def _select_optimizer_cls(self) -> base.OptCls: + optCls: base.OptCls = NGOptBase + funcinfo = self.parametrization.function + if not self.has_noise and self._arity > 0: + return DiscreteLenglerOnePlusOne + if ( + self.fully_continuous + and self.num_workers == 1 + and self.budget is not None + and self.budget < 1000 * self.dimension + and self.budget > 20 * self.dimension + and not self.has_noise + and self.dimension > 1 + and self.dimension < 100 + ): + print(f"budget={self.budget}, dim={self.dimension}, nw={self.num_workers}, Carola2") + return Carola2 + if ( + self.fully_continuous + and self.num_workers == 1 + and self.budget is not None + and self.budget >= 1000 * self.dimension + and not self.has_noise + and self.dimension > 1 + and self.dimension < 50 + ): + print(f"budget={self.budget}, dim={self.dimension}, nw={self.num_workers}, Carola2") + return Carola2 + # Special cases in the bounded case + if self.has_noise and (self.has_discrete_not_softmax or not funcinfo.metrizable): + optCls = RecombiningPortfolioOptimisticNoisyDiscreteOnePlusOne + elif self.dimension >= 60 and not funcinfo.metrizable: + optCls = CMA + print(f"budget={self.budget}, dim={self.dimension}, nw={self.num_workers}, we choose {optCls}") + return optCls + + +@registry.register +class NgIoh6(NGOptBase): + """Nevergrad optimizer by competence map. You might modify this one for designing your own competence map.""" + + def _select_optimizer_cls(self) -> base.OptCls: + optCls: base.OptCls = NGOptBase + funcinfo = self.parametrization.function + if not self.has_noise and self._arity > 0: + return RecombiningPortfolioDiscreteOnePlusOne + if ( + self.fully_continuous + and self.num_workers == 1 + and self.budget is not None + and self.budget < 1000 * self.dimension + and self.budget > 20 * self.dimension + and not self.has_noise + and self.dimension > 1 + and self.dimension < 100 + ): + print(f"budget={self.budget}, dim={self.dimension}, nw={self.num_workers}, Carola2") + return Carola2 + if ( + self.fully_continuous + and self.num_workers == 1 + and self.budget is not None + and self.budget >= 1000 * self.dimension + and not self.has_noise + and self.dimension > 1 + and self.dimension < 50 + ): + print(f"budget={self.budget}, dim={self.dimension}, nw={self.num_workers}, Carola2") + return Carola2 + # Special cases in the bounded case + if self.has_noise and (self.has_discrete_not_softmax or not funcinfo.metrizable): + optCls = RecombiningPortfolioOptimisticNoisyDiscreteOnePlusOne + elif self.dimension >= 60 and not funcinfo.metrizable: + optCls = CMA + print(f"budget={self.budget}, dim={self.dimension}, nw={self.num_workers}, we choose {optCls}") + return optCls + + class _MSR(Portfolio): """This code applies multiple copies of NGOpt with random weights for the different objective functions. diff --git a/nevergrad/optimization/recorded_recommendations.csv b/nevergrad/optimization/recorded_recommendations.csv index e0824271a0..1120087145 100644 --- a/nevergrad/optimization/recorded_recommendations.csv +++ b/nevergrad/optimization/recorded_recommendations.csv @@ -59,6 +59,11 @@ NGOpt4,1.012515477,-0.9138805701,-1.029555946,1.2098418178,,,,,,,,,,,, NGOpt8,1.012515477,-0.9138805701,-1.029555946,1.2098418178,,,,,,,,,,,, NGOptRW,1.0082049151,-0.9099785499,-1.025147209,1.2046460074,,,,,,,,,,,, NgIoh,1.0082049151,-0.9099785499,-1.025147209,1.2046460074,,,,,,,,,,,, +NgIoh2,1.0082049151,-0.9099785499,-1.025147209,1.2046460074,,,,,,,,,,,, +NgIoh3,1.0082049151,-0.9099785499,-1.025147209,1.2046460074,,,,,,,,,,,, +NgIoh4,0.0,-0.3451057176,-0.1327329683,1.9291307781,,,,,,,,,,,, +NgIoh5,0.0,-0.3451057176,-0.1327329683,1.9291307781,,,,,,,,,,,, +NgIoh6,0.0,-0.3451057176,-0.1327329683,1.9291307781,,,,,,,,,,,, NoisyDE,0.7325595717,-0.3250848292,-0.4968122173,1.9884218193,1.8577990761,1.7725922124,-0.966685952,-1.5886443264,,,,,,,, NoisyRL1,-0.3375952501,-0.585268278,-0.1149199408,2.2418177944,,,,,,,,,,,, NoisyRL2,-0.3375952501,-0.585268278,-0.1149199408,2.2418177944,,,,,,,,,,,, diff --git a/requirements/bench.txt b/requirements/bench.txt index eb121f3890..508cb5b4e7 100644 --- a/requirements/bench.txt +++ b/requirements/bench.txt @@ -29,13 +29,16 @@ mixsimulator>=0.3.3 hyperopt>=0.2.5 IOHexperimenter>=0.2.8.7 cdt>=0.5.23 -#koncept>=0.2.2 # removed because sometimes complicated tensorflow-estimator>=2.7.0 +tensorflow>=2.7.0 +keras>=2.4.3 +#koncept>=0.2.2 # removed because sometimes complicated +#tensorflow-estimator>=2.7.0 scikit-learn>=1.0.1 scikit-image==0.18.3 -tensorflow>=2.7.0 image-quality>=1.2.7 -keras>=2.4.3 +#keras>=2.4.3 +#tensorflow>=2.7.0 pymoo==0.5.0 olymp==0.0.1b0 ; sys_platform == "linux" Keras-Preprocessing diff --git a/scripts/dagstuhloid.sh b/scripts/dagstuhloid.sh index 7ffd8a858e..bfff53dadc 100755 --- a/scripts/dagstuhloid.sh +++ b/scripts/dagstuhloid.sh @@ -13,6 +13,11 @@ tasks=(zp_pbbob zp_ms_bbob aquacrop_fao bonnans deceptive double_o_seven fishing task=${tasks[SLURM_ARRAY_TASK_ID]} echo task attribution $SLURM_ARRAY_TASK_ID $task +echo Keras/TF versions: +pip show keras tensorflow tensorflow-estimator + +conda info + echo Starting at date # num_workers is the number of processes. Maybe use a bit more than the number of cores at the line "cpus-per-task" diff --git a/scripts/latexize.sh b/scripts/latexize.sh index a0808fdf3f..81787d129f 100755 --- a/scripts/latexize.sh +++ b/scripts/latexize.sh @@ -54,11 +54,14 @@ bestsr=`cat rnk__${uminus}_plots.cp.txt | grep '^[ ]*algo.*:' | head -n 1 |sed ' echo "\\subsubsection{`echo $u | sed 's/_plots.$//g'` (NSR:$bestsr) (Freq:$bestfreq) (num:$num)}" | sed 's/_/ /g'| sed 's/aquacrop/(RW) &/g' | sed 's/rocket/(RW)&/g' | sed 's/fishing/(RW)&/g' | sed 's/MLDA/(RW)&/g' | sed 's/keras/(RW)&/g' | sed 's/mltuning/(RW)&/g' | sed 's/powersystems/(RW)&/g' | sed 's/mixsimulator/(RW)&/g' | sed 's/olympus/(RW)&/g' | sed 's/double.o.seven/(RW)&/g' timeout 10 cat scripts/txt/`echo $u | sed 's/_plots/.txt/g' | sed 's/\///g'` ( +( + convert ${u}/fight_all_pure.png -trim +repage ${u}/fight_all_pure.pre.png img2pdf -o ${u}/fight_all_pure.png.pdf ${u}/fight_all_pure.pre.png convert ${u}/xpresults_all.png -trim +repage ${u}/xpresults_all.pre.png img2pdf -o ${u}/xpresults_all.png.pdf ${u}/xpresults_all.pre.png ) 2>&1 | cat > logconvert${uminus}.log +) & echo " " echo " " ls ${u}/*all_pure.png ${u}/xpresults_all.png | sed 's/.*/\\includegraphics[width=.99\\textwidth]{{&}}\\\\/g' @@ -83,6 +86,7 @@ done done ) | tee competition.tex ) > dagstuhloid.tex +wait ( echo "\\section{Statistics over all benchmarks}\\label{bigstats}" echo "We point out that NGOpt and variants are wizards (automatic algorithm selectors and combinators) created by the same authors as Nevergrad, and their (good) results might therefore be biased: we do not cheat, but we recognize that common authorship for benchmarks and algorithms imply a bias." @@ -125,6 +129,11 @@ done listalgos=$( grep '^\\item [A-Za-z0-9]* (' dagstuhloid.tex | grep '(' | sed 's/ (.*//g' | sed 's/\\item //g' | sort | uniq ) +if [[ $(find "tmp.tex.tmp" -mtime -10000 -print) ]]; then +echo skipping tmp.tex.tmp, because recent such file found. +else +touch tmp.tex.tmp +rm tmp.tex.tmp ( echo '\section{Pairwise comparisons}' for a in $listalgos @@ -142,12 +151,12 @@ do fi done done ) > tmp.tex.tmp +fi ( echo '\section{Conclusion}' cat scripts/tex/conclusion.tex cat tmp.tex.tmp -rm tmp.tex.tmp #echo '\appendix' #echo '\section{Competence maps}' #for u in $allplots @@ -175,9 +184,6 @@ sed -i 's/\\subsubsection{ranknoisy .*}/\\subsection{Noisy optimization}&/g' $v sed -i 's/(x)/ /g' $v done - -# ==================== - #for u in `grep includegraphics dagstuhloid.tex | sed 's/.*{{//g' | sed 's/}}.*//g' | grep -v pdf | grep png` #do #echo working on $u @@ -207,5 +213,4 @@ echo '' -tar -zcvf texdag.tgz dagstuhloid.tex biblio.bib *plots/*all_pure.png *plots/xpresults_all.png ms_bbob_plots/fight_tran*.png *_plots/*.pdf dagstuhloid.html -#tar -zcvf texdag.tgz competition.tex dagstuhloid.tex biblio.bib *_plots/*.pdf dagstuhloid.html bigstats.tex +tar -zcvf texdag.tgz dagstuhloid.tex biblio.bib *plots/*all_pure.png *plots/xpresults_all.png ms_bbob_plots/fight_tran*.png *_plots/*.pdf dagstuhloid.html competition.tex bigstats.tex dagstuhloid.pdf