diff --git a/.flake8 b/.flake8 index 69880fa1..348e52f6 100644 --- a/.flake8 +++ b/.flake8 @@ -7,6 +7,6 @@ extend-exclude = build extend-ignore = # No whitespace before ':' in [x : y] - E203 + E203, # No lambdas — too strict - E731 + E731, diff --git a/CHANGELOG.md b/CHANGELOG.md index aef9dca8..d9e30361 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## Plugins - Add symbolic explanations plugin (#46). +- It is now possible to view multiple unequal runs at once in Cost over Time and Pareto (#93). +- Runs with unequal objectives cannot be displayed together. +- Added an enum for displaying according warning messages. ## Enhancements - Fix lower bounds of dependency versions. @@ -19,6 +22,9 @@ - Reset inputs to fix error when subsequently selecting runs with different configspaces, objectives or budgets (#106). - Fix errors due to changing inputs before runselection (#64). - For fANOVA, remove constant hyperparameters from configspace (#9). +- When getting budget, objectives etc from multiple runs in Cost over Time and Pareto Front: + - Instead of taking the first run as comparative value, + - take the one with the lowest budget, else the index for the budgets could be out of bounds. ## Version-Updates - Black version from 23.1.0 to 23.3.0 @@ -64,7 +70,7 @@ - SMAC 2.0 ## Dependencies -- Remove SMAC dependency by adding required function directly +- Remove SMAC dependency by adding required function directly. # Version 1.0.1 diff --git a/deepcave/plugins/__init__.py b/deepcave/plugins/__init__.py index 79ed399c..1c74d9de 100644 --- a/deepcave/plugins/__init__.py +++ b/deepcave/plugins/__init__.py @@ -917,7 +917,6 @@ def __call__(self, render_button: bool = False) -> List[Component]: ] else: components += [html.H1(self.name)] - try: self.check_runs_compatibility(self.all_runs) except NotMergeableError as message: diff --git a/deepcave/plugins/dynamic.py b/deepcave/plugins/dynamic.py index bd493c8d..20acaea6 100644 --- a/deepcave/plugins/dynamic.py +++ b/deepcave/plugins/dynamic.py @@ -95,6 +95,7 @@ def plugin_output_update(_: Any, *inputs_list: str) -> Any: runs = self.get_selected_runs(inputs) raw_outputs = {} + rc.clear() for run in runs: run_outputs = rc.get(run, self.id, inputs_key) if run_outputs is None: diff --git a/deepcave/plugins/objective/cost_over_time.py b/deepcave/plugins/objective/cost_over_time.py index d2a5bea2..62775c3c 100644 --- a/deepcave/plugins/objective/cost_over_time.py +++ b/deepcave/plugins/objective/cost_over_time.py @@ -19,9 +19,11 @@ from dash import dcc, html from dash.exceptions import PreventUpdate +from deepcave import notification from deepcave.config import Config from deepcave.plugins.dynamic import DynamicPlugin from deepcave.runs import AbstractRun, check_equality +from deepcave.runs.exceptions import NotMergeableError, RunInequality from deepcave.utils.layout import get_select_options, help_button from deepcave.utils.styled_plotty import ( get_color, @@ -56,6 +58,9 @@ def check_runs_compatibility(self, runs: List[AbstractRun]) -> None: Since this function is called before the layout is created, it can be also used to set common values for the plugin. + If the runs are not mergeable, they still should be displayed + but with a corresponding warning message. + Parameters ---------- runs : List[AbstractRun] @@ -69,18 +74,36 @@ def check_runs_compatibility(self, runs: List[AbstractRun]) -> None: If the budgets of the runs are not equal. If the objective of the runs are not equal. """ - check_equality(runs, objectives=True, budgets=True) + try: + check_equality(runs, objectives=True, budgets=True) + except NotMergeableError as e: + run_inequality = e.args[1] + if run_inequality == RunInequality.INEQ_BUDGET: + notification.update("The budgets of the runs are not equal.", color="warning") + elif run_inequality == RunInequality.INEQ_CONFIGSPACE: + notification.update( + "The configuration spaces of the runs are not equal.", color="warning" + ) + elif run_inequality == RunInequality.INEQ_META: + notification.update("The meta data of the runs is not equal.", color="warning") + elif run_inequality == RunInequality.INEQ_OBJECTIVE: + raise NotMergeableError("The objectives of the selected runs cannot be merged.") # Set some attributes here - run = runs[0] - - objective_names = run.get_objective_names() - objective_ids = run.get_objective_ids() - self.objective_options = get_select_options(objective_names, objective_ids) - - budgets = run.get_budgets(human=True) - budget_ids = run.get_budget_ids() - self.budget_options = get_select_options(budgets, budget_ids) + # It is necessary to get the run with the smallest budget and objective options + # as first comparative value, else there is gonna be an index problem + objective_options = [] + budget_options = [] + for run in runs: + objective_names = run.get_objective_names() + objective_ids = run.get_objective_ids() + objective_options.append(get_select_options(objective_names, objective_ids)) + + budgets = run.get_budgets(human=True) + budget_ids = run.get_budget_ids() + budget_options.append(get_select_options(budgets, budget_ids)) + self.objective_options = min(objective_options, key=len) + self.budget_options = min(budget_options, key=len) @staticmethod def get_input_layout(register: Callable) -> List[dbc.Row]: diff --git a/deepcave/plugins/objective/pareto_front.py b/deepcave/plugins/objective/pareto_front.py index 05b9b13b..49062c34 100644 --- a/deepcave/plugins/objective/pareto_front.py +++ b/deepcave/plugins/objective/pareto_front.py @@ -17,9 +17,11 @@ import plotly.graph_objs as go from dash import dcc, html +from deepcave import notification from deepcave.config import Config from deepcave.plugins.dynamic import DynamicPlugin from deepcave.runs import AbstractRun, Status, check_equality +from deepcave.runs.exceptions import NotMergeableError, RunInequality from deepcave.utils.layout import get_select_options, help_button from deepcave.utils.styled_plot import plt from deepcave.utils.styled_plotty import ( @@ -55,6 +57,9 @@ def check_runs_compatibility(self, runs: List[AbstractRun]) -> None: Since this function is called before the layout is created, it can be also used to set common values for the plugin. + If the runs are not mergeable, they still should be displayed + but with a corresponding warning message + Parameters ---------- runs : List[AbstractRun] @@ -68,18 +73,36 @@ def check_runs_compatibility(self, runs: List[AbstractRun]) -> None: If the budgets of the runs are not equal. If the objective of the runs are not equal. """ - check_equality(runs, objectives=True, budgets=True) + try: + check_equality(runs, objectives=True, budgets=True) + except NotMergeableError as e: + run_inequality = e.args[1] + if run_inequality == RunInequality.INEQ_BUDGET: + notification.update("The budgets of the runs are not equal.", color="warning") + elif run_inequality == RunInequality.INEQ_CONFIGSPACE: + notification.update( + "The configuration spaces of the runs are not equal.", color="warning" + ) + elif run_inequality == RunInequality.INEQ_META: + notification.update("The meta data of the runs is not equal.", color="warning") + elif run_inequality == RunInequality.INEQ_OBJECTIVE: + raise NotMergeableError("The objectives of the selected runs cannot be merged.") # Set some attributes here - run = runs[0] - - objective_names = run.get_objective_names() - objective_ids = run.get_objective_ids() - self.objective_options = get_select_options(objective_names, objective_ids) - - budgets = run.get_budgets(human=True) - budget_ids = run.get_budget_ids() - self.budget_options = get_select_options(budgets, budget_ids) + # It is necessary to get the run with the smallest budget and objective options + # as first comparative value, else there is gonna be an index problem + objective_options = [] + budget_options = [] + for run in runs: + objective_names = run.get_objective_names() + objective_ids = run.get_objective_ids() + objective_options.append(get_select_options(objective_names, objective_ids)) + + budgets = run.get_budgets(human=True) + budget_ids = run.get_budget_ids() + budget_options.append(get_select_options(budgets, budget_ids)) + self.objective_options = min(objective_options, key=len) + self.budget_options = min(budget_options, key=len) @staticmethod def get_input_layout(register: Callable) -> List[Any]: diff --git a/deepcave/runs/__init__.py b/deepcave/runs/__init__.py index 0ee084e9..6a391415 100644 --- a/deepcave/runs/__init__.py +++ b/deepcave/runs/__init__.py @@ -32,7 +32,7 @@ CONSTANT_VALUE, NAN_VALUE, ) -from deepcave.runs.exceptions import NotMergeableError +from deepcave.runs.exceptions import NotMergeableError, RunInequality from deepcave.runs.objective import Objective from deepcave.runs.status import Status from deepcave.runs.trial import Trial @@ -1236,35 +1236,34 @@ def check_equality( if len(runs) == 0: return result - # Check meta - if meta: - ignore = ["objectives", "budgets", "wallclock_limit"] - - m1 = runs[0].get_meta() + # Check if objectives are mergeable + if objectives: + o1 = None for run in runs: - m2 = run.get_meta() - - for k, v in m1.items(): - # Don't check on objectives or budgets - if k in ignore: - continue + o2 = run.get_objectives() - if k not in m2 or m2[k] != v: - raise NotMergeableError("Meta data of runs are not equal.") + if o1 is None: + o1 = o2 + continue - result["meta"] = m1 + if len(o1) != len(o2): + raise NotMergeableError( + "Objectives of runs are not equal.", RunInequality.INEQ_OBJECTIVE + ) - # Make sure the same configspace is used - # Otherwise it does not make sense to merge - # the histories - if configspace: - cs1 = runs[0].configspace - for run in runs: - cs2 = run.configspace - if cs1 != cs2: - raise NotMergeableError("Configspace of runs are not equal.") + for o1_, o2_ in zip(o1, o2): + try: + o1_.merge(o2_) + except NotMergeableError: + raise NotMergeableError( + "Objectives of runs are not equal.", RunInequality.INEQ_OBJECTIVE + ) - result["configspace"] = cs1 + assert o1 is not None + serialized_objectives = [o.to_json() for o in o1] + result["objectives"] = serialized_objectives + if meta: + result["meta"]["objectives"] = serialized_objectives # Also check if budgets are the same if budgets: @@ -1272,32 +1271,44 @@ def check_equality( for run in runs: b2 = run.get_budgets(include_combined=False) if b1 != b2: - raise NotMergeableError("Budgets of runs are not equal.") + raise NotMergeableError("Budgets of runs are not equal.", RunInequality.INEQ_BUDGET) result["budgets"] = b1 if meta: result["meta"]["budgets"] = b1 - # And if objectives are the same - if objectives: - o1 = None + # Make sure the same configspace is used + # Otherwise it does not make sense to merge + # the histories + if configspace: + cs1 = runs[0].configspace for run in runs: - o2 = run.get_objectives() + cs2 = run.configspace + if cs1 != cs2: + raise NotMergeableError( + "Configspace of runs are not equal.", RunInequality.INEQ_CONFIGSPACE + ) - if o1 is None: - o1 = o2 - continue + result["configspace"] = cs1 - if len(o1) != len(o2): - raise NotMergeableError("Objectives of runs are not equal.") + # Check meta + if meta: + ignore = ["objectives", "budgets", "wallclock_limit"] - for o1_, o2_ in zip(o1, o2): - o1_.merge(o2_) + m1 = runs[0].get_meta() + for run in runs: + m2 = run.get_meta() - assert o1 is not None - serialized_objectives = [o.to_json() for o in o1] - result["objectives"] = serialized_objectives - if meta: - result["meta"]["objectives"] = serialized_objectives + for k, v in m1.items(): + # Don't check on objectives or budgets + if k in ignore: + continue + + if k not in m2 or m2[k] != v: + raise NotMergeableError( + "Meta data of runs are not equal.", RunInequality.INEQ_META + ) + + result["meta"] = m1 return result diff --git a/deepcave/runs/exceptions.py b/deepcave/runs/exceptions.py index d36d0e11..b3412884 100644 --- a/deepcave/runs/exceptions.py +++ b/deepcave/runs/exceptions.py @@ -12,6 +12,8 @@ - NotMergeableError: Raised if two or more runs are not mergeable. """ +from enum import Enum + class NotValidRunError(Exception): """Raised if directory is not a valid run.""" @@ -23,3 +25,12 @@ class NotMergeableError(Exception): """Raised if two or more runs are not mergeable.""" pass + + +class RunInequality(Enum): + """Check why runs were not compatible.""" + + INEQ_META = 1 + INEQ_OBJECTIVE = 2 + INEQ_BUDGET = 3 + INEQ_CONFIGSPACE = 4 diff --git a/docs/plugins/cost_over_time.rst b/docs/plugins/cost_over_time.rst index cf60ec6e..7cb580a1 100644 --- a/docs/plugins/cost_over_time.rst +++ b/docs/plugins/cost_over_time.rst @@ -9,9 +9,9 @@ Since multiple runs are supported, you directly see which run performs best to w If you decide to display groups (which are combined runs), you will see the mean and standard deviation too. -.. note:: - The configuration spaces of the selected runs have to be equal. Otherwise, a good comparison - is not possible. +.. note:: + The configuration spaces of the selected runs should be equal. Otherwise, a good comparison + is not possible. They can, however, still be displayed in the same graph. This plugin is capable of answering following questions: diff --git a/docs/plugins/pareto_front.rst b/docs/plugins/pareto_front.rst index f83b2ddf..4f634fca 100644 --- a/docs/plugins/pareto_front.rst +++ b/docs/plugins/pareto_front.rst @@ -9,7 +9,7 @@ configurations for two given objectives. .. note:: You can enable or disable specific runs if you click on the name right to the plot. - If you click on a configuration you a redirected to the configuration plugin to see + If you click on a configuration you are redirected to the configuration plugin to see the configuration in detail. This plugin is capable of answering following questions: