From d150210d09c94f1d0907d4f828e9d30bc66d87d5 Mon Sep 17 00:00:00 2001 From: "alistair.sellar" Date: Wed, 25 Oct 2023 11:03:04 +0100 Subject: [PATCH 1/7] Skip read of example user config file --- .../app/configure/bin/configure.py | 33 ++----------------- 1 file changed, 2 insertions(+), 31 deletions(-) diff --git a/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/app/configure/bin/configure.py b/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/app/configure/bin/configure.py index f6542d1f81..9ea990e6fd 100755 --- a/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/app/configure/bin/configure.py +++ b/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/app/configure/bin/configure.py @@ -17,16 +17,11 @@ def main(): environment for the ``configure`` task and the result is written to the file defined by ``USER_CONFIG_PATH`` in the ``flow.cylc`` file. """ - # Get the default configuration values from the latest version of - # ESMValTool. - config_values = get_latest_default_conf_vals() - # Get the configuration values defined in the environment for the # 'configure' task. - config_values_from_task_env = get_config_values_from_task_env() + config_values = get_config_values_from_task_env() - # Update the default configuration values. - config_values.update(config_values_from_task_env) + # Update the configuration from OS environment. user_config_path = os.environ["USER_CONFIG_PATH"] config_values["config_file"] = user_config_path @@ -38,30 +33,6 @@ def main(): write_yaml(user_config_path, config_values) -def get_latest_default_conf_vals(): - """Return the default configuration values from the latest version of - ESMValTool.""" - latest_esmvaltool_dir = os.environ["ESMVALTOOL_DIR"] - default_config_file = os.path.join(latest_esmvaltool_dir, - "config-user-example.yml") - # A dictionary is needed here to avoid the config object from - # converting paths to 'PosixPath' objects, which causes issues when - # writing the YAML file. - config_values = dict(Config._load_user_config(default_config_file)) - - # Update the type of the value of 'config_developer_file' from a - # 'PosixPath' object to a string, to avoid issues when writing the - # YAML file (all other paths will be overwritten). - config_developer_file = str(config_values["config_developer_file"]) - config_values["config_developer_file"] = config_developer_file - - print("Default configuration values from the latest version of ESMValTool " - f"{esmvaltool.__version__} (from '{default_config_file}'): ") - pprint.PrettyPrinter().pprint(config_values) - - return config_values - - def get_config_values_from_task_env(): """Return the configuration values defined in the environment for the ``configure`` task.""" From 3329a4e73c4fbf07740a0ec3b228bc881cd07114 Mon Sep 17 00:00:00 2001 From: "alistair.sellar" Date: Wed, 25 Oct 2023 12:31:00 +0100 Subject: [PATCH 2/7] Break dependency between clone and configure --- .../recipe_test_workflow/recipe_test_workflow/flow.cylc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/flow.cylc b/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/flow.cylc index 351f71b0c8..c6e2b5d4b6 100644 --- a/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/flow.cylc +++ b/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/flow.cylc @@ -11,9 +11,12 @@ initial cycle point = 1 final cycle point = 10 [[graph]] - R1 = """install_env_file => clone_latest_esmval => configure + R1 = """install_env_file => configure + install_env_file => clone_latest_esmval configure => process => compare - configure => process => compare""" + configure => process => compare + clone_latest_esmval => process + clone_latest_esmval => process""" [runtime] [[root]] From 2436aa18edd6b065dae1aeac8ada9e2802d9e5b7 Mon Sep 17 00:00:00 2001 From: "alistair.sellar" Date: Wed, 25 Oct 2023 19:18:09 +0100 Subject: [PATCH 3/7] Remove unused imports --- .../recipe_test_workflow/app/configure/bin/configure.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/app/configure/bin/configure.py b/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/app/configure/bin/configure.py index 9ea990e6fd..fe75f3658b 100755 --- a/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/app/configure/bin/configure.py +++ b/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/app/configure/bin/configure.py @@ -4,9 +4,6 @@ import pprint import yaml -from esmvalcore.config._config_object import Config - -import esmvaltool def main(): From 67d87f1de08b65b7d4b899594e003c78536cc679 Mon Sep 17 00:00:00 2001 From: Alistair Sellar Date: Thu, 26 Oct 2023 09:26:01 +0100 Subject: [PATCH 4/7] Concise notation for graph dependecy Co-authored-by: Emma Hogan --- .../utils/recipe_test_workflow/recipe_test_workflow/flow.cylc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/flow.cylc b/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/flow.cylc index c6e2b5d4b6..f05c157526 100644 --- a/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/flow.cylc +++ b/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/flow.cylc @@ -11,8 +11,7 @@ initial cycle point = 1 final cycle point = 10 [[graph]] - R1 = """install_env_file => configure - install_env_file => clone_latest_esmval + R1 = """install_env_file => configure & clone_latest_esmval configure => process => compare configure => process => compare clone_latest_esmval => process From 8005532f53c10c97f69b65d8329c8a451870d3f5 Mon Sep 17 00:00:00 2001 From: "alistair.sellar" Date: Thu, 26 Oct 2023 09:31:49 +0100 Subject: [PATCH 5/7] Concise notation for graph dependecies --- .../recipe_test_workflow/recipe_test_workflow/flow.cylc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/flow.cylc b/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/flow.cylc index f05c157526..06cd7fb3d3 100644 --- a/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/flow.cylc +++ b/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/flow.cylc @@ -12,10 +12,8 @@ final cycle point = 10 [[graph]] R1 = """install_env_file => configure & clone_latest_esmval - configure => process => compare - configure => process => compare - clone_latest_esmval => process - clone_latest_esmval => process""" + configure & clone_latest_esmval => process => compare + configure & clone_latest_esmval => process => compare""" [runtime] [[root]] From 33b08020f94dff660a809a9ff5818dd0d370b3d5 Mon Sep 17 00:00:00 2001 From: "alistair.sellar" Date: Thu, 26 Oct 2023 09:34:44 +0100 Subject: [PATCH 6/7] Remove reference to "default configuration values" --- .../recipe_test_workflow/app/configure/bin/configure.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/app/configure/bin/configure.py b/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/app/configure/bin/configure.py index fe75f3658b..61dc67cc21 100755 --- a/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/app/configure/bin/configure.py +++ b/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/app/configure/bin/configure.py @@ -9,10 +9,9 @@ def main(): """Write the required user configuration file for ESMValTool. - The default configuration values from the latest version of - ESMValTool are updated with the configuration values defined in the - environment for the ``configure`` task and the result is written to - the file defined by ``USER_CONFIG_PATH`` in the ``flow.cylc`` file. + The configuration values defined in the environment for the + ``configure`` task are written to the configuration file defined + by ``USER_CONFIG_PATH`` in the ``flow.cylc`` file. """ # Get the configuration values defined in the environment for the # 'configure' task. From 63b4b14b6acfbe74599d189b12390d9351754660 Mon Sep 17 00:00:00 2001 From: "alistair.sellar" Date: Thu, 26 Oct 2023 09:45:20 +0100 Subject: [PATCH 7/7] Reflect PR changes in documentation --- .../doc/source/user_guide/workflow.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/esmvaltool/utils/recipe_test_workflow/doc/source/user_guide/workflow.rst b/esmvaltool/utils/recipe_test_workflow/doc/source/user_guide/workflow.rst index 9ac2129a1d..759101436b 100644 --- a/esmvaltool/utils/recipe_test_workflow/doc/source/user_guide/workflow.rst +++ b/esmvaltool/utils/recipe_test_workflow/doc/source/user_guide/workflow.rst @@ -30,15 +30,14 @@ The |RTW| performs the following steps: ``configure`` :Description: - Creates and modifies the |ESMValTool| user configuration file + Creates the |ESMValTool| user configuration file :Runs on: Localhost :Executes: The ``configure.py`` script from the |Rose| app :Details: - ``configure`` should run at the start of each cycle after ``clone_latest_esmval`` - has completed; in the case where the next cycle doesn't happen for a week, - the clone could potentially be out of date by the time of the task trigger. + ``configure`` should run at the start of each cycle after + ``install_env_file`` has completed. ``process`` :Description: