From beaaab95be6034eec8d499a784e184ac6d9b64dd Mon Sep 17 00:00:00 2001 From: Nadine Wieters Date: Thu, 19 Oct 2023 16:34:20 +0200 Subject: [PATCH 01/12] Implement tar_binary_restarts plugin into awicm3 and fesom-2.5. --- configs/components/fesom/fesom-2.5.yaml | 20 ++++++++++++++++++++ configs/setups/awicm3/awicm3.yaml | 1 + 2 files changed, 21 insertions(+) diff --git a/configs/components/fesom/fesom-2.5.yaml b/configs/components/fesom/fesom-2.5.yaml index eeb2b1a44..9eef6463f 100644 --- a/configs/components/fesom/fesom-2.5.yaml +++ b/configs/components/fesom/fesom-2.5.yaml @@ -19,6 +19,9 @@ icb_code: False with_wiso: false wiso_code: false +required_plugins: +- "git+https://github.com/esm-tools-plugins/tar_binary_restarts" + choose_version: '2.5': branch: "2.5" @@ -554,3 +557,20 @@ grids: further_reading: - fesom/fesom.forcing.yaml +tidy_recipe: +- "tidy_coupler" +- "tar_binary_restarts" +- "assemble" +- "check_for_unknown_files" +- "log_used_files" +- "throw_away_some_infiles" +- "copy_stuff_back_from_work" +- "copy_all_results_to_exp" +#- "_update_chunk_date_file" +- "clean_run_dir" +#- "start_post_job" +#- "signal_tidy_completion" +#- "_increment_date_and_run_number" +#- "_write_date_file" +#- "maybe_resubmit" + diff --git a/configs/setups/awicm3/awicm3.yaml b/configs/setups/awicm3/awicm3.yaml index 3d2c995cf..97d6ee5fa 100644 --- a/configs/setups/awicm3/awicm3.yaml +++ b/configs/setups/awicm3/awicm3.yaml @@ -105,6 +105,7 @@ general: # Load compute recipe from oifs.yaml prepcompute_recipe: ${oifs.prepcompute_recipe} + tidy_recipe: ${fesom.tidy_recipe} From a19e1f5b9c1594a258bf13a8ac4afae09d0bb66d Mon Sep 17 00:00:00 2001 From: Jan Streffing Date: Thu, 19 Oct 2023 20:24:40 +0200 Subject: [PATCH 02/12] copy out into restart folder --- configs/components/fesom/fesom-2.5.yaml | 6 ++++++ .../v3.2/awicm3-v3.2-levante-TCO95L91-CORE2.yaml | 11 +++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/configs/components/fesom/fesom-2.5.yaml b/configs/components/fesom/fesom-2.5.yaml index 9eef6463f..2419558b8 100644 --- a/configs/components/fesom/fesom-2.5.yaml +++ b/configs/components/fesom/fesom-2.5.yaml @@ -167,14 +167,18 @@ restart_out_files: fesom_raw_restart: fesom_raw_restart fesom_bin_restart_info: fesom_bin_restart_info fesom_bin_restart: fesom_bin_restart + fesom_bin_restart_tar: fesom_bin_restart_tar + fesom_raw_restart_tar: fesom_raw_restart_tar restart_out_in_work: par_oce_restart: fesom.${end_date!syear}.oce.restart/*.nc par_ice_restart: fesom.${end_date!syear}.ice.restart/*.nc fesom_raw_restart_info: fesom_raw_restart/*.info fesom_raw_restart: fesom_raw_restart/np${nproc}/*.dump + fesom_raw_restart_tar: fesom_raw_restart.tar.* fesom_bin_restart_info: fesom_bin_restart/*.info fesom_bin_restart: fesom_bin_restart/np${nproc}/* + fesom_bin_restart_tar: fesom_bin_restart.tar.* wiso_restart: fesom.${end_date!syear}.wiso.restart.nc icb_restart: iceberg.restart #.${parent_date!syear} icb_restart_ISM: iceberg.restart.ISM @@ -183,8 +187,10 @@ restart_out_sources: par_ice_restart: fesom.${end_date!syear}.ice.restart/*.nc fesom_raw_restart_info: fesom_raw_restart/*.info fesom_raw_restart: fesom_raw_restart/np${nproc}/*.dump + fesom_raw_restart_tar: fesom_raw_restart.tar.* fesom_bin_restart_info: fesom_bin_restart/*.info fesom_bin_restart: fesom_bin_restart/np${nproc}/* + fesom_bin_restart_tar: fesom_bin_restart.tar.* wiso_restart: fesom.${end_date!syear}.wiso.restart.nc icb_restart: iceberg.restart #.${parent_date!syear} icb_restart_ISM: iceberg.restart.ISM diff --git a/runscripts/awicm3/v3.2/awicm3-v3.2-levante-TCO95L91-CORE2.yaml b/runscripts/awicm3/v3.2/awicm3-v3.2-levante-TCO95L91-CORE2.yaml index b651fb4a5..c069b8183 100644 --- a/runscripts/awicm3/v3.2/awicm3-v3.2-levante-TCO95L91-CORE2.yaml +++ b/runscripts/awicm3/v3.2/awicm3-v3.2-levante-TCO95L91-CORE2.yaml @@ -5,10 +5,10 @@ general: account: "ab0246" compute_time: "00:20:00" initial_date: "2000-01-01" - final_date: "2000-02-01" + final_date: "2000-01-02" base_dir: "/work/ab0246/${user}/runtime/${general.setup_name}-${general.version}/" - nday: 0 - nmonth: 1 + nday: 1 + nmonth: 0 nyear: 0 computer: @@ -20,14 +20,13 @@ awicm3: fesom: resolution: "CORE2" - #pool_dir: "/work/ab0246/a270092/input/fesom2/" - #mesh_dir: "${pool_dir}/core2/" restart_rate: 1 - restart_unit: "m" + restart_unit: "d" restart_first: 1 lresume: false time_step: 1800 nproc: 128 + tar_binary_restarts: true oifs: resolution: "TCO95" From 3d8f1e70d8f83404a05096ceceed3bce647864b1 Mon Sep 17 00:00:00 2001 From: Nadine Wieters <31928930+nwieters@users.noreply.github.com> Date: Fri, 20 Oct 2023 11:29:25 +0200 Subject: [PATCH 03/12] Update configs/components/fesom/fesom-2.5.yaml Co-authored-by: Miguel <63242832+mandresm@users.noreply.github.com> --- configs/components/fesom/fesom-2.5.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/configs/components/fesom/fesom-2.5.yaml b/configs/components/fesom/fesom-2.5.yaml index 2419558b8..e5dc96473 100644 --- a/configs/components/fesom/fesom-2.5.yaml +++ b/configs/components/fesom/fesom-2.5.yaml @@ -572,11 +572,5 @@ tidy_recipe: - "throw_away_some_infiles" - "copy_stuff_back_from_work" - "copy_all_results_to_exp" -#- "_update_chunk_date_file" - "clean_run_dir" -#- "start_post_job" -#- "signal_tidy_completion" -#- "_increment_date_and_run_number" -#- "_write_date_file" -#- "maybe_resubmit" From 6ef347dcc831ced8386e38ae62bac969ebe02130 Mon Sep 17 00:00:00 2001 From: Nadine Wieters Date: Fri, 20 Oct 2023 11:49:48 +0200 Subject: [PATCH 04/12] Integrate plugin tar_binary_restarts into fesom-2.0 --- configs/components/fesom/fesom-2.0.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/configs/components/fesom/fesom-2.0.yaml b/configs/components/fesom/fesom-2.0.yaml index bbebb7f40..02b1d49ed 100644 --- a/configs/components/fesom/fesom-2.0.yaml +++ b/configs/components/fesom/fesom-2.0.yaml @@ -9,6 +9,9 @@ type: ocean comp_command: ${defaults.comp_command} clean_command: ${defaults.clean_command} +required_plugins: +- "git+https://github.com/esm-tools-plugins/tar_binary_restarts" + choose_version: '2.0': branch: 2.0.2 @@ -407,3 +410,16 @@ grids: oasis_grid_type: "U" number_of_overlapping_points: 0 # oasis P-value +# ------------------------ +# RECIPE +# ------------------------ + tidy_recipe: +- "tidy_coupler" +- "tar_binary_restarts" +- "assemble" +- "check_for_unknown_files" +- "log_used_files" +- "throw_away_some_infiles" +- "copy_stuff_back_from_work" +- "copy_all_results_to_exp" +- "clean_run_dir" From b5b676fd6af675ef02c7b6f101515672cc38719f Mon Sep 17 00:00:00 2001 From: Nadine Wieters Date: Fri, 20 Oct 2023 11:56:43 +0200 Subject: [PATCH 05/12] Corrected syntax error in fesom-2.0.yaml --- configs/components/fesom/fesom-2.0.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/configs/components/fesom/fesom-2.0.yaml b/configs/components/fesom/fesom-2.0.yaml index 02b1d49ed..436f1e0e0 100644 --- a/configs/components/fesom/fesom-2.0.yaml +++ b/configs/components/fesom/fesom-2.0.yaml @@ -413,13 +413,13 @@ grids: # ------------------------ # RECIPE # ------------------------ - tidy_recipe: -- "tidy_coupler" -- "tar_binary_restarts" -- "assemble" -- "check_for_unknown_files" -- "log_used_files" -- "throw_away_some_infiles" -- "copy_stuff_back_from_work" -- "copy_all_results_to_exp" -- "clean_run_dir" +tidy_recipe: + - "tidy_coupler" + - "tar_binary_restarts" + - "assemble" + - "check_for_unknown_files" + - "log_used_files" + - "throw_away_some_infiles" + - "copy_stuff_back_from_work" + - "copy_all_results_to_exp" + - "clean_run_dir" From 822f5dfba21df2f16a84552e2117c12e2b9c77a1 Mon Sep 17 00:00:00 2001 From: Nadine Wieters Date: Fri, 20 Oct 2023 12:28:46 +0200 Subject: [PATCH 06/12] Corrected syntax error in fesom-2.1.yaml --- configs/components/fesom/fesom-2.1.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/configs/components/fesom/fesom-2.1.yaml b/configs/components/fesom/fesom-2.1.yaml index 1021f74a0..2bb6e967e 100644 --- a/configs/components/fesom/fesom-2.1.yaml +++ b/configs/components/fesom/fesom-2.1.yaml @@ -9,6 +9,9 @@ type: ocean comp_command: ${defaults.comp_command} clean_command: ${defaults.clean_command} +required_plugins: +- "git+https://github.com/esm-tools-plugins/tar_binary_restarts" + # LA these are both iceberg related use_icebergs: False # turns on icebergs use_icesheet_coupling: False # initializes new icebergs every restart @@ -597,3 +600,17 @@ grids: further_reading: - fesom/fesom.forcing.yaml + +# ------------------------ +# RECIPE +# ------------------------ +tidy_recipe: + - "tidy_coupler" + - "tar_binary_restarts" + - "assemble" + - "check_for_unknown_files" + - "log_used_files" + - "throw_away_some_infiles" + - "copy_stuff_back_from_work" + - "copy_all_results_to_exp" + - "clean_run_dir" From a0b607a3b22f258a48081b872fe2d1dff71b0d9a Mon Sep 17 00:00:00 2001 From: Nadine Wieters Date: Fri, 20 Oct 2023 12:41:22 +0200 Subject: [PATCH 07/12] Added variable tar_binary_restarts with default value False in all fesom yaml files. --- configs/components/fesom/fesom-2.0.yaml | 2 ++ configs/components/fesom/fesom-2.1.yaml | 2 ++ configs/components/fesom/fesom-2.5.yaml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/configs/components/fesom/fesom-2.0.yaml b/configs/components/fesom/fesom-2.0.yaml index 436f1e0e0..2fbb628b5 100644 --- a/configs/components/fesom/fesom-2.0.yaml +++ b/configs/components/fesom/fesom-2.0.yaml @@ -77,6 +77,8 @@ restart_unit: "m" restart_first: 12 restart_flag: "last" +tar_binary_restarts: False + time_step: 1800 resolution: CORE2 diff --git a/configs/components/fesom/fesom-2.1.yaml b/configs/components/fesom/fesom-2.1.yaml index 2bb6e967e..40c2b31b0 100644 --- a/configs/components/fesom/fesom-2.1.yaml +++ b/configs/components/fesom/fesom-2.1.yaml @@ -66,6 +66,8 @@ restart_unit: "m" restart_first: 12 restart_flag: "last" +tar_binary_restarts: False + time_step: 1800 resolution: CORE2 diff --git a/configs/components/fesom/fesom-2.5.yaml b/configs/components/fesom/fesom-2.5.yaml index e5dc96473..b3794e2c5 100644 --- a/configs/components/fesom/fesom-2.5.yaml +++ b/configs/components/fesom/fesom-2.5.yaml @@ -38,6 +38,8 @@ restart_unit: "m" restart_first: 12 restart_flag: "last" +tar_binary_restarts: False + time_step: 1800 resolution: CORE2 From 1f4fd1e5ff0c5a3d25025be08c6b1217668e9b68 Mon Sep 17 00:00:00 2001 From: mandresm Date: Mon, 13 Nov 2023 19:31:22 +0100 Subject: [PATCH 08/12] add automatic installation of missing plugins also for esm_runscripts --- .../esm_runscripts/esm_plugins.yaml | 1 + .../esm_runscripts/esm_runscripts.yaml | 1 + src/esm_master/task.py | 34 +----------- src/esm_plugin_manager/esm_plugin_manager.py | 52 +++++++++++++++++++ src/esm_runscripts/prepare.py | 1 + 5 files changed, 56 insertions(+), 33 deletions(-) diff --git a/configs/esm_software/esm_runscripts/esm_plugins.yaml b/configs/esm_software/esm_runscripts/esm_plugins.yaml index 4c2d2ae94..e80644882 100644 --- a/configs/esm_software/esm_runscripts/esm_plugins.yaml +++ b/configs/esm_software/esm_runscripts/esm_plugins.yaml @@ -9,6 +9,7 @@ core: - "set_prev_date" - "set_parent_info" - "finalize_config" + - "install_missing_plugins" - "add_submission_info" - "initialize_batch_system" - "initialize_coupler" diff --git a/configs/esm_software/esm_runscripts/esm_runscripts.yaml b/configs/esm_software/esm_runscripts/esm_runscripts.yaml index a7a0b3cb4..7d4b022be 100644 --- a/configs/esm_software/esm_runscripts/esm_runscripts.yaml +++ b/configs/esm_software/esm_runscripts/esm_runscripts.yaml @@ -26,6 +26,7 @@ choose_job_type: - "set_prev_date" - "set_parent_info" - "finalize_config" + - "install_missing_plugins" - "assemble_workflow" - "restore_protected_last_minute_changes" - "add_submission_info" diff --git a/src/esm_master/task.py b/src/esm_master/task.py index 536f9c140..3a1266c07 100644 --- a/src/esm_master/task.py +++ b/src/esm_master/task.py @@ -14,38 +14,6 @@ import esm_plugin_manager -# Yes, Type Hints. Python >= 3.5 supports them. Small steps towards stability, -# until Paul goes crazy and redoes everything in Go. Or Rust. Or Brainfuck -# (yes, that's not made up: https://en.wikipedia.org/wiki/Brainfuck) -# Docs for typing: https://docs.python.org/3/library/typing.html -def install(package: str) -> None: - """ - Checks if a package is already installed in the system and if it's not, then it - installs it. - - Parameters - ---------- - package : str - Name of the package or get operation. Can be a package name (e.g. - ``numpy``) or a full pip address (e.g. - ``git@https://github.com/esm-tools/esm_tools.git``) - - Returns - ------- - None - """ - package_name = package.split("/")[-1].replace(".git", "") - installed_packages = esm_plugin_manager.find_installed_plugins() - arg_list = [sys.executable, "-m", "pip", "install", "--user", package] - if os.environ.get("VIRTUAL_ENV"): - arg_list.remove("--user") - if not package_name in installed_packages: - try: - subprocess.check_call(arg_list) - except (OSError, subprocess.CalledProcessError): # PermissionDeniedError would be nicer... - subprocess.check_call(arg_list) - - ###################################################################################### ################################# class "task" ####################################### ###################################################################################### @@ -401,7 +369,7 @@ def assemble_command_list(self): if plugin not in self.already_installed_plugins: # Actually only works because Paul put the gfw_creator # required plugin for awiesm onto PyPI... - install(plugin) + esm_plugin_manager.install(plugin) self.already_installed_plugins.append(plugin) if self.package.kind in ["setups", "couplings"]: diff --git a/src/esm_plugin_manager/esm_plugin_manager.py b/src/esm_plugin_manager/esm_plugin_manager.py index 792d27502..d0f761011 100644 --- a/src/esm_plugin_manager/esm_plugin_manager.py +++ b/src/esm_plugin_manager/esm_plugin_manager.py @@ -1,5 +1,7 @@ """ ESM Framework for organizing python code in plugins / yaml recipes """ +import os +import subprocess import sys import esm_parser from esm_parser import yaml_file_to_dict @@ -157,3 +159,53 @@ def work_through_recipe(recipe, plugins, config): spec.loader.exec_module(thismodule) config = getattr(thismodule, workitem)(config) return config + + +def install(package: str) -> None: + """ + Checks if a package is already installed in the system and if it's not, then it + installs it. + + Parameters + ---------- + package : str + Name of the package or get operation. Can be a package name (e.g. + ``numpy``) or a full pip address (e.g. + ``git@https://github.com/esm-tools/esm_tools.git``) + + Returns + ------- + None + """ + package_name = package.split("/")[-1].replace(".git", "") + installed_packages = find_installed_plugins() + arg_list = [sys.executable, "-m", "pip", "install", "--user", package] + if os.environ.get("VIRTUAL_ENV"): + arg_list.remove("--user") + if not package_name in installed_packages: + try: + subprocess.check_call(arg_list) + except (OSError, subprocess.CalledProcessError): # PermissionDeniedError would be nicer... + subprocess.check_call(arg_list) + + +def install_missing_plugins(config: esm_parser.ConfigSetup) -> esm_parser.ConfigSetup: + """ + Loop through the components and install the missing plugins. This method can be + called from a recipe. + + Parameters + ---------- + config : esm_parser.ConfigSetup + ConfigSetup object containing the information of the current simulation. + + Returns + ------- + config : esm_parser.ConfigSetup + ConfigSetup object containing the information of the current simulation. + """ + for component in config: + for plugin in config[component].get("required_plugins", []): + install(plugin) + + return config diff --git a/src/esm_runscripts/prepare.py b/src/esm_runscripts/prepare.py index 40315067e..2b62bf4fa 100644 --- a/src/esm_runscripts/prepare.py +++ b/src/esm_runscripts/prepare.py @@ -8,6 +8,7 @@ import yaml import esm_utilities from esm_calendar import Calendar, Date +from esm_plugin_manager import install_missing_plugins from loguru import logger from . import batch_system, helpers From 14d87d9661d03747d1b82c0e3a81af62df7b65bc Mon Sep 17 00:00:00 2001 From: mandresm Date: Tue, 14 Nov 2023 19:47:25 +0100 Subject: [PATCH 09/12] complete fesom configurations for moving the binary_tar files --- configs/components/fesom/fesom-2.0.yaml | 13 +++++++++++++ configs/components/fesom/fesom-2.1.yaml | 13 +++++++++++++ configs/components/fesom/fesom-2.5.yaml | 14 ++++++++++++-- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/configs/components/fesom/fesom-2.0.yaml b/configs/components/fesom/fesom-2.0.yaml index 2fbb628b5..3bf784370 100644 --- a/configs/components/fesom/fesom-2.0.yaml +++ b/configs/components/fesom/fesom-2.0.yaml @@ -205,8 +205,10 @@ restart_out_in_work: par_ice_restart: fesom.${end_date!syear}.ice.restart/*.nc fesom_raw_restart_info: fesom_raw_restart/*.info fesom_raw_restart: fesom_raw_restart/np${nproc}/*.dump + fesom_raw_restart_tar: fesom_raw_restart.tar.* fesom_bin_restart_info: fesom_bin_restart/*.info fesom_bin_restart: fesom_bin_restart/np${nproc}/* + fesom_bin_restart_tar: fesom_bin_restart.tar.* restart_out_sources: oce_restart: fesom.${end_date!syear}.oce.restart.nc @@ -215,8 +217,10 @@ restart_out_sources: par_ice_restart: fesom.${end_date!syear}.ice.restart/*.nc fesom_raw_restart_info: fesom_raw_restart/*.info fesom_raw_restart: fesom_raw_restart/np${nproc}/*.dump + fesom_raw_restart_tar: fesom_raw_restart.tar.* fesom_bin_restart_info: fesom_bin_restart/*.info fesom_bin_restart: fesom_bin_restart/np${nproc}/* + fesom_bin_restart_tar: fesom_bin_restart.tar.* outdata_sources: @@ -245,6 +249,8 @@ file_movements: all_directions: move fesom_raw_restart_info_out: all_directions: move + fesom_raw_restart_tar_out: + all_directions: move fesom_bin_restart_in: all_directions: move fesom_bin_restart_out: @@ -253,6 +259,13 @@ file_movements: all_directions: move fesom_bin_restart_info_out: all_directions: move + fesom_bin_restart_tar_out: + all_directions: move + +choose_tar_binary_restarts: + True: + add_restart_out_files: + fesom_raw_restart_tar: fesom_raw_restart_tar # Is it a branchoff experiment? branchoff: "$(( ${lresume} and ${general.run_number}==1 ))" diff --git a/configs/components/fesom/fesom-2.1.yaml b/configs/components/fesom/fesom-2.1.yaml index 40c2b31b0..e1ad13099 100644 --- a/configs/components/fesom/fesom-2.1.yaml +++ b/configs/components/fesom/fesom-2.1.yaml @@ -191,8 +191,10 @@ restart_out_in_work: par_ice_restart: fesom.${end_date!syear}.ice.restart/*.nc fesom_raw_restart_info: fesom_raw_restart/*.info fesom_raw_restart: fesom_raw_restart/np${nproc}/*.dump + fesom_raw_restart_tar: fesom_raw_restart.tar.* fesom_bin_restart_info: fesom_bin_restart/*.info fesom_bin_restart: fesom_bin_restart/np${nproc}/* + fesom_bin_restart_tar: fesom_bin_restart.tar.* wiso_restart: fesom.${end_date!syear}.wiso.restart.nc icb_restart: iceberg.restart #.${parent_date!syear} icb_restart_ISM: iceberg.restart.ISM @@ -203,8 +205,10 @@ restart_out_sources: par_ice_restart: fesom.${end_date!syear}.ice.restart/*.nc fesom_raw_restart_info: fesom_raw_restart/*.info fesom_raw_restart: fesom_raw_restart/np${nproc}/*.dump + fesom_raw_restart_tar: fesom_raw_restart.tar.* fesom_bin_restart_info: fesom_bin_restart/*.info fesom_bin_restart: fesom_bin_restart/np${nproc}/* + fesom_bin_restart_tar: fesom_bin_restart.tar.* wiso_restart: fesom.${end_date!syear}.wiso.restart.nc icb_restart: iceberg.restart #.${parent_date!syear} icb_restart_ISM: iceberg.restart.ISM @@ -304,6 +308,8 @@ file_movements: all_directions: move fesom_raw_restart_info_out: all_directions: move + fesom_raw_restart_tar_out: + all_directions: move fesom_bin_restart_in: all_directions: move fesom_bin_restart_out: @@ -312,6 +318,13 @@ file_movements: all_directions: move fesom_bin_restart_info_out: all_directions: move + fesom_bin_restart_tar_out: + all_directions: move + +choose_tar_binary_restarts: + True: + add_restart_out_files: + fesom_raw_restart_tar: fesom_raw_restart_tar # Is it a branchoff experiment? branchoff: "$(( ${lresume} and ${general.run_number}==1 ))" diff --git a/configs/components/fesom/fesom-2.5.yaml b/configs/components/fesom/fesom-2.5.yaml index b3794e2c5..b5ff35d90 100644 --- a/configs/components/fesom/fesom-2.5.yaml +++ b/configs/components/fesom/fesom-2.5.yaml @@ -169,8 +169,6 @@ restart_out_files: fesom_raw_restart: fesom_raw_restart fesom_bin_restart_info: fesom_bin_restart_info fesom_bin_restart: fesom_bin_restart - fesom_bin_restart_tar: fesom_bin_restart_tar - fesom_raw_restart_tar: fesom_raw_restart_tar restart_out_in_work: par_oce_restart: fesom.${end_date!syear}.oce.restart/*.nc @@ -292,6 +290,8 @@ file_movements: all_directions: move fesom_raw_restart_info_out: all_directions: move + fesom_raw_restart_tar_out: + all_directions: move fesom_bin_restart_in: all_directions: move fesom_bin_restart_out: @@ -300,6 +300,13 @@ file_movements: all_directions: move fesom_bin_restart_info_out: all_directions: move + fesom_bin_restart_tar_out: + all_directions: move + +choose_tar_binary_restarts: + True: + add_restart_out_files: + fesom_raw_restart_tar: fesom_raw_restart_tar # Is it a branchoff experiment? branchoff: "$(( ${lresume} and ${general.run_number}==1 ))" @@ -565,6 +572,9 @@ grids: further_reading: - fesom/fesom.forcing.yaml +# ------------------------ +# RECIPE +# ------------------------ tidy_recipe: - "tidy_coupler" - "tar_binary_restarts" From 2b1934feb1e17483e3e8421d03b5b1ad6d144197 Mon Sep 17 00:00:00 2001 From: Miguel Andres-Martinez Date: Wed, 15 Nov 2023 08:50:35 +0100 Subject: [PATCH 10/12] add a switch to esm_runscripts to control whether missing plugins are installed or not --- docs/esm_variables.rst | 1 + src/esm_plugin_manager/esm_plugin_manager.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/esm_variables.rst b/docs/esm_variables.rst index 4e7e620bf..13421ad98 100644 --- a/docs/esm_variables.rst +++ b/docs/esm_variables.rst @@ -83,6 +83,7 @@ Runtime variables execution_command, "Command for executing the component, including ``${executable}`` and the necessary flags." heterogeneous_parallelization, "A boolean that controls whether the simulation needs to be run with or without heterogeneous parallelization. When ``false`` OpenMP is not used for any component, independently of the value of ``omp_num_threads`` defined in the components. When ``true``, ``open_num_threads`` needs to be specified for each component using OpenMP. ``heterogeneous_parallelization`` variable **needs to be defined inside the** ``computer`` section of the runscript. See :ref:`cookbook:Heterogeneous Parallelization Run (MPI/OpenMP)` for examples." omp_num_threads, "A variable to control the number of OpenMP threads used by a component during an heterogeneous parallelization run. This variable **has to be defined inside the section of the components** for which OpenMP needs to be used. This variable will be ignored if ``computer.heterogeneous_parallelization`` is not set to ``true``." + general.install_missing_plugins, "A boolean to indicate whether ``esm_runscripts`` needs to install missing plugins (``True``, default) or not (``False``). Implemented to solve a problem with the ``esm_tests`` CI in GitHub where we might not have access to some repositories." Calendar variables ------------------ diff --git a/src/esm_plugin_manager/esm_plugin_manager.py b/src/esm_plugin_manager/esm_plugin_manager.py index d0f761011..0f156e818 100644 --- a/src/esm_plugin_manager/esm_plugin_manager.py +++ b/src/esm_plugin_manager/esm_plugin_manager.py @@ -204,8 +204,9 @@ def install_missing_plugins(config: esm_parser.ConfigSetup) -> esm_parser.Config config : esm_parser.ConfigSetup ConfigSetup object containing the information of the current simulation. """ - for component in config: - for plugin in config[component].get("required_plugins", []): - install(plugin) + if config.get("general", {}).get("install_missing_plugins", True) + for component in config: + for plugin in config[component].get("required_plugins", []): + install(plugin) return config From 08c838b895e31c0af3725593ca16e99a1090599b Mon Sep 17 00:00:00 2001 From: Miguel Andres-Martinez Date: Wed, 15 Nov 2023 17:28:00 +0100 Subject: [PATCH 11/12] fix silly bug --- src/esm_plugin_manager/esm_plugin_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/esm_plugin_manager/esm_plugin_manager.py b/src/esm_plugin_manager/esm_plugin_manager.py index 0f156e818..07669442a 100644 --- a/src/esm_plugin_manager/esm_plugin_manager.py +++ b/src/esm_plugin_manager/esm_plugin_manager.py @@ -204,7 +204,7 @@ def install_missing_plugins(config: esm_parser.ConfigSetup) -> esm_parser.Config config : esm_parser.ConfigSetup ConfigSetup object containing the information of the current simulation. """ - if config.get("general", {}).get("install_missing_plugins", True) + if config.get("general", {}).get("install_missing_plugins", True): for component in config: for plugin in config[component].get("required_plugins", []): install(plugin) From df3c399296fe2a1c099295f84b0c287195c469a3 Mon Sep 17 00:00:00 2001 From: BumpVersion Action Date: Thu, 16 Nov 2023 10:53:12 +0000 Subject: [PATCH 12/12] =?UTF-8?q?Bump=20version:=206.25.5=20=E2=86=92=206.?= =?UTF-8?q?25.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- setup.cfg | 2 +- setup.py | 2 +- src/esm_archiving/__init__.py | 2 +- src/esm_calendar/__init__.py | 2 +- src/esm_cleanup/__init__.py | 2 +- src/esm_database/__init__.py | 2 +- src/esm_environment/__init__.py | 2 +- src/esm_master/__init__.py | 2 +- src/esm_motd/__init__.py | 2 +- src/esm_parser/__init__.py | 2 +- src/esm_plugin_manager/__init__.py | 2 +- src/esm_profile/__init__.py | 2 +- src/esm_runscripts/__init__.py | 2 +- src/esm_tests/__init__.py | 2 +- src/esm_tools/__init__.py | 2 +- src/esm_utilities/__init__.py | 2 +- 16 files changed, 16 insertions(+), 16 deletions(-) diff --git a/setup.cfg b/setup.cfg index daac3d8f1..34fe2c384 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 6.25.5 +current_version = 6.25.6 commit = True tag = True diff --git a/setup.py b/setup.py index b863f533d..2df678ac6 100644 --- a/setup.py +++ b/setup.py @@ -103,6 +103,6 @@ test_suite="tests", tests_require=test_requirements, url="https://github.com/esm-tools/esm_tools", - version="6.25.5", + version="6.25.6", zip_safe=False, ) diff --git a/src/esm_archiving/__init__.py b/src/esm_archiving/__init__.py index d63e41555..e91dd773b 100644 --- a/src/esm_archiving/__init__.py +++ b/src/esm_archiving/__init__.py @@ -4,7 +4,7 @@ __author__ = """Paul Gierz""" __email__ = "pgierz@awi.de" -__version__ = "6.25.5" +__version__ = "6.25.6" from .esm_archiving import (archive_mistral, check_tar_lists, delete_original_data, determine_datestamp_location, diff --git a/src/esm_calendar/__init__.py b/src/esm_calendar/__init__.py index 9158e4deb..028537d32 100644 --- a/src/esm_calendar/__init__.py +++ b/src/esm_calendar/__init__.py @@ -2,6 +2,6 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.25.5" +__version__ = "6.25.6" from .esm_calendar import * diff --git a/src/esm_cleanup/__init__.py b/src/esm_cleanup/__init__.py index ac5c52e76..5c2ac1e93 100644 --- a/src/esm_cleanup/__init__.py +++ b/src/esm_cleanup/__init__.py @@ -2,4 +2,4 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.25.5" +__version__ = "6.25.6" diff --git a/src/esm_database/__init__.py b/src/esm_database/__init__.py index ed18b8a1e..f291f760a 100644 --- a/src/esm_database/__init__.py +++ b/src/esm_database/__init__.py @@ -2,4 +2,4 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.25.5" +__version__ = "6.25.6" diff --git a/src/esm_environment/__init__.py b/src/esm_environment/__init__.py index 652f9470f..176cd1e47 100644 --- a/src/esm_environment/__init__.py +++ b/src/esm_environment/__init__.py @@ -2,6 +2,6 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.25.5" +__version__ = "6.25.6" from .esm_environment import * diff --git a/src/esm_master/__init__.py b/src/esm_master/__init__.py index 3b21be03c..6d4bae8ec 100644 --- a/src/esm_master/__init__.py +++ b/src/esm_master/__init__.py @@ -2,7 +2,7 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.25.5" +__version__ = "6.25.6" from . import database diff --git a/src/esm_motd/__init__.py b/src/esm_motd/__init__.py index 657c9b510..90f6dd388 100644 --- a/src/esm_motd/__init__.py +++ b/src/esm_motd/__init__.py @@ -2,6 +2,6 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.25.5" +__version__ = "6.25.6" from .esm_motd import * diff --git a/src/esm_parser/__init__.py b/src/esm_parser/__init__.py index c3bd5a08f..f3ecf8e67 100644 --- a/src/esm_parser/__init__.py +++ b/src/esm_parser/__init__.py @@ -2,7 +2,7 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.25.5" +__version__ = "6.25.6" from .esm_parser import * diff --git a/src/esm_plugin_manager/__init__.py b/src/esm_plugin_manager/__init__.py index 23156f5a0..7dcba47f2 100644 --- a/src/esm_plugin_manager/__init__.py +++ b/src/esm_plugin_manager/__init__.py @@ -2,6 +2,6 @@ __author__ = """Dirk Barbi, Paul Gierz, Sebastian Wahl""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.25.5" +__version__ = "6.25.6" from .esm_plugin_manager import * diff --git a/src/esm_profile/__init__.py b/src/esm_profile/__init__.py index b94dccf1c..be610fe5d 100644 --- a/src/esm_profile/__init__.py +++ b/src/esm_profile/__init__.py @@ -2,6 +2,6 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.25.5" +__version__ = "6.25.6" from .esm_profile import * diff --git a/src/esm_runscripts/__init__.py b/src/esm_runscripts/__init__.py index cffd49a1e..7e668eb56 100644 --- a/src/esm_runscripts/__init__.py +++ b/src/esm_runscripts/__init__.py @@ -2,7 +2,7 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.25.5" +__version__ = "6.25.6" from .batch_system import * from .chunky_parts import * diff --git a/src/esm_tests/__init__.py b/src/esm_tests/__init__.py index a00fe7c9f..87d9be353 100644 --- a/src/esm_tests/__init__.py +++ b/src/esm_tests/__init__.py @@ -2,7 +2,7 @@ __author__ = """Miguel Andres-Martinez""" __email__ = "miguel.andres-martinez@awi.de" -__version__ = "6.25.5" +__version__ = "6.25.6" from .initialization import * from .read_shipped_data import * diff --git a/src/esm_tools/__init__.py b/src/esm_tools/__init__.py index 01f013733..1a97833a9 100644 --- a/src/esm_tools/__init__.py +++ b/src/esm_tools/__init__.py @@ -23,7 +23,7 @@ __author__ = """Dirk Barbi, Paul Gierz""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.25.5" +__version__ = "6.25.6" import functools import inspect diff --git a/src/esm_utilities/__init__.py b/src/esm_utilities/__init__.py index 35a205d78..6e07a52c8 100644 --- a/src/esm_utilities/__init__.py +++ b/src/esm_utilities/__init__.py @@ -2,6 +2,6 @@ __author__ = """Paul Gierz""" __email__ = "pgierz@awi.de" -__version__ = "6.25.5" +__version__ = "6.25.6" from .utils import *