Merge pull request #1075 from esm-tools/integrate_tar_plugin

Implement tar_binary_restarts plugin into awicm3 and fesom-2.5.
esm-tools · Nov 16, 2023 · c56fbfa · c56fbfa
2 parents 1600624 + df3c399
commit c56fbfa
Show file tree

Hide file tree

Showing 27 changed files with 175 additions and 55 deletions.
diff --git a/configs/components/fesom/fesom-2.0.yaml b/configs/components/fesom/fesom-2.0.yaml
@@ -9,6 +9,9 @@ type: ocean
 comp_command: ${defaults.comp_command}
 clean_command: ${defaults.clean_command}
 
+required_plugins:
+- "git+https://github.com/esm-tools-plugins/tar_binary_restarts"
+
 choose_version:
   '2.0':
     branch: 2.0.2
@@ -74,6 +77,8 @@ restart_unit: "m"
 restart_first: 12
 restart_flag: "last"
 
+tar_binary_restarts: False
+
 time_step: 1800
 resolution: CORE2
 
@@ -200,8 +205,10 @@ restart_out_in_work:
         par_ice_restart: fesom.${end_date!syear}.ice.restart/*.nc
         fesom_raw_restart_info: fesom_raw_restart/*.info
         fesom_raw_restart: fesom_raw_restart/np${nproc}/*.dump
+        fesom_raw_restart_tar: fesom_raw_restart.tar.*
         fesom_bin_restart_info: fesom_bin_restart/*.info
         fesom_bin_restart: fesom_bin_restart/np${nproc}/*
+        fesom_bin_restart_tar: fesom_bin_restart.tar.*
 
 restart_out_sources:
         oce_restart: fesom.${end_date!syear}.oce.restart.nc
@@ -210,8 +217,10 @@ restart_out_sources:
         par_ice_restart: fesom.${end_date!syear}.ice.restart/*.nc
         fesom_raw_restart_info: fesom_raw_restart/*.info
         fesom_raw_restart: fesom_raw_restart/np${nproc}/*.dump
+        fesom_raw_restart_tar: fesom_raw_restart.tar.*
         fesom_bin_restart_info: fesom_bin_restart/*.info
         fesom_bin_restart: fesom_bin_restart/np${nproc}/*
+        fesom_bin_restart_tar: fesom_bin_restart.tar.*
 
 
 outdata_sources:
@@ -240,6 +249,8 @@ file_movements:
                 all_directions: move
         fesom_raw_restart_info_out:
                 all_directions: move
+        fesom_raw_restart_tar_out:
+                all_directions: move
         fesom_bin_restart_in:
                 all_directions: move
         fesom_bin_restart_out:
@@ -248,6 +259,13 @@ file_movements:
                 all_directions: move
         fesom_bin_restart_info_out:
                 all_directions: move
+        fesom_bin_restart_tar_out:
+                all_directions: move
+
+choose_tar_binary_restarts:
+    True:
+        add_restart_out_files:
+            fesom_raw_restart_tar: fesom_raw_restart_tar
 
 # Is it a branchoff experiment?
 branchoff: "$(( ${lresume} and ${general.run_number}==1 ))"
@@ -407,3 +425,16 @@ grids:
                 oasis_grid_type: "U"
                 number_of_overlapping_points: 0 # oasis P-value
 
+# ------------------------
+# RECIPE
+# ------------------------
+tidy_recipe:
+    - "tidy_coupler"
+    - "tar_binary_restarts"
+    - "assemble"
+    - "check_for_unknown_files"
+    - "log_used_files"
+    - "throw_away_some_infiles"
+    - "copy_stuff_back_from_work"
+    - "copy_all_results_to_exp"
+    - "clean_run_dir"
diff --git a/configs/components/fesom/fesom-2.1.yaml b/configs/components/fesom/fesom-2.1.yaml
@@ -9,6 +9,9 @@ type: ocean
 comp_command: ${defaults.comp_command}
 clean_command: ${defaults.clean_command}
 
+required_plugins:
+- "git+https://github.com/esm-tools-plugins/tar_binary_restarts"
+
 # LA these are both iceberg related
 use_icebergs: False                 # turns on icebergs
 use_icesheet_coupling: False        # initializes new icebergs every restart
@@ -63,6 +66,8 @@ restart_unit: "m"
 restart_first: 12
 restart_flag: "last"
 
+tar_binary_restarts: False
+
 time_step: 1800
 resolution: CORE2
 
@@ -186,8 +191,10 @@ restart_out_in_work:
         par_ice_restart: fesom.${end_date!syear}.ice.restart/*.nc
         fesom_raw_restart_info: fesom_raw_restart/*.info
         fesom_raw_restart: fesom_raw_restart/np${nproc}/*.dump
+        fesom_raw_restart_tar: fesom_raw_restart.tar.*
         fesom_bin_restart_info: fesom_bin_restart/*.info
         fesom_bin_restart: fesom_bin_restart/np${nproc}/*
+        fesom_bin_restart_tar: fesom_bin_restart.tar.*
         wiso_restart: fesom.${end_date!syear}.wiso.restart.nc
         icb_restart: iceberg.restart #.${parent_date!syear}
         icb_restart_ISM: iceberg.restart.ISM
@@ -198,8 +205,10 @@ restart_out_sources:
         par_ice_restart: fesom.${end_date!syear}.ice.restart/*.nc
         fesom_raw_restart_info: fesom_raw_restart/*.info
         fesom_raw_restart: fesom_raw_restart/np${nproc}/*.dump
+        fesom_raw_restart_tar: fesom_raw_restart.tar.*
         fesom_bin_restart_info: fesom_bin_restart/*.info
         fesom_bin_restart: fesom_bin_restart/np${nproc}/*
+        fesom_bin_restart_tar: fesom_bin_restart.tar.*
         wiso_restart: fesom.${end_date!syear}.wiso.restart.nc
         icb_restart: iceberg.restart #.${parent_date!syear}
         icb_restart_ISM: iceberg.restart.ISM
@@ -299,6 +308,8 @@ file_movements:
                 all_directions: move
         fesom_raw_restart_info_out:
                 all_directions: move
+        fesom_raw_restart_tar_out:
+                all_directions: move
         fesom_bin_restart_in:
                 all_directions: move
         fesom_bin_restart_out:
@@ -307,6 +318,13 @@ file_movements:
                 all_directions: move
         fesom_bin_restart_info_out:
                 all_directions: move
+        fesom_bin_restart_tar_out:
+                all_directions: move
+
+choose_tar_binary_restarts:
+    True:
+        add_restart_out_files:
+            fesom_raw_restart_tar: fesom_raw_restart_tar
 
 # Is it a branchoff experiment?
 branchoff: "$(( ${lresume} and ${general.run_number}==1 ))"
@@ -597,3 +615,17 @@ grids:
 
 further_reading:
         - fesom/fesom.forcing.yaml
+
+# ------------------------
+# RECIPE
+# ------------------------
+tidy_recipe:
+    - "tidy_coupler"
+    - "tar_binary_restarts"
+    - "assemble"
+    - "check_for_unknown_files"
+    - "log_used_files"
+    - "throw_away_some_infiles"
+    - "copy_stuff_back_from_work"
+    - "copy_all_results_to_exp"
+    - "clean_run_dir"
diff --git a/configs/components/fesom/fesom-2.5.yaml b/configs/components/fesom/fesom-2.5.yaml
@@ -19,6 +19,9 @@ icb_code: False
 with_wiso: false
 wiso_code: false
 
+required_plugins:
+- "git+https://github.com/esm-tools-plugins/tar_binary_restarts"
+
 choose_version:
   '2.5':
     branch: "2.5"
@@ -35,6 +38,8 @@ restart_unit: "m"
 restart_first: 12
 restart_flag: "last"
 
+tar_binary_restarts: False
+
 time_step: 1800
 resolution: CORE2
 
@@ -170,8 +175,10 @@ restart_out_in_work:
         par_ice_restart: fesom.${end_date!syear}.ice.restart/*.nc
         fesom_raw_restart_info: fesom_raw_restart/*.info
         fesom_raw_restart: fesom_raw_restart/np${nproc}/*.dump
+        fesom_raw_restart_tar: fesom_raw_restart.tar.*
         fesom_bin_restart_info: fesom_bin_restart/*.info
         fesom_bin_restart: fesom_bin_restart/np${nproc}/*
+        fesom_bin_restart_tar: fesom_bin_restart.tar.*
         wiso_restart: fesom.${end_date!syear}.wiso.restart.nc
         icb_restart: iceberg.restart #.${parent_date!syear}
         icb_restart_ISM: iceberg.restart.ISM
@@ -180,8 +187,10 @@ restart_out_sources:
         par_ice_restart: fesom.${end_date!syear}.ice.restart/*.nc
         fesom_raw_restart_info: fesom_raw_restart/*.info
         fesom_raw_restart: fesom_raw_restart/np${nproc}/*.dump
+        fesom_raw_restart_tar: fesom_raw_restart.tar.*
         fesom_bin_restart_info: fesom_bin_restart/*.info
         fesom_bin_restart: fesom_bin_restart/np${nproc}/*
+        fesom_bin_restart_tar: fesom_bin_restart.tar.*
         wiso_restart: fesom.${end_date!syear}.wiso.restart.nc
         icb_restart: iceberg.restart #.${parent_date!syear}
         icb_restart_ISM: iceberg.restart.ISM
@@ -281,6 +290,8 @@ file_movements:
                 all_directions: move
         fesom_raw_restart_info_out:
                 all_directions: move
+        fesom_raw_restart_tar_out:
+                all_directions: move
         fesom_bin_restart_in:
                 all_directions: move
         fesom_bin_restart_out:
@@ -289,6 +300,13 @@ file_movements:
                 all_directions: move
         fesom_bin_restart_info_out:
                 all_directions: move
+        fesom_bin_restart_tar_out:
+                all_directions: move
+
+choose_tar_binary_restarts:
+    True:
+        add_restart_out_files:
+            fesom_raw_restart_tar: fesom_raw_restart_tar
 
 # Is it a branchoff experiment?
 branchoff: "$(( ${lresume} and ${general.run_number}==1 ))"
@@ -554,3 +572,17 @@ grids:
 further_reading:
         - fesom/fesom.forcing.yaml
 
+# ------------------------
+# RECIPE
+# ------------------------
+tidy_recipe:
+- "tidy_coupler"
+- "tar_binary_restarts"
+- "assemble"
+- "check_for_unknown_files"
+- "log_used_files"
+- "throw_away_some_infiles"
+- "copy_stuff_back_from_work"
+- "copy_all_results_to_exp"
+- "clean_run_dir"
+
diff --git a/configs/esm_software/esm_runscripts/esm_plugins.yaml b/configs/esm_software/esm_runscripts/esm_plugins.yaml
@@ -9,6 +9,7 @@ core:
                         - "set_prev_date"
                         - "set_parent_info"
                         - "finalize_config"
+                        - "install_missing_plugins"
                         - "add_submission_info"
                         - "initialize_batch_system"
                         - "initialize_coupler"

diff --git a/configs/esm_software/esm_runscripts/esm_runscripts.yaml b/configs/esm_software/esm_runscripts/esm_runscripts.yaml
@@ -26,6 +26,7 @@ choose_job_type:
                         - "set_prev_date"
                         - "set_parent_info"
                         - "finalize_config"
+                        - "install_missing_plugins"
                         - "assemble_workflow"
                         - "restore_protected_last_minute_changes"
                         - "add_submission_info"

diff --git a/configs/setups/awicm3/awicm3.yaml b/configs/setups/awicm3/awicm3.yaml
@@ -105,6 +105,7 @@ general:
 
         # Load compute recipe from oifs.yaml
         prepcompute_recipe: ${oifs.prepcompute_recipe}
+        tidy_recipe: ${fesom.tidy_recipe}
 
 
 

diff --git a/docs/esm_variables.rst b/docs/esm_variables.rst
@@ -83,6 +83,7 @@ Runtime variables
    execution_command,   "Command for executing the component, including ``${executable}`` and the necessary flags."
    heterogeneous_parallelization,   "A boolean that controls whether the simulation needs to be run with or without heterogeneous parallelization. When ``false`` OpenMP is not used for any component, independently of the value of ``omp_num_threads`` defined in the components. When ``true``, ``open_num_threads`` needs to be specified for each component using OpenMP. ``heterogeneous_parallelization`` variable **needs to be defined inside the** ``computer`` section of the runscript. See :ref:`cookbook:Heterogeneous Parallelization Run (MPI/OpenMP)` for examples."
    omp_num_threads,     "A variable to control the number of OpenMP threads used by a component during an heterogeneous parallelization run. This variable **has to be defined inside the section of the components** for which OpenMP needs to be used. This variable will be ignored if ``computer.heterogeneous_parallelization`` is not set to ``true``."
+   general.install_missing_plugins, "A boolean to indicate whether ``esm_runscripts`` needs to install missing plugins (``True``, default) or not (``False``). Implemented to solve a problem with the ``esm_tests`` CI in GitHub where we might not have access to some repositories."
 
 Calendar variables
 ------------------

diff --git a/runscripts/awicm3/v3.2/awicm3-v3.2-levante-TCO95L91-CORE2.yaml b/runscripts/awicm3/v3.2/awicm3-v3.2-levante-TCO95L91-CORE2.yaml
@@ -5,10 +5,10 @@ general:
     account: "ab0246"
     compute_time: "00:20:00"
     initial_date: "2000-01-01"
-    final_date: "2000-02-01"
+    final_date: "2000-01-02"
     base_dir: "/work/ab0246/${user}/runtime/${general.setup_name}-${general.version}/"
-    nday: 0
-    nmonth: 1
+    nday: 1
+    nmonth: 0
     nyear: 0
 
 computer:
@@ -20,14 +20,13 @@ awicm3:
 
 fesom:
     resolution: "CORE2"
-    #pool_dir: "/work/ab0246/a270092/input/fesom2/"
-    #mesh_dir: "${pool_dir}/core2/"
     restart_rate: 1
-    restart_unit: "m"
+    restart_unit: "d"
     restart_first: 1
     lresume: false
     time_step: 1800
     nproc: 128
+    tar_binary_restarts: true
 
 oifs:
     resolution: "TCO95"

diff --git a/setup.cfg b/setup.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 6.25.5
+current_version = 6.25.6
 commit = True
 tag = True
 

diff --git a/setup.py b/setup.py
@@ -103,6 +103,6 @@
     test_suite="tests",
     tests_require=test_requirements,
     url="https://github.com/esm-tools/esm_tools",
-    version="6.25.5",
+    version="6.25.6",
     zip_safe=False,
 )
diff --git a/src/esm_archiving/__init__.py b/src/esm_archiving/__init__.py
@@ -4,7 +4,7 @@
 
 __author__ = """Paul Gierz"""
 __email__ = "[email protected]"
-__version__ = "6.25.5"
+__version__ = "6.25.6"
 
 from .esm_archiving import (archive_mistral, check_tar_lists,
                             delete_original_data, determine_datestamp_location,

diff --git a/src/esm_calendar/__init__.py b/src/esm_calendar/__init__.py
@@ -2,6 +2,6 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.25.5"
+__version__ = "6.25.6"
 
 from .esm_calendar import *
diff --git a/src/esm_cleanup/__init__.py b/src/esm_cleanup/__init__.py
@@ -2,4 +2,4 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.25.5"
+__version__ = "6.25.6"
diff --git a/src/esm_database/__init__.py b/src/esm_database/__init__.py
@@ -2,4 +2,4 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.25.5"
+__version__ = "6.25.6"
diff --git a/src/esm_environment/__init__.py b/src/esm_environment/__init__.py
@@ -2,6 +2,6 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.25.5"
+__version__ = "6.25.6"
 
 from .esm_environment import *
diff --git a/src/esm_master/__init__.py b/src/esm_master/__init__.py
@@ -2,7 +2,7 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.25.5"
+__version__ = "6.25.6"
 
 
 from . import database
Original file line number	Diff line number	Diff line change
Expand Up		@@ -105,6 +105,7 @@ general:

		# Load compute recipe from oifs.yaml
		prepcompute_recipe: ${oifs.prepcompute_recipe}
		tidy_recipe: ${fesom.tidy_recipe}



Expand Down