New config and ready for coupled mode (#127)

* start new config * Update suites-hofx.yaml * Update suites.yaml * Add new class framework for yamls * restructure default * end of day * add more config options * add fixed options * reorganize the config directories * add more options and improve checking * reorg the static config files * towards yaml comments * towards writing commented dict * comment written to dictionary * finalize location of the prep_config files * coding norms * when files are used still add to config * Optionally pass model to task * jinja stuff * pure string write * generate a coupled flow.cylc with different times * templating of flow.cylc * rename config files * rename sat channel files * refactor the config class * make model names consistent * stage working * getbackground working * remove jedi_config task and put into the runjediexe task * all tasks working manually * remove old suites directory * allow swell_create to call the prep config in one step * suite file bugs * hofx flow working * routine clean up * clean up and codestyle * add a welcome message * move minor version up * YAML norms * YAML norms * YAML norms * YAML norms * use env variables for CiCd id and root * coding norms * clean up some files Co-authored-by: danholdaway <[email protected]>
GEOS-ESM · Nov 1, 2022 · 7a8e2b0 · 7a8e2b0
1 parent 716a6a2
commit 7a8e2b0
Show file tree

Hide file tree

Showing 150 changed files with 2,053 additions and 2,254 deletions.
diff --git a/setup.cfg b/setup.cfg
@@ -0,0 +1,2 @@
+[metadata]
+version = attr: swell.__version__
diff --git a/setup.py b/setup.py
@@ -12,14 +12,14 @@
 
 # --------------------------------------------------------------------------------------------------
 
+import os.path
 import setuptools
 
 setuptools.setup(
     name='swell',
-    version='1.0.8',
     author='NASA Global Modeling and Assimilation Office',
     description='Workflow suites, tasks and configuration for coupled data assimilation',
-    url='https://github.com/danholdaway/swell',
+    url='https://github.com/geos-esm/swell',
     package_dir={'': 'src'},
     packages=setuptools.find_packages(where='src'),
     classifiers=[
@@ -33,6 +33,7 @@
     python_requires='>=3.6',
     install_requires=[
         'click',
+        'jinja2>=3.0.3',
         'pyyaml>=6.0',
         'pycodestyle>=2.8.0',
         'pandas>=1.4.0',
@@ -42,17 +43,23 @@
     package_data={
         '': [
                'deployment/platforms/*/modules*',
-               'deployment/platforms/*/r2d2_config.yaml',
+               'deployment/platforms/*/*.yaml',
+               'suites/*',
                'suites/*/*',
-               'configuration/*.yaml',
+               'suites/*/*/*',
+               'configuration/*',
                'configuration/*/*',
+               'configuration/*/*/*',
+               'configuration/*/*/*/*',
+               'configuration/*/*/*/*/*',
              ],
     },
     include_package_data=True,
     entry_points={
         'console_scripts': [
             'swell_task = swell.tasks.base.task_base:main',
             'swell_create_experiment = swell.deployment.bin.swell_create_experiment:main',
+            'swell_prepare_experiment_config = swell.deployment.bin.swell_prepare_config:main',
             'swell_launch_experiment = swell.deployment.bin.swell_launch_experiment:main',
             'swell_sat_db_processing = swell.deployment.bin.swell_sat_db_processing:main',
         ],

diff --git a/src/swell/__init__.py b/src/swell/__init__.py
@@ -7,3 +7,6 @@
 import os
 
 repo_directory = os.path.dirname(__file__)
+
+# Set the version for swell
+__version__ = '1.1.0'
diff --git a/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/background.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/background.yaml
@@ -0,0 +1,10 @@
+datetime: '{{local_background_time_iso}}'
+filetype: cube sphere history
+provider: geos
+datapath: ''
+filenames: ['{{cycle_dir}}/bkg.%yyyy%mm%ddT%hh%MM%ssZ.nc4',
+            '{{experiment_dir}}/stage/Data/bkg/geos.crtmsrf.{{horizontal_resolution}}.nc4']
+state variables: [u,v,ua,va,t,delp,q,qi,ql,qr,qs,o3ppmv,phis,
+                  qls,qcn,cfcn,frocean,frland,varflt,ustar,bstar,
+                  zpbl,cm,ct,cq,kcbl,tsm,khl,khu,frlake,frseaice,vtype,
+                  stype,vfrac,sheleg,ts,soilt,soilm,u10m,v10m]
diff --git a/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/geometry.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/geometry.yaml
@@ -0,0 +1,9 @@
+fms initialization:
+  namelist filename: '{{experiment_dir}}/stage/Data/fv3files/fmsmpp.nml'
+  field table filename: '{{experiment_dir}}/stage/Data/fv3files/field_table_gmao'
+akbk: '{{experiment_dir}}/stage/Data/fv3files/akbk{{vertical_resolution}}.nc4'
+layout: [{{npx_proc}},{{npy_proc}}]
+npx: {{horizontal_resolution}}
+npy: {{horizontal_resolution}}
+npz: {{vertical_resolution}}
+field metadata override: '{{experiment_dir}}/stage/Data/fieldmetadata/geos.yaml'
diff --git a/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/getvalues.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/getvalues.yaml
@@ -0,0 +1,2 @@
+variable change:
+  variable change name: Model2GeoVaLs
diff --git a/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/pseudo-model.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/pseudo-model.yaml
@@ -0,0 +1,10 @@
+name: PSEUDO
+tstep: {{background_frequency}}
+filetype: cube sphere history
+provider: geos
+datapath: ''
+filenames: ['{{cycle_dir}}/bkg.%yyyy%mm%ddT%hh%MM%ssZ.nc4',
+            '{{experiment_dir}}/stage/Data/bkg/geos.crtmsrf.{{horizontal_resolution}}.nc4']
+model variables: [u,v,ua,va,t,delp,q,qi,ql,qr,qs,o3ppmv,phis,qls,qcn,cfcn,frocean,frland,varflt,ustar,
+                  bstar,zpbl,cm,ct,cq,kcbl,tsm,khl,khu,frlake,frseaice,vtype,stype,vfrac,sheleg,
+                  ts,soilt,soilm,u10m,v10m]
diff --git a/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/r2d2.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/r2d2.yaml
@@ -0,0 +1,10 @@
+fetch:
+  an:
+    - file_type: [bkg]
+      filename: '{{cycle_dir}}/bkg.%Y%m%dT%H%M%SZ.nc4'
+  fc:
+    - file_type: [bkg]
+      filename: '{{cycle_dir}}/bkg.%Y%m%dT%H%M%SZ.nc4'
+store:
+  fc:
+    - file_type: [bkg]
diff --git a/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/stage.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/stage.yaml
@@ -0,0 +1,7 @@
+- copy_files:
+    directories:
+      - ['/discover/nobackup/drholdaw/JediSwell/bundle/latest/fv3-jedi/test/Data/fieldmetadata/*', '{{experiment_dir}}/stage/Data/fieldmetadata/']
+      - ['/discover/nobackup/drholdaw/JediSwell/bundle/latest/fv3-jedi/test/Data/fv3files/*', '{{experiment_dir}}/stage/Data/fv3files/']
+  link_files:
+    directories:
+      - ['/discover/nobackup/drholdaw/JediData/GEOS_CRTM_Surface/geos.crtmsrf.{{horizontal_resolution}}.nc4', '{{experiment_dir}}/stage/Data/bkg/']
diff --git a/...ation/observation_operators/aircraft.yaml → ...eos_atmosphere/observations/aircraft.yaml b/...ation/observation_operators/aircraft.yaml → ...eos_atmosphere/observations/aircraft.yaml
@@ -5,15 +5,15 @@ obs space:
   obsdatain:
     engine:
       type: H5File
-      obsfile: $(cycle_dir)/aircraft.{{window_begin}}.nc4
+      obsfile: '{{cycle_dir}}/aircraft.{{window_begin}}.nc4'
     obsgrouping:
       group variables: ["station_id"]
       sort variable: "air_pressure"
       sort order: "descending"
   obsdataout:
     engine:
       type: H5File
-      obsfile: $(cycle_dir)/$(experiment_id).aircraft.{{window_begin}}.nc4
+      obsfile: '{{cycle_dir}}/{{experiment_id}}.aircraft.{{window_begin}}.nc4'
   simulated variables: [eastward_wind, northward_wind, air_temperature]
 #obs filters:
 ##--------------------------------------------------------------------------------------------------------------------

diff --git a/...tion/observation_operators/airs_aqua.yaml → ...os_atmosphere/observations/airs_aqua.yaml b/...tion/observation_operators/airs_aqua.yaml → ...os_atmosphere/observations/airs_aqua.yaml
@@ -3,11 +3,11 @@ obs space:
   obsdatain:
     engine:
       type: H5File
-      obsfile: $(cycle_dir)/airs_aqua.{{window_begin}}.nc4
+      obsfile: '{{cycle_dir}}/airs_aqua.{{window_begin}}.nc4'
   obsdataout:
     engine:
       type: H5File
-      obsfile: $(cycle_dir)/$(experiment_id).airs_aqua.{{window_begin}}.nc4
+      obsfile: '{{cycle_dir}}/{{experiment_id}}.airs_aqua.{{window_begin}}.nc4'
   simulated variables: [brightness_temperature]
   channels: &airs_aqua_channels 1, 6, 7, 10, 11, 15, 16, 17, 20, 21, 22, 24,
             27, 28, 30, 36, 39, 40, 42, 51, 52, 54, 55, 56, 59, 62, 63, 68, 69, 71,
@@ -37,15 +37,15 @@ obs operator:
   obs options:
     Sensor_ID: airs_aqua
     EndianType: little_endian
-    CoefficientPath: $(crtm_coeff_dir)
+    CoefficientPath: '{{crtm_coeff_dir}}'
 obs bias:
-  input file: $(cycle_dir)/airs_aqua.{{background_time}}.satbias.nc4
+  input file: '{{cycle_dir}}/airs_aqua.{{background_time}}.satbias.nc4'
   variational bc:
     predictors:
     - name: constant
     - name: lapse_rate
       order: 2
-      tlapse: &airs_aqua_tlapse $(cycle_dir)/airs_aqua.{{background_time}}.tlapse.txt
+      tlapse: &airs_aqua_tlapse '{{cycle_dir}}/airs_aqua.{{background_time}}.tlapse.txt'
     - name: lapse_rate
       tlapse: *airs_aqua_tlapse
     - name: emissivity

diff --git a/.../observation_operators/amsr2_gcom-w1.yaml → ...tmosphere/observations/amsr2_gcom-w1.yaml b/.../observation_operators/amsr2_gcom-w1.yaml → ...tmosphere/observations/amsr2_gcom-w1.yaml
@@ -3,11 +3,11 @@ obs space:
   obsdatain:
     engine:
       type: H5File
-      obsfile: $(cycle_dir)/amsr2_gcom-w1.{{window_begin}}.nc4
+      obsfile: '{{cycle_dir}}/amsr2_gcom-w1.{{window_begin}}.nc4'
   obsdataout:
     engine:
       type: H5File
-      obsfile: $(cycle_dir)/$(experiment_id).amsr2_gcom-w1.{{window_begin}}.nc4
+      obsfile: '{{cycle_dir}}/{{experiment_id}}.amsr2_gcom-w1.{{window_begin}}.nc4'
   simulated variables: [brightness_temperature]
   channels: &amsr2_gcom-w1_channels 1-14
 obs operator:
@@ -21,15 +21,15 @@ obs operator:
   obs options:
     Sensor_ID: amsr2_gcom-w1
     EndianType: little_endian
-    CoefficientPath: $(crtm_coeff_dir)
+    CoefficientPath: '{{crtm_coeff_dir}}'
 obs bias:
-  input file: $(cycle_dir)/amsr2_gcom-w1.{{background_time}}.satbias.nc4
+  input file: '{{cycle_dir}}/amsr2_gcom-w1.{{background_time}}.satbias.nc4'
   variational bc:
     predictors:
     - name: constant
     - name: lapse_rate
       order: 2
-      tlapse: &amsr2_gcom-w1_tlapse $(cycle_dir)/amsr2_gcom-w1.{{background_time}}.tlapse.txt
+      tlapse: &amsr2_gcom-w1_tlapse '{{cycle_dir}}/amsr2_gcom-w1.{{background_time}}.tlapse.txt'
     - name: lapse_rate
       tlapse: *amsr2_gcom-w1_tlapse
     - name: emissivity

diff --git a/...ion/observation_operators/amsua_aqua.yaml → ...s_atmosphere/observations/amsua_aqua.yaml b/...ion/observation_operators/amsua_aqua.yaml → ...s_atmosphere/observations/amsua_aqua.yaml
@@ -3,11 +3,11 @@ obs space:
   obsdatain:
     engine:
       type: H5File
-      obsfile: $(cycle_dir)/amsua_aqua.{{window_begin}}.nc4
+      obsfile: '{{cycle_dir}}/amsua_aqua.{{window_begin}}.nc4'
   obsdataout:
     engine:
       type: H5File
-      obsfile: $(cycle_dir)/$(experiment_id).amsua_aqua.{{window_begin}}.nc4
+      obsfile: '{{cycle_dir}}/{{experiment_id}}.amsua_aqua.{{window_begin}}.nc4'
   simulated variables: [brightness_temperature]
   channels: &amsua_aqua_channels 1-15
 obs operator:
@@ -18,9 +18,9 @@ obs operator:
   obs options:
     Sensor_ID: &Sensor_ID amsua_aqua
     EndianType: little_endian
-    CoefficientPath: $(crtm_coeff_dir)
+    CoefficientPath: '{{crtm_coeff_dir}}'
 obs bias:
-  input file: $(cycle_dir)/amsua_aqua.{{background_time}}.satbias.nc4
+  input file: '{{cycle_dir}}/amsua_aqua.{{background_time}}.satbias.nc4'
   variational bc:
     predictors:
     - name: constant
@@ -30,7 +30,7 @@ obs bias:
       clwdif_ch314: 2
     - name: lapse_rate
       order: 2
-      tlapse: &amsua_aqua_tlapse $(cycle_dir)/amsua_aqua.{{background_time}}.tlapse.txt
+      tlapse: &amsua_aqua_tlapse '{{cycle_dir}}/amsua_aqua.{{background_time}}.tlapse.txt'
     - name: lapse_rate
       tlapse: *amsua_aqua_tlapse
     - name: emissivity

diff --git a/.../observation_operators/amsua_metop-a.yaml → ...tmosphere/observations/amsua_metop-a.yaml b/.../observation_operators/amsua_metop-a.yaml → ...tmosphere/observations/amsua_metop-a.yaml
@@ -3,11 +3,11 @@ obs space:
   obsdatain:
     engine:
       type: H5File
-      obsfile: $(cycle_dir)/amsua_metop-a.{{window_begin}}.nc4
+      obsfile: '{{cycle_dir}}/amsua_metop-a.{{window_begin}}.nc4'
   obsdataout:
     engine:
       type: H5File
-      obsfile: $(cycle_dir)/$(experiment_id).amsua_metop-a.{{window_begin}}.nc4
+      obsfile: '{{cycle_dir}}/{{experiment_id}}.amsua_metop-a.{{window_begin}}.nc4'
   simulated variables: [brightness_temperature]
   channels: &amsua_metop-a_channels 1-15
 obs operator:
@@ -18,9 +18,9 @@ obs operator:
   obs options:
     Sensor_ID: &Sensor_ID amsua_metop-a
     EndianType: little_endian
-    CoefficientPath: $(crtm_coeff_dir)
+    CoefficientPath: '{{crtm_coeff_dir}}'
 obs bias:
-  input file: $(cycle_dir)/amsua_metop-a.{{background_time}}.satbias.nc4
+  input file: '{{cycle_dir}}/amsua_metop-a.{{background_time}}.satbias.nc4'
   channels without bc: 14
   variational bc:
     predictors:
@@ -31,7 +31,7 @@ obs bias:
       clwdif_ch314: 2
     - name: lapse_rate
       order: 2
-      tlapse: &amsua_metop-a_tlapse $(cycle_dir)/amsua_metop-a.{{background_time}}.tlapse.txt
+      tlapse: &amsua_metop-a_tlapse '{{cycle_dir}}/amsua_metop-a.{{background_time}}.tlapse.txt'
     - name: lapse_rate
       tlapse: *amsua_metop-a_tlapse
     - name: emissivity

diff --git a/.../observation_operators/amsua_metop-b.yaml → ...tmosphere/observations/amsua_metop-b.yaml b/.../observation_operators/amsua_metop-b.yaml → ...tmosphere/observations/amsua_metop-b.yaml
@@ -3,11 +3,11 @@ obs space:
   obsdatain:
     engine:
       type: H5File
-      obsfile: $(cycle_dir)/amsua_metop-b.{{window_begin}}.nc4
+      obsfile: '{{cycle_dir}}/amsua_metop-b.{{window_begin}}.nc4'
   obsdataout:
     engine:
       type: H5File
-      obsfile: $(cycle_dir)/$(experiment_id).amsua_metop-b.{{window_begin}}.nc4
+      obsfile: '{{cycle_dir}}/{{experiment_id}}.amsua_metop-b.{{window_begin}}.nc4'
   simulated variables: [brightness_temperature]
   channels: &amsua_metop-b_channels 1-15
 obs operator:
@@ -18,9 +18,9 @@ obs operator:
   obs options:
     Sensor_ID: &Sensor_ID amsua_metop-b
     EndianType: little_endian
-    CoefficientPath: $(crtm_coeff_dir)
+    CoefficientPath: '{{crtm_coeff_dir}}'
 obs bias:
-  input file: $(cycle_dir)/amsua_metop-b.{{background_time}}.satbias.nc4
+  input file: '{{cycle_dir}}/amsua_metop-b.{{background_time}}.satbias.nc4'
   channels without bc: 14
   variational bc:
     predictors:
@@ -31,7 +31,7 @@ obs bias:
       clwdif_ch314: 2
     - name: lapse_rate
       order: 2
-      tlapse: &amsua_metop-b_tlapse $(cycle_dir)/amsua_metop-b.{{background_time}}.tlapse.txt
+      tlapse: &amsua_metop-b_tlapse '{{cycle_dir}}/amsua_metop-b.{{background_time}}.tlapse.txt'
     - name: lapse_rate
       tlapse: *amsua_metop-b_tlapse
     - name: emissivity

diff --git a/.../observation_operators/amsua_metop-c.yaml → ...tmosphere/observations/amsua_metop-c.yaml b/.../observation_operators/amsua_metop-c.yaml → ...tmosphere/observations/amsua_metop-c.yaml
@@ -3,11 +3,11 @@ obs space:
   obsdatain:
     engine:
       type: H5File
-      obsfile: $(cycle_dir)/amsua_metop-c.{{window_begin}}.nc4
+      obsfile: '{{cycle_dir}}/amsua_metop-c.{{window_begin}}.nc4'
   obsdataout:
     engine:
       type: H5File
-      obsfile: $(cycle_dir)/$(experiment_id).amsua_metop-c.{{window_begin}}.nc4
+      obsfile: '{{cycle_dir}}/{{experiment_id}}.amsua_metop-c.{{window_begin}}.nc4'
   simulated variables: [brightness_temperature]
   channels: &amsua_metop-c_channels 1-15
 obs operator:
@@ -18,9 +18,9 @@ obs operator:
   obs options:
     Sensor_ID: &Sensor_ID amsua_metop-c
     EndianType: little_endian
-    CoefficientPath: $(crtm_coeff_dir)
+    CoefficientPath: '{{crtm_coeff_dir}}'
 obs bias:
-  input file: $(cycle_dir)/amsua_metop-c.{{background_time}}.satbias.nc4
+  input file: '{{cycle_dir}}/amsua_metop-c.{{background_time}}.satbias.nc4'
   channels without bc: 14
   variational bc:
     predictors:
@@ -31,7 +31,7 @@ obs bias:
       clwdif_ch314: 2
     - name: lapse_rate
       order: 2
-      tlapse: &amsua_metop-c_tlapse $(cycle_dir)/amsua_metop-c.{{background_time}}.tlapse.txt
+      tlapse: &amsua_metop-c_tlapse '{{cycle_dir}}/amsua_metop-c.{{background_time}}.tlapse.txt'
     - name: lapse_rate
       tlapse: *amsua_metop-c_tlapse
     - name: emissivity

diff --git a/...tion/observation_operators/amsua_n15.yaml → ...os_atmosphere/observations/amsua_n15.yaml b/...tion/observation_operators/amsua_n15.yaml → ...os_atmosphere/observations/amsua_n15.yaml
@@ -3,11 +3,11 @@ obs space:
   obsdatain:
     engine:
       type: H5File
-      obsfile: $(cycle_dir)/amsua_n15.{{window_begin}}.nc4
+      obsfile: '{{cycle_dir}}/amsua_n15.{{window_begin}}.nc4'
   obsdataout:
     engine:
       type: H5File
-      obsfile: $(cycle_dir)/$(experiment_id).amsua_n15.{{window_begin}}.nc4
+      obsfile: '{{cycle_dir}}/{{experiment_id}}.amsua_n15.{{window_begin}}.nc4'
   simulated variables: [brightness_temperature]
   channels: &amsua_n15_channels 1-15
 obs operator:
@@ -18,9 +18,9 @@ obs operator:
   obs options:
     Sensor_ID: &Sensor_ID amsua_n15
     EndianType: little_endian
-    CoefficientPath: $(crtm_coeff_dir)
+    CoefficientPath: '{{crtm_coeff_dir}}'
 obs bias:
-  input file: $(cycle_dir)/amsua_n15.{{background_time}}.satbias.nc4
+  input file: '{{cycle_dir}}/amsua_n15.{{background_time}}.satbias.nc4'
   channels without bc: 14
   variational bc:
     predictors:
@@ -31,7 +31,7 @@ obs bias:
       clwdif_ch314: 2
     - name: lapse_rate
       order: 2
-      tlapse: &amsua_n15_tlapse $(cycle_dir)/amsua_n15.{{background_time}}.tlapse.txt
+      tlapse: &amsua_n15_tlapse '{{cycle_dir}}/amsua_n15.{{background_time}}.tlapse.txt'
     - name: lapse_rate
       tlapse: *amsua_n15_tlapse
     - name: emissivity
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		variable change:
		variable change name: Model2GeoVaLs