From 7a8e2b0fc1d10de26dd2c66923c7bc731f34b1d3 Mon Sep 17 00:00:00 2001 From: Dan Holdaway <27729500+danholdaway@users.noreply.github.com> Date: Tue, 1 Nov 2022 12:23:48 -0400 Subject: [PATCH] New config and ready for coupled mode (#127) * start new config * Update suites-hofx.yaml * Update suites.yaml * Add new class framework for yamls * restructure default * end of day * add more config options * add fixed options * reorganize the config directories * add more options and improve checking * reorg the static config files * towards yaml comments * towards writing commented dict * comment written to dictionary * finalize location of the prep_config files * coding norms * when files are used still add to config * Optionally pass model to task * jinja stuff * pure string write * generate a coupled flow.cylc with different times * templating of flow.cylc * rename config files * rename sat channel files * refactor the config class * make model names consistent * stage working * getbackground working * remove jedi_config task and put into the runjediexe task * all tasks working manually * remove old suites directory * allow swell_create to call the prep config in one step * suite file bugs * hofx flow working * routine clean up * clean up and codestyle * add a welcome message * move minor version up * YAML norms * YAML norms * YAML norms * YAML norms * use env variables for CiCd id and root * coding norms * clean up some files Co-authored-by: danholdaway --- setup.cfg | 2 + setup.py | 15 +- src/swell/__init__.py | 3 + .../geos_atmosphere/model/background.yaml | 10 + .../geos_atmosphere/model/geometry.yaml | 9 + .../geos_atmosphere/model/getvalues.yaml | 2 + .../geos_atmosphere/model/pseudo-model.yaml | 10 + .../fv3-jedi/geos_atmosphere/model/r2d2.yaml | 10 + .../fv3-jedi/geos_atmosphere/model/stage.yaml | 7 + .../observations}/aircraft.yaml | 4 +- .../observations}/airs_aqua.yaml | 10 +- .../observations}/amsr2_gcom-w1.yaml | 10 +- .../observations}/amsua_aqua.yaml | 10 +- .../observations}/amsua_metop-a.yaml | 10 +- .../observations}/amsua_metop-b.yaml | 10 +- .../observations}/amsua_metop-c.yaml | 10 +- .../observations}/amsua_n15.yaml | 10 +- .../observations}/amsua_n18.yaml | 10 +- .../observations}/amsua_n19.yaml | 10 +- .../observations}/atms_n20.yaml | 10 +- .../observations}/atms_npp.yaml | 10 +- .../observations}/avhrr3_metop-a.yaml | 10 +- .../observations}/avhrr3_n18.yaml | 10 +- .../observations}/cris-fsr_n20.yaml | 10 +- .../observations}/cris-fsr_npp.yaml | 10 +- .../observations}/gmi_gpm.yaml | 10 +- .../observations}/gnssrobndnbam.yaml | 4 +- .../observations}/iasi_metop-a.yaml | 10 +- .../observations}/iasi_metop-b.yaml | 10 +- .../observations}/mhs_metop-b.yaml | 10 +- .../observations}/mhs_metop-c.yaml | 10 +- .../observations}/mhs_n19.yaml | 10 +- .../observations}/mls55_aura.yaml | 4 +- .../observations}/omi_aura.yaml | 4 +- .../observations}/ompslpnc_npp.yaml | 4 +- .../observations}/ompsnm_npp.yaml | 4 +- .../observations}/rass_tv.yaml | 4 +- .../observations}/satwind.yaml | 4 +- .../observations}/scatwind.yaml | 4 +- .../observations}/seviri_m11.yaml | 10 +- .../geos_atmosphere/observations}/sfc.yaml | 4 +- .../observations}/sfcship.yaml | 4 +- .../geos_atmosphere/observations}/sondes.yaml | 4 +- .../observations}/ssmis_f17.yaml | 10 +- .../observations}/vadwind.yaml | 4 +- .../{ => jedi}/observation_ioda_names.yaml | 0 src/swell/configuration/jedi/oops/hofx4D.yaml | 14 + .../soca/geos_ocean/model/background.yaml} | 0 .../jedi/soca/geos_ocean/model/geometry.yaml | 0 .../jedi/soca/geos_ocean/model/getvalues.yaml | 0 .../soca/geos_ocean/model/pseudo-model.yaml | 0 .../jedi/soca/geos_ocean/model/r2d2.yaml | 0 .../jedi/soca/geos_ocean/model/stage.yaml | 0 .../soca/geos_ocean/observations/adt_3a.yaml | 0 .../geos_atmosphere/amsua_n19.yaml | 24 ++ .../{ => geos_atmosphere}/aqua.yaml | 0 .../{ => geos_atmosphere}/f08.yaml | 0 .../{ => geos_atmosphere}/f10.yaml | 0 .../{ => geos_atmosphere}/f11.yaml | 0 .../{ => geos_atmosphere}/f13.yaml | 0 .../{ => geos_atmosphere}/f14.yaml | 0 .../{ => geos_atmosphere}/f15.yaml | 0 .../{ => geos_atmosphere}/f16.yaml | 0 .../{ => geos_atmosphere}/f17.yaml | 0 .../{ => geos_atmosphere}/f18.yaml | 0 .../{ => geos_atmosphere}/g08.yaml | 0 .../{ => geos_atmosphere}/g10.yaml | 0 .../{ => geos_atmosphere}/g11.yaml | 0 .../{ => geos_atmosphere}/g12.yaml | 0 .../{ => geos_atmosphere}/g13.yaml | 0 .../{ => geos_atmosphere}/g14.yaml | 0 .../{ => geos_atmosphere}/g15.yaml | 0 .../{ => geos_atmosphere}/gcom-w1.yaml | 0 .../{ => geos_atmosphere}/gpm.yaml | 0 .../geos_atmosphere/iasi_metop-a.yaml | 56 ++++ .../{ => geos_atmosphere}/m09.yaml | 0 .../{ => geos_atmosphere}/m10.yaml | 0 .../{ => geos_atmosphere}/metop-a.yaml | 0 .../{ => geos_atmosphere}/metop-b.yaml | 0 .../{ => geos_atmosphere}/metop-c.yaml | 0 .../{ => geos_atmosphere}/n06.yaml | 0 .../{ => geos_atmosphere}/n07.yaml | 0 .../{ => geos_atmosphere}/n08.yaml | 0 .../{ => geos_atmosphere}/n09.yaml | 0 .../{ => geos_atmosphere}/n10.yaml | 0 .../{ => geos_atmosphere}/n11.yaml | 0 .../{ => geos_atmosphere}/n12.yaml | 0 .../{ => geos_atmosphere}/n14.yaml | 0 .../{ => geos_atmosphere}/n15.yaml | 0 .../{ => geos_atmosphere}/n16.yaml | 0 .../{ => geos_atmosphere}/n17.yaml | 0 .../{ => geos_atmosphere}/n18.yaml | 0 .../{ => geos_atmosphere}/n19.yaml | 0 .../{ => geos_atmosphere}/n20.yaml | 0 .../{ => geos_atmosphere}/npp.yaml | 0 .../{ => geos_atmosphere}/tirosn.yaml | 0 .../{ => geos_atmosphere}/trmm.yaml | 0 .../deployment/bin/swell_create_experiment.py | 217 ++++++-------- .../deployment/bin/swell_launch_experiment.py | 6 +- .../deployment/bin/swell_prepare_config.py | 42 +++ .../platforms/nccs_discover/experiment.yaml | 6 + .../platforms/nccs_discover/modules | 4 +- .../platforms/nccs_discover/r2d2_config.yaml | 8 +- src/swell/deployment/prep_config.py | 138 +++++++++ src/swell/deployment/prep_config_base.py | 205 ++++++++++++++ .../prep_config_cli.py} | 11 +- src/swell/deployment/prep_config_defaults.py | 106 +++++++ .../prep_config_gui.py} | 11 +- src/swell/deployment/prep_exp_dirs.py | 79 ++---- src/swell/deployment/prep_suite.py | 94 ++++--- src/swell/deployment/yaml_exploder.py | 56 ---- src/swell/suites/hofx/eva.yaml | 42 +-- src/swell/suites/hofx/experiment.yaml | 234 ---------------- src/swell/suites/hofx/flow.cylc | 100 ++++--- .../suites-hofx-geos_atmosphere.yaml | 111 ++++++++ .../geos_ocean/suites-hofx-geos_ocean.yaml | 46 +++ src/swell/suites/hofx/jedi_config.yaml | 14 - src/swell/suites/hofx/suite_page_hofx.yaml | 55 ---- src/swell/suites/hofx/suites-hofx.yaml | 61 ++++ src/swell/suites/process_obs/experiment.yaml | 74 ----- src/swell/suites/process_obs/flow.cylc | 61 ---- .../experiment.yaml | 61 ---- .../store_backgrounds_geos_run/flow.cylc | 60 ---- src/swell/suites/suite_page_start.yaml | 7 - src/swell/suites/suites.yaml | 26 ++ src/swell/suites/swell_gui.py | 233 --------------- src/swell/{suites/suites.py => swell_path.py} | 2 +- src/swell/tasks/base/config.py | 265 ++++++++---------- src/swell/tasks/base/task_base.py | 165 +++++++++-- src/swell/tasks/{ => base}/task_registry.py | 5 +- src/swell/tasks/{utilities => base}/utils.py | 5 - src/swell/tasks/build_jedi.py | 138 +++------ src/swell/tasks/clean_cycle.py | 8 +- src/swell/tasks/eva_driver.py | 56 ++-- src/swell/tasks/get_background.py | 55 ++-- src/swell/tasks/get_observations.py | 44 +-- src/swell/tasks/jedi_config.py | 232 --------------- src/swell/tasks/merge_ioda_files.py | 33 +-- src/swell/tasks/run_jedi_executable.py | 56 ---- src/swell/tasks/run_jedi_hofx_executable.py | 128 +++++++++ src/swell/tasks/save_obs_diags.py | 21 +- src/swell/tasks/{stage.py => stage_jedi.py} | 12 +- src/swell/tasks/utilities/__init__.py | 9 - src/swell/utilities/dictionary.py | 106 +++++++ src/swell/utilities/dictionary_utilities.py | 119 -------- src/swell/utilities/jinja2.py | 36 +++ src/swell/utilities/logger.py | 61 +++- src/swell/utilities/observations.py | 46 +-- src/swell/utilities/string_utils.py | 65 ----- src/swell/utilities/welcome_message.py | 28 ++ 150 files changed, 2053 insertions(+), 2254 deletions(-) create mode 100644 setup.cfg create mode 100755 src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/background.yaml create mode 100755 src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/geometry.yaml create mode 100755 src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/getvalues.yaml create mode 100755 src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/pseudo-model.yaml create mode 100755 src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/r2d2.yaml create mode 100755 src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/stage.yaml rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/aircraft.yaml (98%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/airs_aqua.yaml (97%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/amsr2_gcom-w1.yaml (93%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/amsua_aqua.yaml (93%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/amsua_metop-a.yaml (93%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/amsua_metop-b.yaml (93%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/amsua_metop-c.yaml (93%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/amsua_n15.yaml (94%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/amsua_n18.yaml (94%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/amsua_n19.yaml (94%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/atms_n20.yaml (97%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/atms_npp.yaml (97%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/avhrr3_metop-a.yaml (92%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/avhrr3_n18.yaml (92%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/cris-fsr_n20.yaml (98%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/cris-fsr_npp.yaml (98%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/gmi_gpm.yaml (91%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/gnssrobndnbam.yaml (86%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/iasi_metop-a.yaml (98%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/iasi_metop-b.yaml (98%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/mhs_metop-b.yaml (95%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/mhs_metop-c.yaml (95%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/mhs_n19.yaml (64%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/mls55_aura.yaml (82%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/omi_aura.yaml (90%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/ompslpnc_npp.yaml (82%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/ompsnm_npp.yaml (87%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/rass_tv.yaml (58%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/satwind.yaml (98%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/scatwind.yaml (95%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/seviri_m11.yaml (90%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/sfc.yaml (95%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/sfcship.yaml (95%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/sondes.yaml (98%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/ssmis_f17.yaml (92%) rename src/swell/configuration/{observation_operators => jedi/fv3-jedi/geos_atmosphere/observations}/vadwind.yaml (96%) rename src/swell/configuration/{ => jedi}/observation_ioda_names.yaml (100%) create mode 100644 src/swell/configuration/jedi/oops/hofx4D.yaml rename src/swell/{suites/hofx/suite_page_hofx.py => configuration/jedi/soca/geos_ocean/model/background.yaml} (100%) mode change 100644 => 100755 create mode 100755 src/swell/configuration/jedi/soca/geos_ocean/model/geometry.yaml create mode 100755 src/swell/configuration/jedi/soca/geos_ocean/model/getvalues.yaml create mode 100755 src/swell/configuration/jedi/soca/geos_ocean/model/pseudo-model.yaml create mode 100755 src/swell/configuration/jedi/soca/geos_ocean/model/r2d2.yaml create mode 100755 src/swell/configuration/jedi/soca/geos_ocean/model/stage.yaml create mode 100755 src/swell/configuration/jedi/soca/geos_ocean/observations/adt_3a.yaml create mode 100755 src/swell/configuration/satellite_channels/geos_atmosphere/amsua_n19.yaml rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/aqua.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/f08.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/f10.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/f11.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/f13.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/f14.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/f15.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/f16.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/f17.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/f18.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/g08.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/g10.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/g11.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/g12.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/g13.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/g14.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/g15.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/gcom-w1.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/gpm.yaml (100%) create mode 100755 src/swell/configuration/satellite_channels/geos_atmosphere/iasi_metop-a.yaml rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/m09.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/m10.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/metop-a.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/metop-b.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/metop-c.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/n06.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/n07.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/n08.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/n09.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/n10.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/n11.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/n12.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/n14.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/n15.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/n16.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/n17.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/n18.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/n19.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/n20.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/npp.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/tirosn.yaml (100%) rename src/swell/configuration/satellite_channels/{ => geos_atmosphere}/trmm.yaml (100%) create mode 100644 src/swell/deployment/bin/swell_prepare_config.py create mode 100644 src/swell/deployment/platforms/nccs_discover/experiment.yaml create mode 100644 src/swell/deployment/prep_config.py create mode 100644 src/swell/deployment/prep_config_base.py rename src/swell/{configuration/configuration.py => deployment/prep_config_cli.py} (64%) create mode 100644 src/swell/deployment/prep_config_defaults.py rename src/swell/{install_path.py => deployment/prep_config_gui.py} (64%) delete mode 100644 src/swell/deployment/yaml_exploder.py delete mode 100644 src/swell/suites/hofx/experiment.yaml create mode 100644 src/swell/suites/hofx/geos_atmosphere/suites-hofx-geos_atmosphere.yaml create mode 100644 src/swell/suites/hofx/geos_ocean/suites-hofx-geos_ocean.yaml delete mode 100644 src/swell/suites/hofx/jedi_config.yaml delete mode 100644 src/swell/suites/hofx/suite_page_hofx.yaml create mode 100644 src/swell/suites/hofx/suites-hofx.yaml delete mode 100644 src/swell/suites/process_obs/experiment.yaml delete mode 100644 src/swell/suites/process_obs/flow.cylc delete mode 100644 src/swell/suites/store_backgrounds_geos_run/experiment.yaml delete mode 100644 src/swell/suites/store_backgrounds_geos_run/flow.cylc delete mode 100644 src/swell/suites/suite_page_start.yaml create mode 100644 src/swell/suites/suites.yaml delete mode 100644 src/swell/suites/swell_gui.py rename src/swell/{suites/suites.py => swell_path.py} (96%) rename src/swell/tasks/{ => base}/task_registry.py (91%) rename src/swell/tasks/{utilities => base}/utils.py (91%) delete mode 100644 src/swell/tasks/jedi_config.py delete mode 100644 src/swell/tasks/run_jedi_executable.py create mode 100644 src/swell/tasks/run_jedi_hofx_executable.py rename src/swell/tasks/{stage.py => stage_jedi.py} (75%) delete mode 100644 src/swell/tasks/utilities/__init__.py create mode 100644 src/swell/utilities/dictionary.py delete mode 100644 src/swell/utilities/dictionary_utilities.py create mode 100644 src/swell/utilities/jinja2.py delete mode 100644 src/swell/utilities/string_utils.py create mode 100644 src/swell/utilities/welcome_message.py diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..858a86c7 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +version = attr: swell.__version__ diff --git a/setup.py b/setup.py index 40190a95..e7683e48 100644 --- a/setup.py +++ b/setup.py @@ -12,14 +12,14 @@ # -------------------------------------------------------------------------------------------------- +import os.path import setuptools setuptools.setup( name='swell', - version='1.0.8', author='NASA Global Modeling and Assimilation Office', description='Workflow suites, tasks and configuration for coupled data assimilation', - url='https://github.com/danholdaway/swell', + url='https://github.com/geos-esm/swell', package_dir={'': 'src'}, packages=setuptools.find_packages(where='src'), classifiers=[ @@ -33,6 +33,7 @@ python_requires='>=3.6', install_requires=[ 'click', + 'jinja2>=3.0.3', 'pyyaml>=6.0', 'pycodestyle>=2.8.0', 'pandas>=1.4.0', @@ -42,10 +43,15 @@ package_data={ '': [ 'deployment/platforms/*/modules*', - 'deployment/platforms/*/r2d2_config.yaml', + 'deployment/platforms/*/*.yaml', + 'suites/*', 'suites/*/*', - 'configuration/*.yaml', + 'suites/*/*/*', + 'configuration/*', 'configuration/*/*', + 'configuration/*/*/*', + 'configuration/*/*/*/*', + 'configuration/*/*/*/*/*', ], }, include_package_data=True, @@ -53,6 +59,7 @@ 'console_scripts': [ 'swell_task = swell.tasks.base.task_base:main', 'swell_create_experiment = swell.deployment.bin.swell_create_experiment:main', + 'swell_prepare_experiment_config = swell.deployment.bin.swell_prepare_config:main', 'swell_launch_experiment = swell.deployment.bin.swell_launch_experiment:main', 'swell_sat_db_processing = swell.deployment.bin.swell_sat_db_processing:main', ], diff --git a/src/swell/__init__.py b/src/swell/__init__.py index ac1c0bc4..c448ce6f 100644 --- a/src/swell/__init__.py +++ b/src/swell/__init__.py @@ -7,3 +7,6 @@ import os repo_directory = os.path.dirname(__file__) + +# Set the version for swell +__version__ = '1.1.0' diff --git a/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/background.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/background.yaml new file mode 100755 index 00000000..86377762 --- /dev/null +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/background.yaml @@ -0,0 +1,10 @@ +datetime: '{{local_background_time_iso}}' +filetype: cube sphere history +provider: geos +datapath: '' +filenames: ['{{cycle_dir}}/bkg.%yyyy%mm%ddT%hh%MM%ssZ.nc4', + '{{experiment_dir}}/stage/Data/bkg/geos.crtmsrf.{{horizontal_resolution}}.nc4'] +state variables: [u,v,ua,va,t,delp,q,qi,ql,qr,qs,o3ppmv,phis, + qls,qcn,cfcn,frocean,frland,varflt,ustar,bstar, + zpbl,cm,ct,cq,kcbl,tsm,khl,khu,frlake,frseaice,vtype, + stype,vfrac,sheleg,ts,soilt,soilm,u10m,v10m] diff --git a/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/geometry.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/geometry.yaml new file mode 100755 index 00000000..91ff1482 --- /dev/null +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/geometry.yaml @@ -0,0 +1,9 @@ +fms initialization: + namelist filename: '{{experiment_dir}}/stage/Data/fv3files/fmsmpp.nml' + field table filename: '{{experiment_dir}}/stage/Data/fv3files/field_table_gmao' +akbk: '{{experiment_dir}}/stage/Data/fv3files/akbk{{vertical_resolution}}.nc4' +layout: [{{npx_proc}},{{npy_proc}}] +npx: {{horizontal_resolution}} +npy: {{horizontal_resolution}} +npz: {{vertical_resolution}} +field metadata override: '{{experiment_dir}}/stage/Data/fieldmetadata/geos.yaml' diff --git a/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/getvalues.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/getvalues.yaml new file mode 100755 index 00000000..dff4cd5d --- /dev/null +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/getvalues.yaml @@ -0,0 +1,2 @@ +variable change: + variable change name: Model2GeoVaLs diff --git a/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/pseudo-model.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/pseudo-model.yaml new file mode 100755 index 00000000..4755c806 --- /dev/null +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/pseudo-model.yaml @@ -0,0 +1,10 @@ +name: PSEUDO +tstep: {{background_frequency}} +filetype: cube sphere history +provider: geos +datapath: '' +filenames: ['{{cycle_dir}}/bkg.%yyyy%mm%ddT%hh%MM%ssZ.nc4', + '{{experiment_dir}}/stage/Data/bkg/geos.crtmsrf.{{horizontal_resolution}}.nc4'] +model variables: [u,v,ua,va,t,delp,q,qi,ql,qr,qs,o3ppmv,phis,qls,qcn,cfcn,frocean,frland,varflt,ustar, + bstar,zpbl,cm,ct,cq,kcbl,tsm,khl,khu,frlake,frseaice,vtype,stype,vfrac,sheleg, + ts,soilt,soilm,u10m,v10m] diff --git a/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/r2d2.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/r2d2.yaml new file mode 100755 index 00000000..7e786dde --- /dev/null +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/r2d2.yaml @@ -0,0 +1,10 @@ +fetch: + an: + - file_type: [bkg] + filename: '{{cycle_dir}}/bkg.%Y%m%dT%H%M%SZ.nc4' + fc: + - file_type: [bkg] + filename: '{{cycle_dir}}/bkg.%Y%m%dT%H%M%SZ.nc4' +store: + fc: + - file_type: [bkg] diff --git a/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/stage.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/stage.yaml new file mode 100755 index 00000000..c672a202 --- /dev/null +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/stage.yaml @@ -0,0 +1,7 @@ +- copy_files: + directories: + - ['/discover/nobackup/drholdaw/JediSwell/bundle/latest/fv3-jedi/test/Data/fieldmetadata/*', '{{experiment_dir}}/stage/Data/fieldmetadata/'] + - ['/discover/nobackup/drholdaw/JediSwell/bundle/latest/fv3-jedi/test/Data/fv3files/*', '{{experiment_dir}}/stage/Data/fv3files/'] + link_files: + directories: + - ['/discover/nobackup/drholdaw/JediData/GEOS_CRTM_Surface/geos.crtmsrf.{{horizontal_resolution}}.nc4', '{{experiment_dir}}/stage/Data/bkg/'] diff --git a/src/swell/configuration/observation_operators/aircraft.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/aircraft.yaml similarity index 98% rename from src/swell/configuration/observation_operators/aircraft.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/aircraft.yaml index 81343305..b98656db 100644 --- a/src/swell/configuration/observation_operators/aircraft.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/aircraft.yaml @@ -5,7 +5,7 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/aircraft.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/aircraft.{{window_begin}}.nc4' obsgrouping: group variables: ["station_id"] sort variable: "air_pressure" @@ -13,7 +13,7 @@ obs space: obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).aircraft.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.aircraft.{{window_begin}}.nc4' simulated variables: [eastward_wind, northward_wind, air_temperature] #obs filters: ##-------------------------------------------------------------------------------------------------------------------- diff --git a/src/swell/configuration/observation_operators/airs_aqua.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/airs_aqua.yaml similarity index 97% rename from src/swell/configuration/observation_operators/airs_aqua.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/airs_aqua.yaml index 27e9792e..9e390056 100644 --- a/src/swell/configuration/observation_operators/airs_aqua.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/airs_aqua.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/airs_aqua.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/airs_aqua.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).airs_aqua.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.airs_aqua.{{window_begin}}.nc4' simulated variables: [brightness_temperature] channels: &airs_aqua_channels 1, 6, 7, 10, 11, 15, 16, 17, 20, 21, 22, 24, 27, 28, 30, 36, 39, 40, 42, 51, 52, 54, 55, 56, 59, 62, 63, 68, 69, 71, @@ -37,15 +37,15 @@ obs operator: obs options: Sensor_ID: airs_aqua EndianType: little_endian - CoefficientPath: $(crtm_coeff_dir) + CoefficientPath: '{{crtm_coeff_dir}}' obs bias: - input file: $(cycle_dir)/airs_aqua.{{background_time}}.satbias.nc4 + input file: '{{cycle_dir}}/airs_aqua.{{background_time}}.satbias.nc4' variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &airs_aqua_tlapse $(cycle_dir)/airs_aqua.{{background_time}}.tlapse.txt + tlapse: &airs_aqua_tlapse '{{cycle_dir}}/airs_aqua.{{background_time}}.tlapse.txt' - name: lapse_rate tlapse: *airs_aqua_tlapse - name: emissivity diff --git a/src/swell/configuration/observation_operators/amsr2_gcom-w1.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsr2_gcom-w1.yaml similarity index 93% rename from src/swell/configuration/observation_operators/amsr2_gcom-w1.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsr2_gcom-w1.yaml index 2b0ac717..e1feb31f 100644 --- a/src/swell/configuration/observation_operators/amsr2_gcom-w1.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsr2_gcom-w1.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/amsr2_gcom-w1.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/amsr2_gcom-w1.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).amsr2_gcom-w1.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.amsr2_gcom-w1.{{window_begin}}.nc4' simulated variables: [brightness_temperature] channels: &amsr2_gcom-w1_channels 1-14 obs operator: @@ -21,15 +21,15 @@ obs operator: obs options: Sensor_ID: amsr2_gcom-w1 EndianType: little_endian - CoefficientPath: $(crtm_coeff_dir) + CoefficientPath: '{{crtm_coeff_dir}}' obs bias: - input file: $(cycle_dir)/amsr2_gcom-w1.{{background_time}}.satbias.nc4 + input file: '{{cycle_dir}}/amsr2_gcom-w1.{{background_time}}.satbias.nc4' variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &amsr2_gcom-w1_tlapse $(cycle_dir)/amsr2_gcom-w1.{{background_time}}.tlapse.txt + tlapse: &amsr2_gcom-w1_tlapse '{{cycle_dir}}/amsr2_gcom-w1.{{background_time}}.tlapse.txt' - name: lapse_rate tlapse: *amsr2_gcom-w1_tlapse - name: emissivity diff --git a/src/swell/configuration/observation_operators/amsua_aqua.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsua_aqua.yaml similarity index 93% rename from src/swell/configuration/observation_operators/amsua_aqua.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsua_aqua.yaml index 0784f548..69a2d517 100644 --- a/src/swell/configuration/observation_operators/amsua_aqua.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsua_aqua.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/amsua_aqua.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/amsua_aqua.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).amsua_aqua.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.amsua_aqua.{{window_begin}}.nc4' simulated variables: [brightness_temperature] channels: &amsua_aqua_channels 1-15 obs operator: @@ -18,9 +18,9 @@ obs operator: obs options: Sensor_ID: &Sensor_ID amsua_aqua EndianType: little_endian - CoefficientPath: $(crtm_coeff_dir) + CoefficientPath: '{{crtm_coeff_dir}}' obs bias: - input file: $(cycle_dir)/amsua_aqua.{{background_time}}.satbias.nc4 + input file: '{{cycle_dir}}/amsua_aqua.{{background_time}}.satbias.nc4' variational bc: predictors: - name: constant @@ -30,7 +30,7 @@ obs bias: clwdif_ch314: 2 - name: lapse_rate order: 2 - tlapse: &amsua_aqua_tlapse $(cycle_dir)/amsua_aqua.{{background_time}}.tlapse.txt + tlapse: &amsua_aqua_tlapse '{{cycle_dir}}/amsua_aqua.{{background_time}}.tlapse.txt' - name: lapse_rate tlapse: *amsua_aqua_tlapse - name: emissivity diff --git a/src/swell/configuration/observation_operators/amsua_metop-a.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsua_metop-a.yaml similarity index 93% rename from src/swell/configuration/observation_operators/amsua_metop-a.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsua_metop-a.yaml index c87793c0..896af8df 100644 --- a/src/swell/configuration/observation_operators/amsua_metop-a.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsua_metop-a.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/amsua_metop-a.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/amsua_metop-a.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).amsua_metop-a.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.amsua_metop-a.{{window_begin}}.nc4' simulated variables: [brightness_temperature] channels: &amsua_metop-a_channels 1-15 obs operator: @@ -18,9 +18,9 @@ obs operator: obs options: Sensor_ID: &Sensor_ID amsua_metop-a EndianType: little_endian - CoefficientPath: $(crtm_coeff_dir) + CoefficientPath: '{{crtm_coeff_dir}}' obs bias: - input file: $(cycle_dir)/amsua_metop-a.{{background_time}}.satbias.nc4 + input file: '{{cycle_dir}}/amsua_metop-a.{{background_time}}.satbias.nc4' channels without bc: 14 variational bc: predictors: @@ -31,7 +31,7 @@ obs bias: clwdif_ch314: 2 - name: lapse_rate order: 2 - tlapse: &amsua_metop-a_tlapse $(cycle_dir)/amsua_metop-a.{{background_time}}.tlapse.txt + tlapse: &amsua_metop-a_tlapse '{{cycle_dir}}/amsua_metop-a.{{background_time}}.tlapse.txt' - name: lapse_rate tlapse: *amsua_metop-a_tlapse - name: emissivity diff --git a/src/swell/configuration/observation_operators/amsua_metop-b.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsua_metop-b.yaml similarity index 93% rename from src/swell/configuration/observation_operators/amsua_metop-b.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsua_metop-b.yaml index 27961a4e..5bed4520 100644 --- a/src/swell/configuration/observation_operators/amsua_metop-b.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsua_metop-b.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/amsua_metop-b.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/amsua_metop-b.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).amsua_metop-b.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.amsua_metop-b.{{window_begin}}.nc4' simulated variables: [brightness_temperature] channels: &amsua_metop-b_channels 1-15 obs operator: @@ -18,9 +18,9 @@ obs operator: obs options: Sensor_ID: &Sensor_ID amsua_metop-b EndianType: little_endian - CoefficientPath: $(crtm_coeff_dir) + CoefficientPath: '{{crtm_coeff_dir}}' obs bias: - input file: $(cycle_dir)/amsua_metop-b.{{background_time}}.satbias.nc4 + input file: '{{cycle_dir}}/amsua_metop-b.{{background_time}}.satbias.nc4' channels without bc: 14 variational bc: predictors: @@ -31,7 +31,7 @@ obs bias: clwdif_ch314: 2 - name: lapse_rate order: 2 - tlapse: &amsua_metop-b_tlapse $(cycle_dir)/amsua_metop-b.{{background_time}}.tlapse.txt + tlapse: &amsua_metop-b_tlapse '{{cycle_dir}}/amsua_metop-b.{{background_time}}.tlapse.txt' - name: lapse_rate tlapse: *amsua_metop-b_tlapse - name: emissivity diff --git a/src/swell/configuration/observation_operators/amsua_metop-c.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsua_metop-c.yaml similarity index 93% rename from src/swell/configuration/observation_operators/amsua_metop-c.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsua_metop-c.yaml index 04fca734..d7a4da97 100644 --- a/src/swell/configuration/observation_operators/amsua_metop-c.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsua_metop-c.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/amsua_metop-c.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/amsua_metop-c.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).amsua_metop-c.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.amsua_metop-c.{{window_begin}}.nc4' simulated variables: [brightness_temperature] channels: &amsua_metop-c_channels 1-15 obs operator: @@ -18,9 +18,9 @@ obs operator: obs options: Sensor_ID: &Sensor_ID amsua_metop-c EndianType: little_endian - CoefficientPath: $(crtm_coeff_dir) + CoefficientPath: '{{crtm_coeff_dir}}' obs bias: - input file: $(cycle_dir)/amsua_metop-c.{{background_time}}.satbias.nc4 + input file: '{{cycle_dir}}/amsua_metop-c.{{background_time}}.satbias.nc4' channels without bc: 14 variational bc: predictors: @@ -31,7 +31,7 @@ obs bias: clwdif_ch314: 2 - name: lapse_rate order: 2 - tlapse: &amsua_metop-c_tlapse $(cycle_dir)/amsua_metop-c.{{background_time}}.tlapse.txt + tlapse: &amsua_metop-c_tlapse '{{cycle_dir}}/amsua_metop-c.{{background_time}}.tlapse.txt' - name: lapse_rate tlapse: *amsua_metop-c_tlapse - name: emissivity diff --git a/src/swell/configuration/observation_operators/amsua_n15.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsua_n15.yaml similarity index 94% rename from src/swell/configuration/observation_operators/amsua_n15.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsua_n15.yaml index 668b1f7b..12897cec 100644 --- a/src/swell/configuration/observation_operators/amsua_n15.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsua_n15.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/amsua_n15.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/amsua_n15.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).amsua_n15.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.amsua_n15.{{window_begin}}.nc4' simulated variables: [brightness_temperature] channels: &amsua_n15_channels 1-15 obs operator: @@ -18,9 +18,9 @@ obs operator: obs options: Sensor_ID: &Sensor_ID amsua_n15 EndianType: little_endian - CoefficientPath: $(crtm_coeff_dir) + CoefficientPath: '{{crtm_coeff_dir}}' obs bias: - input file: $(cycle_dir)/amsua_n15.{{background_time}}.satbias.nc4 + input file: '{{cycle_dir}}/amsua_n15.{{background_time}}.satbias.nc4' channels without bc: 14 variational bc: predictors: @@ -31,7 +31,7 @@ obs bias: clwdif_ch314: 2 - name: lapse_rate order: 2 - tlapse: &amsua_n15_tlapse $(cycle_dir)/amsua_n15.{{background_time}}.tlapse.txt + tlapse: &amsua_n15_tlapse '{{cycle_dir}}/amsua_n15.{{background_time}}.tlapse.txt' - name: lapse_rate tlapse: *amsua_n15_tlapse - name: emissivity diff --git a/src/swell/configuration/observation_operators/amsua_n18.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsua_n18.yaml similarity index 94% rename from src/swell/configuration/observation_operators/amsua_n18.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsua_n18.yaml index b77400fc..bee9be3b 100644 --- a/src/swell/configuration/observation_operators/amsua_n18.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsua_n18.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/amsua_n18.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/amsua_n18.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).amsua_n18.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.amsua_n18.{{window_begin}}.nc4' simulated variables: [brightness_temperature] channels: &amsua_n18_channels 1-15 obs operator: @@ -18,9 +18,9 @@ obs operator: obs options: Sensor_ID: &Sensor_ID amsua_n18 EndianType: little_endian - CoefficientPath: $(crtm_coeff_dir) + CoefficientPath: '{{crtm_coeff_dir}}' obs bias: - input file: $(cycle_dir)/amsua_n18.{{background_time}}.satbias.nc4 + input file: '{{cycle_dir}}/amsua_n18.{{background_time}}.satbias.nc4' channels without bc: 14 variational bc: predictors: @@ -31,7 +31,7 @@ obs bias: clwdif_ch314: 2 - name: lapse_rate order: 2 - tlapse: &amsua_n18_tlapse $(cycle_dir)/amsua_n18.{{background_time}}.tlapse.txt + tlapse: &amsua_n18_tlapse '{{cycle_dir}}/amsua_n18.{{background_time}}.tlapse.txt' - name: lapse_rate tlapse: *amsua_n18_tlapse - name: emissivity diff --git a/src/swell/configuration/observation_operators/amsua_n19.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsua_n19.yaml similarity index 94% rename from src/swell/configuration/observation_operators/amsua_n19.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsua_n19.yaml index c8cc26c3..2135c89e 100644 --- a/src/swell/configuration/observation_operators/amsua_n19.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsua_n19.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/amsua_n19.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/amsua_n19.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).amsua_n19.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.amsua_n19.{{window_begin}}.nc4' simulated variables: [brightness_temperature] channels: &amsua_n19_channels 1-15 obs operator: @@ -18,9 +18,9 @@ obs operator: obs options: Sensor_ID: &Sensor_ID amsua_n19 EndianType: little_endian - CoefficientPath: $(crtm_coeff_dir) + CoefficientPath: '{{crtm_coeff_dir}}' obs bias: - input file: $(cycle_dir)/amsua_n19.{{background_time}}.satbias.nc4 + input file: '{{cycle_dir}}/amsua_n19.{{background_time}}.satbias.nc4' channels without bc: 14 variational bc: predictors: @@ -31,7 +31,7 @@ obs bias: clwdif_ch314: 2 - name: lapse_rate order: 2 - tlapse: &amsua_n19_tlapse $(cycle_dir)/amsua_n19.{{background_time}}.tlapse.txt + tlapse: &amsua_n19_tlapse '{{cycle_dir}}/amsua_n19.{{background_time}}.tlapse.txt' - name: lapse_rate tlapse: *amsua_n19_tlapse - name: emissivity diff --git a/src/swell/configuration/observation_operators/atms_n20.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/atms_n20.yaml similarity index 97% rename from src/swell/configuration/observation_operators/atms_n20.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/atms_n20.yaml index ce8ac7e5..409998c7 100644 --- a/src/swell/configuration/observation_operators/atms_n20.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/atms_n20.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/atms_n20.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/atms_n20.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).atms_n20.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.atms_n20.{{window_begin}}.nc4' simulated variables: [brightness_temperature] channels: &atms_n20_channels 1-22 obs operator: @@ -18,15 +18,15 @@ obs operator: obs options: Sensor_ID: atms_n20 EndianType: little_endian - CoefficientPath: $(crtm_coeff_dir) + CoefficientPath: '{{crtm_coeff_dir}}' obs bias: - input file: $(cycle_dir)/atms_n20.{{background_time}}.satbias.nc4 + input file: '{{cycle_dir}}/atms_n20.{{background_time}}.satbias.nc4' variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &atms_n20_tlapse $(cycle_dir)/atms_n20.{{background_time}}.tlapse.txt + tlapse: &atms_n20_tlapse '{{cycle_dir}}/atms_n20.{{background_time}}.tlapse.txt' - name: lapse_rate tlapse: *atms_n20_tlapse - name: emissivity diff --git a/src/swell/configuration/observation_operators/atms_npp.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/atms_npp.yaml similarity index 97% rename from src/swell/configuration/observation_operators/atms_npp.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/atms_npp.yaml index aac9ea58..a96789a1 100644 --- a/src/swell/configuration/observation_operators/atms_npp.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/atms_npp.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/atms_npp.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/atms_npp.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).atms_npp.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.atms_npp.{{window_begin}}.nc4' simulated variables: [brightness_temperature] channels: &atms_npp_channels 1-22 obs operator: @@ -18,15 +18,15 @@ obs operator: obs options: Sensor_ID: atms_npp EndianType: little_endian - CoefficientPath: $(crtm_coeff_dir) + CoefficientPath: '{{crtm_coeff_dir}}' obs bias: - input file: $(cycle_dir)/atms_npp.{{background_time}}.satbias.nc4 + input file: '{{cycle_dir}}/atms_npp.{{background_time}}.satbias.nc4' variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &atms_npp_tlapse $(cycle_dir)/atms_npp.{{background_time}}.tlapse.txt + tlapse: &atms_npp_tlapse '{{cycle_dir}}/atms_npp.{{background_time}}.tlapse.txt' - name: lapse_rate tlapse: *atms_npp_tlapse - name: emissivity diff --git a/src/swell/configuration/observation_operators/avhrr3_metop-a.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/avhrr3_metop-a.yaml similarity index 92% rename from src/swell/configuration/observation_operators/avhrr3_metop-a.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/avhrr3_metop-a.yaml index 414692a4..0d7570d2 100644 --- a/src/swell/configuration/observation_operators/avhrr3_metop-a.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/avhrr3_metop-a.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/avhrr3_metop-a.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/avhrr3_metop-a.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).avhrr3_metop-a.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.avhrr3_metop-a.{{window_begin}}.nc4' simulated variables: [brightness_temperature] channels: &avhrr3_metop-a_channels 3,4,5 obs operator: @@ -16,15 +16,15 @@ obs operator: obs options: Sensor_ID: avhrr3_metop-a EndianType: little_endian - CoefficientPath: $(crtm_coeff_dir) + CoefficientPath: '{{crtm_coeff_dir}}' obs bias: - input file: $(cycle_dir)/avhrr3_metop-a.{{background_time}}.satbias.nc4 + input file: '{{cycle_dir}}/avhrr3_metop-a.{{background_time}}.satbias.nc4' variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &avhrr3_metop-a_tlapse $(cycle_dir)/avhrr3_metop-a.{{background_time}}.tlapse.txt + tlapse: &avhrr3_metop-a_tlapse '{{cycle_dir}}/avhrr3_metop-a.{{background_time}}.tlapse.txt' - name: lapse_rate tlapse: *avhrr3_metop-a_tlapse - name: emissivity diff --git a/src/swell/configuration/observation_operators/avhrr3_n18.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/avhrr3_n18.yaml similarity index 92% rename from src/swell/configuration/observation_operators/avhrr3_n18.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/avhrr3_n18.yaml index cb1fa482..9e034db1 100644 --- a/src/swell/configuration/observation_operators/avhrr3_n18.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/avhrr3_n18.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/avhrr3_n18.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/avhrr3_n18.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).avhrr3_n18.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.avhrr3_n18.{{window_begin}}.nc4' simulated variables: [brightness_temperature] channels: &avhrr3_n18_channels 3,4,5 obs operator: @@ -16,15 +16,15 @@ obs operator: obs options: Sensor_ID: avhrr3_n18 EndianType: little_endian - CoefficientPath: $(crtm_coeff_dir) + CoefficientPath: '{{crtm_coeff_dir}}' obs bias: - input file: $(cycle_dir)/avhrr3_n18.{{background_time}}.satbias.nc4 + input file: '{{cycle_dir}}/avhrr3_n18.{{background_time}}.satbias.nc4' variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &avhrr3_n18_tlapse $(cycle_dir)/avhrr3_n18.{{background_time}}.tlapse.txt + tlapse: &avhrr3_n18_tlapse '{{cycle_dir}}/avhrr3_n18.{{background_time}}.tlapse.txt' - name: lapse_rate tlapse: *avhrr3_n18_tlapse - name: emissivity diff --git a/src/swell/configuration/observation_operators/cris-fsr_n20.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/cris-fsr_n20.yaml similarity index 98% rename from src/swell/configuration/observation_operators/cris-fsr_n20.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/cris-fsr_n20.yaml index fa9b9a0b..af188e10 100644 --- a/src/swell/configuration/observation_operators/cris-fsr_n20.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/cris-fsr_n20.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/cris-fsr_n20.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/cris-fsr_n20.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).cris-fsr_n20.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.cris-fsr_n20.{{window_begin}}.nc4' simulated variables: [brightness_temperature] channels: &cris-fsr_n20_channels 19, 24, 26, 27, 28, 31, 32, 33, 37, 39, 42, 44, 47, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, @@ -47,15 +47,15 @@ obs operator: obs options: Sensor_ID: cris-fsr_n20 EndianType: little_endian - CoefficientPath: $(crtm_coeff_dir) + CoefficientPath: '{{crtm_coeff_dir}}' obs bias: - input file: $(cycle_dir)/cris-fsr_n20.{{background_time}}.satbias.nc4 + input file: '{{cycle_dir}}/cris-fsr_n20.{{background_time}}.satbias.nc4' variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &cris-fsr_n20_tlapse $(cycle_dir)/cris-fsr_n20.{{background_time}}.tlapse.txt + tlapse: &cris-fsr_n20_tlapse '{{cycle_dir}}/cris-fsr_n20.{{background_time}}.tlapse.txt' - name: lapse_rate tlapse: *cris-fsr_n20_tlapse - name: emissivity diff --git a/src/swell/configuration/observation_operators/cris-fsr_npp.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/cris-fsr_npp.yaml similarity index 98% rename from src/swell/configuration/observation_operators/cris-fsr_npp.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/cris-fsr_npp.yaml index 0967f75f..97e4481b 100644 --- a/src/swell/configuration/observation_operators/cris-fsr_npp.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/cris-fsr_npp.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/cris-fsr_npp.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/cris-fsr_npp.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).cris-fsr_npp.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.cris-fsr_npp.{{window_begin}}.nc4' simulated variables: [brightness_temperature] channels: &cris-fsr_npp_channels 19, 24, 26, 27, 28, 31, 32, 33, 37, 39, 42, 44, 47, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, @@ -47,15 +47,15 @@ obs operator: obs options: Sensor_ID: cris-fsr_npp EndianType: little_endian - CoefficientPath: $(crtm_coeff_dir) + CoefficientPath: '{{crtm_coeff_dir}}' obs bias: - input file: $(cycle_dir)/cris-fsr_npp.{{background_time}}.satbias.nc4 + input file: '{{cycle_dir}}/cris-fsr_npp.{{background_time}}.satbias.nc4' variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &cris-fsr_npp_tlapse $(cycle_dir)/cris-fsr_npp.{{background_time}}.tlapse.txt + tlapse: &cris-fsr_npp_tlapse '{{cycle_dir}}/cris-fsr_npp.{{background_time}}.tlapse.txt' - name: lapse_rate tlapse: *cris-fsr_npp_tlapse - name: emissivity diff --git a/src/swell/configuration/observation_operators/gmi_gpm.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/gmi_gpm.yaml similarity index 91% rename from src/swell/configuration/observation_operators/gmi_gpm.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/gmi_gpm.yaml index 9c2c57a0..ad161c76 100644 --- a/src/swell/configuration/observation_operators/gmi_gpm.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/gmi_gpm.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/gmi_gpm.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/gmi_gpm.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).gmi_gpm.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.gmi_gpm.{{window_begin}}.nc4' simulated variables: [brightness_temperature] channels: &gmi_gpm_channels 1-13 obs operator: @@ -22,15 +22,15 @@ obs operator: obs options: Sensor_ID: gmi_gpm EndianType: little_endian - CoefficientPath: $(crtm_coeff_dir) + CoefficientPath: '{{crtm_coeff_dir}}' obs bias: - input file: $(cycle_dir)/gmi_gpm.{{background_time}}.satbias.nc4 + input file: '{{cycle_dir}}/gmi_gpm.{{background_time}}.satbias.nc4' variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &gmi_gpm_tlapse $(cycle_dir)/gmi_gpm.{{background_time}}.tlapse.txt + tlapse: &gmi_gpm_tlapse '{{cycle_dir}}/gmi_gpm.{{background_time}}.tlapse.txt' - name: lapse_rate tlapse: *gmi_gpm_tlapse - name: emissivity diff --git a/src/swell/configuration/observation_operators/gnssrobndnbam.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/gnssrobndnbam.yaml similarity index 86% rename from src/swell/configuration/observation_operators/gnssrobndnbam.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/gnssrobndnbam.yaml index 06abbbb3..de8160ea 100644 --- a/src/swell/configuration/observation_operators/gnssrobndnbam.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/gnssrobndnbam.yaml @@ -3,7 +3,7 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/gnssrobndnbam.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/gnssrobndnbam.{{window_begin}}.nc4' obsgrouping: group variables: [ 'record_number' ] sort variable: 'impact_height' @@ -11,7 +11,7 @@ obs space: obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).gnssrobndnbam.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.gnssrobndnbam.{{window_begin}}.nc4' simulated variables: [bending_angle] obs operator: name: GnssroBndNBAM diff --git a/src/swell/configuration/observation_operators/iasi_metop-a.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/iasi_metop-a.yaml similarity index 98% rename from src/swell/configuration/observation_operators/iasi_metop-a.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/iasi_metop-a.yaml index f7780f4a..269c87d2 100644 --- a/src/swell/configuration/observation_operators/iasi_metop-a.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/iasi_metop-a.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/iasi_metop-a.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/iasi_metop-a.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).iasi_metop-a.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.iasi_metop-a.{{window_begin}}.nc4' simulated variables: [brightness_temperature] channels: &iasi_metop-a_channels 16, 29, 32, 35, 38, 41, 44, 47, 49, 50, 51, 53, 55, 56, 57, 59, 61, 62, 63, 66, 68, 70, 72, 74, 76, 78, 79, 81, 82, 83, @@ -64,15 +64,15 @@ obs operator: obs options: Sensor_ID: iasi_metop-a EndianType: little_endian - CoefficientPath: $(crtm_coeff_dir) + CoefficientPath: '{{crtm_coeff_dir}}' obs bias: - input file: $(cycle_dir)/iasi_metop-a.{{background_time}}.satbias.nc4 + input file: '{{cycle_dir}}/iasi_metop-a.{{background_time}}.satbias.nc4' variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &iasi_metop-a_tlapse $(cycle_dir)/iasi_metop-a.{{background_time}}.tlapse.txt + tlapse: &iasi_metop-a_tlapse '{{cycle_dir}}/iasi_metop-a.{{background_time}}.tlapse.txt' - name: lapse_rate tlapse: *iasi_metop-a_tlapse - name: emissivity diff --git a/src/swell/configuration/observation_operators/iasi_metop-b.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/iasi_metop-b.yaml similarity index 98% rename from src/swell/configuration/observation_operators/iasi_metop-b.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/iasi_metop-b.yaml index b5ecae72..d4f5a4c1 100644 --- a/src/swell/configuration/observation_operators/iasi_metop-b.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/iasi_metop-b.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/iasi_metop-b.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/iasi_metop-b.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).iasi_metop-b.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.iasi_metop-b.{{window_begin}}.nc4' simulated variables: [brightness_temperature] channels: &iasi_metop-b_channels 16, 29, 32, 35, 38, 41, 44, 47, 49, 50, 51, 53, 55, 56, 57, 59, 61, 62, 63, 66, 68, 70, 72, 74, 76, 78, 79, 81, 82, 83, @@ -64,15 +64,15 @@ obs operator: obs options: Sensor_ID: iasi_metop-b EndianType: little_endian - CoefficientPath: $(crtm_coeff_dir) + CoefficientPath: '{{crtm_coeff_dir}}' obs bias: - input file: $(cycle_dir)/iasi_metop-b.{{background_time}}.satbias.nc4 + input file: '{{cycle_dir}}/iasi_metop-b.{{background_time}}.satbias.nc4' variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &iasi_metop-b_tlapse $(cycle_dir)/iasi_metop-b.{{background_time}}.tlapse.txt + tlapse: &iasi_metop-b_tlapse '{{cycle_dir}}/iasi_metop-b.{{background_time}}.tlapse.txt' - name: lapse_rate tlapse: *iasi_metop-b_tlapse - name: emissivity diff --git a/src/swell/configuration/observation_operators/mhs_metop-b.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/mhs_metop-b.yaml similarity index 95% rename from src/swell/configuration/observation_operators/mhs_metop-b.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/mhs_metop-b.yaml index f84ee324..5fe1a172 100644 --- a/src/swell/configuration/observation_operators/mhs_metop-b.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/mhs_metop-b.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/mhs_metop-b.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/mhs_metop-b.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).mhs_metop-b.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.mhs_metop-b.{{window_begin}}.nc4' simulated variables: [brightness_temperature] channels: &mhs_metop-b_channels 1-5 obs operator: @@ -21,15 +21,15 @@ obs operator: obs options: Sensor_ID: &Sensor_ID mhs_metop-b EndianType: little_endian - CoefficientPath: $(crtm_coeff_dir) + CoefficientPath: '{{crtm_coeff_dir}}' obs bias: - input file: $(cycle_dir)/mhs_metop-b.{{background_time}}.satbias.nc4 + input file: '{{cycle_dir}}/mhs_metop-b.{{background_time}}.satbias.nc4' variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &mhs_metop-b_tlapse $(cycle_dir)/mhs_metop-b.{{background_time}}.tlapse.txt + tlapse: &mhs_metop-b_tlapse '{{cycle_dir}}/mhs_metop-b.{{background_time}}.tlapse.txt' - name: lapse_rate tlapse: *mhs_metop-b_tlapse - name: emissivity diff --git a/src/swell/configuration/observation_operators/mhs_metop-c.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/mhs_metop-c.yaml similarity index 95% rename from src/swell/configuration/observation_operators/mhs_metop-c.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/mhs_metop-c.yaml index 6bc8a804..8fa21ebd 100644 --- a/src/swell/configuration/observation_operators/mhs_metop-c.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/mhs_metop-c.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/mhs_metop-c.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/mhs_metop-c.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).mhs_metop-c.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.mhs_metop-c.{{window_begin}}.nc4' simulated variables: [brightness_temperature] channels: &mhs_metop-c_channels 1-5 obs operator: @@ -21,15 +21,15 @@ obs operator: obs options: Sensor_ID: &Sensor_ID mhs_metop-c EndianType: little_endian - CoefficientPath: $(crtm_coeff_dir) + CoefficientPath: '{{crtm_coeff_dir}}' obs bias: - input file: $(cycle_dir)/mhs_metop-c.{{background_time}}.satbias.nc4 + input file: '{{cycle_dir}}/mhs_metop-c.{{background_time}}.satbias.nc4' variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &mhs_metop-c_tlapse $(cycle_dir)/mhs_metop-c.{{background_time}}.tlapse.txt + tlapse: &mhs_metop-c_tlapse '{{cycle_dir}}/mhs_metop-c.{{background_time}}.tlapse.txt' - name: lapse_rate tlapse: *mhs_metop-c_tlapse - name: emissivity diff --git a/src/swell/configuration/observation_operators/mhs_n19.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/mhs_n19.yaml similarity index 64% rename from src/swell/configuration/observation_operators/mhs_n19.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/mhs_n19.yaml index 9890b228..30e33e72 100644 --- a/src/swell/configuration/observation_operators/mhs_n19.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/mhs_n19.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/mhs_n19.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/mhs_n19.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).mhs_n19.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.mhs_n19.{{window_begin}}.nc4' simulated variables: [brightness_temperature] channels: 1-5 obs operator: @@ -16,15 +16,15 @@ obs operator: obs options: Sensor_ID: mhs_n19 EndianType: little_endian - CoefficientPath: $(crtm_coeff_dir) + CoefficientPath: '{{crtm_coeff_dir}}' obs bias: - input file: $(cycle_dir)/mhs_n19.{{background_time}}.satbias.nc4 + input file: '{{cycle_dir}}/mhs_n19.{{background_time}}.satbias.nc4' variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &mhs_n19_tlapse $(cycle_dir)/mhs_n19.{{background_time}}.tlapse.txt + tlapse: &mhs_n19_tlapse '{{cycle_dir}}/mhs_n19.{{background_time}}.tlapse.txt' - name: lapse_rate tlapse: *mhs_n19_tlapse - name: emissivity diff --git a/src/swell/configuration/observation_operators/mls55_aura.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/mls55_aura.yaml similarity index 82% rename from src/swell/configuration/observation_operators/mls55_aura.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/mls55_aura.yaml index c787b65e..e5007187 100644 --- a/src/swell/configuration/observation_operators/mls55_aura.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/mls55_aura.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/mls55_aura.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/mls55_aura.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).mls55_aura.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.mls55_aura.{{window_begin}}.nc4' simulated variables: [mole_fraction_of_ozone_in_air] obs operator: name: VertInterp diff --git a/src/swell/configuration/observation_operators/omi_aura.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/omi_aura.yaml similarity index 90% rename from src/swell/configuration/observation_operators/omi_aura.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/omi_aura.yaml index 7528728c..57f5dfa2 100644 --- a/src/swell/configuration/observation_operators/omi_aura.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/omi_aura.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/omi_aura.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/omi_aura.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).omi_aura.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.omi_aura.{{window_begin}}.nc4' simulated variables: [integrated_layer_ozone_in_air] obs operator: name: AtmVertInterpLay diff --git a/src/swell/configuration/observation_operators/ompslpnc_npp.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/ompslpnc_npp.yaml similarity index 82% rename from src/swell/configuration/observation_operators/ompslpnc_npp.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/ompslpnc_npp.yaml index b63529dc..30c45cd2 100644 --- a/src/swell/configuration/observation_operators/ompslpnc_npp.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/ompslpnc_npp.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/ompslpnc_npp.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/ompslpnc_npp.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).ompslpnc_npp.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.ompslpnc_npp.{{window_begin}}.nc4' simulated variables: [mole_fraction_of_ozone_in_air] obs operator: name: VertInterp diff --git a/src/swell/configuration/observation_operators/ompsnm_npp.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/ompsnm_npp.yaml similarity index 87% rename from src/swell/configuration/observation_operators/ompsnm_npp.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/ompsnm_npp.yaml index e941dd5b..cb7d4ce8 100644 --- a/src/swell/configuration/observation_operators/ompsnm_npp.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/ompsnm_npp.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/ompsnm_npp.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/ompsnm_npp.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).ompsnm_npp.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.ompsnm_npp.{{window_begin}}.nc4' simulated variables: [integrated_layer_ozone_in_air] obs operator: name: AtmVertInterpLay diff --git a/src/swell/configuration/observation_operators/rass_tv.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/rass_tv.yaml similarity index 58% rename from src/swell/configuration/observation_operators/rass_tv.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/rass_tv.yaml index c0956f70..b0342290 100644 --- a/src/swell/configuration/observation_operators/rass_tv.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/rass_tv.yaml @@ -5,9 +5,9 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/rass_tv.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/rass_tv.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).rass_tv.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.rass_tv.{{window_begin}}.nc4' simulated variables: [virtual_temperature] diff --git a/src/swell/configuration/observation_operators/satwind.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/satwind.yaml similarity index 98% rename from src/swell/configuration/observation_operators/satwind.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/satwind.yaml index 266b96cd..d79c206a 100644 --- a/src/swell/configuration/observation_operators/satwind.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/satwind.yaml @@ -5,11 +5,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/satwind.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/satwind.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).satwind.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.satwind.{{window_begin}}.nc4' simulated variables: [eastward_wind, northward_wind] obs filters: # diff --git a/src/swell/configuration/observation_operators/scatwind.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/scatwind.yaml similarity index 95% rename from src/swell/configuration/observation_operators/scatwind.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/scatwind.yaml index e666778e..f360f634 100644 --- a/src/swell/configuration/observation_operators/scatwind.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/scatwind.yaml @@ -5,11 +5,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/scatwind.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/scatwind.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).scatwind.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.scatwind.{{window_begin}}.nc4' simulated variables: [eastward_wind, northward_wind] obs filters: # Reject all obs with PreQC mark already set above 3 diff --git a/src/swell/configuration/observation_operators/seviri_m11.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/seviri_m11.yaml similarity index 90% rename from src/swell/configuration/observation_operators/seviri_m11.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/seviri_m11.yaml index 0fa0e5c5..42e02d2f 100644 --- a/src/swell/configuration/observation_operators/seviri_m11.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/seviri_m11.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/seviri_m11.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/seviri_m11.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).seviri_m11.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.seviri_m11.{{window_begin}}.nc4' simulated variables: [brightness_temperature] channels: &seviri_m11_channels 4-11 obs operator: @@ -16,15 +16,15 @@ obs operator: obs options: Sensor_ID: seviri_m11 EndianType: little_endian - CoefficientPath: $(crtm_coeff_dir) + CoefficientPath: '{{crtm_coeff_dir}}' obs bias: - input file: $(cycle_dir)/seviri_m11.{{background_time}}.satbias.nc4 + input file: '{{cycle_dir}}/seviri_m11.{{background_time}}.satbias.nc4' variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &seviri_m11_tlapse $(cycle_dir)/seviri_m11.{{background_time}}.tlapse.txt + tlapse: &seviri_m11_tlapse '{{cycle_dir}}/seviri_m11.{{background_time}}.tlapse.txt' - name: lapse_rate tlapse: *seviri_m11_tlapse - name: emissivity diff --git a/src/swell/configuration/observation_operators/sfc.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/sfc.yaml similarity index 95% rename from src/swell/configuration/observation_operators/sfc.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/sfc.yaml index aab481ab..deade53a 100644 --- a/src/swell/configuration/observation_operators/sfc.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/sfc.yaml @@ -8,11 +8,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/sfc.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/sfc.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).sfc.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.sfc.{{window_begin}}.nc4' simulated variables: [surface_pressure] #, air_temperature] obs filters: # Observation Range Sanity Check diff --git a/src/swell/configuration/observation_operators/sfcship.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/sfcship.yaml similarity index 95% rename from src/swell/configuration/observation_operators/sfcship.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/sfcship.yaml index c46a8b31..a4d3bd69 100644 --- a/src/swell/configuration/observation_operators/sfcship.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/sfcship.yaml @@ -6,11 +6,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/sfcship.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/sfcship.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).sfcship.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.sfcship.{{window_begin}}.nc4' simulated variables: [surface_pressure] #, air_temperature, specific_humidity] obs filters: # Observation Range Sanity Check diff --git a/src/swell/configuration/observation_operators/sondes.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/sondes.yaml similarity index 98% rename from src/swell/configuration/observation_operators/sondes.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/sondes.yaml index be3e076e..3e98c31c 100644 --- a/src/swell/configuration/observation_operators/sondes.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/sondes.yaml @@ -3,7 +3,7 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/sondes.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/sondes.{{window_begin}}.nc4' obsgrouping: group variables: ["station_id", "LaunchTime"] sort variable: "air_pressure" @@ -11,7 +11,7 @@ obs space: obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).sondes.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.sondes.{{window_begin}}.nc4' simulated variables: [air_temperature, specific_humidity, eastward_wind, northward_wind, surface_pressure] obs operator: name: Composite diff --git a/src/swell/configuration/observation_operators/ssmis_f17.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/ssmis_f17.yaml similarity index 92% rename from src/swell/configuration/observation_operators/ssmis_f17.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/ssmis_f17.yaml index 9b555195..97d857bc 100644 --- a/src/swell/configuration/observation_operators/ssmis_f17.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/ssmis_f17.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/ssmis_f17.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/ssmis_f17.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).ssmis_f17.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.ssmis_f17.{{window_begin}}.nc4' simulated variables: [brightness_temperature] channels: &ssmis_f17_channels 1-24 obs operator: @@ -16,9 +16,9 @@ obs operator: obs options: Sensor_ID: ssmis_f17 EndianType: little_endian - CoefficientPath: $(crtm_coeff_dir) + CoefficientPath: '{{crtm_coeff_dir}}' obs bias: - input file: $(cycle_dir)/ssmis_f17.{{background_time}}.satbias.nc4 + input file: '{{cycle_dir}}/ssmis_f17.{{background_time}}.satbias.nc4' variational bc: predictors: - name: constant @@ -35,7 +35,7 @@ obs bias: - name: sine_of_latitude - name: lapse_rate order: 2 - tlapse: &ssmis_f17_tlapse $(cycle_dir)/ssmis_f17.{{background_time}}.tlapse.txt + tlapse: &ssmis_f17_tlapse '{{cycle_dir}}/ssmis_f17.{{background_time}}.tlapse.txt' - name: lapse_rate tlapse: *ssmis_f17_tlapse - name: emissivity diff --git a/src/swell/configuration/observation_operators/vadwind.yaml b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/vadwind.yaml similarity index 96% rename from src/swell/configuration/observation_operators/vadwind.yaml rename to src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/vadwind.yaml index 3b90d306..74784485 100644 --- a/src/swell/configuration/observation_operators/vadwind.yaml +++ b/src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/vadwind.yaml @@ -5,7 +5,7 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(cycle_dir)/vadwind.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/vadwind.{{window_begin}}.nc4' obsgrouping: group variables: ["station_id", "datetime"] sort variable: "air_pressure" @@ -13,7 +13,7 @@ obs space: obsdataout: engine: type: H5File - obsfile: $(cycle_dir)/$(experiment_id).vadwind.{{window_begin}}.nc4 + obsfile: '{{cycle_dir}}/{{experiment_id}}.vadwind.{{window_begin}}.nc4' simulated variables: [eastward_wind, northward_wind] #-------------------------------------------------------------------------------------------------------------------- obs filters: diff --git a/src/swell/configuration/observation_ioda_names.yaml b/src/swell/configuration/jedi/observation_ioda_names.yaml similarity index 100% rename from src/swell/configuration/observation_ioda_names.yaml rename to src/swell/configuration/jedi/observation_ioda_names.yaml diff --git a/src/swell/configuration/jedi/oops/hofx4D.yaml b/src/swell/configuration/jedi/oops/hofx4D.yaml new file mode 100644 index 00000000..f7e0a157 --- /dev/null +++ b/src/swell/configuration/jedi/oops/hofx4D.yaml @@ -0,0 +1,14 @@ +window begin: '{{window_begin_iso}}' +window length: {{window_length}} +forecast length: {{window_length}} +geometry: + TASKFILLgeometry +model: + TASKFILLmodel +initial condition: + TASKFILLbackground +observations: + get values: + TASKFILLgetvalues + observers: + TASKFILLobservations diff --git a/src/swell/suites/hofx/suite_page_hofx.py b/src/swell/configuration/jedi/soca/geos_ocean/model/background.yaml old mode 100644 new mode 100755 similarity index 100% rename from src/swell/suites/hofx/suite_page_hofx.py rename to src/swell/configuration/jedi/soca/geos_ocean/model/background.yaml diff --git a/src/swell/configuration/jedi/soca/geos_ocean/model/geometry.yaml b/src/swell/configuration/jedi/soca/geos_ocean/model/geometry.yaml new file mode 100755 index 00000000..e69de29b diff --git a/src/swell/configuration/jedi/soca/geos_ocean/model/getvalues.yaml b/src/swell/configuration/jedi/soca/geos_ocean/model/getvalues.yaml new file mode 100755 index 00000000..e69de29b diff --git a/src/swell/configuration/jedi/soca/geos_ocean/model/pseudo-model.yaml b/src/swell/configuration/jedi/soca/geos_ocean/model/pseudo-model.yaml new file mode 100755 index 00000000..e69de29b diff --git a/src/swell/configuration/jedi/soca/geos_ocean/model/r2d2.yaml b/src/swell/configuration/jedi/soca/geos_ocean/model/r2d2.yaml new file mode 100755 index 00000000..e69de29b diff --git a/src/swell/configuration/jedi/soca/geos_ocean/model/stage.yaml b/src/swell/configuration/jedi/soca/geos_ocean/model/stage.yaml new file mode 100755 index 00000000..e69de29b diff --git a/src/swell/configuration/jedi/soca/geos_ocean/observations/adt_3a.yaml b/src/swell/configuration/jedi/soca/geos_ocean/observations/adt_3a.yaml new file mode 100755 index 00000000..e69de29b diff --git a/src/swell/configuration/satellite_channels/geos_atmosphere/amsua_n19.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/amsua_n19.yaml new file mode 100755 index 00000000..1756015a --- /dev/null +++ b/src/swell/configuration/satellite_channels/geos_atmosphere/amsua_n19.yaml @@ -0,0 +1,24 @@ +available_channels: [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] +activation_date: 2009-04-14T00:00:00 +decommission_date: 2100-01-01T0:00:00 + +# Channel revisions +# ----------------- +revisions: +- date: 2009-04-14T00:00:00 + note: 'Start' + operation: + action_type: set_active_channels + channels: [4,5,6,7,8,9,10,11,12,13,14] + +- date: 2009-12-22T00:00:00 + note: 'ch8 noisy (noise started on 12/21/2009)' + operation: + action_type: remove_channels + channels: [8] + +- date: 2014-01-29T00:00:00 + note: 'per NCEP r35918' + operation: + action_type: remove_channels + channels: [7] diff --git a/src/swell/configuration/satellite_channels/aqua.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/aqua.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/aqua.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/aqua.yaml diff --git a/src/swell/configuration/satellite_channels/f08.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/f08.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/f08.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/f08.yaml diff --git a/src/swell/configuration/satellite_channels/f10.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/f10.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/f10.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/f10.yaml diff --git a/src/swell/configuration/satellite_channels/f11.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/f11.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/f11.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/f11.yaml diff --git a/src/swell/configuration/satellite_channels/f13.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/f13.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/f13.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/f13.yaml diff --git a/src/swell/configuration/satellite_channels/f14.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/f14.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/f14.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/f14.yaml diff --git a/src/swell/configuration/satellite_channels/f15.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/f15.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/f15.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/f15.yaml diff --git a/src/swell/configuration/satellite_channels/f16.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/f16.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/f16.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/f16.yaml diff --git a/src/swell/configuration/satellite_channels/f17.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/f17.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/f17.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/f17.yaml diff --git a/src/swell/configuration/satellite_channels/f18.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/f18.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/f18.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/f18.yaml diff --git a/src/swell/configuration/satellite_channels/g08.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/g08.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/g08.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/g08.yaml diff --git a/src/swell/configuration/satellite_channels/g10.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/g10.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/g10.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/g10.yaml diff --git a/src/swell/configuration/satellite_channels/g11.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/g11.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/g11.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/g11.yaml diff --git a/src/swell/configuration/satellite_channels/g12.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/g12.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/g12.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/g12.yaml diff --git a/src/swell/configuration/satellite_channels/g13.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/g13.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/g13.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/g13.yaml diff --git a/src/swell/configuration/satellite_channels/g14.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/g14.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/g14.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/g14.yaml diff --git a/src/swell/configuration/satellite_channels/g15.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/g15.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/g15.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/g15.yaml diff --git a/src/swell/configuration/satellite_channels/gcom-w1.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/gcom-w1.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/gcom-w1.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/gcom-w1.yaml diff --git a/src/swell/configuration/satellite_channels/gpm.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/gpm.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/gpm.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/gpm.yaml diff --git a/src/swell/configuration/satellite_channels/geos_atmosphere/iasi_metop-a.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/iasi_metop-a.yaml new file mode 100755 index 00000000..d4965b99 --- /dev/null +++ b/src/swell/configuration/satellite_channels/geos_atmosphere/iasi_metop-a.yaml @@ -0,0 +1,56 @@ +available_channels: [16,29,32,35,38,41,44,47,49,50,51,53,55,56,57,59,61,62,63,66,68,70,72,74,76,78, + 79,81,82,83,84,85,86,87,89,92,93,95,97,99,101,103,104,106,109,110,111,113,116, + 119,122,125,128,131,133,135,138,141,144,146,148,150,151,154,157,159,160,161, + 163,167,170,173,176,179,180,185,187,191,193,197,199,200,202,203,205,207,210, + 212,213,214,217,218,219,222,224,225,226,228,230,231,232,236,237,239,243,246, + 249,252,254,259,260,262,265,267,269,275,279,282,285,294,296,299,300,303,306, + 309,313,320,323,326,327,329,332,335,345,347,350,354,356,360,363,366,371,372, + 373,375,377,379,381,383,386,389,398,401,404,405,407,408,410,411,414,416,418, + 423,426,428,432,433,434,439,442,445,450,457,459,472,477,483,509,515,546,552, + 559,566,571,573,578,584,594,625,646,662,668,705,739,756,797,867,906,921,1027, + 1046,1090,1098,1121,1133,1173,1191,1194,1222,1271,1283,1338,1409,1414,1420, + 1424,1427,1430,1434,1440,1442,1445,1450,1454,1460,1463,1469,1474,1479,1483, + 1487,1494,1496,1502,1505,1509,1510,1513,1518,1521,1526,1529,1532,1536,1537, + 1541,1545,1548,1553,1560,1568,1574,1579,1583,1585,1587,1606,1626,1639,1643, + 1652,1658,1659,1666,1671,1675,1681,1694,1697,1710,1786,1791,1805,1839,1884, + 1913,1946,1947,1991,2019,2094,2119,2213,2239,2271,2289,2321,2333,2346,2349, + 2352,2359,2367,2374,2398,2426,2562,2701,2741,2745,2760,2819,2889,2907,2910, + 2919,2921,2939,2944,2945,2948,2951,2958,2971,2977,2985,2988,2990,2991,2993, + 3002,3008,3014,3027,3029,3030,3036,3047,3049,3052,3053,3055,3058,3064,3069, + 3087,3093,3098,3105,3107,3110,3116,3127,3129,3136,3146,3151,3160,3165,3168, + 3175,3178,3189,3207,3228,3244,3248,3252,3256,3263,3281,3295,3303,3309,3312, + 3322,3326,3354,3366,3375,3378,3411,3416,3432,3438,3440,3442,3444,3446,3448, + 3450,3452,3454,3458,3467,3476,3484,3491,3497,3499,3504,3506,3509,3518,3527, + 3555,3575,3577,3580,3582,3586,3589,3599,3610,3626,3638,3646,3653,3658,3661, + 3673,3689,3700,3710,3726,3763,3814,3841,3888,4032,4059,4068,4082,4095,4160, + 4234,4257,4411,4498,4520,4552,4567,4608,4646,4698,4808,4849,4920,4939,4947, + 4967,4991,4996,5015,5028,5056,5128,5130,5144,5170,5178,5183,5188,5191,5368, + 5371,5379,5381,5383,5397,5399,5401,5403,5405,5446,5455,5472,5480,5483,5485, + 5492,5497,5502,5507,5509,5517,5528,5558,5697,5714,5749,5766,5785,5798,5799, + 5801,5817,5833,5834,5836,5849,5851,5852,5865,5869,5881,5884,5897,5900,5916, + 5932,5948,5963,5968,5978,5988,5992,5994,5997,6003,6008,6023,6026,6039,6053, + 6056,6067,6071,6082,6085,6098,6112,6126,6135,6140,6149,6154,6158,6161,6168, + 6174,6182,6187,6205,6209,6213,6317,6339,6342,6366,6381,6391,6489,6962,6966, + 6970,6975,6977,6982,6985,6987,6989,6991,6993,6995,6997,6999,7000,7004,7008, + 7013,7016,7021,7024,7027,7029,7032,7038,7043,7046,7049,7069,7072,7076,7081, + 7084,7089,7099,7209,7222,7231,7235,7247,7267,7269,7284,7389,7419,7423,7424, + 7426,7428,7431,7436,7444,7475,7549,7584,7665,7666,7831,7836,7853,7865,7885, + 7888,7912,7950,7972,7980,7995,8007,8015,8055,8078] +activation_date: 2008-08-01T00:00:00 +decommission_date: 2021-09-08T06:00:00 + +# Channel revisions +# ----------------- +revisions: +- date: 2009-04-14T00:00:00 + note: 'Start' + operation: + action_type: set_active_channels + channels: [16,38,49,51,55,57,59,61,63,66,70,72,74,79,81,83,85,87,104,106,109,111,113,116,119, + 122,125,128,131,133,135,138,141,144,146,148,151,154,157,159,161,163,167,170,173,176, + 180,185,187,193,199,205,207,210,212,214,217,219,222,224,226,230,232,236,239,243,246, + 249,252,254,260,262,275,282,294,296,299,303,306,323,327,329,335,345,347,350,354,356, + 360,366,371,373,375,377,379,381,383,386,389,398,401,404,407,410,414,416,426,428,432, + 434,439,445,457,515,546,552,559,566,571,573,646,662,668,756,867,906,921,1027,1046, + 1121,1133,1191,1194,1271,1427,1536,1579,1585,1626,1643,1671] + diff --git a/src/swell/configuration/satellite_channels/m09.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/m09.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/m09.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/m09.yaml diff --git a/src/swell/configuration/satellite_channels/m10.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/m10.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/m10.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/m10.yaml diff --git a/src/swell/configuration/satellite_channels/metop-a.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/metop-a.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/metop-a.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/metop-a.yaml diff --git a/src/swell/configuration/satellite_channels/metop-b.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/metop-b.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/metop-b.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/metop-b.yaml diff --git a/src/swell/configuration/satellite_channels/metop-c.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/metop-c.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/metop-c.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/metop-c.yaml diff --git a/src/swell/configuration/satellite_channels/n06.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/n06.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/n06.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/n06.yaml diff --git a/src/swell/configuration/satellite_channels/n07.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/n07.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/n07.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/n07.yaml diff --git a/src/swell/configuration/satellite_channels/n08.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/n08.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/n08.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/n08.yaml diff --git a/src/swell/configuration/satellite_channels/n09.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/n09.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/n09.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/n09.yaml diff --git a/src/swell/configuration/satellite_channels/n10.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/n10.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/n10.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/n10.yaml diff --git a/src/swell/configuration/satellite_channels/n11.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/n11.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/n11.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/n11.yaml diff --git a/src/swell/configuration/satellite_channels/n12.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/n12.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/n12.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/n12.yaml diff --git a/src/swell/configuration/satellite_channels/n14.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/n14.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/n14.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/n14.yaml diff --git a/src/swell/configuration/satellite_channels/n15.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/n15.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/n15.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/n15.yaml diff --git a/src/swell/configuration/satellite_channels/n16.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/n16.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/n16.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/n16.yaml diff --git a/src/swell/configuration/satellite_channels/n17.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/n17.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/n17.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/n17.yaml diff --git a/src/swell/configuration/satellite_channels/n18.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/n18.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/n18.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/n18.yaml diff --git a/src/swell/configuration/satellite_channels/n19.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/n19.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/n19.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/n19.yaml diff --git a/src/swell/configuration/satellite_channels/n20.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/n20.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/n20.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/n20.yaml diff --git a/src/swell/configuration/satellite_channels/npp.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/npp.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/npp.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/npp.yaml diff --git a/src/swell/configuration/satellite_channels/tirosn.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/tirosn.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/tirosn.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/tirosn.yaml diff --git a/src/swell/configuration/satellite_channels/trmm.yaml b/src/swell/configuration/satellite_channels/geos_atmosphere/trmm.yaml similarity index 100% rename from src/swell/configuration/satellite_channels/trmm.yaml rename to src/swell/configuration/satellite_channels/geos_atmosphere/trmm.yaml diff --git a/src/swell/deployment/bin/swell_create_experiment.py b/src/swell/deployment/bin/swell_create_experiment.py index 128d217d..c9604e01 100644 --- a/src/swell/deployment/bin/swell_create_experiment.py +++ b/src/swell/deployment/bin/swell_create_experiment.py @@ -6,179 +6,128 @@ # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + + import click import os import shutil import yaml -from swell.install_path import swell_install_path -from swell.utilities.logger import Logger -from swell.utilities.git_utils import git_got -from swell.utilities.string_utils import replace_vars -from swell.utilities.dictionary_utilities import resolve_definitions -from swell.deployment.prep_exp_dirs import add_dir_to_conf_mkdir, copy_suite_files, \ +from swell.deployment.prep_config import prepare_config +from swell.deployment.prep_exp_dirs import copy_suite_and_platform_files, \ set_swell_path_in_modules, create_modules_csh -from swell.deployment.yaml_exploder import recursive_yaml_expansion -from swell.deployment.prep_suite import prepare_suite +from swell.deployment.prep_suite import prepare_cylc_suite_jinja2 +from swell.swell_path import get_swell_path +from swell.utilities.dictionary import dict_get +from swell.utilities.jinja2 import template_string_jinja2 +from swell.utilities.logger import Logger +from swell.utilities.welcome_message import write_welcome_message # -------------------------------------------------------------------------------------------------- @click.command() -@click.argument('config') -@click.option('--clean', '-c', is_flag=True, help='Remove any existing experiment directory') -def main(config, clean): +@click.option('-m', '--method', 'method', default='defaults', + help='Method for configuration: [\'defaults\'] or \'tui\'. If the config argument ' + + 'is present then this argument will be ignored in favor of using the existing ' + + 'configuration file.') +@click.option('-c', '--config', 'config', default=None, + help='Directory containing the suite file needed by the workflow manager') +@click.option('-t', '--cidi', 'ci_cd', default=False, + help='Setup experiment using continuous integration parameters') +def main(method, config, ci_cd): + + # Welcome message + # --------------- + write_welcome_message('Create Experiment') # Create a logger # --------------- logger = Logger('SwellCreateExperiment') + # Check arguments + # --------------- + method_options = ['defaults', 'tui'] + logger.assert_abort(method in method_options, f'Method \'{method}\' is not one of the valid ' + + f'options {method_options}.') + + # Generate the configuration file + # ------------------------------- + if config is None: + config_file = prepare_config(method, ci_cd) + else: + config_file = config + # Load experiment file # -------------------- - with open(config, 'r') as ymlfile: + with open(config_file, 'r') as ymlfile: experiment_dict = yaml.safe_load(ymlfile) - # Set experiment directory + # Extract from the config + # ----------------------- + experiment_id = dict_get(logger, experiment_dict, 'experiment_id') + experiment_root = dict_get(logger, experiment_dict, 'experiment_root') + platform = dict_get(logger, experiment_dict, 'platform', None) + suite_to_run = dict_get(logger, experiment_dict, 'suite_to_run') + model_components = dict_get(logger, experiment_dict, 'model_components') + + # Make the suite directory # ------------------------ + exp_path = os.path.join(experiment_root, experiment_id) + exp_suite_path = os.path.join(exp_path, experiment_id+'-suite') + os.makedirs(exp_suite_path, 0o755, exist_ok=True) - # Create experiment directory with user specified experiment ID and location, user can keep - # ${USER} in experiment.yaml - user = os.environ['USER'] - user_template = '${USER}' - - # replace all instances of ${USER} - for key, value in experiment_dict.items(): - if isinstance(value, str) and user_template in value: - experiment_dict[key] = experiment_dict[key].replace(user_template, user) - - exp_root = experiment_dict['experiment_root'] - exp_id = experiment_dict['experiment_id'] - experiment_dir = os.path.join(exp_root, exp_id) - - # Add to dictionary - experiment_dict.update({'experiment_dir': experiment_dir}) - experiment_dict.update({'USERNAME': user}) - - # Optionally clean up any existing directory - # ------------------------------------------ - if clean: - logger.input('Removing existing experiment directory ' + experiment_dir) - logger.info('removing' + experiment_dir) - try: - shutil.rmtree(experiment_dir) - except Exception as e: - logger.info(f'Failed to remove the existing directory, with excpetion: {e}. Continuing') - - # Create the experiment directory - # ------------------------------- - logger.info('Creating experiment directory: '+experiment_dir) - if not os.path.exists(experiment_dir): - os.makedirs(experiment_dir, 0o755) - else: - logger.info('Experiment directory is already present, overwriting files') - - # Copy experiment.yaml to the experiment directory with the experiment id name - # ---------------------------------------------------------------------------- - shutil.copy(config, os.path.join(experiment_dir, 'experiment_{}.yaml'.format(exp_id))) - logger.info('Experiment yaml copied to working directory...') - - # Create directories within experiment directory and add key to dictionary - # ------------------------------------------------------------------------ - add_dir_to_conf_mkdir(logger, experiment_dict, 'bundle_dir', 'bundle') - add_dir_to_conf_mkdir(logger, experiment_dict, 'jedi_build_dir', 'bundle/build', False) - add_dir_to_conf_mkdir(logger, experiment_dict, 'stage_dir', 'stage') - add_dir_to_conf_mkdir(logger, experiment_dict, 'suite_dir', exp_id+'-suite') - add_dir_to_conf_mkdir(logger, experiment_dict, 'cycle_dir', 'run/{{current_cycle}}', False) - add_dir_to_conf_mkdir(logger, experiment_dict, 'geos_dir', 'geos') - - # Put the swell install path in to the config - # ------------------------------------------- - swell_dir = swell_install_path() - add_dir_to_conf_mkdir(logger, experiment_dict, 'swell_dir', swell_dir, False) - - # Resolve all dictionary definitions - # ---------------------------------- - experiment_dict = resolve_definitions(experiment_dict) - - # Copy files to the suite directory + # Copy experiment file to suite dir # --------------------------------- - copy_suite_files(logger, experiment_dict) - - # Set the swell paths in the modules file - # --------------------------------------- - set_swell_path_in_modules(logger, experiment_dict) - - # Clone the git repos needed for the yaml file explosion - # ------------------------------------------------------ - # User chosen clones - if 'build jedi' in experiment_dict: - if 'bundle repos' in experiment_dict['build jedi']: - needed_repos = [] - bundle_repos = experiment_dict['build jedi']['bundle repos'] - for bundle_repo in bundle_repos: - if 'clone on create' in bundle_repo.keys(): - if bundle_repo['clone on create']: - needed_repos.append(bundle_repo['project']) - - # Remove duplicates - needed_repos = list(set(needed_repos)) - - # Clone only the needed repos - repo_dict = experiment_dict['build jedi']['bundle repos'] - for d in repo_dict: - if d['project'] in needed_repos: - project = d['project'] - git_url = d['git url'] - branch = d['branch'] - proj_dir = os.path.join(experiment_dict['bundle_dir'], project) - git_got(git_url, branch, proj_dir, logger) - - # Expand yaml - # ----------- - recursive_yaml_expansion(experiment_dict) - - # Resolve all dictionary definitions for full yaml - # ------------------------------------------------ - experiment_dict = resolve_definitions(experiment_dict) - - # Prepare the suite driver file - # ----------------------------- - prepare_suite(logger, experiment_dict) - - # Write the complete experiment yaml to suite directory - # ----------------------------------------------------- - output_file_name = os.path.join(experiment_dict['suite_dir'], 'experiment-filled.yaml') - with open(output_file_name, 'w') as output_file: - yaml.dump(experiment_dict, output_file, default_flow_style=False) - - # Create full path to the r2d2 config file - # ---------------------------------------- - r2d2_conf_path = os.path.join(experiment_dict['suite_dir'], 'r2d2_config.yaml') + shutil.copyfile(config_file, os.path.join(exp_suite_path, 'experiment.yaml')) + + # Copy suite and platform files to experiment suite directory + # ----------------------------------------------------------- + swell_suite_path = os.path.join(get_swell_path(), 'suites', suite_to_run) + copy_suite_and_platform_files(logger, swell_suite_path, exp_suite_path, platform) + + # Create R2D2 database file + # ------------------------- + r2d2_conf_path = os.path.join(exp_suite_path, 'r2d2_config.yaml') # Write R2D2_CONFIG to modules - # ---------------------------- - with open(os.path.join(experiment_dict['suite_dir'], 'modules'), 'a') as module_file: - module_file.write('export R2D2_CONFIG={}'.format(r2d2_conf_path)) + with open(os.path.join(exp_suite_path, 'modules'), 'a') as module_file: + module_file.write(f'export R2D2_CONFIG={r2d2_conf_path}') # Open the r2d2 file to dictionary - # ------------------------------------ with open(r2d2_conf_path, 'r') as r2d2_file_open: r2d2_file_str = r2d2_file_open.read() - r2d2_file_str = replace_vars(r2d2_file_str, **experiment_dict) + r2d2_file_str = template_string_jinja2(logger, r2d2_file_str, experiment_dict) + r2d2_file_str = os.path.expandvars(r2d2_file_str) with open(r2d2_conf_path, 'w') as r2d2_file_open: r2d2_file_open.write(r2d2_file_str) - # Create csh modules file for csh users to use when debugging - # ----------------------------------------------------------- - create_modules_csh(logger, experiment_dict) + # Set the swell paths in the modules file and create csh versions + # --------------------------------------------------------------- + set_swell_path_in_modules(logger, exp_suite_path) + create_modules_csh(logger, exp_suite_path) + + # Set the jinja2 file for cylc + # ---------------------------- + prepare_cylc_suite_jinja2(logger, swell_suite_path, exp_suite_path, experiment_dict) + + # Copy config directory to experiment + # ----------------------------------- + src = os.path.join(get_swell_path(), 'configuration') + dst = os.path.join(exp_path, 'configuration') + if os.path.exists(dst) and os.path.isdir(dst): + shutil.rmtree(dst) + shutil.copytree(src, dst, ignore=shutil.ignore_patterns('*.py*', '*__*')) # Write out launch command for convenience # ---------------------------------------- logger.info(' ') logger.info(' Experiment successfully installed. To launch experiment use: ') - logger.info(' swell_launch_experiment --suite_path ' + experiment_dict['suite_dir']) + logger.info(' swell_launch_experiment --suite_path ' + exp_suite_path, False) logger.info(' ') diff --git a/src/swell/deployment/bin/swell_launch_experiment.py b/src/swell/deployment/bin/swell_launch_experiment.py index 66401e82..fd5bf485 100644 --- a/src/swell/deployment/bin/swell_launch_experiment.py +++ b/src/swell/deployment/bin/swell_launch_experiment.py @@ -16,7 +16,7 @@ # local imports from swell.utilities.logger import Logger - +from swell.utilities.welcome_message import write_welcome_message # -------------------------------------------------------------------------------------------------- @@ -102,6 +102,10 @@ def cylc_run_experiment(self): # NB: Could be a factory based on workflow_manag help='Directory to receive workflow manager logging output') def main(suite_path, workflow_manager, no_detach, log_path): + # Welcome message + # --------------- + write_welcome_message('Launch Experiment') + # Get the path to where the suite files are located # ------------------------------------------------- if suite_path is None: diff --git a/src/swell/deployment/bin/swell_prepare_config.py b/src/swell/deployment/bin/swell_prepare_config.py new file mode 100644 index 00000000..354a5b15 --- /dev/null +++ b/src/swell/deployment/bin/swell_prepare_config.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +# (C) Copyright 2022 United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import click + +from swell.deployment.prep_config import prepare_config +from swell.utilities.welcome_message import write_welcome_message + + +# -------------------------------------------------------------------------------------------------- + + +@click.command() +@click.argument('method') +def main(method): + + # Welcome message + # --------------- + write_welcome_message('Prepare Config') + + # Create suites object + # -------------------- + config_file = prepare_config(method) + + +# -------------------------------------------------------------------------------------------------- + + +if __name__ == '__main__': + main() + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/deployment/platforms/nccs_discover/experiment.yaml b/src/swell/deployment/platforms/nccs_discover/experiment.yaml new file mode 100644 index 00000000..cd9e83b7 --- /dev/null +++ b/src/swell/deployment/platforms/nccs_discover/experiment.yaml @@ -0,0 +1,6 @@ +default: + experiment_id: swell-{{suite_to_run}} + experiment_root: /discover/nobackup/${USER}/SwellExperiments +ci_cd: + experiment_id: ${CICD_EXPERIMENT_ID}-{{datetime}} + experiment_root: ${CICD_EXPERIMENT_ROOT} diff --git a/src/swell/deployment/platforms/nccs_discover/modules b/src/swell/deployment/platforms/nccs_discover/modules index 1e45cbd7..c896fadd 100644 --- a/src/swell/deployment/platforms/nccs_discover/modules +++ b/src/swell/deployment/platforms/nccs_discover/modules @@ -32,5 +32,5 @@ module load eva/1.2.2 # Set the swell paths # ------------------- -PATH=$(swell_bin_path):$PATH -PYTHONPATH=$(swell_lib_path):$PYTHONPATH +PATH={{swell_bin_path}}:$PATH +PYTHONPATH={{swell_lib_path}}:$PYTHONPATH diff --git a/src/swell/deployment/platforms/nccs_discover/r2d2_config.yaml b/src/swell/deployment/platforms/nccs_discover/r2d2_config.yaml index 1330274b..493e97f3 100755 --- a/src/swell/deployment/platforms/nccs_discover/r2d2_config.yaml +++ b/src/swell/deployment/platforms/nccs_discover/r2d2_config.yaml @@ -1,8 +1,8 @@ databases: - $(USERNAME): + ${USER}: class: LocalDB - root: $(R2D2_LOCAL_PATH) + root: {{r2d2_local_path}} cache_fetch: false gmao-shared: @@ -16,6 +16,6 @@ fetch_order: # when storing data, in which order should the databases accessed? store_order: - - $(USERNAME) + - ${USER} -cache_name: $(USERNAME) +cache_name: ${USER} diff --git a/src/swell/deployment/prep_config.py b/src/swell/deployment/prep_config.py new file mode 100644 index 00000000..1d39269a --- /dev/null +++ b/src/swell/deployment/prep_config.py @@ -0,0 +1,138 @@ +# (C) Copyright 2022 United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import datetime +import os +import importlib +import ruamel.yaml as ry +import sys +import yaml + +from swell.utilities.logger import Logger +from swell.swell_path import get_swell_path +from swell.utilities.dictionary import dict_get, add_comments_to_dictionary +from swell.utilities.jinja2 import template_string_jinja2 + + +# -------------------------------------------------------------------------------------------------- + + +def platform_fill(logger, experiment_dict, ci_cd): + + # Get platform + platform = experiment_dict['platform'] + + # Open platform experiment.yaml + platform_exp_file = os.path.join(get_swell_path(), 'deployment', 'platforms', platform, + 'experiment.yaml') + + with open(platform_exp_file, 'r') as platform_exp_file_open: + platform_exp_templated = platform_exp_file_open.read() + + # Create a template dictionary + temp_experiment_dict = experiment_dict + temp_experiment_dict['datetime'] = datetime.datetime.today().strftime("%Y%m%d_%H%M%SZ") + + # Resolve any templates + platform_exp_str = template_string_jinja2(logger, platform_exp_templated, temp_experiment_dict) + + # Load to dictionary + platform_dict = yaml.safe_load(platform_exp_str) + + # Set platform dictionary to use + dict_to_use = 'default' + if ci_cd: + dict_to_use = 'ci_cd' + + # Ensure environment variables are set + logger.assert_abort(os.environ.get('CICD_EXPERIMENT_ID') is not None, + 'If running with CI/CD the environment variable ' + + '${CICD_EXPERIMENT_ID} must be set.') + logger.assert_abort(os.environ.get('CICD_EXPERIMENT_ROOT') is not None, + 'If running with CI/CD the environment variable ' + + '${CICD_EXPERIMENT_ROOT} must be set.') + + # Update experiment dict + experiment_dict_new = experiment_dict + experiment_dict_new['experiment_id'] = platform_dict[dict_to_use]['experiment_id'] + experiment_dict_new['experiment_root'] = platform_dict[dict_to_use]['experiment_root'] + + return experiment_dict_new + + +# -------------------------------------------------------------------------------------------------- + + +def prepare_config(method, ci_cd=False): + + # Create a logger + # --------------- + logger = Logger('SwellPrepSuiteConfig') + + # Starting point for configuration generation + # ------------------------------------------- + config_file = os.path.join(get_swell_path(), 'suites', 'suites.yaml') + + # Assert valid method + # ------------------- + valid_tasks = ['defaults', 'gui', 'cli'] + if method not in valid_tasks: + logger.abort(f'In Suites constructor method \'{method}\' not one of the valid ' + + f'tasks {valid_tasks}') + + # Set the object that will be used to populate dictionary options + # --------------------------------------------------------------- + PrepUsing = getattr(importlib.import_module('swell.deployment.prep_config_'+method), + 'PrepConfig'+method.capitalize()) + prep_using = PrepUsing(logger, config_file) + + # Call the config prep step + # ------------------------- + prep_using.execute() + + # Set platform specific entires + # ----------------------------- + experiment_dict = platform_fill(logger, prep_using.experiment_dict, ci_cd) + + # Write final experiment dictionary + # --------------------------------- + experiment_id = dict_get(logger, experiment_dict, 'experiment_id') + experiment_rt = dict_get(logger, experiment_dict, 'experiment_root') + + experiment_id = os.path.expandvars(experiment_id) + experiment_rt = os.path.expandvars(experiment_rt) + experiment_dict['experiment_id'] = experiment_id + experiment_dict['experiment_root'] = experiment_rt + + # Make directory + # -------------- + experiment_root_id = os.path.join(experiment_rt, experiment_id) + os.makedirs(experiment_root_id, exist_ok=True) + + # Add comments to dictionary + # -------------------------- + experiment_dict_string = yaml.dump(experiment_dict, default_flow_style=False, sort_keys=False) + + experiment_dict_string_comments = add_comments_to_dictionary(experiment_dict_string, + prep_using.comment_dict) + + # Dictionary file to write + exp_dict_file = os.path.join(experiment_root_id, 'experiment.yaml') + + # Write dictionary to YAML file + exp_dict_file_open = open(exp_dict_file, "w") + n = exp_dict_file_open.write(experiment_dict_string_comments) + exp_dict_file_open.close() + logger.info(f'Prepared configuration file written to {exp_dict_file}', False) + + return exp_dict_file + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/deployment/prep_config_base.py b/src/swell/deployment/prep_config_base.py new file mode 100644 index 00000000..22641573 --- /dev/null +++ b/src/swell/deployment/prep_config_base.py @@ -0,0 +1,205 @@ +# (C) Copyright 2022 United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +from abc import ABC, abstractmethod +import datetime +import os +import yaml + +from swell.swell_path import get_swell_path + +# -------------------------------------------------------------------------------------------------- + + +class PrepConfigBase(ABC): + + def __init__(self, logger, dictionary_file): + + # Store a logger for all to use + self.logger = logger + + # Get the path and filename of the dictionary + self.directory = os.path.join(get_swell_path(), 'suites') + self.filename = os.path.splitext(os.path.basename(dictionary_file))[0] + + # Dictionary + self.dictionary = self.read_dictionary_file(dictionary_file) + + # Keep track of the model, atmosphere, ocean etc + self.model = None + + # Extension to use for dictionary files + self.dictionary_extension = '.yaml' + + # Experiment dictionary to be created and used in swell + self.experiment_dict = {} + + # Comment dictionary to be created and used to add comments to config file + self.comment_dict = {} + + # Dictionary validation things + self.valid_types = ['string', 'iso-datetime', 'iso-duration', 'drop-list-string', + 'check-list-string', 'file-drop-list', 'file-check-list', 'boolean'] + + # Disallowed element types + self.dis_elem_types = [datetime.datetime, datetime.date] + + # ---------------------------------------------------------------------------------------------- + + def append_directory_and_filename(self, sub_dict_name): + + self.directory = os.path.join(self.directory, sub_dict_name) + self.filename = self.filename + '-' + sub_dict_name + + # ---------------------------------------------------------------------------------------------- + + def subtract_directory_and_filename(self): + + self.directory = os.path.dirname(self.directory) + self.filename = '-'.join(self.filename.split('-')[0:-1]) + + # ---------------------------------------------------------------------------------------------- + + def open_dictionary(self): + + # Append the filename according the type of files + filename_ext = os.path.join(self.directory, self.filename + self.dictionary_extension) + + # Open file into dictionary + dictionary = self.read_dictionary_file(filename_ext) + + # Check that dictionary contained something + if dictionary is None: + self.logger.abort(f'Dictionary at {filename_ext} returned {None} when opened') + + return dictionary + + # ---------------------------------------------------------------------------------------------- + + def read_dictionary_file(self, dictionary_file): + + # Open file and load as dictionary + with open(dictionary_file, 'r') as dictionary_file_open: + dictionary = yaml.safe_load(dictionary_file_open) + + return dictionary + + # ---------------------------------------------------------------------------------------------- + + def validate_dictionary(self, dictionary): + + # Check for required key + required_keys = ['default_value', 'prompt', 'type'] + for required_key in required_keys: + if required_key not in dictionary.keys(): + self.logger.abort(f'Each section of the suites config files must contain the key ' + + f'\'{required_key}\'. Offending dictionary: \n {dictionary}') + + # Check that type is supported + type = dictionary['type'] + if type not in self.valid_types: + self.logger.abort(f'Dictionary has type \'{type}\' that is not one of the supported ' + + f'types: {self.valid_types}. Offending dictionary: \n {dictionary}') + + # ---------------------------------------------------------------------------------------------- + + def update_model(self, model): + + if model is None: + self.model = None + else: + self.model = model + + # If models list not already in the dictionary added + if 'models' not in self.experiment_dict.keys(): + self.experiment_dict['models'] = {} + + # If specific model dictionary not added to the list of model then add it + if self.model not in self.experiment_dict['models'].keys(): + self.experiment_dict['models'][self.model] = {} + + # ---------------------------------------------------------------------------------------------- + + def add_to_experiment_dictionary(self, key, element_dict): + + # Set the element + # --------------- + element = element_dict['default_value'] + prompt = element_dict['prompt'] + + # Validate the element + # -------------------- + + # Ensure always a list to make following logic not need to check if list or not + if not isinstance(element, list): + element_items = [element] + else: + element_items = element + + # Check for disallowed element types + for element_item in element_items: + element_item_type = type(element_item) + for dis_elem_type in self.dis_elem_types: + if isinstance(element_item, dis_elem_type): + self.logger.abort(f'Element \'{element}\' has a type that is not permitted. ' + + f'Type is \'{dis_elem_type}\'. Try replacing with a string ' + + f'in the configuration file.') + + # Validate the key + # ---------------- + + # Ensure there are no spaces in the key + if ' ' in key: + self.logger.abort(f'Key \'{key}\' contains a space. For consistency across the ' + + f'configurations please avoid spaces and instead use _ if needed.') + + # Check that dictionary does not already contain the key + if key in self.experiment_dict.keys(): + self.logger.abort(f'Key \'{key}\' is already in the experiment dictionary.') + + # Make sure the element was not already added + # ------------------------------------------- + if self.model is None: + if key in self.experiment_dict.keys(): + self.logger.abort(f'Key \'{key}\' is already in the experiment dictionary.') + else: + if key in self.experiment_dict['models'][self.model].keys(): + self.logger.abort(f'Key \'{key}\' is already in the experiment dictionary.') + + # Add element + # ----------- + if self.model is None: + self.experiment_dict[key] = element + else: + self.experiment_dict['models'][self.model][key] = element + + # Add option + # ---------- + if self.model is None: + option_key = key + else: + if 'models' not in self.comment_dict.keys(): + self.comment_dict['models'] = 'Options for individual model components' + if 'models.' + self.model not in self.comment_dict.keys(): + self.comment_dict['models.' + self.model] = f'Options for the {self.model} ' + \ + f'model component' + option_key = 'models.' + self.model + '.' + key + self.comment_dict[option_key] = prompt + + # ---------------------------------------------------------------------------------------------- + + @abstractmethod + def execute(self): + pass + # The subclass has to implement an execute method since this is how it is called into + # action. + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/configuration/configuration.py b/src/swell/deployment/prep_config_cli.py similarity index 64% rename from src/swell/configuration/configuration.py rename to src/swell/deployment/prep_config_cli.py index bde1f323..61ad4ac4 100644 --- a/src/swell/configuration/configuration.py +++ b/src/swell/deployment/prep_config_cli.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2022 United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 @@ -8,14 +8,17 @@ # -------------------------------------------------------------------------------------------------- -import os +from swell.deployment.prep_config_base import PrepConfigBase # -------------------------------------------------------------------------------------------------- -def return_configuration_path(): - return os.path.split(__file__)[0] +class PrepConfigCli(PrepConfigBase): + + def execute(self): + + self.logger.abort('This is where a cli interface will do its work') # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/deployment/prep_config_defaults.py b/src/swell/deployment/prep_config_defaults.py new file mode 100644 index 00000000..74650933 --- /dev/null +++ b/src/swell/deployment/prep_config_defaults.py @@ -0,0 +1,106 @@ +# (C) Copyright 2022 United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import os +import yaml + +from swell.deployment.prep_config_base import PrepConfigBase + + +# -------------------------------------------------------------------------------------------------- + + +class PrepConfigDefaults(PrepConfigBase): + + # ---------------------------------------------------------------------------------------------- + + def execute(self, dictionary=None): + + # Set dictionary to use in this scope + if dictionary is None: + dictionary = self.dictionary + + for key in dictionary: + + # Element dictionary + el_dict = dictionary[key] + + if key != 'fixed_options': + + # Validate the element dictionary + self.validate_dictionary(el_dict) + + # Extract type + type = el_dict['type'] + + # If the type is not another file add to dictionary + if 'file' not in type: + + # Check that the key does not have a dependency + depends_flag = True + if 'depends' in el_dict.keys(): + dep_key = el_dict['depends']['key'] + dep_val = el_dict['depends']['value'] + if self.experiment_dict[dep_key] != dep_val: + depends_flag = False + + # In this case the key is not expected to refer to a sub dictionary but have + # everything needed in the elements dictionary + if depends_flag: + self.add_to_experiment_dictionary(key, el_dict) + + elif 'file-drop-list' in type: + + # Add the choice to the dictionary + self.add_to_experiment_dictionary(key, el_dict) + + # In this case the key refers to a single sub dictionary that involves opening + # that dictionary and recursively calling this routine. + + # First append the directory and filename to denote moving to the sub dictionary + self.append_directory_and_filename(el_dict['default_value']) + + # Open next level down dictionary and recursively add + self.execute(self.open_dictionary()) + + # As we come back from the sub dictionary subtract the directory and filename + self.subtract_directory_and_filename() + + elif 'file-check-list' in type: + + # Add the choice to the dictionary + self.add_to_experiment_dictionary(key, el_dict) + + # In this case the key asks the user to provide a list of items that correspond + # to sub dictionaries. Inside a loop this method is called recursively. + options = el_dict['default_value'] + for option in options: + + # If the key is models change the internal model to match this model + if key == 'model_components': + self.update_model(option) + + self.append_directory_and_filename(option) + self.execute(self.open_dictionary()) + self.subtract_directory_and_filename() + + if key == 'model_components': + self.update_model(None) + + else: + + for fixed_key in el_dict: + + self.add_to_experiment_dictionary(fixed_key, el_dict[fixed_key]) + + return + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/install_path.py b/src/swell/deployment/prep_config_gui.py similarity index 64% rename from src/swell/install_path.py rename to src/swell/deployment/prep_config_gui.py index 70f487d5..9e4e1549 100644 --- a/src/swell/install_path.py +++ b/src/swell/deployment/prep_config_gui.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2022 United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 @@ -8,14 +8,17 @@ # -------------------------------------------------------------------------------------------------- -import os +from swell.deployment.prep_config_base import PrepConfigBase # -------------------------------------------------------------------------------------------------- -def swell_install_path(): - return os.path.split(__file__)[0] +class PrepConfigGui(PrepConfigBase): + + def execute(self): + + self.logger.abort('This is where a gui interface will do its work') # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/deployment/prep_exp_dirs.py b/src/swell/deployment/prep_exp_dirs.py index 1701b1d5..5003f77b 100644 --- a/src/swell/deployment/prep_exp_dirs.py +++ b/src/swell/deployment/prep_exp_dirs.py @@ -13,64 +13,28 @@ import pathlib import shutil -from swell.install_path import swell_install_path -from swell.suites.suites import return_suite_path -from swell.utilities.string_utils import replace_vars +from swell.swell_path import get_swell_path +from swell.utilities.jinja2 import template_string_jinja2 # -------------------------------------------------------------------------------------------------- -def add_dir_to_conf_mkdir(logger, experiment_dict, experiment_dict_key, experiment_sub_dir, - make_dir=True): - - # Get experiment directory - experiment_dir = experiment_dict['experiment_dir'] - experiment_sub_dir_full = os.path.join(experiment_dir, experiment_sub_dir) - - if make_dir: - # Make the new directory - os.makedirs(experiment_sub_dir_full, exist_ok=True) - - # Set permissions - os.chmod(experiment_sub_dir_full, 0o755) - - # Add the associated key to the dictionary - experiment_dict.update({experiment_dict_key: experiment_sub_dir_full}) - - -# -------------------------------------------------------------------------------------------------- - - -def copy_suite_files(logger, experiment_dict): - - # Extract config - # -------------- - suite_dir = experiment_dict['suite_dir'] - - suite_dict = experiment_dict['suite'] - suite_name = suite_dict['suite name'] +def copy_suite_and_platform_files(logger, swell_suite_path, exp_suite_path, platform=None): # Copy suite related files to the suite directory # ----------------------------------------------- - suite_files = [ - 'jedi_config.yaml', - 'flow.cylc', - 'eva.yaml', - ] - - suite_path = return_suite_path() + suite_files = ['eva.yaml'] for suite_file in suite_files: - src_path_file = os.path.join(suite_path, suite_name, suite_file) - dst_path_file = os.path.join(suite_dir, suite_file) + src_path_file = os.path.join(swell_suite_path, suite_file) + dst_path_file = os.path.join(exp_suite_path, suite_file) if os.path.exists(src_path_file): - logger.trace('Copying {} to {}'.format(src_path_file, dst_path_file)) + logger.trace(f'Copying {src_path_file} to {dst_path_file}') shutil.copy(src_path_file, dst_path_file) # Copy platform related files to the suite directory # -------------------------------------------------- - if 'platform' in suite_dict: - platform = suite_dict['platform'] + if platform is not None: plat_mod = importlib.import_module('swell.deployment.platforms.'+platform+'.install_path') return_platform_install_path_call = getattr(plat_mod, 'return_platform_install_path') platform_path = return_platform_install_path_call() @@ -78,7 +42,7 @@ def copy_suite_files(logger, experiment_dict): for s in ['modules', 'r2d2_config.yaml']: src_file = os.path.split(s)[1] src_path_file = os.path.join(platform_path, os.path.split(s)[0], src_file) - dst_path_file = os.path.join(suite_dir, '{}'.format(src_file)) + dst_path_file = os.path.join(exp_suite_path, '{}'.format(src_file)) if os.path.exists(src_path_file): logger.trace('Copying {} to {}'.format(src_path_file, dst_path_file)) shutil.copy(src_path_file, dst_path_file) @@ -87,15 +51,11 @@ def copy_suite_files(logger, experiment_dict): # -------------------------------------------------------------------------------------------------- -def set_swell_path_in_modules(logger, experiment_dict): - - # Extract config - # -------------- - suite_dir = experiment_dict['suite_dir'] +def set_swell_path_in_modules(logger, exp_suite_path): # Modules file # ------------ - modules_file = os.path.join(suite_dir, 'modules') + modules_file = os.path.join(exp_suite_path, 'modules') # Only do if the suite needs modules # ---------------------------------- @@ -108,12 +68,12 @@ def set_swell_path_in_modules(logger, experiment_dict): # Swell lib path # -------------- - swell_lib_path = swell_install_path() + swell_lib_path = get_swell_path() swell_lib_path = os.path.split(swell_lib_path)[0] # Swell suite path # ---------------- - swell_sui_path = return_suite_path() + swell_sui_path = os.path.join(get_swell_path(), 'suites') # Dictionary of definitions # ------------------------- @@ -126,7 +86,10 @@ def set_swell_path_in_modules(logger, experiment_dict): # ------------- with open(modules_file, 'r') as modules_file_open: modules_file_str = modules_file_open.read() - modules_file_str = replace_vars(modules_file_str, **swell_paths) + + # Resolve templates + # ----------------- + modules_file_str = template_string_jinja2(logger, modules_file_str, swell_paths) # Overwrite the file # ------------------ @@ -137,15 +100,11 @@ def set_swell_path_in_modules(logger, experiment_dict): # -------------------------------------------------------------------------------------------------- -def create_modules_csh(logger, experiment_dict): - - # Extract config - # -------------- - suite_dir = experiment_dict['suite_dir'] +def create_modules_csh(logger, exp_suite_path): # Modules file # ------------ - modules_file = os.path.join(suite_dir, 'modules') + modules_file = os.path.join(exp_suite_path, 'modules') # Only do if the suite needs modules # ---------------------------------- diff --git a/src/swell/deployment/prep_suite.py b/src/swell/deployment/prep_suite.py index 2653f787..3f5c929f 100644 --- a/src/swell/deployment/prep_suite.py +++ b/src/swell/deployment/prep_suite.py @@ -8,55 +8,63 @@ # -------------------------------------------------------------------------------------------------- -import importlib import os +from swell.utilities.jinja2 import template_string_jinja2 -# -------------------------------------------------------------------------------------------------- - - -def prepare_suite(logger, experiment_dict): - - # Configuration things - # -------------------- - suite_dir = experiment_dict['suite_dir'] - - suite_dict = experiment_dict['suite'] - - # Setup the platform specific - if 'platform' in suite_dict and 'scheduling' in suite_dict: - platform = suite_dict['platform'] - scheduling_dicts = suite_dict['scheduling'] - # Loop and get default scheduling dictionary - # ------------------------------------------ - for scheduling_dict in scheduling_dicts: - if scheduling_dict['task'] == 'Default': - def_scheduling_dict = scheduling_dict - - # Get default tasks per node depending on platform/constraint - plat_mod = importlib.import_module('swell.deployment.platforms.'+platform+'.scheduling') - get_tasks_per_node_call = getattr(plat_mod, 'get_tasks_per_node') - dtpn = get_tasks_per_node_call(platform, def_scheduling_dict['constraint']) - def_scheduling_dict['ntasks_per_node'] = dtpn - - # Loop and set task dictionaries - # ------------------------------ - for scheduling_dict in scheduling_dicts: - - if scheduling_dict['task'] != 'Default': - # Merge the default and task specific values - task_merged = ({**def_scheduling_dict, **scheduling_dict}) - suite_dict[task_merged['task']] = task_merged - - del suite_dict['scheduling'] +# -------------------------------------------------------------------------------------------------- - # Convert dictionary to jinja2 like string - suite_dict_jinja2_str = '{% set suite_properties = '+str(suite_dict)+'%}' - # Write jinja2 dictionary to file in suite directory - with open(os.path.join(suite_dir, 'suite.jinja2'), 'w') as suite_jinja2: - suite_jinja2.write(suite_dict_jinja2_str) +def prepare_cylc_suite_jinja2(logger, swell_suite_path, exp_suite_path, experiment_dict): + + # Open suite file from swell + # -------------------------- + with open(os.path.join(swell_suite_path, 'flow.cylc'), 'r') as file: + suite_file = file.read() + + # Copy the experiment dictionary to the rendering dictionary + # ---------------------------------------------------------- + render_dictionary = experiment_dict + + # Get unique list of cycle times with model flags to render dictionary + # -------------------------------------------------------------------- + cycle_times = [] + for model in experiment_dict['model_components']: + cycle_times = list(set(cycle_times + experiment_dict['models'][model]['cycle_times'])) + cycle_times.sort() + + cycle_times_dict_list = [] + for cycle_time in cycle_times: + cycle_time_dict = {} + cycle_time_dict['cycle_time'] = cycle_time + for model in experiment_dict['model_components']: + cycle_time_dict[model] = False + if cycle_time in experiment_dict['models'][model]['cycle_times']: + cycle_time_dict[model] = True + cycle_times_dict_list.append(cycle_time_dict) + + render_dictionary['cycle_times'] = cycle_times_dict_list + + # Add scheduling to the render dictionary (TODO: do not hard code this) + # --------------------------------------- + render_dictionary['scheduling'] = {} + render_dictionary['scheduling']['RunJediHofxExecutable'] = {} + render_dictionary['scheduling']['RunJediHofxExecutable']['execution_time_limit'] = 'PT1H' + render_dictionary['scheduling']['RunJediHofxExecutable']['account'] = 'g0613' + render_dictionary['scheduling']['RunJediHofxExecutable']['qos'] = 'debug' + render_dictionary['scheduling']['RunJediHofxExecutable']['constraint'] = 'hasw' + render_dictionary['scheduling']['RunJediHofxExecutable']['nodes'] = 1 + render_dictionary['scheduling']['RunJediHofxExecutable']['ntasks_per_node'] = 24 + + # Render the template + # ------------------- + new_suite_file = template_string_jinja2(logger, suite_file, render_dictionary) + + # Write suite file to experiment + # ------------------------------ + with open(os.path.join(exp_suite_path, 'flow.cylc'), 'w') as file: + file.write(new_suite_file) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/deployment/yaml_exploder.py b/src/swell/deployment/yaml_exploder.py deleted file mode 100644 index 38088216..00000000 --- a/src/swell/deployment/yaml_exploder.py +++ /dev/null @@ -1,56 +0,0 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - - -import yaml - - -# -------------------------------------------------------------------------------------------------- - - -def recursive_yaml_expansion(experiment_dict): - ''' - Replace paths to yaml with the dictionary defined in that yaml - ''' - # Loop over experiment yaml and expand - for k in experiment_dict.keys(): - - # Get the dictionary element (might be a list) - experiment_item = experiment_dict[k] - - # Create list of items - params = [] - if isinstance(experiment_item, list): - params = params + experiment_item - else: - params.append(experiment_item) - - # Check if the item list contains the key string 'yaml::' - if 'yaml::' in str(params): - - # Create empty list of dictionaries to be filled - exp_list = [] - - for param in params: - - # Read yaml into dictionary and append - with open(param.split('yaml::')[1], 'r') as yaml_file: - sub_yaml_dict = yaml.safe_load(yaml_file) - - # Append the dictionary to the expanded list - exp_list.append(sub_yaml_dict) - - # Write the expanded list element to the original dictionary key - if len(exp_list) > 1 or isinstance(experiment_item, list): - experiment_dict[k] = exp_list - else: - experiment_dict[k] = exp_list[0] - - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/hofx/eva.yaml b/src/swell/suites/hofx/eva.yaml index 192e3104..220f8744 100644 --- a/src/swell/suites/hofx/eva.yaml +++ b/src/swell/suites/hofx/eva.yaml @@ -6,11 +6,11 @@ diagnostics: - name: experiment filenames: - - ${obs_path_file} - channels: &channels $(channels) + - {{obs_path_file}} + channels: &channels {{channels}} groups: - name: ObsValue - variables: &variables $[simulated_variables] + variables: &variables {{simulated_variables}} - name: GsiHofXBc #- name: GsiEffectiveQC - name: hofx @@ -87,8 +87,8 @@ diagnostics: channels: *channels figure: layout: [1,1] - title: 'Observations vs. JEDI h(x) | $(instrument_title) | ${variable_title}' - output name: $(cycle_dir)/eva/$(instrument)/correlation_scatter/${variable}${channel}/jedi_hofx_vs_obs_$(instrument)_${variable}${channel}.png + title: 'Observations vs. JEDI h(x) | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/correlation_scatter/${variable}${channel}/jedi_hofx_vs_obs_{{instrument}}_${variable}${channel}.png' plots: - add_xlabel: 'Observation Value' add_ylabel: 'JEDI h(x)' @@ -121,8 +121,8 @@ diagnostics: channels: *channels figure: layout: [1,1] - title: 'Observations vs. GSI h(x) | $(instrument_title) | ${variable_title}' - output name: $(cycle_dir)/eva/$(instrument)/correlation_scatter/${variable}${channel}/gsi_hofx_vs_obs_$(instrument)_${variable}${channel}.png + title: 'Observations vs. GSI h(x) | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/correlation_scatter/${variable}${channel}/gsi_hofx_vs_obs_{{instrument}}_${variable}${channel}.png' plots: - add_xlabel: 'Observation Value' add_ylabel: 'GSI h(x)' @@ -155,8 +155,8 @@ diagnostics: channels: *channels figure: layout: [1,1] - title: 'JEDI h(x) vs. GSI h(x) | $(instrument_title) | ${variable_title}' - output name: $(cycle_dir)/eva/$(instrument)/correlation_scatter/${variable}${channel}/gsi_hofx_vs_jedi_hofx_$(instrument)_${variable}${channel}.png + title: 'JEDI h(x) vs. GSI h(x) | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/correlation_scatter/${variable}${channel}/gsi_hofx_vs_jedi_hofx_{{instrument}}_${variable}${channel}.png' plots: - add_xlabel: 'GSI h(x)' add_ylabel: 'JEDI h(x)' @@ -189,8 +189,8 @@ diagnostics: channels: *channels figure: layout: [1,1] - title: 'JEDI omb vs. GSI omb | $(instrument_title) | ${variable_title}' - output name: $(cycle_dir)/eva/$(instrument)/correlation_scatter/${variable}${channel}/gsi_omb_vs_jedi_omb_$(instrument)_${variable}${channel}.png + title: 'JEDI omb vs. GSI omb | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/correlation_scatter/${variable}${channel}/gsi_omb_vs_jedi_omb_{{instrument}}_${variable}${channel}.png' plots: - add_xlabel: 'GSI observation minus h(x)' add_ylabel: 'JEDI observation minus h(x)' @@ -231,8 +231,8 @@ diagnostics: figure: figure size: [20,10] layout: [1,1] - title: 'Observations | $(instrument_title) | Obs Value' - output name: $(cycle_dir)/eva/$(instrument)/map_plots/${variable}${channel}/observations_$(instrument)_${variable}${channel}.png + title: 'Observations | {{instrument_title}} | Obs Value' + output name: '{{cycle_dir}}/eva/{{instrument}}/map_plots/${variable}${channel}/observations_{{instrument}}_${variable}${channel}.png' plots: - mapping: projection: plcarr @@ -268,8 +268,8 @@ diagnostics: figure: figure size: [20,10] layout: [1,1] - title: 'JEDI OmB | $(instrument_title) | ${variable_title}' - output name: $(cycle_dir)/eva/$(instrument)/map_plots/${variable}${channel}/omb_jedi_$(instrument)_${variable}${channel}.png + title: 'JEDI OmB | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/map_plots/${variable}${channel}/omb_jedi_{{instrument}}_${variable}${channel}.png' plots: - mapping: projection: plcarr @@ -305,8 +305,8 @@ diagnostics: figure: figure size: [20,10] layout: [1,1] - title: 'GSI OmB | $(instrument_title) | ${variable_title}' - output name: $(cycle_dir)/eva/$(instrument)/map_plots/${variable}${channel}/omb_gsi_$(instrument)_${variable}${channel}.png + title: 'GSI OmB | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/map_plots/${variable}${channel}/omb_gsi_{{instrument}}_${variable}${channel}.png' plots: - mapping: projection: plcarr @@ -342,8 +342,8 @@ diagnostics: figure: figure size: [20,10] layout: [1,1] - title: 'Hofx Difference | $(instrument_title) | ${variable_title}' - output name: $(cycle_dir)/eva/$(instrument)/map_plots/${variable}${channel}/hofx_difference_$(instrument)_${variable}${channel}.png + title: 'Hofx Difference | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/map_plots/${variable}${channel}/hofx_difference_{{instrument}}_${variable}${channel}.png' plots: - mapping: projection: plcarr @@ -382,8 +382,8 @@ diagnostics: data variable: experiment::ObsValueMinusHofx::${variable} figure: layout: [1,1] - title: 'JEDI omb vs. GSI omb | $(instrument_title) | ${variable_title}' - output name: $(cycle_dir)/eva/$(instrument)/histograms/${variable}${channel}/gsi_omb_vs_jedi_omb_$(instrument)_${variable}${channel}.png + title: 'JEDI omb vs. GSI omb | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/histograms/${variable}${channel}/gsi_omb_vs_jedi_omb_{{instrument}}_${variable}${channel}.png' plots: - add_xlabel: 'Observation minus h(x)' add_ylabel: 'Count' diff --git a/src/swell/suites/hofx/experiment.yaml b/src/swell/suites/hofx/experiment.yaml deleted file mode 100644 index c3a7824a..00000000 --- a/src/swell/suites/hofx/experiment.yaml +++ /dev/null @@ -1,234 +0,0 @@ -# Experiment name and location -experiment_id: swell-experiment -experiment_root: /discover/nobackup/${USER}/SwellExperiments/ - -# suite and executable -executable: fv3jedi_hofx.x - -# suite configuration -suite: - platform: nccs_discover - intial cylc point: 2020-12-15T00 - final cylc point: 2020-12-15T06 - runahead limit: PT24H - suite name: hofx - scheduling: - - task: Default - nodes: 1 - account: g0613 - execution_time_limit: PT1H - qos: debug - constraint: hasw - platform: nccs_discover - - task: BuildJedi - execution_time_limit: PT1H - ntasks_per_node: 28 - - task: RunJediExecutable - ntasks_per_node: 24 - execution_time_limit: PT1H - -# data assimilation window -window_type: 4D -window_length: PT6H -window_offset: PT3H -analysis_forecast_window_offset: -PT3H -pseudo_model_tstep: PT1H - -# backgrounds -backgrounds: - background experiment: x0044 # Experiment that provides the background - background source: file # Could be file or model - background schema: geos # Schema for r2d2 - background frequency: PT1H # Frequency of background if coming from file - -# resolution -horizontal_resolution: c360 -npx: 361 -npy: 361 -npz: 72 -npx_proc: 2 -npy_proc: 2 -processors: - - $(npx_proc) - - $(npy_proc) - - 6 - -build jedi: - existing build directory: /discover/nobackup/drholdaw/JediSwell/bundle/latest/build-intel-release - build options: - - release # release, debug, relwithdebug - packages: - - fv3-jedi - bundle repos: - - git url: https://github.com/jcsda/crtm - project: crtm - branch: 6938cc1a9 - - git url: https://github.com/jcsda/femps - project: femps - branch: 75544fc - - git url: https://github.com/jcsda/FMS - project: fms - branch: 1f739141 - - git url: https://github.com/jcsda/GFDL_atmos_cubed_sphere - project: fv3 - branch: f4c1b7e - - git url: https://github.com/jcsda/fv3-jedi - project: fv3-jedi - branch: 5649cad5 - clone on create: false - - git url: https://github.com/jcsda/fv3-jedi-linearmodel - project: fv3-jedi-lm - branch: 6bb36ce - - git url: https://github.com/jcsda/ioda - project: ioda - branch: 9d8f983a - - git url: https://github.com/jcsda/jedi-cmake - project: jedicmake - branch: 93402f2 - - git url: https://github.com/jcsda/oops - project: oops - branch: 1eefd50c - - git url: https://github.com/jcsda/saber - project: saber - branch: 6d4d7ed3 - - git url: https://github.com/jcsda/ufo - project: ufo - branch: 5e7253cf - -# Fix file staging -# ---------------- -STAGE: - - copy_files: - directories: - - [/discover/nobackup/drholdaw/JediSwell/bundle/1.0.5/fv3-jedi/test/Data/fieldmetadata/*, $(stage_dir)/Data/fieldmetadata/] - - [/discover/nobackup/drholdaw/JediSwell/bundle/1.0.5/fv3-jedi/test/Data/fv3files/*, $(stage_dir)/Data/fv3files/] - link_files: - directories: - - [/discover/nobackup/drholdaw/JediData/GEOS_CRTM_Surface/geos.crtmsrf.$(horizontal_resolution).nc4, $(stage_dir)/Data/bkg/] - -# JEDI Geometry -# ------------- -GEOMETRY: - fms initialization: - namelist filename: '$(stage_dir)/Data/fv3files/fmsmpp.nml' - field table filename: '$(stage_dir)/Data/fv3files/field_table_gmao' - akbk: '$(stage_dir)/Data/fv3files/akbk$(npz).nc4' - layout: [$(npx_proc),$(npy_proc)] - npx: $(npx) - npy: $(npy) - npz: $(npz) - field metadata override: '$(stage_dir)/Data/fieldmetadata/geos.yaml' - -# JEDI Background -# --------------- -BACKGROUND: - datetime: '{{local_background_time_iso}}' - filetype: cube sphere history - provider: geos - datapath: '' - filenames: [$(cycle_dir)/%yyyy%mm%dd.%hh%MM%ss.bkg.nc4, - $(stage_dir)/Data/bkg/geos.crtmsrf.$(horizontal_resolution).nc4] - state variables: [u,v,ua,va,t,delp,q,qi,ql,qr,qs,o3ppmv,phis, - qls,qcn,cfcn,frocean,frland,varflt,ustar,bstar, - zpbl,cm,ct,cq,kcbl,tsm,khl,khu,frlake,frseaice,vtype, - stype,vfrac,sheleg,ts,soilt,soilm,u10m,v10m] - filename: $(cycle_dir)/$(valid_date).$(file_type).nc4 # R2D2 - -# JEDI Forecast model -# ------------------- -MODEL: - name: PSEUDO - tstep: '$(pseudo_model_tstep)' - filetype: cube sphere history - provider: geos - datapath: '' - filenames: [$(cycle_dir)/%yyyy%mm%dd.%hh%MM%ss.bkg.nc4, - $(stage_dir)/Data/bkg/geos.crtmsrf.$(horizontal_resolution).nc4] - model variables: [u,v,ua,va,t,delp,q,qi,ql,qr,qs,o3ppmv,phis,qls,qcn,cfcn,frocean,frland,varflt,ustar, - bstar,zpbl,cm,ct,cq,kcbl,tsm,khl,khu,frlake,frseaice,vtype,stype,vfrac,sheleg, - ts,soilt,soilm,u10m,v10m] - -# Variable change between model and GeoVars -# ----------------------------------------- -MODELGETVALUES: - variable change: - variable change name: Model2GeoVaLs - -# R2D2 schema -# ----------- -R2D2_LOCAL_PATH: /discover/nobackup/${USER}/R2D2DataStore/Local/ -R2D2: - fetch: - an: - - file_type: [bkg] - user_date_format: '%Y%m%d.%H%M%S' - fc: - - file_type: [bkg] - user_date_format: '%Y%m%d.%H%M%S' - bc: - - file_type: satbias - target_file: $(cycle_dir)/$(obs_type).$(date).$(file_type).nc4 - - file_type: tlapse - target_file: $(cycle_dir)/$(obs_type).$(date).$(file_type).txt - store: - fc: - - file_type: [bkg] - user_date_format: '%Y%m%d.%H%M%S' - - -# Observations -# ------------ -obs_experiment: x0044_jjin_20220520 -update channels from database: false -use geos satellite channel database: true -crtm_coeff_dir: /discover/nobackup/projects/gmao/share/gmao_ops/fvInput_4dvar/gsi/etc/fix_ncep20210525/REL-2.2.3-r60152_local-rev_5/CRTM_Coeffs/Little_Endian/ -save_geovals: false -time_interpolation: linear # linear or nearest - -OBSERVATIONS: - - # Conventional - - yaml::$(swell_dir)/configuration/observation_operators/aircraft.yaml - - # Radiances - - yaml::$(swell_dir)/configuration/observation_operators/amsua_aqua.yaml - - yaml::$(swell_dir)/configuration/observation_operators/amsua_metop-a.yaml - - yaml::$(swell_dir)/configuration/observation_operators/amsua_metop-b.yaml - - yaml::$(swell_dir)/configuration/observation_operators/amsua_metop-c.yaml - - yaml::$(swell_dir)/configuration/observation_operators/amsua_n15.yaml - - yaml::$(swell_dir)/configuration/observation_operators/amsua_n18.yaml - - yaml::$(swell_dir)/configuration/observation_operators/amsua_n19.yaml - - yaml::$(swell_dir)/configuration/observation_operators/gmi_gpm.yaml - - yaml::$(swell_dir)/configuration/observation_operators/amsr2_gcom-w1.yaml - - # Ozone - - yaml::$(swell_dir)/configuration/observation_operators/mls55_aura.yaml - - yaml::$(swell_dir)/configuration/observation_operators/omi_aura.yaml - - yaml::$(swell_dir)/configuration/observation_operators/ompsnm_npp.yaml - #- yaml::$(swell_dir)/configuration/observation_operators/ompslpnc_npp.yaml - - - #- yaml::$(swell_dir)/configuration/observation_operators/airs_aqua.yaml - #- yaml::$(swell_dir)/configuration/observation_operators/cris-fsr_n20.yaml - #- yaml::$(swell_dir)/configuration/observation_operators/cris-fsr_npp.yaml - #- yaml::$(swell_dir)/configuration/observation_operators/iasi_metop-a.yaml - #- yaml::$(swell_dir)/configuration/observation_operators/iasi_metop-b.yaml - #- yaml::$(swell_dir)/configuration/observation_operators/mhs_metop-b.yaml - #- yaml::$(swell_dir)/configuration/observation_operators/mhs_metop-c.yaml - #- yaml::$(swell_dir)/configuration/observation_operators/mhs_n19.yaml - #- yaml::$(swell_dir)/configuration/observation_operators/rass_tv.yaml - #- yaml::$(swell_dir)/configuration/observation_operators/satwind.yaml - #- yaml::$(swell_dir)/configuration/observation_operators/sfc.yaml - #- yaml::$(swell_dir)/configuration/observation_operators/sfcship.yaml - #- yaml::$(swell_dir)/configuration/observation_operators/vadwind.yaml - -# Diagnostics -# ----------- -EVA: - yaml::$(suite_dir)/eva.yaml - -# Clean up -# -------- -CLEAN: - - '*.nc4' - - '*.txt' diff --git a/src/swell/suites/hofx/flow.cylc b/src/swell/suites/hofx/flow.cylc index fc4b399a..ecf62377 100644 --- a/src/swell/suites/hofx/flow.cylc +++ b/src/swell/suites/hofx/flow.cylc @@ -8,34 +8,46 @@ # -------------------------------------------------------------------------------------------------- -# Cylc suite for executing JEDI-based 4D h(x) - -# -------------------------------------------------------------------------------------------------- - -%include "suite.jinja2" +# Cylc suite for executing JEDI-based h(x) # -------------------------------------------------------------------------------------------------- [scheduler] UTC mode = True + allow implicit tasks = False # -------------------------------------------------------------------------------------------------- [scheduling] - initial cycle point = {{suite_properties["intial cylc point"]}} - final cycle point = {{suite_properties["final cylc point"]}} - runahead limit = {{suite_properties["runahead limit"]}} + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + runahead limit = {{runahead_limit}} [[graph]] R1 = """ - BuildJedi & Stage - """ - T00,T06,T12,T18 = """ - GetBackground & GetObservations & JediConfig & BuildJedi[^] & Stage[^] - => RunJediExecutable => MergeObsDiags => - EvaDriver & SaveObsDiags => CleanCycle - """ + {% for model_component in model_components %} + BuildJedi & StageJedi-{{model_component}} + {% endfor %} + """ + + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ + {% for model_component in model_components %} + {% if cycle_time[model_component] %} + GetBackground-{{model_component}} & + GetObservations-{{model_component}} & + StageJedi-{{model_component}}[^] & + BuildJedi[^] + => RunJediHofxExecutable-{{model_component}} + => MergeIodaFiles-{{model_component}} + => EvaDriver-{{model_component}} & SaveObsDiags-{{model_component}} + => CleanCycle-{{model_component}} + {% endif %} + {% endfor %} + """ + + {% endfor %} # -------------------------------------------------------------------------------------------------- @@ -48,47 +60,47 @@ [[[environment]]] datetime = $CYLC_TASK_CYCLE_POINT - config = $CYLC_SUITE_DEF_PATH/experiment-filled.yaml + config = $CYLC_SUITE_DEF_PATH/experiment.yaml # Tasks # ----- - [[Stage]] - script = "swell_task Stage $config" - [[BuildJedi]] script = "swell_task BuildJedi $config" - [[GetBackground]] - script = "swell_task GetBackground $config -d $datetime" + {% for model_component in model_components %} + [[StageJedi-{{model_component}}]] + script = "swell_task StageJedi $config -m {{model_component}}" + + [[ GetBackground-{{model_component}} ]] + script = "swell_task GetBackground $config -d $datetime -m {{model_component}}" - [[GetObservations]] - script = "swell_task GetObservations $config -d $datetime" + [[GetObservations-{{model_component}}]] + script = "swell_task GetObservations $config -d $datetime -m {{model_component}}" - [[JediConfig]] - script = "swell_task JediConfig $config -d $datetime" + [[RunJediHofxExecutable-{{model_component}}]] + script = "swell_task RunJediHofxExecutable $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["RunJediHofxExecutable"]["execution_time_limit"]}} + [[[directives]]] + --account = {{scheduling["RunJediHofxExecutable"]["account"]}} + --qos = {{scheduling["RunJediHofxExecutable"]["qos"]}} + --constraint = {{scheduling["RunJediHofxExecutable"]["constraint"]}} + --job-name = RunJediHofxExecutable + --nodes={{scheduling["RunJediHofxExecutable"]["nodes"]}} + --ntasks-per-node={{scheduling["RunJediHofxExecutable"]["ntasks_per_node"]}} - [[RunJediExecutable]] - script = "swell_task RunJediExecutable $config -d $datetime" - platform = {{suite_properties["RunJediExecutable"]["platform"]}} - execution time limit = {{suite_properties["RunJediExecutable"]["execution_time_limit"]}} - [[[directives]]] - --account = {{suite_properties["RunJediExecutable"]["account"]}} - --qos = {{suite_properties["RunJediExecutable"]["qos"]}} - --constraint = {{suite_properties["RunJediExecutable"]["constraint"]}} - --job-name = RunJediExecutable - --nodes={{suite_properties["RunJediExecutable"]["nodes"]}} - --ntasks-per-node={{suite_properties["RunJediExecutable"]["ntasks_per_node"]}} + [[MergeIodaFiles-{{model_component}}]] + script = "swell_task MergeIodaFiles $config -d $datetime -m {{model_component}}" - [[MergeObsDiags]] - script = "swell_task MergeIodaFiles $config -d $datetime" + [[EvaDriver-{{model_component}}]] + script = "swell_task EvaDriver $config -d $datetime -m {{model_component}}" - [[EvaDriver]] - script = "swell_task EvaDriver $config -d $datetime" + [[SaveObsDiags-{{model_component}}]] + script = "swell_task SaveObsDiags $config -d $datetime -m {{model_component}}" - [[SaveObsDiags]] - script = "swell_task SaveObsDiags $config -d $datetime" + [[CleanCycle-{{model_component}}]] + script = "swell_task CleanCycle $config -d $datetime -m {{model_component}}" - [[CleanCycle]] - script = "swell_task CleanCycle $config -d $datetime" + {% endfor %} # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/hofx/geos_atmosphere/suites-hofx-geos_atmosphere.yaml b/src/swell/suites/hofx/geos_atmosphere/suites-hofx-geos_atmosphere.yaml new file mode 100644 index 00000000..dd1a982f --- /dev/null +++ b/src/swell/suites/hofx/geos_atmosphere/suites-hofx-geos_atmosphere.yaml @@ -0,0 +1,111 @@ +# Cycle frequency +cycle_times: + default_value: ['T00', 'T06', 'T12', 'T18'] + prompt: List all cycle times, by middle of the window? + type: iso-duration + +# Window length +window_length: + default_value: 'PT6H' + prompt: Window length + type: iso-duration + +# Horizontal resolution +horizontal_resolution: + default_value: '361' + prompt: What resolution for the atmospheric background (For c360 choose 361)? + options: ['181', '361', '721'] + type: drop-list-string + +# Vertical resolution +vertical_resolution: + default_value: '72' + prompt: What vertical resolution for the atmospheric background? + options: ['72'] + type: drop-list-string + +# Processor layout +npx_proc: + default_value: '2' + prompt: What processor layout (x-direction) per cube face? + type: string +npy_proc: + default_value: '2' + prompt: What processor layout (y-direction) per cube face? + type: string + +# Backgrounds +background_experiment: + default_value: x0044 + prompt: Experiment providing the background files? + type: string + +# Observation experiment +obs_experiment: + default_value: x0044_jjin_20220520 + prompt: Database providing the observations? + type: string + +# Observations +observations: + default_value: + # Conventional + - aircraft + # Radiances + - amsua_n19 + - amsua_aqua + - amsua_metop-a + - amsua_metop-b + - amsua_metop-c + - amsua_n15 + - amsua_n18 + - amsua_n19 + - gmi_gpm + - amsr2_gcom-w1 + # Ozone + - mls55_aura + - omi_aura + - ompsnm_npp + prompt: Select atmospheric observations. + options: use_method + type: check-list-string + +# Fixed options (user not prompted for these) +# ------------------------------------------- +fixed_options: + jedi_interface: + default_value: fv3-jedi + prompt: Name of the JEDI model interface repo + jedi_executable: + default_value: fv3jedi_hofx.x + prompt: Name of the executable from the interface + window_type: + default_value: 4D + prompt: Window type (3D or 4D) + background_frequency: + default_value: PT1H + prompt: 'Frequency of 4D backgrounds' + background_source: + default_value: file + prompt: 'Source of 4D background files (file or from model)' + crtm_coeff_dir: + default_value: /discover/nobackup/projects/gmao/share/gmao_ops/fvInput_4dvar/gsi/etc/fix_ncep20210525/REL-2.2.3-r60152_local-rev_5/CRTM_Coeffs/Little_Endian/ + prompt: 'Directory containing the CRTM coefficient files' + window_offset: + default_value: PT3H + prompt: Time from beginning to middle of the window + analysis_forecast_window_offset: + default_value: -PT3H + prompt: Time from the middle of the window when forecasts start + total_processors: + default_value: '{{npx_proc}} * {{npy_proc}} * 6' + prompt: Equation to compute total number of processors + model: + default_value: pseudo-model + prompt: Type of model to use for 4D runs. + background_time_offset: + default_value: PT9H + prompt: Time before the middle of the window that the background providing forecast began + clean_patterns: + default_value: ['*.nc4','*.txt'] + prompt: 'Patterns for the files to remove after completing a cycle' diff --git a/src/swell/suites/hofx/geos_ocean/suites-hofx-geos_ocean.yaml b/src/swell/suites/hofx/geos_ocean/suites-hofx-geos_ocean.yaml new file mode 100644 index 00000000..3a5366e3 --- /dev/null +++ b/src/swell/suites/hofx/geos_ocean/suites-hofx-geos_ocean.yaml @@ -0,0 +1,46 @@ +# Cycle frequency +cycle_times: + default_value: ['T00', 'T12'] + prompt: List all cycle times, by middle of the window? + type: string + +# Window length +window_length: + default_value: PT6H + prompt: Window length + type: iso-duration + +# Observations +horizontal_resolution: + default_value: '1' + prompt: What resolution for the ocean background (degrees)? + options: ['1/4', '1/2', '1'] + type: drop-list-string + +# Observations +observations: + default_value: [adt_3a] + prompt: Select ocean observations. + options: use_method + type: check-list-string + +# Options that are not included in any user input +fixed_options: + jedi_interface: + default_value: soca + prompt: Name of the JEDI model interface repo + jedi_executable: + default_value: soca_hofx.x + prompt: soca-mom6-geos + window_type: + default_value: 3D + prompt: Window type (3D or 4D) + window_offset: + default_value: PT12H + prompt: Time from beginning to middle of the window + analysis_forecast_window_offset: + default_value: -PT12H + prompt: Time from the middle of the window when forecasts start + background_time_offset: + default_value: PT9H + prompt: Time before the middle of the window that the background providing forecast began diff --git a/src/swell/suites/hofx/jedi_config.yaml b/src/swell/suites/hofx/jedi_config.yaml deleted file mode 100644 index ef59e475..00000000 --- a/src/swell/suites/hofx/jedi_config.yaml +++ /dev/null @@ -1,14 +0,0 @@ -window begin: '{{window_begin_iso}}' -window length: ${window_length} -forecast length: ${window_length} -geometry: - ${GEOMETRY} -model: - ${MODEL} -initial condition: - '{{BACKGROUND}}' -observations: - get values: - ${MODELGETVALUES} - observers: - ${OBSERVATIONS} diff --git a/src/swell/suites/hofx/suite_page_hofx.yaml b/src/swell/suites/hofx/suite_page_hofx.yaml deleted file mode 100644 index 20f49e74..00000000 --- a/src/swell/suites/hofx/suite_page_hofx.yaml +++ /dev/null @@ -1,55 +0,0 @@ -elements: - - name: Experiment ID - key: experiment_id - widget type: entry - default: None - - name: Experiment Root - key: experiment_root - widget type: entry - - name: Compute Platform - key: - suite: - scheduling: platform - widget type: dropdown - options: [nccs_discover] - - name: Intital time - key: - suite: intial cylc point - widget type: entry - - name: Final time - key: - suite: final cylc point - widget type: entry - - name: Experiment that provides backgrounds - key: - backgrounds: background experiment - widget type: entry - - name: Horizontal resolution - key: horizontal_resolution - widget type: dropdown - options: [c12, c360] - - name: Number of processors in the x direction - key: npx_proc - widget type: entry - - name: Number of processors in the y direction - key: npy_proc - widget type: entry - - name: Jedi Build directory - key: - build jedi: existing build directory - widget type: entry - - name: Observation provider - key: obs_experiment - widget type: entry - - name: Select satellite channels from database? - key: update channels from database - widget type: radio button - options: - - name: 'No' - value: 1 - - name: 'Yes' - value: 2 - - name: Observation operators - key: OBSERVATIONS - widget type: check button - options: [aircraft, airs_aqua, amsr2_gcom-w1, amsua_aqua, amsua_metop-a, amsua_metop-b, amsua_metop-c, amsua_n15, amsua_n18, amsua_n19] diff --git a/src/swell/suites/hofx/suites-hofx.yaml b/src/swell/suites/hofx/suites-hofx.yaml new file mode 100644 index 00000000..029432c7 --- /dev/null +++ b/src/swell/suites/hofx/suites-hofx.yaml @@ -0,0 +1,61 @@ +# Data assimilation run +data_assimilation_run: + default_value: True + prompt: Is this a run involving data assimilation? + type: boolean + +# Cycle start point +start_cycle_point: + default_value: '2020-12-15T00:00:00Z' + prompt: What is the time of the first cycle (middle of the window)? + type: iso-datetime + +# Cycle final point +final_cycle_point: + default_value: '2020-12-15T06:00:00Z' + prompt: What is the time of the final cycle (middle of the window)? + type: iso-datetime + +# Run ahead limit for workflow +runahead_limit: + default_value: 'PT6H' + prompt: Since this suite is non-cycling choose how many hours the workflow can run ahead? + type: iso-duration + +# Jedi build system +jedi_build_method: + default_value: use_existing + prompt: Create a new JEDI build or use an existing build? + options: ['create', 'use_existing'] + type: drop-list-string + +# Existing JEDI build +existing_build_directory: + default_value: /discover/nobackup/drholdaw/JediSwell/bundle/latest/build-intel-release/ + prompt: Provide the path to an existing JEDI build directory + type: string + depends: + key: jedi_build_method + value: use_existing + +# R2D2 Configuration +r2d2_local_path: + default_value: /discover/nobackup/${USER}/R2D2DataStore/Local/ + prompt: Enter the path where R2D2 will store experiment output + type: string + +# What kind of coupling is needed +coupling_style: + default_value: uncoupled + prompt: What kind of coupling would you like to run with? + options: ['uncoupled', 'weakly coupled', 'strongly coupled'] + type: check-list-string + +# Models to use +model_components: + default_value: ['geos_atmosphere'] + prompt: Select models to use + options: use_method + type: file-check-list + + diff --git a/src/swell/suites/process_obs/experiment.yaml b/src/swell/suites/process_obs/experiment.yaml deleted file mode 100644 index f573cf14..00000000 --- a/src/swell/suites/process_obs/experiment.yaml +++ /dev/null @@ -1,74 +0,0 @@ -# Experiment name and location -experiment_id: x0044-jedi-process_obs -experiment_root: /discover/nobackup/${USER}/SwellExperiments/ - -# suite configuration -suite: - intial cylc point: 2020-12-14T00 - final cylc point: 2020-12-15T00 - runahead limit: PT24H - suite name: process_obs - platform: nccs_discover - -# GEOS experiment to get backgrounds from -geos_experiment: x0044 -iodabin: /discover/nobackup/drholdaw/JediSwell/bundle/1.0.7/build-intel-release/bin/ -geos_obs_dir_template: /discover/nobackup/projects/gmao/dadev/rtodling/archive/$(geos_experiment)/obs/Y%Y/M%m/D%d/H%H/ -satbias_dir_template: /discover/nobackup/projects/gmao/dadev/rtodling/archive/$(geos_experiment)/rs/Y%Y/M%m/ -combined_obs_template: obs_%Y%m%d%H.nc4 -sondes_obs_template: sondes_obs_%Y%m%d%H.nc4 -sondes_obs_template_rename: radiosonde_obs_%Y%m%d%H.nc4 -sfcship_obs_template: sfcship_obs_%Y%m%d%H.nc4 -sfcship_obs_template_rename: ship_obs_%Y%m%d%H.nc4 -r2d2_obs_standard_template: .%Y-%m-%dT%H:00:00Z.nc4 -geos_obs_offset: PT3H -geos_satbias_offset: PT9H -#gnssro_inputbufr: /archive/input/dao_ops/obs/flk/ncep_g5obs/bufr/GPSRO/Y%Y/M%m/gdas1.${yyyy:2:4}%m%d.t%Hz.gpsro.tm00.bufr_d #FIX YEAR -satbias_org_template: $(geos_experiment).rst.%Y%m%d_%Hz.tar -satbias_template: $(geos_experiment).ana_satbias_rst.%Y%m%d_%Hz.txt -satbiaspc_template: $(geos_experiment).ana_satbiaspc_rst.%Y%m%d_%Hz.txt -tlapse_template: .%Y-%m-%dT%H:00:00Z.tlapse -r2d2_satbias_template: .%Y-%m-%dT%H:00:00Z.satbias - -# data assimilation window (to determine which backgrounds are needed) -window_type: 4D -window_length: PT6H -window_offset: PT3H -analysis_forecast_window_offset: -PT3H -pseudo_model_tstep: PT1H - -# backgrounds -backgrounds: - background experiment: x0044 # Experiment that provides the background - background source: file # Could be file or model - background schema: geos # Schema for r2d2 - background frequency: PT1H # Frequency of background if coming from file - -# resolution -horizontal_resolution: c360 - -# JEDI Background -# --------------- -BACKGROUND: - filename: $(cycle_dir)/$(valid_date).$(file_type).nc4 # R2D2 - -# R2D2 schema -# ----------- -R2D2_LOCAL_PATH: /discover/nobackup/${USER}/R2D2DataStore/Local/ -R2D2: - fetch: - an: - - file_type: [bkg] - user_date_format: '%Y%m%d.%H%M%S' - fc: - - file_type: [bkg] - user_date_format: '%Y%m%d.%H%M%S' - bc: - - file_type: satbias - target_file: $(cycle_dir)/$(obs_type).$(date).$(file_type).nc4 - - file_type: tlapse - target_file: $(cycle_dir)/$(obs_type).$(date).$(file_type).txt - store: - fc: - - file_type: [bkg] - user_date_format: '%Y%m%d.%H%M%S' diff --git a/src/swell/suites/process_obs/flow.cylc b/src/swell/suites/process_obs/flow.cylc deleted file mode 100644 index 2f63630d..00000000 --- a/src/swell/suites/process_obs/flow.cylc +++ /dev/null @@ -1,61 +0,0 @@ -#!jinja2 - -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -# -------------------------------------------------------------------------------------------------- - -# Cylc suite for executing a workflow that retrieves backgrounds from a GEOS experiment and -# stores them in R2D2 - -# -------------------------------------------------------------------------------------------------- - -%include "suite.jinja2" - -# -------------------------------------------------------------------------------------------------- - -[scheduler] - UTC mode = True - -# -------------------------------------------------------------------------------------------------- - -[scheduling] - - initial cycle point = {{suite_properties["intial cylc point"]}} - final cycle point = {{suite_properties["final cylc point"]}} - runahead limit = {{suite_properties["runahead limit"]}} - - [[graph]] - T00,T06,T12,T18 = """ - ObsProcessSetup - #GetBackgroundGeosExperiment => StoreBackground #=> CleanCycle - """ - -# -------------------------------------------------------------------------------------------------- - -[runtime] - - # Task defaults - # ------------- - [[root]] - pre-script = "source $CYLC_SUITE_DEF_PATH/modules" - - [[[environment]]] - config = $CYLC_SUITE_DEF_PATH/experiment-filled.yaml - datetime = $CYLC_TASK_CYCLE_POINT - - # Tasks - # ----- - [[ObsProcessSetup]] - script = "swell_task ObsProcessSetup $config -d $datetime" - - #[[StoreBackground]] - # script = "swell_task StoreBackground $config -d $datetime" - - #[[CleanCycle]] - # script = "swell_task CleanCycle $config -d $datetime" - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/store_backgrounds_geos_run/experiment.yaml b/src/swell/suites/store_backgrounds_geos_run/experiment.yaml deleted file mode 100644 index a5bc5ca4..00000000 --- a/src/swell/suites/store_backgrounds_geos_run/experiment.yaml +++ /dev/null @@ -1,61 +0,0 @@ -# Experiment name and location -experiment_id: x0044-jedi-get_background -experiment_root: /discover/nobackup/${USER}/JediExperiments/ - -# suite configuration -suite: - intial cylc point: 2020-12-14T06 - final cylc point: 2020-12-16T12 - runahead limit: PT24H - suite name: store_backgrounds_geos_run - platform: nccs_discover - -# GEOS experiment to get backgrounds from -geos_experiment: x0044 -geos_bkg_tar_filename_template: /discover/nobackup/projects/gmao/dadev/rtodling/archive/$(geos_experiment)/rs/Y%Y/M%m/Revised/$(geos_experiment).trajrst.%Y%m%d_%Hz.tar -geos_background_restart_offset: PT9H # Time before middle of analysis window that bkg forecast began -geos_bkg_filename_template: $(geos_experiment).traj_lcv_rst.%Y%m%d_%H%Mz.nc4 # Name for bkg files from GEOS -jedi_bkg_filename_template: '%Y%m%d.%H%M%S.bkg.nc4' - -# data assimilation window (to determine which backgrounds are needed) -window_type: 4D -window_length: PT6H -window_offset: PT3H -analysis_forecast_window_offset: -PT3H -pseudo_model_tstep: PT1H - -# backgrounds -backgrounds: - background experiment: x0044 # Experiment that provides the background - background source: file # Could be file or model - background schema: geos # Schema for r2d2 - background frequency: PT1H # Frequency of background if coming from file - -# resolution -horizontal_resolution: c360 - -# JEDI Background -# --------------- -BACKGROUND: - filename: $(cycle_dir)/$(valid_date).$(file_type).nc4 # R2D2 - -# R2D2 schema -# ----------- -R2D2_LOCAL_PATH: /discover/nobackup/${USER}/R2D2DataStore/Local/ -R2D2: - fetch: - an: - - file_type: [bkg] - user_date_format: '%Y%m%d.%H%M%S' - fc: - - file_type: [bkg] - user_date_format: '%Y%m%d.%H%M%S' - bc: - - file_type: satbias - target_file: $(cycle_dir)/$(obs_type).$(date).$(file_type).nc4 - - file_type: tlapse - target_file: $(cycle_dir)/$(obs_type).$(date).$(file_type).txt - store: - fc: - - file_type: [bkg] - user_date_format: '%Y%m%d.%H%M%S' diff --git a/src/swell/suites/store_backgrounds_geos_run/flow.cylc b/src/swell/suites/store_backgrounds_geos_run/flow.cylc deleted file mode 100644 index 640a1651..00000000 --- a/src/swell/suites/store_backgrounds_geos_run/flow.cylc +++ /dev/null @@ -1,60 +0,0 @@ -#!jinja2 - -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -# -------------------------------------------------------------------------------------------------- - -# Cylc suite for executing a workflow that retrieves backgrounds from a GEOS experiment and -# stores them in R2D2 - -# -------------------------------------------------------------------------------------------------- - -%include "suite.jinja2" - -# -------------------------------------------------------------------------------------------------- - -[scheduler] - UTC mode = True - -# -------------------------------------------------------------------------------------------------- - -[scheduling] - - initial cycle point = {{suite_properties["intial cylc point"]}} - final cycle point = {{suite_properties["final cylc point"]}} - runahead limit = {{suite_properties["runahead limit"]}} - - [[graph]] - T00,T06,T12,T18 = """ - GetBackgroundGeosExperiment => StoreBackground #=> CleanCycle - """ - -# -------------------------------------------------------------------------------------------------- - -[runtime] - - # Task defaults - # ------------- - [[root]] - pre-script = "source $CYLC_SUITE_DEF_PATH/modules" - - [[[environment]]] - config = $CYLC_SUITE_DEF_PATH/experiment-filled.yaml - datetime = $CYLC_TASK_CYCLE_POINT - - # Tasks - # ----- - [[GetBackgroundGeosExperiment]] - script = "swell_task GetBackgroundGeosExperiment $config -d $datetime" - - [[StoreBackground]] - script = "swell_task StoreBackground $config -d $datetime" - - #[[CleanCycle]] - # script = "swell_task CleanCycle $config -d $datetime" - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/suite_page_start.yaml b/src/swell/suites/suite_page_start.yaml deleted file mode 100644 index 2e960311..00000000 --- a/src/swell/suites/suite_page_start.yaml +++ /dev/null @@ -1,7 +0,0 @@ -elements: - - name: "Choose workflow to run" - widget type: dropdown - default: '' - options: - - hofx - - store_backgrounds_geos_run diff --git a/src/swell/suites/suites.yaml b/src/swell/suites/suites.yaml new file mode 100644 index 00000000..d4f14c54 --- /dev/null +++ b/src/swell/suites/suites.yaml @@ -0,0 +1,26 @@ +# Experiment name +experiment_id: + default_value: swell-experiment + prompt: Enter the experiment ID + type: string + +# Location to store the experiment +experiment_root: + default_value: /home/$USER/SwellExperiments + prompt: Enter the path where experiment will be staged + type: string + +# Compute platform to use +platform: + default_value: nccs_discover + prompt: Which compute platform do you wish to use? + options: use_method + type: string + +# Type of suite to run +suite_to_run: + default_value: hofx + prompt: Choose the suite to run + options: use_method + type: file-drop-list + diff --git a/src/swell/suites/swell_gui.py b/src/swell/suites/swell_gui.py deleted file mode 100644 index 93008d54..00000000 --- a/src/swell/suites/swell_gui.py +++ /dev/null @@ -1,233 +0,0 @@ -import tkinter as tk -from tkinter import ttk -from tkinter import messagebox -import yaml - -with open('hofx/suite_page_hofx.yaml', 'r') as yml: - widget_dict = yaml.safe_load(yml) - -with open('hofx/experiment.yaml', 'r') as yml: - default_exp_dict = yaml.safe_load(yml) - - -class ScrollableFrame(ttk.Frame): - def __init__(self, container, *args, **kwargs): - super().__init__(container, *args, **kwargs) - canvas = tk.Canvas(self, height=500, width=550) - scrollbar = ttk.Scrollbar(self, - orient="vertical", - command=canvas.yview) - self.scrollable_frame = ttk.Frame(canvas) - - self.scrollable_frame.bind( - "", - lambda e: canvas.configure( - scrollregion=canvas.bbox("all") - ) - ) - - canvas.create_window((0, 0), window=self.scrollable_frame, anchor="nw") - - canvas.configure(yscrollcommand=scrollbar.set) - - canvas.pack(side="left", fill="both", expand=True) - scrollbar.pack(side="right", fill="y") - - -class tkinterApp(tk.Tk): - - def __init__(self, *args, **kwargs): - # __init__ function for class Tk - tk.Tk.__init__(self, *args, **kwargs) - - # creating a container - container = tk.Frame(self) - container.pack(side="top", fill="both", expand=True) - - # Set aesthetics - - self.geometry("600x500") - self.LARGEFONT = 18 - self.XL_FONT = 22 - - # initializing frames to an empty array - self.frames = {} - - # iterating through a tuple consisting - # of the different page layouts - for F in (StartPage, Model): - frame = F(container, self) - # initializing frame of that object from - # startpage, page1, page2 respectively with for loop - self.frames[F] = frame - frame.grid(row=0, column=0, sticky="nsew") - - self.show_frame(StartPage) - - # to display the current frame passed as - # parameter - def show_frame(self, cont): - frame = self.frames[cont] - frame.tkraise() - - -# Landing Page -class StartPage(tk.Frame): - def __init__(self, parent, controller): - tk.Frame.__init__(self, parent) - - # Button to go to HofX Model - label = ttk.Label(self, text="Welcome to SWELL\n", - font=('Arial', controller.XL_FONT)).pack() - label = ttk.Label(self, text="Choose a model", - font=('Arial', controller.LARGEFONT)).pack() - button1 = ttk.Button(self, text="HofX", - command=lambda: - controller.show_frame(Model)).pack() - - -# Main Application for Model Component -class Model(tk.Frame): - def __init__(self, parent, controller): - tk.Frame.__init__(self, parent) - - self.frame = ScrollableFrame(self, width=600) - - self.widget_inputs = [] - for widget in widget_dict['elements']: - self.check_widget_type(widget) - - # Submit Button - submit = ttk.Button(self.frame.scrollable_frame, text='Generate YAML', - command=self.send_to_file) - submit.pack(side=tk.LEFT, padx=5, pady=5) - # Back Button - backbutton = ttk.Button(self.frame.scrollable_frame, text="Back", - command=lambda: - controller.show_frame(StartPage)) - backbutton.pack(side=tk.LEFT, padx=5, pady=5) - - self.frame.pack() - - def send_to_file(self): - self.yaml_dict = {} - self.cb_dict = {} - for widget in self.widget_inputs: - field = widget[0]['name'] - if widget[0]['widget type'] == 'check button': - self.check_checks(widget) - value = self.cb_dict - else: - value = widget[1].get() - self.yaml_dict[field] = value - self.yaml_dict[field] = value - with open('gui_experiment.yaml', 'w') as outfile: - yaml.dump(self.yaml_dict, outfile, default_flow_style=False) - - def check_widget_type(self, widget): - self.get_defaults(widget) - self.widget = widget - if widget['widget type'] == 'entry': - self.make_entry() - elif widget['widget type'] == 'radio button': - self.make_radio_btn() - elif widget['widget type'] == 'dropdown': - self.make_dropdown() - elif widget['widget type'] == 'check button': - self.make_check_button() - else: - print('Widget not defined') - - def make_entry(self): - row = tk.Frame(self.frame.scrollable_frame) - lab = tk.Label(row, width=20, text=self.widget['name'], anchor='w') - ent = tk.Entry(row) - ent.insert(0, self.default) - row.pack(side=tk.TOP, fill=tk.X, padx=5, pady=5) - lab.pack(side=tk.LEFT) - ent.pack(side=tk.RIGHT, expand=tk.YES, fill=tk.X) - self.widget_inputs.append((self.widget, ent)) - - def make_radio_btn(self): - tk.Label(self.frame.scrollable_frame, text=self.widget['name'], - justify=tk.LEFT, padx=5).pack() - v = tk.IntVar() - v.set(1) - options = self.widget['options'] - for option in options: - tk.Radiobutton(self.frame.scrollable_frame, - text=option['name'], - padx=20, - variable=v, - value=option['value']).pack(anchor=tk.W) - - self.widget_inputs.append((self.widget, v)) - - def make_dropdown(self): - tk.Label(self.frame.scrollable_frame, - text=self.widget['name'], - justify=tk.LEFT, - padx=5).pack() - clicked = tk.StringVar() - clicked.set(self.widget['options'][0]) - tk.OptionMenu(self.frame.scrollable_frame, - clicked, - *self.widget['options']).pack() - self.widget_inputs.append((self.widget, clicked)) - - def make_check_button(self): - name = self.widget.get('name', '') - options = self.widget.get('options', []) - - lab = tk.Label(self.frame.scrollable_frame, - text=name, - justify=tk.LEFT, - padx=5) - lab.pack(side=tk.TOP, anchor=tk.W) - - vlist = [] - for option in options: - v = tk.IntVar() - option_val = 0 - for default in self.default: - if option in default: - option_val = 1 - break - else: - pass - v.set(option_val) - w = tk.Checkbutton(self.frame.scrollable_frame, - text=option, - variable=v) - w.pack(side=tk.TOP, padx=5, pady=5, anchor=tk.W) - vlist.append(v) - - self.widget_inputs.append((self.widget, vlist)) - - def check_checks(self, check_button): - for i, opt in enumerate(check_button[0]['options']): - self.cb_dict[opt] = check_button[1][i].get() - - def get_defaults(self, widget): - w_val = widget['key'] - self.default = '' - if isinstance(w_val, dict): - default_check = default_exp_dict - while isinstance(w_val, dict): - w_key = list(w_val.keys())[0] - w_val = w_val[w_key] - default_check = default_check[w_key] - if isinstance(default_check, list): - for item in default_check: - if w_val in item: - default_check = item - break - self.default = default_check[w_val] - else: - default_check = default_exp_dict[w_val] - self.default = default_check - - -# Driver Code -app = tkinterApp() -app.mainloop() diff --git a/src/swell/suites/suites.py b/src/swell/swell_path.py similarity index 96% rename from src/swell/suites/suites.py rename to src/swell/swell_path.py index 8f6a5c57..d1adeb09 100644 --- a/src/swell/suites/suites.py +++ b/src/swell/swell_path.py @@ -14,7 +14,7 @@ # -------------------------------------------------------------------------------------------------- -def return_suite_path(): +def get_swell_path(): return os.path.split(__file__)[0] diff --git a/src/swell/tasks/base/config.py b/src/swell/tasks/base/config.py index 29ca1aa8..1b8adba9 100644 --- a/src/swell/tasks/base/config.py +++ b/src/swell/tasks/base/config.py @@ -6,18 +6,12 @@ # -------------------------------------------------------------------------------------------------- - -import copy -import datetime as pydatetime -import glob +import datetime import isodate import os -import string -import re -import json import yaml -from swell.utilities.string_utils import replace_vars +from swell.utilities.jinja2 import template_string_jinja2 # -------------------------------------------------------------------------------------------------- # @package configuration @@ -29,7 +23,7 @@ # -------------------------------------------------------------------------------------------------- -class Config(dict): +class Config(): """Provides methods for reading YAML files and managing configuration parameters. @@ -50,103 +44,133 @@ class Config(dict): # ---------------------------------------------------------------------------------------------- - def __init__(self, input, logger): - """Reads YAML file(s) as a dictionary. + def __init__(self, input_file, logger, **kwargs): - Environment definitions and root-level YAML parameters are extracted to be - used for variable interpolation within strings (see replace_vars()). + # Keep track of the input config file + self.__input_file__ = input_file + + # Keep copy of owner's logger + self.__logger__ = logger + + # Read the configuration yaml file + with open(self.__input_file__, 'r') as ymlfile: + self.__config__ = yaml.safe_load(ymlfile) + + # Get model part of the config + self.__model__ = kwargs['model'] + if self.__model__ is not None: + # Assert the model name is found in the config + if self.__model__ not in self.__config__['models'].keys(): + self.__logger__.abort(f'Did not find the model \'{self.__model__}\' in the ' + + f'experiment configuration') + # Extract the model specific part of the config + model_config = self.__config__['models'][self.__model__] + else: + model_config = {} - Parameters - ---------- - input : string, required Name of YAML file(s) + # Remove the model specific part from the full config + if 'models' in self.__config__.keys(): + del self.__config__['models'] + if 'model_components' in self.__config__.keys(): + del self.__config__['model_components'] - Returns - ------- - config : Config, dict - Config object - """ + # Assert that the full and model level configs have only unique keys + for key in self.__config__.keys(): + if key in model_config.keys(): + self.__logger__.abort(f'Model config contains the key \'{key}\'. Which is ' + + f'also contained in the top level config.') - # Keep track of the input config file - self.input = input + # Now merge the top level config and the model specific parts of the config. This prevents + # tasks from accessing the config associated with any model other than the one they are + # supposed to act upon. + self.__config__.update(model_config) - # Read the configuration yaml file(s) - with open(self.input, 'r') as ymlfile: - config = yaml.safe_load(ymlfile) + # Add the experiment directory to the configuration + experiment_root = self.get('experiment_root') + experiment_id = self.get('experiment_id') + experiment_dir = os.path.join(experiment_root, experiment_id) + self.__config__['experiment_dir'] = experiment_dir - # Initialize the parent class with the config - super().__init__(config) + # Swell datetime format (avoid colons in paths and filenames) + self.__datetime_swl_format__ = "%Y%m%dT%H%M%SZ" - # Standard datetime format for config - self.dt_format = "%Y%m%dT%H%M%SZ" - self.dt_foriso = "%Y-%m-%dT%H:%M:%SZ" + # ISO datetime format + self.__datetime_iso_format__ = "%Y-%m-%dT%H:%M:%SZ" - # Create list of definitions from top level of dictionary - self.defs = {} - self.defs.update({k: str(v) for k, v in iter(self.items()) - if not isinstance(v, dict) and not isinstance(v, list)}) + # If datetime passed add some extra datetime parameters to config + if 'datetime_in' in kwargs and kwargs['datetime_in'] is not None: + self.add_cycle_time_parameter(kwargs['datetime_in'].datetime) - # Keep copy of logger - self.logger = logger + if self.get('data_assimilation_run', False): + self.add_data_assimilation_window_parameters() # ---------------------------------------------------------------------------------------------- - def merge(self, other): - """ Merge another dictionary with self + def get(self, key, default='NODEFAULT'): - Parameters - ---------- - other : dictionary, required - other dictionary to merge - """ + if key in self.__config__.keys(): + return self.__config__[key] + else: + if default == 'NODEFAULT': + self.__logger__.abort(f'In config.get the key \'{key}\' was not found in the ' + + f'configuration and no default was provided.') + else: + return default - # Merge the other dictionary into self - self.update(other) + # ---------------------------------------------------------------------------------------------- - # Overwrite the top level definitions - self.defs.update({k: str(v) for k, v in iter(self.items()) - if not isinstance(v, dict) and not isinstance(v, list)}) + def put(self, key, value): + self.__config__[key] = value # ---------------------------------------------------------------------------------------------- - def add_cyle_time_parameter(self, cycle_dt): - """ Add cyle time to the configuration + def use_config_to_template_string(self, string_in): - Parameters - ---------- - cycle_dt : datetime, required - Current cycle date/time as datetime object - """ + return template_string_jinja2(self.__logger__, string_in, self.__config__) - # Create new dictionary to hold cycle time - cycle_dict = {} - cycle_dict['current_cycle'] = cycle_dt.strftime(self.dt_format) + # ---------------------------------------------------------------------------------------------- - # Merge with self - self.merge(cycle_dict) + def get_datetime_format(self): + return self.__datetime_swl_format__ -# -------------------------------------------------------------------------------------------------- + # ---------------------------------------------------------------------------------------------- - def add_data_assimilation_window_parameters(self): - """ Defines cycle dependent parameters for the data assimilation window + def add_cycle_time_parameter(self, cycle_dt): + """ + Defines cycle time parameter and adds to config + """ + + # Add current cycle to the config + # ------------------------------- + current_cycle = cycle_dt.strftime(self.__datetime_swl_format__) + self.put('current_cycle', current_cycle) + + # Add cycle directory to config + # ----------------------------- + cycle_dir = current_cycle + if self.__model__ is not None: + cycle_dir = cycle_dir + '-' + self.__model__ + cycle_dir = os.path.join(self.__config__['experiment_dir'], 'run', cycle_dir) - Parameters defined by this method are needed for resolving - time-dependent variables using the replace_vars() method. + self.put('cycle_dir', cycle_dir) - Parameters - ---------- - cycle_dt : datetime, required - Current cycle date/time as datetime object + # ---------------------------------------------------------------------------------------------- + + def add_data_assimilation_window_parameters(self): + """ + Defines cycle dependent parameters for the data assimilation window and adds to config """ # Current cycle datetime object - current_cycle_dto = pydatetime.datetime.strptime(self.get('current_cycle'), self.dt_format) + current_cycle_dto = datetime.datetime.strptime(self.get('current_cycle'), + self.__datetime_swl_format__) # Type of data assimilation window (3D or 4D) - window_type = self.get('window_type', '4D') + window_type = self.get('window_type') # Extract window information and convert to duration - window_length = self.get('window_length', 'PT6H') - window_offset = self.get('window_offset', 'PT3H') + window_length = self.get('window_length') + window_offset = self.get('window_offset') window_offset_dur = isodate.parse_duration(window_offset) @@ -154,7 +178,7 @@ def add_data_assimilation_window_parameters(self): window_begin_dto = current_cycle_dto - window_offset_dur # Background time for satbias files - background_time_offset = self.get('background_time_offset', 'PT9H') + background_time_offset = self.get('background_time_offset') background_time_offset_dur = isodate.parse_duration(background_time_offset) background_time_dto = current_cycle_dto - background_time_offset_dur @@ -165,83 +189,24 @@ def add_data_assimilation_window_parameters(self): elif window_type == '3D': local_background_time = current_cycle_dto else: - self.logger.abort("add_data_assimilation_window_parameters: window type must be " + - "either 4D or 3D") + self.__logger__.abort('add_data_assimilation_window_parameters: window type must be ' + + 'either 4D or 3D') - # Create new dictionary with these items - window_dict = {} - window_dict['window_type'] = window_type - window_dict['window_length'] = window_length - window_dict['window_offset'] = window_offset - window_dict['window_begin'] = window_begin_dto.strftime(self.dt_format) - window_dict['window_begin_iso'] = window_begin_dto.strftime(self.dt_foriso) - window_dict['background_time'] = background_time_dto.strftime(self.dt_format) + window_begin = window_begin_dto.strftime(self.__datetime_swl_format__) + window_begin_iso = window_begin_dto.strftime(self.__datetime_iso_format__) + background_time = background_time_dto.strftime(self.__datetime_swl_format__) + local_background_time_iso = local_background_time.strftime(self.__datetime_iso_format__) + local_background_time = local_background_time.strftime(self.__datetime_swl_format__) - window_dict['local_background_time'] = local_background_time.strftime(self.dt_format) - window_dict['local_background_time_iso'] = local_background_time.strftime(self.dt_foriso) - - # Merge with self - self.merge(window_dict) - - # -------------------------------------------------------------------------------------------------- - - def resolve_config_file(self): - """Resolves/interpolates all defined variables in the base configuration. - - Returns - ------- - d: dict - YAML dictionary with all defined variables interpolated. - """ - - # Read input file as text file - with open(self.input) as f: - text = f.read() - - # Replace any unresolved variables in the file - text = replace_vars(text, **self.defs) - - # Return a yaml - resolved_dict = yaml.safe_load(text) - - # Merge dictionary - self.merge(resolved_dict) - - # ---------------------------------------------------------------------------------------------- - - def overlay(self, hash, override=False, root=None): - """Combines two dictionaries. - - This method recursively traverses the nodes of the dictionaries to - locate the appropriate insertion point at the leaf-nodes. - - Parameters - ---------- - hash : dict, required - New dictionary to be added - - root : dict, private - Root node to add new values. This is set during recursion. - - override : boolean, optional - Indicates whether existing dictionary entries should be overwritten. - """ - - if root is None: - root = self - - for key in hash: - - if key not in root: - if isinstance(hash[key], dict): - root[key] = copy.deepcopy(hash[key]) - else: - root[key] = hash[key] - elif isinstance(hash[key], dict) and isinstance(root[key], dict): - self.overlay(hash[key], override, root[key]) - else: - if override: - root[key] = hash[key] + # Create new dictionary with these items + self.put('window_type', window_type) + self.put('window_length', window_length) + self.put('window_offset', window_offset) + self.put('window_begin', window_begin) + self.put('window_begin_iso', window_begin_iso) + self.put('background_time', background_time) + self.put('local_background_time', local_background_time) + self.put('local_background_time_iso', local_background_time_iso) # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index 5b1f8cbf..58b4ef65 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -13,15 +13,17 @@ from abc import ABC, abstractmethod import click import importlib +import os import sys import time +import yaml # local imports from swell.tasks.base.config import Config from swell.tasks.base.datetime import Datetime from swell.utilities.logger import Logger -from swell.tasks.task_registry import valid_tasks -from swell.tasks.utilities.utils import camelcase_to_underscore +from swell.tasks.base.task_registry import valid_tasks +from swell.tasks.base.utils import camelcase_to_underscore # -------------------------------------------------------------------------------------------------- @@ -30,7 +32,7 @@ class taskBase(ABC): # Base class constructor - def __init__(self, config_input, datetime_input, task_name): + def __init__(self, config_input, datetime_input, model, task_name): # Create message logger # --------------------- @@ -41,30 +43,25 @@ def __init__(self, config_input, datetime_input, task_name): self.logger.info(' Initializing task with the following parameters:') self.logger.info(' Task name: ' + task_name) self.logger.info(' Configuration: ' + config_input) - - # Create a configuration object - # ----------------------------- - self.config = Config(config_input, self.logger) - - # If task receives a datetime create the object and update the config - # ------------------------------------------------------------------- if (datetime_input is not None): - - # Write out the datetime self.logger.info(' Date and time: ' + datetime_input + '\n') - # Create a datetime object - self.datetime = Datetime(datetime_input) + # Create datetime + # --------------- + self.__datetime__ = None + if datetime_input is not None: + self.__datetime__ = Datetime(datetime_input) - # Augment configuration with cycle time. - self.config.add_cyle_time_parameter(self.datetime.datetime) + # Keep copy of model directive + # ---------------------------- + self.__model__ = model - # Add data assimilation window paramters to config - self.config.add_data_assimilation_window_parameters() + # Create a configuration object + # ----------------------------- + self.__config__ = Config(config_input, self.logger, datetime_in=self.__datetime__, + model=self.__model__) - # Resolve all variables that can be resolved - # ------------------------------------------ - self.config.resolve_config_file() + # ---------------------------------------------------------------------------------------------- # Execute is the place where a task does its work. It's defined as abstract in the base class # in order to force the sub classes (tasks) to implement it. @@ -72,13 +69,122 @@ def __init__(self, config_input, datetime_input, task_name): def execute(self): pass + # ---------------------------------------------------------------------------------------------- + + # Method to get something from config (with fail if not existing) + def config_get(self, key, default='NODEFAULT'): + return self.__config__.get(key, default) + + # ---------------------------------------------------------------------------------------------- + + # Method to get the Swell experiment path + def get_swell_exp_path(self): + experiment_root = self.config_get('experiment_root') + experiment_id = self.config_get('experiment_id') + return os.path.join(experiment_root, experiment_id) + + # ---------------------------------------------------------------------------------------------- + + # Method to get the Swell experiment configuration path + def get_swell_exp_config_path(self): + swell_exp_path = self.get_swell_exp_path() + return os.path.join(swell_exp_path, 'configuration') + + # ---------------------------------------------------------------------------------------------- + + # Method to get the Swell experiment configuration path + def get_datetime_format(self): + return self.__config__.get_datetime_format() + + # ---------------------------------------------------------------------------------------------- + + # Method to open a specific configuration file + def __open_jedi_interface_config_file(self, model_or_obs, config_name): + + # Assert that the task has a model associated with it + self.logger.assert_abort(self.__model__ is not None, + 'Task must have a model associated with it.') + + # JEDI interface name + jedi_interface = self.config_get('jedi_interface') + + # Get experiment configuration path + swell_exp_config_path = self.get_swell_exp_config_path() + + # Path to configuration file + config_file = os.path.join(swell_exp_config_path, 'jedi', jedi_interface, + self.__model__, model_or_obs, config_name + '.yaml') + + # Open file as a string + with open(config_file, 'r') as config_file_open: + config_file_str_templated = config_file_open.read() + + # Fill templates in the configuration file using the config + config_file_str = self.__config__.use_config_to_template_string(config_file_str_templated) + + # Convert string to dictionary + return yaml.safe_load(config_file_str) + + # ---------------------------------------------------------------------------------------------- + + # Method to open a specific model configuration file + def open_jedi_oops_config_file(self, config_name): + + # Get experiment configuration path + swell_exp_config_path = self.get_swell_exp_config_path() + + # Path to configuration file + config_file = os.path.join(swell_exp_config_path, 'jedi', 'oops', config_name + '.yaml') + + # Open file as a string + with open(config_file, 'r') as config_file_open: + config_file_str_templated = config_file_open.read() + + # Fill templates in the configuration file using the config + config_file_str = self.__config__.use_config_to_template_string(config_file_str_templated) + + # Convert string to dictionary + return yaml.safe_load(config_file_str) + + # ---------------------------------------------------------------------------------------------- + + # Method to open a specific model configuration file + def open_jedi_interface_model_config_file(self, config_name): + return self.__open_jedi_interface_config_file('model', config_name) + + # ---------------------------------------------------------------------------------------------- + + # Method to open a specific observation configuration file + def open_jedi_interface_obs_config_file(self, config_name): + obs_dict = self.__open_jedi_interface_config_file('observations', config_name) + + # If 4D window then add time interpolation to the dictionary + if self.config_get('window_type') == '4D': + obs_dict['get values'] = {} + obs_dict['get values']['time interpolation'] = 'linear' + + # Placeholder to add GeoVaLs saver filter + + # Placeholder for IO pool things + + return obs_dict + + # ---------------------------------------------------------------------------------------------- + + def use_config_to_template_string(self, string_in): + return self.__config__.use_config_to_template_string(string_in) + + # ---------------------------------------------------------------------------------------------- + + def get_model(self): + return self.__model__ # -------------------------------------------------------------------------------------------------- class taskFactory(): - def create_task(self, task, config, datetime): + def create_task(self, task, config, datetime, model): # Convert capitilized string to one with underscores task_lower = camelcase_to_underscore(task) @@ -87,13 +193,13 @@ def create_task(self, task, config, datetime): task_class = getattr(importlib.import_module('swell.tasks.'+task_lower), task) # Return task object - return task_class(config, datetime, task) + return task_class(config, datetime, model, task) # -------------------------------------------------------------------------------------------------- -def task_main(task, config, datetime): +def task_main(task, config, datetime, model): # For security check that task is in the registry if task not in valid_tasks: @@ -104,13 +210,12 @@ def task_main(task, config, datetime): for valid_task in valid_tasks: valid_task_logger.info(' ' + valid_task) valid_task_logger.info(' ') - valid_task_logger.info('ABORT: Task not found in task registry.') - sys.exit() + valid_task_logger.abort('ABORT: Task not found in task registry.') # Create the object constrc_start = time.perf_counter() creator = taskFactory() - task_object = creator.create_task(task, config, datetime) + task_object = creator.create_task(task, config, datetime, model) constrc_final = time.perf_counter() constrc_time = f'Constructed in {constrc_final - constrc_start:0.4f} seconds' @@ -137,9 +242,10 @@ def task_main(task, config, datetime): @click.argument('task') @click.argument('config') @click.option('-d', '--datetime', 'datetime', default=None) -def main(task, config, datetime): +@click.option('-m', '--model', 'model', default=None) +def main(task, config, datetime, model): - task_main(task, config, datetime) + task_main(task, config, datetime, model) # -------------------------------------------------------------------------------------------------- @@ -148,4 +254,5 @@ def main(task, config, datetime): if __name__ == '__main__': main() + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/task_registry.py b/src/swell/tasks/base/task_registry.py similarity index 91% rename from src/swell/tasks/task_registry.py rename to src/swell/tasks/base/task_registry.py index db5bb511..ae8ff9ca 100644 --- a/src/swell/tasks/task_registry.py +++ b/src/swell/tasks/base/task_registry.py @@ -5,12 +5,11 @@ # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. valid_tasks = [ - 'Stage', + 'StageJedi', 'GetObservations', 'BuildJedi', - 'JediConfig', 'GetBackground', - 'RunJediExecutable', + 'RunJediHofxExecutable', 'EvaDriver', 'MergeIodaFiles', 'GetBackgroundGeosExperiment', diff --git a/src/swell/tasks/utilities/utils.py b/src/swell/tasks/base/utils.py similarity index 91% rename from src/swell/tasks/utilities/utils.py rename to src/swell/tasks/base/utils.py index c911c114..1aaf8b32 100644 --- a/src/swell/tasks/utilities/utils.py +++ b/src/swell/tasks/base/utils.py @@ -4,11 +4,6 @@ # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -# -------------------------------------------------------------------------------------------------- - - -import subprocess - # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/build_jedi.py b/src/swell/tasks/build_jedi.py index 988a6208..3ec90cb2 100644 --- a/src/swell/tasks/build_jedi.py +++ b/src/swell/tasks/build_jedi.py @@ -4,6 +4,10 @@ # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + + import os import shutil import subprocess @@ -18,107 +22,57 @@ class BuildJedi(taskBase): def execute(self): - """ - Comment here - """ - cfg = self.config + # Get the build method + # -------------------- + jedi_build_method = self.config_get('jedi_build_method') - # Path to JEDI build - jedi_build_dir = self.config.get('jedi_build_dir') + # Get the experiment/jedi_bundle directory + # ---------------------------------------- + swell_exp_path = self.get_swell_exp_path() + jedi_bundle_path = os.path.join(swell_exp_path, 'jedi_bundle') + jedi_bundle_build_path = os.path.join(jedi_bundle_path, 'build') - # Get path to jedi executable - if 'executable' in cfg.keys() and 'existing build directory' in cfg['build jedi'].keys(): + # Make jedi_bundle directory + # -------------------------- + os.makedirs(jedi_bundle_path, 0o755, exist_ok=True) - # Check for presence of required executable and link build directory - # ------------------------------------------------------------------ - ex_build_dir = os.path.join(cfg['build jedi']['existing build directory']) + # Choice to link to existing build or build JEDI using jedi_bundle + # ---------------------------------------------------------------- + if jedi_build_method == 'use_existing': - if os.path.exists(os.path.join(ex_build_dir, 'bin', cfg['executable'])): - self.logger.info("Suitable JEDI build found, linking build directory. Warning: " + - "problems will follow if the loaded modules are not consistent " + - "with those used to build this version of JEDI.") + # Get the existing build directory from the dictionary + existing_build_directory = self.config_get('existing_build_directory') - # Remove trailing slash if needed - if jedi_build_dir.endswith('/'): - jedi_build_dir = jedi_build_dir[:-1] + # Assert that the existing build directory contains a bin directory + if not os.path.exists(os.path.join(existing_build_directory, 'bin')): + self.logger.abort(f'Existing JEDI build directory is provided but a bin ' + + f'directory is not found in the path ' + + f'\'{existing_build_directory}\'') - # Remove existing build path if present - if os.path.islink(jedi_build_dir): # Is a link - os.remove(jedi_build_dir) - elif os.path.isdir(jedi_build_dir): # Is a directory - shutil.rmtree(jedi_build_dir) + # Write warning to user + self.logger.info('Suitable JEDI build found, linking build directory. Warning: ' + + 'problems will follow if the loaded modules are not consistent ' + + 'with those used to build this version of JEDI. Also note that ' + + 'this experiment may not be reproducible if the build changes.') - # Link directory - os.symlink(ex_build_dir, jedi_build_dir) + # Remove trailing slash if needed + if jedi_bundle_build_path.endswith('/'): + jedi_bundle_build_path = jedi_bundle_build_path[:-1] - else: + # Remove existing build path if present + if os.path.islink(jedi_bundle_build_path): # Is a link + os.remove(jedi_bundle_build_path) + elif os.path.isdir(jedi_bundle_build_path): # Is a directory + shutil.rmtree(jedi_bundle_build_path) - self.logger.abort('Existing JEDI build directory is provided but the executable' + - ' is not found in that directory') + # Link existing build into the directory + os.symlink(existing_build_directory, jedi_bundle_build_path) - else: + elif jedi_build_method == 'create': + + self.logger.abort(f'Building JEDI is not yet supported') - # Build the JEDI code - # ------------------- - # No jedi executable was found - self.logger.info('No jedi executable found... \nBuilding Jedi...') - - # Get bundle and suite directories - bundle_dir = cfg['bundle'] - suite_dir = cfg['suite_dir'] - - # Get the repos from the repo dict and clone them if they haven't already been cloned - repo_dict = cfg['build jedi']['bundle repos'] - - # Prep ecbuild string - ecb_tmp = 'ecbuild_bundle( PROJECT {} GIT "https://github.com/{}/{}.git" ' + \ - 'BRANCH {} UPDATE )\n' - - # Prepare CMakeLists file - cmake_dst = os.path.join(bundle_dir, 'CMakeLists.txt') - - # Template CMakeLists.txt - cmake = ['cmake_minimum_required( VERSION 3.12 FATAL_ERROR )\n', - 'find_package( ecbuild 3.5 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ' + - '${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild)\n', - 'project( jedi-bundle VERSION 1.1.0 LANGUAGES C CXX Fortran )\n', - 'list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")\n', - 'include( ecbuild_bundle )\n', - 'set( ECBUILD_DEFAULT_BUILD_TYPE Release )\n', - 'set( ENABLE_MPI ON CACHE BOOL "Compile with MPI")\n', - 'ecbuild_bundle_initialize()\n', - 'ecbuild_bundle_finalize()\n'] - - # Iterate over repo dictionary - for d in repo_dict: - project = d['project'] - org = d['git org'] - repo = d['repo'] - branch = d['branch'] - proj_dir = os.path.join(bundle_dir, project) - cmake.insert(-4, ecb_tmp.format(project, org, repo, branch)) - if not os.path.exists(proj_dir): - self.logger.info('Cloning into {}...'.format(project)) - git_url = 'https://github.com/{}/{}.git'.format(org, repo) - git_got(git_url, branch, proj_dir, self.logger) - else: - self.logger.info('{} has already been cloned into bundle'.format(project)) - - # Write out new cmake lists file in bundle directory - with open(cmake_dst, 'w') as f: - f.writelines(cmake) - - # Create and change into the build directory - if not os.path.exists(jedi_build_dir): - os.mkdir(jedi_build_dir) - os.chdir(jedi_build_dir) - self.logger.info('Starting Jedi build at {}'.format(os.getcwd())) - - # Commands to build jedi - subprocess.run(['ecbuild -DMPIEXEC=$MPIEXEC {}'.format(bundle_dir)], shell=True) - # Format string used to be '../' so we need to point it directly to bundle - os.chdir('fv3-jedi/') - subprocess.run(['make -j6'], shell=True) - - self.logger.info('Build Jedi is complete!') + else: + self.logger.abort(f'Found \'{jedi_build_method}\' for jedi_build_method in the ' + f'experiment dictionary. Must be \'use_existing\' or \'create\'.') diff --git a/src/swell/tasks/clean_cycle.py b/src/swell/tasks/clean_cycle.py index cc9a6c4e..44f48ea9 100644 --- a/src/swell/tasks/clean_cycle.py +++ b/src/swell/tasks/clean_cycle.py @@ -27,11 +27,8 @@ class CleanCycle(taskBase): def execute(self): - cfg = self.config - logger = self.logger - cycle_dir = self.config.get("cycle_dir") - clean_list = cfg.get('CLEAN') - current_dir = os.getcwd() + cycle_dir = self.config_get("cycle_dir") + clean_list = self.config_get('clean_patterns') if os.path.isdir(cycle_dir): os.chdir(cycle_dir) @@ -50,4 +47,3 @@ def execute(self): filename = 'cycle_done' cmd = 'touch ' + os.path.join(cycle_dir, filename) os.system(cmd) - os.chdir(current_dir) diff --git a/src/swell/tasks/eva_driver.py b/src/swell/tasks/eva_driver.py index df09fcb7..143bdce9 100644 --- a/src/swell/tasks/eva_driver.py +++ b/src/swell/tasks/eva_driver.py @@ -11,11 +11,13 @@ import os import yaml -from swell.tasks.base.task_base import taskBase -from swell.utilities.dictionary_utilities import remove_matching_keys, replace_vars_dict -from swell.utilities.observations import find_instrument_from_string, ioda_name_to_long_name from eva.eva_base import eva +from swell.tasks.base.task_base import taskBase +from swell.utilities.dictionary import remove_matching_keys +from swell.utilities.jinja2 import template_string_jinja2 +from swell.utilities.observations import ioda_name_to_long_name + # -------------------------------------------------------------------------------------------------- @@ -26,29 +28,32 @@ def execute(self): # Parse config for jedi_config # ---------------------------- - cycle_dir = self.config.get('cycle_dir') - jedi_config_file = os.path.join(cycle_dir, 'jedi_config.yaml') - with open(jedi_config_file, 'r') as jedi_config_string: - config = yaml.safe_load(jedi_config_string) - - # Dictionary with observation config (from config used in JEDI) - # ------------------------------------------------------------- - observers = config['observations']['observers'] - - # Eva configuration - # ----------------- - eva_dict_template = self.config.get('EVA') - - # Loop over observers + cycle_dir = self.config_get('cycle_dir') + experiment_root = self.config_get('experiment_root') + experiment_id = self.config_get('experiment_id') + observations = self.config_get('observations') + + # Read Eva template file into dictionary + # -------------------------------------- + exp_path = os.path.join(experiment_root, experiment_id) + exp_suite_path = os.path.join(exp_path, experiment_id+'-suite') + eva_config_file = os.path.join(exp_suite_path, 'eva.yaml') + with open(eva_config_file, 'r') as eva_config_file_open: + eva_str_template = eva_config_file_open.read() + + # Loop over observations # ------------------- - for observer in observers: + for observation in observations: + + # Load the observation dictionary + observation_dict = self.open_jedi_interface_obs_config_file(observation) # Split the full path into path and filename - obs_path_file = observer['obs space']['obsdataout']['engine']['obsfile'] + obs_path_file = observation_dict['obs space']['obsdataout']['engine']['obsfile'] cycle_dir, obs_file = os.path.split(obs_path_file) # Get instrument ioda and full name - ioda_name = find_instrument_from_string(obs_file, self.logger) + ioda_name = observation full_name = ioda_name_to_long_name(ioda_name, self.logger) # Log the operator being worked on @@ -60,21 +65,24 @@ def execute(self): # Create dictionary used to override the eva config eva_override = {} + eva_override['cycle_dir'] = cycle_dir eva_override['obs_path_file'] = obs_path_file eva_override['instrument'] = ioda_name eva_override['instrument_title'] = full_name - eva_override['simulated_variables'] = observer['obs space']['simulated variables'] + eva_override['simulated_variables'] = \ + observation_dict['obs space']['simulated variables'] - if 'channels' in observer['obs space']: + if 'channels' in observation_dict['obs space']: need_channels = True - eva_override['channels'] = observer['obs space']['channels'] + eva_override['channels'] = observation_dict['obs space']['channels'] else: need_channels = False eva_override['channels'] = '' eva_override['channel'] = '' # Override the eva dictionary - eva_dict = replace_vars_dict(eva_dict_template, **eva_override) + eva_str = template_string_jinja2(self.logger, eva_str_template, eva_override) + eva_dict = yaml.safe_load(eva_str) # Remove channel keys if not needed if not need_channels: diff --git a/src/swell/tasks/get_background.py b/src/swell/tasks/get_background.py index 654c83ba..b2da1722 100644 --- a/src/swell/tasks/get_background.py +++ b/src/swell/tasks/get_background.py @@ -32,27 +32,23 @@ def execute(self): See the taskBase constructor for more information. """ - # Shortcuts to base objects - # ------------------------- - cfg = self.config - logger = self.logger - # Current cycle time object # ------------------------- - current_cycle = cfg.get('current_cycle') - current_cycle_dto = dt.strptime(current_cycle, cfg.dt_format) + current_cycle = self.config_get('current_cycle') + current_cycle_dto = dt.strptime(current_cycle, self.get_datetime_format()) # Get duration into forecast for first background file # ---------------------------------------------------- bkg_steps = [] # Parse config - window_type = cfg.get('window_type') - window_length = cfg.get('window_length') - window_offset = cfg.get('window_offset') - - # Position relative to center of the window where forecast starts - forecast_offset = cfg.get('analysis_forecast_window_offset') + window_type = self.config_get('window_type') + window_length = self.config_get('window_length') + window_offset = self.config_get('window_offset') + background_source = self.config_get('background_source', 'file') + background_experiment = self.config_get('background_experiment') + horizontal_resolution = self.config_get('horizontal_resolution') + forecast_offset = self.config_get('analysis_forecast_window_offset') # Convert to datetime durations window_length_dur = isodate.parse_duration(window_length) @@ -75,16 +71,14 @@ def execute(self): # If background is provided though files get all backgrounds # ---------------------------------------------------------- - bkg_info = cfg.get('backgrounds') - - if window_type == "4D" and bkg_info['background source'] == 'file': + if window_type == "4D" and background_source == 'file': - bkg_freq = bkg_info['background frequency'] + bkg_freq = self.config_get('background_frequency') bkg_freq_dur = isodate.parse_duration(bkg_freq) # Check for a sensible frequency if (window_length_dur/bkg_freq_dur) % 2: - logger.abort('Window length not divisible by background frequency') + self.logger.abort('Window length not divisible by background frequency') # Loop over window start_date = current_cycle_dto - window_offset_dur @@ -99,39 +93,28 @@ def execute(self): # Loop over background files in the R2D2 config and fetch # ------------------------------------------------------- - logger.info('Background steps being fetched: '+' '.join(str(e) for e in bkg_steps)) - - # Background dictionary from config - background_dict = cfg.get('BACKGROUND') + self.logger.info('Background steps being fetched: '+' '.join(str(e) for e in bkg_steps)) # Get r2d2 dictionary - r2d2_dict = cfg.get('R2D2') + r2d2_dict = self.open_jedi_interface_model_config_file('r2d2') # Loop over fc for fc in r2d2_dict['fetch']['fc']: # Reset target file - target_file_template = os.path.split(background_dict['filename'])[1] - - # Datetime format to use - user_date_format = fc['user_date_format'] + target_file_template = fc['filename'] # Loop over file types for file_type in fc['file_type']: - # Replace filetype in target_file_template - target_file_type_template = target_file_template.replace("$(file_type)", file_type) - # Looop over background steps for bkg_step in bkg_steps: # Set the datetime format for the output files background_time = forecast_start_time + isodate.parse_duration(bkg_step) - valid_time_str = background_time.strftime(user_date_format) - # Set the target file name - target_file = target_file_type_template.replace("$(valid_date)", valid_time_str) - target_file = os.path.join(cfg.get('cycle_dir'), target_file) + # Set the datetime templating in the target file name + target_file = background_time.strftime(target_file_template) # Perform the fetch fetch(date=forecast_start_time, @@ -140,9 +123,9 @@ def execute(self): file_type='bkg', fc_date_rendering='analysis', step=bkg_step, - resolution=cfg.get('horizontal_resolution'), + resolution=horizontal_resolution, type='fc', - experiment=bkg_info['background experiment']) + experiment=background_experiment) # Change permission os.chmod(target_file, 0o644) diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index b3e03035..f5d57960 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -33,30 +33,30 @@ def execute(self): "tlapse" files need to be fetched. """ - cfg = self.config - logger = self.logger - # Parse config # ------------ - experiment = cfg.get('obs_experiment') - window_begin = cfg.get('window_begin') - background_time = cfg.get('background_time') - obs = cfg.get('OBSERVATIONS') + experiment = self.config_get('obs_experiment') + window_begin = self.config_get('window_begin') + background_time = self.config_get('background_time') + observations = self.config_get('observations') # Loop over observation operators # ------------------------------- - for ob in obs: + for observation in observations: + + # Open the observation operator dictionary + # ---------------------------------------- + observation_dict = self.open_jedi_interface_obs_config_file(observation) # Fetch observation files # ----------------------- - name = ob['obs space']['name'] - target_file = ob['obs space']['obsdatain']['engine']['obsfile'] - logger.info("Processing observation file "+target_file) + target_file = observation_dict['obs space']['obsdatain']['engine']['obsfile'] + self.logger.info("Processing observation file "+target_file) fetch(date=window_begin, target_file=target_file, provider='ncdiag', - obs_type=name, + obs_type=observation, type='ob', experiment=experiment) @@ -65,17 +65,17 @@ def execute(self): # Fetch bias correction files # --------------------------- - if 'obs bias' not in ob: + if 'obs bias' not in observation_dict: continue # Satbias - target_file = ob['obs bias']['input file'] - logger.info("Processing satbias file "+target_file) + target_file = observation_dict['obs bias']['input file'] + self.logger.info("Processing satbias file "+target_file) fetch(date=background_time, target_file=target_file, provider='gsi', - obs_type=name, + obs_type=observation, type='bc', experiment=experiment, file_type='satbias') @@ -84,14 +84,14 @@ def execute(self): os.chmod(target_file, 0o644) # Tlapse - for target_file in self.get_tlapse_files(ob): + for target_file in self.get_tlapse_files(observation_dict): - logger.info("Processing tlapse file "+target_file) + self.logger.info("Processing tlapse file "+target_file) fetch(date=background_time, target_file=target_file, provider='gsi', - obs_type=name, + obs_type=observation, type='bc', experiment=experiment, file_type='tlapse') @@ -101,11 +101,11 @@ def execute(self): # ---------------------------------------------------------------------------------------------- - def get_tlapse_files(self, ob): + def get_tlapse_files(self, observation_dict): - # Function to locate instanses of tlapse in the obs operator config + # Function to locate instances of tlapse in the obs operator config - hash = ob + hash = observation_dict if 'obs bias' not in hash: return diff --git a/src/swell/tasks/jedi_config.py b/src/swell/tasks/jedi_config.py deleted file mode 100644 index 66dff238..00000000 --- a/src/swell/tasks/jedi_config.py +++ /dev/null @@ -1,232 +0,0 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - - -from swell.tasks.base.task_base import taskBase -from swell.configuration.configuration import return_configuration_path -from swell.utilities.observations import find_instrument_from_string -from swell.utilities.sat_db_utils import run_sat_db_process - -import copy -import os -import re -import yaml -import itertools -from datetime import datetime as dt - -# -------------------------------------------------------------------------------------------------- - - -def ranges(i): - for _, group in itertools.groupby(enumerate(i), lambda pair: pair[1] - pair[0]): - group = list(group) - yield group[0][1], group[-1][1] - - -# -------------------------------------------------------------------------------------------------- - - -class JediConfig(taskBase): - - def replace_jedi_conf(self, jedi_config): - - # Loop over dictionary and replace if element_template is a dictionary - for key, element_template in jedi_config.items(): - if isinstance(element_template, dict): - self.replace_jedi_conf(element_template) - else: - # Strip special characters from element - element = element_template - element = element.replace("$", "") - element = element.replace("{", "") - element = element.replace("}", "") - - try: - # Replace with element from filled config - jedi_config[key] = self.config.get(element) - except KeyError: - # If element not in experiment config remove - del element_template - - def execute(self): - - """Generates the yaml configuration needed to run the JEDI executable. - - Parameters - ---------- - All inputs are extracted from the JEDI experiment file configuration. - See the taskBase constructor for more information. - - """ - - # Parse config - # ------------ - time_window = self.config.get('current_cycle') - update_channels = self.config.get('update channels from database') - use_geos_sat_db = self.config.get('use geos satellite channel database') - obs = self.config.get('OBSERVATIONS') - - sat_db_yaml_loc = os.path.join(return_configuration_path(), 'satellite_channels') - - cycle_dt = dt.strptime(time_window, '%Y%m%dT%H%M%SZ') - - if update_channels: - - # Manage satellite database preparation - # ------------------------------------- - if use_geos_sat_db: - - # Clone GEOSana_GridComp and generate database data frame - sat_db = run_sat_db_process(self.config.get('geos_dir'), self.logger) - - # Loop over observation operators to update instrument channels in loaded config - # ------------------------------------------------------------------------------ - for ob in obs: - - name = ob['obs space']['name'] - - # Delete obs filters - # ------------------ - del ob['obs filters'] - - if '_' in name: - sat_instr_list = name.split('_') - instr = sat_instr_list[0] - sat = sat_instr_list[1] - - try: - if not use_geos_sat_db: - - # Open observation yaml file - # -------------------------- - instr_file = os.path.join(sat_db_yaml_loc, sat + '.yaml') - with open(instr_file, 'r') as file: - sat_dict = yaml.full_load(file) - instr_dict = sat_dict[instr] - - # Find instrument channel ranges for given cycle time - # --------------------------------------------------- - instr_ind = 999 - for ind in range(len(instr_dict)): - begin = dt.strptime(instr_dict[ind]['begin date'], - '%Y%m%dT%H%M%S') - end = dt.strptime(instr_dict[ind]['end date'], '%Y%m%dT%H%M%S') - if ((cycle_dt >= begin) and (cycle_dt < end)): - instr_ind = ind - assert (instr_ind != 999) - instr_ch_list = instr_dict[ind]['channels'] - - else: - - # Extract instrument dataframe from loaded satellite database - # ----------------------------------------------------------- - sat_df = sat_db.loc[sat_db['sat'] == sat] - instr_df = sat_df.loc[sat_df['instr'] == instr] - - # Find instrument channel ranges for given cycle time - # --------------------------------------------------- - instr_ind = 999 - for ind in range(len(instr_df)): - begin = dt.strptime(instr_df['start'][ind], '%Y%m%d%H%M%S') - end = dt.strptime(instr_df['end'][ind], '%Y%m%d%H%M%S') - if ((cycle_dt >= begin) and (cycle_dt < end)): - instr_ind = ind - assert (instr_ind != 999) - instr_ch_list = instr_df['channels'][ind] - - # Process instrument channel list into ranges - # ------------------------------------------- - instr_ch_list = [int(i) for i in instr_ch_list] - ch_ranges = list(ranges(instr_ch_list)) - new_ch_str = '' - for ch_range in ch_ranges: - if (ch_range[0] != ch_range[1]): - new_ch_str = new_ch_str + str(ch_range[0]) + '-' + \ - str(ch_range[1]) + ',' - else: - new_ch_str += str(ch_range[0]) + ',' - new_ch_str = new_ch_str[0:-1] - - # Update channel range directly in loaded config - # ---------------------------------------------- - ob['obs space']['channels'] = new_ch_str - - except AssertionError: - self.logger.info('sat {} and instr {} are not in the sat database'. - format(sat, instr)) - continue - - # Add section for saving the GeoVaLs if necessary - # ----------------------------------------------- - save_geovals = self.config.get("save_geovals", False) - if save_geovals: - save_geovals_dict = {} - save_geovals_dict['filter'] = 'GOMsaver' - - for ob in obs: - # Extract normal output filename - cycle_dir, obs_file = os.path.split( - ob['obs space']['obsdataout']['engine']['obsfile']) - # Append instrument name with geovals - instrument = find_instrument_from_string(obs_file, self.logger) - geovals_file = obs_file.replace(instrument, instrument+'.geovals') - # Update the filter dictionary - save_geovals_dict['filename'] = os.path.join(cycle_dir, geovals_file) - # Extract filters and append - filters = ob.get('obs filters', []) - filters.append(save_geovals_dict) - # Put dictionary into the filter config - ob['obs filters'] = copy.deepcopy(filters) - - # Add section for time interpolation - # ---------------------------------- - time_interpolation_type = 'nearest' - window_type = self.config.get('window_type', '4D') - if window_type == '4D': - time_interpolation_type = 'linear' - - # User can override the window guided behavior (if 4D) - time_interpolation_type = self.config.get('time_interpolation', time_interpolation_type) - - get_values_dict = {} - get_values_dict['time interpolation'] = time_interpolation_type - - # Add to obs operator config - for ob in obs: - ob['get values'] = get_values_dict - - # Set full path to the templated config file - # ------------------------------------------ - jedi_conf_path = os.path.join(self.config.get("suite_dir"), "jedi_config.yaml") - - # Open the template file to dictionary - # ------------------------------------ - with open(jedi_conf_path, 'r') as ymlfile: - jedi_conf = yaml.safe_load(ymlfile) - - # Loop over the dictionary and replace elements - # --------------------------------------------- - self.replace_jedi_conf(jedi_conf) - - # Remove some keys that do not pass yaml validation - # ------------------------------------------------- - del jedi_conf['initial condition']['filename'] - - # Filename for output yaml - # ------------------------ - cycle_dir = self.config.get("cycle_dir") - if not os.path.exists(cycle_dir): - os.makedirs(cycle_dir, 0o755, exist_ok=True) - - jedi_conf_output = os.path.join(cycle_dir, "jedi_config.yaml") - - # Write out the final yaml file - # ----------------------------- - with open(jedi_conf_output, 'w') as outfile: - yaml.dump(jedi_conf, outfile, default_flow_style=False) diff --git a/src/swell/tasks/merge_ioda_files.py b/src/swell/tasks/merge_ioda_files.py index 23e3fe77..ffe3b6e8 100644 --- a/src/swell/tasks/merge_ioda_files.py +++ b/src/swell/tasks/merge_ioda_files.py @@ -16,7 +16,6 @@ import yaml from swell.tasks.base.task_base import taskBase -from swell.utilities.observations import find_instrument_from_string # -------------------------------------------------------------------------------------------------- @@ -31,21 +30,15 @@ class MergeIodaFiles(taskBase): def execute(self): # Parse config - cycle_dir = self.config.get('cycle_dir') - save_geovals = self.config.get("save_geovals", False) - - # Config file used with jedi executable - jedi_config_file = os.path.join(cycle_dir, 'jedi_config.yaml') - - # Read into dictionary - with open(jedi_config_file, 'r') as jedi_config_string: - config = yaml.safe_load(jedi_config_string) - - # Dictionary with observation config - observers = config['observations']['observers'] + cycle_dir = self.config_get('cycle_dir') + save_geovals = self.config_get('save_geovals', False) + observations = self.config_get('observations') # Loop over observations - for observer in observers: + for observation in observations: + + # Load the observation dictionary + observation_dict = self.open_jedi_interface_obs_config_file(observation) # Dataset to hold the concatenated files for both observations and geovals ds_all = xr.Dataset() @@ -60,14 +53,14 @@ def execute(self): # Split the full obs output path into path and filename cycle_dir, obs_file = os.path.split( - observer['obs space']['obsdataout']['engine']['obsfile']) + observation_dict['obs space']['obsdataout']['engine']['obsfile']) # Read in the observation output files if the pool is larger than 1 # ----------------------------------------------------------------- # Check IO pool and skip merge if ioda output a single file try: - max_pool_size = observer['obs space']['io pool']['max pool size'] + max_pool_size = observation_dict['obs space']['io pool']['max pool size'] except Exception: max_pool_size = None @@ -76,8 +69,7 @@ def execute(self): else: # Write info - instrument = find_instrument_from_string(obs_file, self.logger) - self.logger.info('Combining IODA output files for '+instrument+'.') + self.logger.info('Combining IODA output files for '+observation+'.') # Split base and extension part of filename obs_file_bse = os.path.splitext(obs_file)[0] @@ -163,14 +155,13 @@ def execute(self): if save_geovals: # Loop over filters, find geoval saver and extract filename - for obs_filter in observer['obs filters']: + for obs_filter in observation_dict['obs filters']: if obs_filter['filter'] == 'GOMsaver': cycle_dir, geovals_file = os.path.split(obs_filter['filename']) break # Write info - instrument = find_instrument_from_string(geovals_file, self.logger) - self.logger.info('Adding GeoVaLs for '+instrument+'.') + self.logger.info('Adding GeoVaLs for '+observation+'.') # Split base and extension part of filename gvals_file_bse = os.path.splitext(geovals_file)[0] diff --git a/src/swell/tasks/run_jedi_executable.py b/src/swell/tasks/run_jedi_executable.py deleted file mode 100644 index f808774f..00000000 --- a/src/swell/tasks/run_jedi_executable.py +++ /dev/null @@ -1,56 +0,0 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -import os -import subprocess -import sys - -from swell.tasks.base.task_base import taskBase - -# -------------------------------------------------------------------------------------------------- - - -class RunJediExecutable(taskBase): - - def execute(self): - - # Path to executable being run - # ---------------------------- - jedi_build_path = self.config.get('jedi_build_dir') - jedi_executable = self.config.get('executable') - jedi_exe_path = os.path.join(jedi_build_path, 'bin', jedi_executable) - - # Compute number of processors - # ---------------------------- - processors = self.config.get('processors') - np = 1 - for processor in processors: - np = np * processor - - # Jedi configuration file - # ----------------------- - jedi_conf_output = os.path.join(self.config.get("cycle_dir"), "jedi_config.yaml") - - # Run the JEDI executable - # ----------------------- - self.logger.info('Running '+jedi_exe_path+' with '+str(np)+' processors.') - - command = ['mpirun', '-np', str(np), jedi_exe_path, jedi_conf_output] - - process = subprocess.Popen(command, stdout=subprocess.PIPE) - while True: - output = process.stdout.readline().decode() - if output == '' and process.poll() is not None: - break - if output: - print(output.strip()) - rc = process.poll() - - # Abort if the executable did not run successfully - if rc != 0: - command_string = ' '.join(command) - self.logger.abort('subprocess.run with command ' + command_string + - ' failed to execute.') diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py new file mode 100644 index 00000000..b5feef0d --- /dev/null +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -0,0 +1,128 @@ +# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import os +import subprocess +import sys +import yaml + +from swell.tasks.base.task_base import taskBase + + +# -------------------------------------------------------------------------------------------------- + + +interface_executable = { + 'fv3-jedi-4D': 'fv3jedi_hofx.x', + 'fv3-jedi-3D': 'fv3jedi_hofx_nomodel.x', + 'soca-4D': 'soca_hofx.x', + 'soca-3D': 'soca_hofx3d.x', +} + + +# -------------------------------------------------------------------------------------------------- + + +class RunJediHofxExecutable(taskBase): + + def jedi_dictionary_iterator(self, jedi_config_dict): + + # Loop over dictionary and replace if value is a dictionary + for key, value in jedi_config_dict.items(): + if isinstance(value, dict): + self.jedi_dictionary_iterator(value) + else: + if 'TASKFILL' in value: + value_file = value.replace('TASKFILL', '') + value_dict = self.open_jedi_interface_model_config_file(value_file) + jedi_config_dict[key] = value_dict + + # ---------------------------------------------------------------------------------------------- + + def generate_jedi_config(self, window_type): + + # Create dictionary from the templated JEDI config file + jedi_config_dict = self.open_jedi_oops_config_file('hofx' + window_type) + + # Observations is a special case + observations = [] + obs = self.config_get('observations') + for ob in obs: + # Get observation dictionary + observations.append(self.open_jedi_interface_obs_config_file(ob)) + jedi_config_dict['observations']['observers'] = observations + + # Forecast model is a special case + model = self.config_get('model') + model_dict = self.open_jedi_interface_model_config_file(model) + jedi_config_dict['model'] = model_dict + + # Read configs for the rest of the dictionary + self.jedi_dictionary_iterator(jedi_config_dict) + + return jedi_config_dict + + # ---------------------------------------------------------------------------------------------- + + def execute(self): + + # Path to executable being run + # ---------------------------- + cycle_dir = self.config_get('cycle_dir') + experiment_dir = self.config_get('experiment_dir') + jedi_interface = self.config_get('jedi_interface') + window_type = self.config_get('window_type') + model = self.config_get('window_type') + total_processors = self.config_get('total_processors') + + # Jedi configuration file + # ----------------------- + jedi_config_file = os.path.join(cycle_dir, 'jedi_hofx_config.yaml') + + # Generate the JEDI configuration file for running the executable + # --------------------------------------------------------------- + jedi_config_dict = self.generate_jedi_config(window_type) + + with open(jedi_config_file, 'w') as jedi_config_file_open: + yaml.dump(jedi_config_dict, jedi_config_file_open, default_flow_style=False) + + # Jedi executable name + # -------------------- + jedi_executable = interface_executable[jedi_interface + '-' + window_type] + jedi_executable_path = os.path.join(experiment_dir, 'jedi_bundle', 'build', 'bin', + jedi_executable) + + # Compute number of processors + # ---------------------------- + np_string = self.use_config_to_template_string(total_processors) + np = eval(np_string) + + # Run the JEDI executable + # ----------------------- + self.logger.info('Running '+jedi_executable_path+' with '+str(np)+' processors.') + + command = ['mpirun', '-np', str(np), jedi_executable_path, jedi_config_file] + + process = subprocess.Popen(command, stdout=subprocess.PIPE) + while True: + output = process.stdout.readline().decode() + if output == '' and process.poll() is not None: + break + if output: + print(output.strip()) + rc = process.poll() + + # Abort if the executable did not run successfully + if rc != 0: + command_string = ' '.join(command) + self.logger.abort('subprocess.run with command ' + command_string + + ' failed to execute.', False) + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py index 7b6c5fc2..d5c7295d 100644 --- a/src/swell/tasks/save_obs_diags.py +++ b/src/swell/tasks/save_obs_diags.py @@ -22,28 +22,27 @@ class SaveObsDiags(taskBase): def execute(self): - cfg = self.config - logger = self.logger - # Parse config # ------------ - experiment = cfg.get('experiment_id') - window_begin = cfg.get('window_begin') - background_time = cfg.get('background_time') - obs = cfg.get('OBSERVATIONS') + experiment_id = self.config_get('experiment_id') + window_begin = self.config_get('window_begin') + observations = self.config_get('observations') # Loop over observation operators # ------------------------------- - for ob in obs: + for observation in observations: + + # Load the observation dictionary + observation_dict = self.open_jedi_interface_obs_config_file(observation) # Store observation files # ----------------------- - name = ob['obs space']['name'] - source_file = ob['obs space']['obsdataout']['engine']['obsfile'] + name = observation_dict['obs space']['name'] + source_file = observation_dict['obs space']['obsdataout']['engine']['obsfile'] store(date=window_begin, provider='ncdiag', source_file=source_file, obs_type=name, type='ob', - experiment=experiment) + experiment=experiment_id) diff --git a/src/swell/tasks/stage.py b/src/swell/tasks/stage_jedi.py similarity index 75% rename from src/swell/tasks/stage.py rename to src/swell/tasks/stage_jedi.py index ceed173e..90e4949b 100644 --- a/src/swell/tasks/stage.py +++ b/src/swell/tasks/stage_jedi.py @@ -22,10 +22,10 @@ # -------------------------------------------------------------------------------------------------- -class Stage(taskBase): +class StageJedi(taskBase): def execute(self): - """Acquires listed files under the YAML STAGE directive. + """Acquires listed files under the configuration/jedi/interface/model/stage.yaml file. Parameters ---------- @@ -33,8 +33,14 @@ def execute(self): See the taskBase constructor for more information. """ + # Open the stage configuration file + # --------------------------------- + stage_dict = self.open_jedi_interface_model_config_file('stage') + + # Run the file handler + # -------------------- try: - fh = get_file_handler(self.config['STAGE']) + fh = get_file_handler(stage_dict) if not fh.is_ready(): self.logger.abort('One or more files not ready') else: diff --git a/src/swell/tasks/utilities/__init__.py b/src/swell/tasks/utilities/__init__.py deleted file mode 100644 index ac1c0bc4..00000000 --- a/src/swell/tasks/utilities/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -import os - -repo_directory = os.path.dirname(__file__) diff --git a/src/swell/utilities/dictionary.py b/src/swell/utilities/dictionary.py new file mode 100644 index 00000000..ed792269 --- /dev/null +++ b/src/swell/utilities/dictionary.py @@ -0,0 +1,106 @@ +# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + + +import re +import string +import yaml +from collections.abc import Hashable + + +# -------------------------------------------------------------------------------------------------- + + +def dict_get(logger, dictionary, key, default='NODEFAULT'): + + if key in dictionary.keys(): + + return dictionary[key] + + else: + + if default == 'NODEFAULT': + logger.abort(f'In dict_get the key \'{key}\' was not found in the dictionary and no ' + + f'default was provided.') + else: + return default + + +# -------------------------------------------------------------------------------------------------- + + +def remove_matching_keys(d, key): + """ + Recursively locates and removes all dictionary items matching the supplied key. + Parameters + ---------- + d : root node, required + traversable data structure (dictionary or list) to be searched. + key: string, required + Key to be removed + """ + + if isinstance(d, dict): + + d.pop(key, None) + + for k, v in iter(d.items()): + if not isinstance(v, Hashable): + remove_matching_keys(v, key) + + elif isinstance(d, list): + + for v in d: + if not isinstance(v, Hashable): + remove_matching_keys(v, key) + + +# -------------------------------------------------------------------------------------------------- + + +def add_comments_to_dictionary(dictionary_string, comment_dictionary): + + dict_str_items = dictionary_string.split('\n') + + for key in comment_dictionary.keys(): + + keys_hierarchy = key.split('.') + indent_ind = len(key.split('.')) - 1 + indent = indent_ind*2*' ' + + if indent_ind == 0: + + for ind, dict_str_item in enumerate(dict_str_items): + + if key + ':' in dict_str_item: + + dict_str_items.insert(max(0, ind), '\n# ' + comment_dictionary[key]) + break + + else: + + index_of_key = 0 + for key_hierarchy in keys_hierarchy: + + for line in range(index_of_key, len(dict_str_items)): + + if key_hierarchy + ':' in dict_str_items[line]: + + index_of_key = line + + break + + dict_str_items.insert(max(0, index_of_key), '\n' + indent + '# ' + + comment_dictionary[key]) + + dictionary_string_with_comments = '\n'.join(dict_str_items) + + return dictionary_string_with_comments + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/dictionary_utilities.py b/src/swell/utilities/dictionary_utilities.py deleted file mode 100644 index 07752bc8..00000000 --- a/src/swell/utilities/dictionary_utilities.py +++ /dev/null @@ -1,119 +0,0 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -# -------------------------------------------------------------------------------------------------- - - -import re -import string -import yaml -from collections.abc import Hashable - -from swell.utilities.string_utils import replace_vars - -# -------------------------------------------------------------------------------------------------- - - -def resolve_definitions(dictionary): - """ - At the highest level of the dictionary are definitions, such as swell_dir: /path/to/swell. - Elsewhere in the dictionary is use of these definitions, such as key: $(swell_dir)/some/file.ext - In this script variables like $(swell_dir) are replaced everywhere in the dictionary using the - definition. - - Parameters - ---------- - dictionary : dictionary, required - Dictionary to be modified - - Returns - ------- - dictionary: dictionary - Dictionary with any definitions resolved - """ - - # Convert dictionary to string representation in yaml form - dictionary_string = yaml.dump(dictionary) - - # Get definitions in dictionary - defs = {} - defs.update({k: str(v) for k, v in iter(dictionary.items()) - if not isinstance(v, dict) and not isinstance(v, list)}) - - # Replace the definitions everywhere in the dictionary - dictionary_string = replace_vars(dictionary_string, **defs) - - # Convert back to dictionary - dictionary = yaml.safe_load(dictionary_string) - - return dictionary - - -# -------------------------------------------------------------------------------------------------- - - -def replace_vars_dict(d, **defs): - """ - At the highest level of the dictionary are definitions, such as swell_dir: /path/to/swell. - Elsewhere in the dictionary is use of these definitions, such as key: $(swell_dir)/some/file.ext - In this script variables like $(swell_dir) are replaced everywhere in the dictionary using the - definition. - - Parameters - ---------- - d : dictionary, required - Dictionary to be modified - defs: dictionary, required - Dictionary of definitions for resolving variables expressed as key-word arguments. - - Returns - ------- - d_interp: dictionary - Dictionary with any definitions resolved - """ - - # Convert dictionary to string representation in yaml form - d_string = yaml.dump(d) - - # Replace the definitions everywhere in the dictionary - d_string = replace_vars(d_string, **defs) - - # Convert back to dictionary - d_interp = yaml.safe_load(d_string) - - return d_interp - - -# -------------------------------------------------------------------------------------------------- - - -def remove_matching_keys(d, key): - """ - Recursively locates and removes all dictionary items matching the supplied key. - Parameters - ---------- - d : root node, required - traversable data structure (dictionary or list) to be searched. - key: string, required - Key to be removed - """ - - if isinstance(d, dict): - - d.pop(key, None) - - for k, v in iter(d.items()): - if not isinstance(v, Hashable): - remove_matching_keys(v, key) - - elif isinstance(d, list): - - for v in d: - if not isinstance(v, Hashable): - remove_matching_keys(v, key) - - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/jinja2.py b/src/swell/utilities/jinja2.py new file mode 100644 index 00000000..ce93199b --- /dev/null +++ b/src/swell/utilities/jinja2.py @@ -0,0 +1,36 @@ +# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + + +import jinja2 + + +# -------------------------------------------------------------------------------------------------- + + +def template_string_jinja2(logger, templated_string, dictionary_of_templates): + + # Load the templated string + t = jinja2.Template(templated_string, trim_blocks=True, lstrip_blocks=True, + undefined=jinja2.StrictUndefined) + + # Render the templates using the dictionary + string_rendered = t.render(dictionary_of_templates) + + logger.assert_abort('{{' not in string_rendered, f'In use_config_to_template_string ' + + f'the output string still contains template directives. ' + + f'{string_rendered}') + + logger.assert_abort('}}' not in string_rendered, f'In use_config_to_template_string ' + + f'the output string still contains template directives. ' + + f'{string_rendered}') + + return string_rendered + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/logger.py b/src/swell/utilities/logger.py index 760a573f..d4bc15e9 100644 --- a/src/swell/utilities/logger.py +++ b/src/swell/utilities/logger.py @@ -9,6 +9,7 @@ import os import sys +import textwrap # -------------------------------------------------------------------------------------------------- @@ -24,8 +25,12 @@ def __init__(self, task_name): self.task_name = task_name + # Maximum length of lines + self.__maxlen__ = 100 + # Set default logging levels - self.loggerdict = {'INFO': True, + self.loggerdict = {'BLANK': True, + 'INFO': True, 'TRACE': False, 'DEBUG': False, } @@ -41,38 +46,72 @@ def __init__(self, task_name): # ---------------------------------------------------------------------------------------------- - def send_message(self, level, message): + def send_message(self, level, message, wrap): + + # Wrap the message if needed + if wrap: + message_items = textwrap.wrap(message, self.__maxlen__, break_long_words=True) + for i in range(0, len(message_items)-1): + message_items[i] = message_items[i] + ' ...' + else: + message_items = [] + message_items.append(message) + + # Include level in the message + level_show = '' + if level != 'BLANK': + level_show = level_show+' '+self.task_name+': ' if level == 'ABORT' or self.loggerdict[level]: - print(level+' '+self.task_name+': '+message) + first_line = True + for message_item in message_items: + if not first_line: + message_item = ' ' + message_item + print(level_show+message_item) + first_line = False # ---------------------------------------------------------------------------------------------- - def info(self, message): + def info(self, message, wrap=True): - self.send_message('INFO', message) + self.send_message('INFO', message, wrap) # ---------------------------------------------------------------------------------------------- - def trace(self, message): + def trace(self, message, wrap=True): - self.send_message('TRACE', message) + self.send_message('TRACE', message, wrap) # ---------------------------------------------------------------------------------------------- - def debug(self, message): + def debug(self, message, wrap=True): - self.send_message('DEBUG', message) + self.send_message('DEBUG', message, wrap) # ---------------------------------------------------------------------------------------------- - def abort(self, message): + def blank(self, message, wrap=True): + + self.send_message('BLANK', message, wrap) + + # ---------------------------------------------------------------------------------------------- - self.send_message('ABORT', message) + def abort(self, message, wrap=True): + + self.send_message('ABORT', message, wrap) sys.exit('ABORTING\n') # ---------------------------------------------------------------------------------------------- + def assert_abort(self, condition, message, wrap=True): + + if condition: + return + else: + self.abort(message, wrap) + + # ---------------------------------------------------------------------------------------------- + def input(self, message): input('INPUT '+self.task_name+': '+message + ". Press any key to continue...") diff --git a/src/swell/utilities/observations.py b/src/swell/utilities/observations.py index 080c2bc5..961353bd 100644 --- a/src/swell/utilities/observations.py +++ b/src/swell/utilities/observations.py @@ -10,45 +10,7 @@ import os import yaml -from swell.configuration.configuration import return_configuration_path - - -# -------------------------------------------------------------------------------------------------- - - -def find_instrument_from_string(full_string, logger): - - # Get configuration path - configuration_path = return_configuration_path() - - # Insturment list yaml - obs_ioda_names_file = os.path.join(configuration_path, 'observation_ioda_names.yaml') - - # Open file and convert to dictionary - with open(obs_ioda_names_file, 'r') as obs_ioda_names_str: - obs_ioda_names_dict = yaml.safe_load(obs_ioda_names_str) - - # Get the list of ioda instrument names - obs_ioda_names = obs_ioda_names_dict['ioda instrument names'] - - # Set default outputs - key = None - variable = None - - # Loop over list of valid names - obs_ioda_name_found = None - for obs_ioda_name in obs_ioda_names: - if obs_ioda_name['ioda name'] in full_string: - obs_ioda_name_found = obs_ioda_name['ioda name'] - - # Check something found - if obs_ioda_name_found is None: - logger.abort('In find_instrument_from_string the string ' + full_string + 'does not ' + - f'contain one of the valid instruments: {obs_ioda_names} . Additional ' + - 'instruments can be added to swell/configuration/observation_ioda_names.yaml') - - # Return found value - return obs_ioda_name_found +from swell.swell_path import get_swell_path # -------------------------------------------------------------------------------------------------- @@ -57,10 +19,10 @@ def find_instrument_from_string(full_string, logger): def ioda_name_to_long_name(ioda_name, logger): # Get configuration path - configuration_path = return_configuration_path() + jedi_configuration_path = os.path.join(get_swell_path(), 'configuration', 'jedi') # Insturment list yaml - obs_ioda_names_file = os.path.join(configuration_path, 'observation_ioda_names.yaml') + obs_ioda_names_file = os.path.join(jedi_configuration_path, 'observation_ioda_names.yaml') # Open file and convert to dictionary with open(obs_ioda_names_file, 'r') as obs_ioda_names_str: @@ -79,7 +41,7 @@ def ioda_name_to_long_name(ioda_name, logger): # Check something found if not found: - logger.abort('In find_instrument_from_string the string ' + full_string + 'does not ' + + logger.abort('In ioda_name_to_long_name the string ' + full_string + 'does not ' + f'contain one of the valid instruments: {obs_ioda_names} . Additional ' + 'instruments can be added to swell/configuration/observation_ioda_names.yaml') diff --git a/src/swell/utilities/string_utils.py b/src/swell/utilities/string_utils.py deleted file mode 100644 index 6e9670a7..00000000 --- a/src/swell/utilities/string_utils.py +++ /dev/null @@ -1,65 +0,0 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -# -------------------------------------------------------------------------------------------------- - - -import re -import string - - -# -------------------------------------------------------------------------------------------------- - - -def replace_vars(s, **defs): - """Interpolate/replace variables in string - - Resolved variable formats are: $var, {{var}} and $(var). Undefined - variables remain unchanged in the returned string. This method will - recursively resolve variables of variables. - - Parameters - ---------- - s : string, required - Input string containing variables to be resolved. - defs: dict, required - dictionary of definitions for resolving variables expressed - as key-word arguments. - - Returns - ------- - s_interp: string - Interpolated string. Undefined variables are left unchanged. - """ - - expr = s - - # Resolve special variables: {{var}} - for var in re.findall(r'{{(\w+)}}', expr): - if var in defs: - expr = re.sub(r'{{'+var+r'}}', defs[var], expr) - - # Resolve special variables: $(var) - for var in re.findall(r'\$\((\w+)\)', expr): - if var in defs: - expr = re.sub(r'\$\('+var+r'\)', defs[var], expr) - - # Resolve special variables: $[var] (list) - for var in re.findall(r'\$\[(\w+)\]', expr): - if var in defs: - list2str = ','.join(map(str, defs[var])) - expr = re.sub(r'\$\['+var+r'\]', '['+list2str+']', expr) - - # Resolve defs - s_interp = string.Template(expr).safe_substitute(defs) - - # Recurse until no substitutions remain - if s_interp != s: - s_interp = replace_vars(s_interp, **defs) - - return s_interp - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/welcome_message.py b/src/swell/utilities/welcome_message.py new file mode 100644 index 00000000..2da1323f --- /dev/null +++ b/src/swell/utilities/welcome_message.py @@ -0,0 +1,28 @@ +# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + + +from swell import __version__ +from swell.utilities.logger import Logger + + +# -------------------------------------------------------------------------------------------------- + + +def write_welcome_message(application): + + logger = Logger('') + + logger.blank(f" _ _ ", False) # noqa + logger.blank(f" _____ _____| | | Swell workflow deployment manager", False) # noqa + logger.blank(f"/ __\ \ /\ / / _ \ | | NASA Global Modelling and Assimilation Office", False) # noqa + logger.blank(f"\__ \\\ V V / __/ | | Version {__version__}", False) # noqa + logger.blank(f"|___/ \_/\_/ \___|_|_| \x1B[4m\x1B[34mhttps://geos-esm.github.io/swell\033[0m", False) # noqa + logger.blank(f" ", False) # noqa + logger.blank(f"Executing swell \'{application.lower()}\' application", False) # noqa + logger.blank('', False) # noqa