diff --git a/driver/runObsMon.sh b/driver/runObsMon.sh index 4267482..9f07e5b 100755 --- a/driver/runObsMon.sh +++ b/driver/runObsMon.sh @@ -89,7 +89,7 @@ export COMOUT=${COMOUT}/${NET} jobname="PlotObsMon" jobfile="${JOBSobsmon}/JMON_PLOT_OBS" -logdir="${OM_LOGS}/${MODEL}" +LOGDIR="${OM_LOGS}/${MODEL}" if [[ ! -d ${logdir} ]]; then mkdir -p ${logdir}; fi logfile="${OM_LOGS}/${MODEL}/OM_log" @@ -101,9 +101,13 @@ case ${MACHINE_ID} in -J ${jobname} --partition service -o ${logfile} ${jobfile} ;; - wcoss2) # NOTE: this has not been tested; eva doesn't yet run on wcoss2 - $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${logfile} \ - -V -l select=1:mem=500M -l walltime=0:05:00 -N ${jobname} ${jobfile} + wcoss2) + $SUB -q ${JOB_QUEUE} -A ${ACCOUNT} -o ${logfile} -e ${logfile} \ + -v "PYTHONPATH=${PYTHONPATH}, PATH=${PATH}, HOMEobsmon=${HOMEobsmon}, COMOUT=${COMOUT}, \ + MODEL=${MODEL}, PDY=${PDY}, cyc=${cyc}, DATAROOT=${DATAROOT}, APRUN_PY=${APRUN_PY}, \ + MACHINE_ID=${MACHINE_ID}, ACCOUNT=${ACCOUNT}, JOB_QUEUE=${JOB_QUEUE}, SUB=${SUB}, + OM_LOGS=${OM_LOGS}" \ + -l select=1:mem=500M -l walltime=0:05:00 -N ${jobname} ${jobfile} ;; esac diff --git a/jobs/JMON_PLOT_OBS b/jobs/JMON_PLOT_OBS index 84f2c6f..73cd3dd 100644 --- a/jobs/JMON_PLOT_OBS +++ b/jobs/JMON_PLOT_OBS @@ -1,5 +1,38 @@ #!/bin/bash +echo "PYTHONPATH: $PYTHONPATH" +#echo "HOMEobsmon: $HOMEobsmon" +#echo "COMOUT: $COMOUT" +#echo "MODEL: $MODEL" +#echo "PDY: $PDY" +#echo "cyc: $cyc" +#echo "DATAROOT: $DATAROOT" +#echo "APRUN_PY: $APRUN_PY" +echo "PATH: $PATH" +echo "initial module list" +module list +echo "" +echo "" + +module purge +echo "after module purge" +module list + +module load PrgEnv-intel/8.3.3 +module load craype/2.7.17 +module load cray-pals/1.2.2 +module load git/2.29.0 +module load intel/19.1.3.304 +module load python/3.10.4 +module load ve/evs/1.0 +module load libfabric/1.11.0.0 +echo "" +echo "reloaded modules" +`module list` + +export PATH=$PATH:/lfs/h2/emc/da/noscrub/edward.safford/eva/opt/bin +export PYTHONPATH=$PYTHONPATH:/lfs/h2/emc/da/noscrub/edward.safford/eva/opt/ + ########################### # J-job for Mon Plot Obs ########################### @@ -48,8 +81,8 @@ fi #################### # Remove workspace #################### -KEEPDATA=${KEEPDATA:-"NO"} -cd ${DATAROOT} -[[ ${KEEPDATA} = "NO" ]] && rm -rf ${DATA} +#KEEPDATA=${KEEPDATA:-"NO"} +#cd ${DATAROOT} +#[[ ${KEEPDATA} = "NO" ]] && rm -rf ${DATA} report-mem diff --git a/parm/gfs/gfs_plots.yaml b/parm/gfs/gfs_plots.yaml index c79f7dc..e4ea6d5 100644 --- a/parm/gfs/gfs_plots.yaml +++ b/parm/gfs/gfs_plots.yaml @@ -2,519 +2,7 @@ model: gfs cycle_interval: 6 -data: '/scratch1/NCEPDEV/da/Edward.Safford/noscrub/test_data/gfs' - -satellites: - - name: aura - instruments: - - name: omi - plot_list: - - plot: ozn horiz - times: 4 - levels: '1' - component: ges - - - plot: ozn time - times: 121 - levels: '1' - component: ges - - - name: npp - instruments: - - name: ompsnp - plot_list: - - plot: ozn horiz - times: 4 - levels: '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22' - component: ges - - - plot: ozn summary - times: 121 - levels: '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22' - component: ges - - - plot: ozn time - times: 121 - levels: '1' - component: ges - - - - name: ompslp - plot_list: - - plot: ozn horiz - times: 4 - levels: '1' - component: ges - - - plot: ozn time - times: 121 - levels: '1' - component: ges - - - name: ompstc8 - plot_list: - - plot: ozn horiz - times: 4 - levels: '1' - component: ges - - - plot: ozn time - times: 121 - levels: '1' - component: ges - - - name: n20 - instruments: - - name: ompsnp - plot_list: - - plot: ozn horiz - times: 4 - levels: '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22' - component: ges - - - plot: ozn time - times: 121 - levels: '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22' - component: ges - - - plot: ozn summary - times: 121 - levels: '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22' - component: ges - - - name: ompstc8 - plot_list: - - plot: ozn horiz - times: 4 - levels: '1' - component: ges - - - plot: ozn time - times: 121 - levels: '1' - component: ges - -# Radiance - - - name: g16 - instruments: - - name: abi - plot_list: - - plot: rad time - times: 121 - channels: 'all' - component: ges - - plot: rad bcoef - times: 121 - channels: 'all' - component: ges - - plot: rad angle - times: 121 - channels: 'all' - component: ges - - - name: g18 - instruments: - - name: abi - plot_list: - - plot: rad time - times: 121 - channels: 'all' - component: ges - - plot: rad bcoef - times: 121 - channels: 'all' - component: ges - - plot: rad angle - times: 121 - channels: 'all' - component: ges - - - name: himawari9 - instruments: - - name: ahi - plot_list: - - plot: rad time - times: 121 - channels: 'all' - component: ges - - plot: rad bcoef - times: 121 - channels: 'all' - component: ges - - plot: rad angle - times: 121 - channels: 'all' - component: ges - - - name: metop-b - instruments: - - name: amsua - plot_list: - - plot: rad time - times: 121 - channels: 'all' - component: ges - - plot: rad bcoef - times: 121 - channels: 'all' - component: ges - - plot: rad angle - times: 121 - channels: 'all' - component: ges - - - name: iasi - plot_list: - - plot: rad time - times: 121 - channels: 'all' - component: ges - - plot: rad bcoef - times: 121 - channels: 'all' - component: ges -# file corruption here? -# - plot: rad angle -# times: 121 -# channels: 'all' -# component: ges - - - name: mhs - plot_list: - - plot: rad time - times: 121 - channels: 'all' - component: ges - - plot: rad bcoef - times: 121 - channels: '1,2,3,4,5' - component: ges - - plot: rad angle - times: 121 - channels: 'all' - component: ges - - - name: avhrr - plot_list: - - plot: rad time - times: 121 - channels: 'all' - component: ges - - plot: rad bcoef - times: 121 - channels: 'all' - component: ges - - plot: rad angle - times: 121 - channels: 'all' - component: ges - - - name: metop-c - instruments: - - name: amsua - plot_list: - - plot: rad time - times: 121 - channels: 'all' - component: ges - - plot: rad bcoef - times: 121 - channels: 'all' - component: ges - - plot: rad angle - times: 121 - channels: 'all' - component: ges - - - name: iasi - plot_list: - - plot: rad time - times: 121 - channels: 'all' - component: ges - - plot: rad bcoef - times: 121 - channels: 'all' - component: ges - - plot: rad angle - times: 121 - channels: 'all' - component: ges - - - name: avhrr - plot_list: - - plot: rad time - times: 121 - channels: 'all' - component: ges - - plot: rad bcoef - times: 121 - channels: 'all' - component: ges - - plot: rad angle - times: 121 - channels: 'all' - component: ges - - - name: mhs - plot_list: - - plot: rad time - times: 121 - channels: 'all' - component: ges - - plot: rad bcoef - times: 121 - channels: 'all' - component: ges - - plot: rad angle - times: 121 - channels: 'all' - component: ges - - - name: n15 - instruments: - - name: amsua - plot_list: - - plot: rad time - times: 121 - channels: 'all' - component: ges - - plot: rad bcoef - times: 121 - channels: 'all' - component: ges - - plot: rad angle - times: 121 - channels: 'all' - component: ges - - - name: n18 - instruments: - - name: amsua - plot_list: - - plot: rad time - times: 121 - channels: 'all' - component: ges - - plot: rad bcoef - times: 121 - channels: 'all' - component: ges - - plot: rad angle - times: 121 - channels: 'all' - component: ges - - - name: avhrr - plot_list: - - plot: rad time - times: 121 - channels: 'all' - component: ges - - plot: rad bcoef - times: 121 - channels: 'all' - component: ges - - plot: rad angle - times: 121 - channels: 'all' - component: ges - -# file corruption? -# - name: n19 -# instruments: -# - name: amsua -# plot_list: -# - plot: rad time -# times: 121 -# channels: 'all' -# component: ges -# - plot: rad bcoef -# times: 121 -# channels: 'all' -# component: ges -# - plot: rad angle -# times: 121 -# channels: 'all' -# component: ges -# -# - name: avhrr -# plot_list: -# - plot: rad time -# times: 121 -# channels: 'all' -# component: ges -# - plot: rad bcoef -# times: 121 -# channels: 'all' -# component: ges -# - plot: rad angle -# times: 121 -# channels: 'all' -# component: ges - -# - name: mhs -# plot_list: -# - plot: rad time -# times: 121 -# channels: 'all' -# component: ges -# - plot: rad bcoef -# times: 121 -# channels: 'all' -# component: ges -# - plot: rad angle -# times: 121 - - - name: n20 - instruments: - - name: atms - plot_list: - - plot: rad time - times: 121 - channels: 'all' - component: ges - - plot: rad bcoef - times: 121 - channels: 'all' - component: ges - - plot: rad angle - times: 121 - channels: 'all' - component: ges - - - name: cris-fsr - plot_list: - - plot: rad time - times: 121 - channels: 'all' - component: ges - - plot: rad bcoef - times: 121 - channels: 'all' - component: ges - - plot: rad angle - times: 121 - channels: 'all' - component: ges - - - name: n21 - instruments: - - name: atms - plot_list: - - plot: rad time - times: 121 - channels: 'all' - component: ges - - plot: rad bcoef - times: 121 - channels: 'all' - component: ges - - plot: rad angle - times: 121 - channels: 'all' - component: ges - - - name: cris-fsr - plot_list: - - plot: rad time - times: 121 - channels: 'all' - component: ges - - plot: rad bcoef - times: 121 - channels: 'all' - component: ges - - plot: rad angle - times: 121 - channels: 'all' - component: ges - - - name: npp - instruments: - - name: atms - plot_list: - - plot: rad time - times: 121 - channels: 'all' - component: ges - - plot: rad bcoef - times: 121 - channels: 'all' - component: ges - - plot: rad angle - times: 121 - channels: 'all' - component: ges - - - name: viirs-m - plot_list: - - plot: rad time - times: 121 - channels: 'all' - component: ges - - plot: rad bcoef - times: 121 - channels: 'all' - component: ges - - plot: rad angle - times: 121 - channels: 'all' - component: ges - - - name: j1 - instruments: - - name: viirs-m - plot_list: - - plot: rad time - times: 121 - channels: 'all' - component: ges - - plot: rad bcoef - times: 121 - channels: 'all' - component: ges - - plot: rad angle - times: 121 - channels: 'all' - component: ges - - - name: f17 - instruments: - - name: ssmis - plot_list: - - plot: rad time - times: 121 - channels: 'all' - component: ges - - plot: rad bcoef - times: 121 - channels: 'all' - component: ges - - plot: rad angle - times: 121 - channels: 'all' - component: ges - - - name: f18 - instruments: - - name: ssmis - plot_list: - - plot: rad time - times: 121 - channels: 'all' - component: ges - - plot: rad bcoef - times: 121 - channels: 'all' - component: ges - - plot: rad angle - times: 121 - channels: 'all' - component: ges - - +data: '/lfs/h2/emc/da/noscrub/edward.safford/test_data/gfs' minimization: - model: gfs @@ -530,59 +18,3 @@ minimization: - plot: min gnorm one cycle run: gdas times: 28 - - -observations: - - obstype: ps - plot_list: - - plot: con time - times: 121 - datatypes: 'all' - - obstype: gps - plot_list: - - plot: con time - times: 121 - datatypes: 'all' - - plot: con vert - times: 1 - datatypes: 'all' - - obstype: q - plot_list: - - plot: con time - times: 121 - datatypes: 'all' - - plot: con vert - times: 1 - datatypes: 'all' - - obstype: t - plot_list: - - plot: con time - times: 121 - datatypes: 'all' - - plot: con vert - times: 1 - datatypes: 'all' - - obstype: u - plot_list: - - plot: con time - times: 121 - datatypes: 'all' - - plot: con vert - times: 1 - datatypes: 'all' - - obstype: uv - plot_list: - - plot: con time - times: 121 - datatypes: 'all' - - plot: con vert - times: 1 - datatypes: 'all' - - obstype: v - plot_list: - - plot: con time - times: 121 - datatypes: 'all' - - plot: con vert - times: 1 - datatypes: 'all' diff --git a/parm/gfs/gfs_plots.yaml.orig b/parm/gfs/gfs_plots.yaml.orig new file mode 100644 index 0000000..c79f7dc --- /dev/null +++ b/parm/gfs/gfs_plots.yaml.orig @@ -0,0 +1,588 @@ +# DA Monitoring source dictionary + +model: gfs +cycle_interval: 6 +data: '/scratch1/NCEPDEV/da/Edward.Safford/noscrub/test_data/gfs' + +satellites: + - name: aura + instruments: + - name: omi + plot_list: + - plot: ozn horiz + times: 4 + levels: '1' + component: ges + + - plot: ozn time + times: 121 + levels: '1' + component: ges + + - name: npp + instruments: + - name: ompsnp + plot_list: + - plot: ozn horiz + times: 4 + levels: '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22' + component: ges + + - plot: ozn summary + times: 121 + levels: '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22' + component: ges + + - plot: ozn time + times: 121 + levels: '1' + component: ges + + + - name: ompslp + plot_list: + - plot: ozn horiz + times: 4 + levels: '1' + component: ges + + - plot: ozn time + times: 121 + levels: '1' + component: ges + + - name: ompstc8 + plot_list: + - plot: ozn horiz + times: 4 + levels: '1' + component: ges + + - plot: ozn time + times: 121 + levels: '1' + component: ges + + - name: n20 + instruments: + - name: ompsnp + plot_list: + - plot: ozn horiz + times: 4 + levels: '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22' + component: ges + + - plot: ozn time + times: 121 + levels: '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22' + component: ges + + - plot: ozn summary + times: 121 + levels: '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22' + component: ges + + - name: ompstc8 + plot_list: + - plot: ozn horiz + times: 4 + levels: '1' + component: ges + + - plot: ozn time + times: 121 + levels: '1' + component: ges + +# Radiance + + - name: g16 + instruments: + - name: abi + plot_list: + - plot: rad time + times: 121 + channels: 'all' + component: ges + - plot: rad bcoef + times: 121 + channels: 'all' + component: ges + - plot: rad angle + times: 121 + channels: 'all' + component: ges + + - name: g18 + instruments: + - name: abi + plot_list: + - plot: rad time + times: 121 + channels: 'all' + component: ges + - plot: rad bcoef + times: 121 + channels: 'all' + component: ges + - plot: rad angle + times: 121 + channels: 'all' + component: ges + + - name: himawari9 + instruments: + - name: ahi + plot_list: + - plot: rad time + times: 121 + channels: 'all' + component: ges + - plot: rad bcoef + times: 121 + channels: 'all' + component: ges + - plot: rad angle + times: 121 + channels: 'all' + component: ges + + - name: metop-b + instruments: + - name: amsua + plot_list: + - plot: rad time + times: 121 + channels: 'all' + component: ges + - plot: rad bcoef + times: 121 + channels: 'all' + component: ges + - plot: rad angle + times: 121 + channels: 'all' + component: ges + + - name: iasi + plot_list: + - plot: rad time + times: 121 + channels: 'all' + component: ges + - plot: rad bcoef + times: 121 + channels: 'all' + component: ges +# file corruption here? +# - plot: rad angle +# times: 121 +# channels: 'all' +# component: ges + + - name: mhs + plot_list: + - plot: rad time + times: 121 + channels: 'all' + component: ges + - plot: rad bcoef + times: 121 + channels: '1,2,3,4,5' + component: ges + - plot: rad angle + times: 121 + channels: 'all' + component: ges + + - name: avhrr + plot_list: + - plot: rad time + times: 121 + channels: 'all' + component: ges + - plot: rad bcoef + times: 121 + channels: 'all' + component: ges + - plot: rad angle + times: 121 + channels: 'all' + component: ges + + - name: metop-c + instruments: + - name: amsua + plot_list: + - plot: rad time + times: 121 + channels: 'all' + component: ges + - plot: rad bcoef + times: 121 + channels: 'all' + component: ges + - plot: rad angle + times: 121 + channels: 'all' + component: ges + + - name: iasi + plot_list: + - plot: rad time + times: 121 + channels: 'all' + component: ges + - plot: rad bcoef + times: 121 + channels: 'all' + component: ges + - plot: rad angle + times: 121 + channels: 'all' + component: ges + + - name: avhrr + plot_list: + - plot: rad time + times: 121 + channels: 'all' + component: ges + - plot: rad bcoef + times: 121 + channels: 'all' + component: ges + - plot: rad angle + times: 121 + channels: 'all' + component: ges + + - name: mhs + plot_list: + - plot: rad time + times: 121 + channels: 'all' + component: ges + - plot: rad bcoef + times: 121 + channels: 'all' + component: ges + - plot: rad angle + times: 121 + channels: 'all' + component: ges + + - name: n15 + instruments: + - name: amsua + plot_list: + - plot: rad time + times: 121 + channels: 'all' + component: ges + - plot: rad bcoef + times: 121 + channels: 'all' + component: ges + - plot: rad angle + times: 121 + channels: 'all' + component: ges + + - name: n18 + instruments: + - name: amsua + plot_list: + - plot: rad time + times: 121 + channels: 'all' + component: ges + - plot: rad bcoef + times: 121 + channels: 'all' + component: ges + - plot: rad angle + times: 121 + channels: 'all' + component: ges + + - name: avhrr + plot_list: + - plot: rad time + times: 121 + channels: 'all' + component: ges + - plot: rad bcoef + times: 121 + channels: 'all' + component: ges + - plot: rad angle + times: 121 + channels: 'all' + component: ges + +# file corruption? +# - name: n19 +# instruments: +# - name: amsua +# plot_list: +# - plot: rad time +# times: 121 +# channels: 'all' +# component: ges +# - plot: rad bcoef +# times: 121 +# channels: 'all' +# component: ges +# - plot: rad angle +# times: 121 +# channels: 'all' +# component: ges +# +# - name: avhrr +# plot_list: +# - plot: rad time +# times: 121 +# channels: 'all' +# component: ges +# - plot: rad bcoef +# times: 121 +# channels: 'all' +# component: ges +# - plot: rad angle +# times: 121 +# channels: 'all' +# component: ges + +# - name: mhs +# plot_list: +# - plot: rad time +# times: 121 +# channels: 'all' +# component: ges +# - plot: rad bcoef +# times: 121 +# channels: 'all' +# component: ges +# - plot: rad angle +# times: 121 + + - name: n20 + instruments: + - name: atms + plot_list: + - plot: rad time + times: 121 + channels: 'all' + component: ges + - plot: rad bcoef + times: 121 + channels: 'all' + component: ges + - plot: rad angle + times: 121 + channels: 'all' + component: ges + + - name: cris-fsr + plot_list: + - plot: rad time + times: 121 + channels: 'all' + component: ges + - plot: rad bcoef + times: 121 + channels: 'all' + component: ges + - plot: rad angle + times: 121 + channels: 'all' + component: ges + + - name: n21 + instruments: + - name: atms + plot_list: + - plot: rad time + times: 121 + channels: 'all' + component: ges + - plot: rad bcoef + times: 121 + channels: 'all' + component: ges + - plot: rad angle + times: 121 + channels: 'all' + component: ges + + - name: cris-fsr + plot_list: + - plot: rad time + times: 121 + channels: 'all' + component: ges + - plot: rad bcoef + times: 121 + channels: 'all' + component: ges + - plot: rad angle + times: 121 + channels: 'all' + component: ges + + - name: npp + instruments: + - name: atms + plot_list: + - plot: rad time + times: 121 + channels: 'all' + component: ges + - plot: rad bcoef + times: 121 + channels: 'all' + component: ges + - plot: rad angle + times: 121 + channels: 'all' + component: ges + + - name: viirs-m + plot_list: + - plot: rad time + times: 121 + channels: 'all' + component: ges + - plot: rad bcoef + times: 121 + channels: 'all' + component: ges + - plot: rad angle + times: 121 + channels: 'all' + component: ges + + - name: j1 + instruments: + - name: viirs-m + plot_list: + - plot: rad time + times: 121 + channels: 'all' + component: ges + - plot: rad bcoef + times: 121 + channels: 'all' + component: ges + - plot: rad angle + times: 121 + channels: 'all' + component: ges + + - name: f17 + instruments: + - name: ssmis + plot_list: + - plot: rad time + times: 121 + channels: 'all' + component: ges + - plot: rad bcoef + times: 121 + channels: 'all' + component: ges + - plot: rad angle + times: 121 + channels: 'all' + component: ges + + - name: f18 + instruments: + - name: ssmis + plot_list: + - plot: rad time + times: 121 + channels: 'all' + component: ges + - plot: rad bcoef + times: 121 + channels: 'all' + component: ges + - plot: rad angle + times: 121 + channels: 'all' + component: ges + + + +minimization: + - model: gfs + plot_list: + - plot: min summary + run: gdas + times: 28 + + - plot: min gnorm four cycle + run: gdas + times: 28 + + - plot: min gnorm one cycle + run: gdas + times: 28 + + +observations: + - obstype: ps + plot_list: + - plot: con time + times: 121 + datatypes: 'all' + - obstype: gps + plot_list: + - plot: con time + times: 121 + datatypes: 'all' + - plot: con vert + times: 1 + datatypes: 'all' + - obstype: q + plot_list: + - plot: con time + times: 121 + datatypes: 'all' + - plot: con vert + times: 1 + datatypes: 'all' + - obstype: t + plot_list: + - plot: con time + times: 121 + datatypes: 'all' + - plot: con vert + times: 1 + datatypes: 'all' + - obstype: u + plot_list: + - plot: con time + times: 121 + datatypes: 'all' + - plot: con vert + times: 1 + datatypes: 'all' + - obstype: uv + plot_list: + - plot: con time + times: 121 + datatypes: 'all' + - plot: con vert + times: 1 + datatypes: 'all' + - obstype: v + plot_list: + - plot: con time + times: 121 + datatypes: 'all' + - plot: con vert + times: 1 + datatypes: 'all' diff --git a/scripts/exobsmon_plot.sh b/scripts/exobsmon_plot.sh index 078974a..5506941 100755 --- a/scripts/exobsmon_plot.sh +++ b/scripts/exobsmon_plot.sh @@ -8,7 +8,6 @@ # locate $model_plots.yaml and instrument_channels.yaml files #------------------------------------------------------------- plot_yaml=${plot_yaml:-${PARMobsmon}/${MODEL}/${MODEL}_plots.yaml} - if [[ ! -e ${plot_yaml} ]]; then echo "ERROR: yaml plot file ${plot_yaml} NOT FOUND" exit 1 @@ -25,7 +24,6 @@ fi # ${APRUN_PY} ${USHobsmon}/splitPlotYaml.py -i ${plot_yaml} -c ${chan_yaml} - #-------------------------------------------------------------- # Submit OM_sat_plots job if split yields any sat_*.yaml files # @@ -40,20 +38,45 @@ if compgen -G "${DATA}/sat_*.yaml" > /dev/null; then ctr=0 for yaml in ${DATA}/sat_*.yaml; do - echo "${ctr} $yaml" - echo "${ctr} ${APRUN_PY} ${USHobsmon}/plotObsMon.py -i ${yaml} -p ${PDATE}" >> $cmdfile - ((ctr+=1)) + case ${MACHINE_ID} in + hera) + echo "${ctr} ${APRUN_PY} ${USHobsmon}/plotObsMon.py -i ${yaml} -p ${PDATE}" >> $cmdfile + ((ctr+=1)) + ;; + wcoss2) + echo "${APRUN_PY} ${USHobsmon}/plotObsMon.py -i ${yaml} -p ${PDATE}" >> $cmdfile + ;; + esac done + cat $cmdfile chmod 755 $cmdfile echo "ctr: $ctr" echo "submitting job ${jobname}" + exit if [[ ${ctr} > 0 ]]; then - $SUB --account ${ACCOUNT} -n ${ctr} -o ${logfile} -D . -J ${jobname} --time=1:00:00 \ - --mem=80000M --wrap "srun -l --multi-prog ${cmdfile}" + case ${MACHINE_ID} in + hera) + ${SUB} --account ${ACCOUNT} --ntasks=1 --mem=400M --time=0:05:00 \ + -J ${jobname} --partition service -o ${logfile} ${cmdfile} + ;; + + wcoss2) + ${SUB} -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${logfile} \ + -v "PYTHONPATH=${PYTHONPATH}, PATH=${PATH}, HOMEobsmon=${HOMEobsmon}, COMOUT=${COMOUT}, \ + MODEL=${MODEL}, PDY=${PDY}, cyc=${cyc}, DATAROOT=${DATAROOT}, APRUN_PY=${APRUN_PY}, \ + MACHINE_ID=${MACHINE_ID}" \ + -l select=1:mem=500M -l walltime=0:10:00 -N ${jobname} ${cmdfile} + ;; + esac fi +# if [[ ${ctr} > 0 ]]; then +# $SUB --account ${ACCOUNT} -n ${ctr} -o ${logfile} -D . -J ${jobname} --time=1:00:00 \ +# --mem=80000M --wrap "srun -l --multi-prog ${cmdfile}" +# fi + fi #------------------------------------------------------------------ @@ -62,17 +85,28 @@ fi if compgen -G "${DATA}/minimization.yaml" > /dev/null; then jobname="OM_min_plots" + echo "submitting job ${jobname}" logfile="${OM_LOGS}/${MODEL}/OM_min_plot.log" - if [[ -e ${logfile} ]]; then rm ${logfile}; fi + if [[ -e ${logfile} ]]; then rm ${logfile}; fi cmdfile="OM_min_jobscript" - echo "0 ${APRUN_PY} ${USHobsmon}/plotObsMon.py -i ${DATA}/minimization.yaml -p ${PDATE}" > $cmdfile - chmod 755 $cmdfile - - echo "submitting job ${jobname}" - $SUB --account ${ACCOUNT} -n 1 -o ${logfile} -D . -J ${jobname} --time=0:05:00 \ - --mem=80000M --wrap "srun -l --multi-prog ${cmdfile}" + case ${MACHINE_ID} in + hera) + echo "0 ${APRUN_PY} ${USHobsmon}/plotObsMon.py -i ${DATA}/minimization.yaml -p ${PDATE}" > $cmdfile +# chmod 755 $cmdfile + ${SUB} --account ${ACCOUNT} -n 1 -o ${logfile} -D . -J ${jobname} --time=0:05:00 \ + --mem=80000M --wrap "srun -l --multi-prog ${cmdfile}" + ;; + wcoss2) + echo "${APRUN_PY} ${USHobsmon}/plotObsMon.py -i ${DATA}/minimization.yaml -p ${PDATE}" > $cmdfile +# chmod 755 $cmdfile + ${SUB} -q ${JOB_QUEUE} -A ${ACCOUNT} -o ${logfile} -e ${logfile} \ + -v "PYTHONPATH=${PYTHONPATH}, PATH=${PATH}, HOMEobsmon=${HOMEobsmon}, COMOUT=${COMOUT}, \ + MODEL=${MODEL}, DATAROOT=${DATAROOT}, APRUN_PY=${APRUN_PY}, MACHINE_ID=${MACHINE_ID}" \ + -l select=1:mem=500M -l walltime=0:10:00 -N ${jobname} ${cmdfile} + ;; + esac fi #------------------------------------------------------------------ diff --git a/ush/plotObsMon.py b/ush/plotObsMon.py index 5b2d166..f371387 100755 --- a/ush/plotObsMon.py +++ b/ush/plotObsMon.py @@ -198,6 +198,7 @@ def loadConfig(satname, instrument, obstype, plot, cycle_tm, cycle_interval, model = min.get('model') for plot in min.get('plot_list'): + logger.info(f"cycle_interval: {cycle_interval}") config = loadConfig(satname, instrument, obstype, plot, cycle_tm, cycle_interval, data_location, model) diff --git a/ush/splitPlotYaml.py b/ush/splitPlotYaml.py index 8d4ccb9..65c7bde 100644 --- a/ush/splitPlotYaml.py +++ b/ush/splitPlotYaml.py @@ -43,6 +43,7 @@ def removeKey(d, keys): """ logger = Logger('splitPlotYaml') + logger.info('test message') parser = argparse.ArgumentParser() parser.add_argument('-i', '--input', type=str, help='Input YAML plot file', required=True) @@ -50,6 +51,8 @@ def removeKey(d, keys): help='Input YAML instrument channel file', required=True) args = parser.parse_args() + logger.info(f'args: {args}') + try: mon_sources = args.input with open(mon_sources, 'r') as mon_sources_opened: @@ -63,7 +66,7 @@ def removeKey(d, keys): with open(chan_data, 'r') as chan_data_opened: chan_dict = yaml.safe_load(chan_data_opened) except Exception as e: - logger.abort('plotObsMon is expecting a valid satellite channel file, but it encountered ' + + logger.abort('splitPlotYaml is expecting a valid satellite channel file, but encountered ' + f'errors when attempting to load: {chan_data}, error: {e}') model = mon_dict.get('model') @@ -112,6 +115,7 @@ def removeKey(d, keys): file.close() if 'minimization' in mon_dict.keys(): + logger.info(f'minimization is in the house') md = removeKey(mon_dict, ['satellites', 'observations']) fname = f'minimization.yaml' file = open(fname, "w")