Add capability to run the JEDI LETKF DA in land-offline_workflow

.gitmodules

@@ -10,3 +10,7 @@
 	path = vector2tile
 	url = https://github.com/NOAA-PSL/land-vector2tile.git
         branch = develop
+[submodule "stochastic_physics"]
+	path = stochastic_physics
+	url = https://github.com/tsga/stochastic_physics
+	branch = lnd_ens

README.md

@@ -40,6 +40,11 @@ COMPILING and TESTING.
 > build_all.sh 
 > cd .. 
 
+2e.
+> cd stochastic_physics 
+> ./compile_ens.hera_intel 
+> cd ..
+
 3. Run the test.
 
  in settings_cycle_test check WORKDIR and OUTDIR are OK

check_test.sh

@@ -10,6 +10,7 @@ module load nccmp
 CMP="nccmp -d"
 
 TEST_BASEDIR=/scratch2/BMC/gsienkf/Clara.Draper/DA_test_cases/land-offline_workflow/DA_test_era5/mem000/restarts/vector/
+#TEST_BASEDIR=${ICSDIR}/mem000/restarts/vector/
 
 for TEST_DATE in 2019-12-01_00-00-00 2019-12-02_00-00-00 
 do

do_submit_cycle.sh

@@ -20,13 +20,15 @@ source $config_file
 
 export KEEPWORKDIR="YES"
 
-export CYCLEDIR=$(pwd) 
+CYCLEDIR=${CYCLEDIR:-$(pwd)} 
 
 ############################
 # set executables
 
 export vec2tileexec=${CYCLEDIR}/vector2tile/vector2tile_converter.exe
-export LSMexec=${CYCLEDIR}/ufs-land-driver/run/ufsLand.exe 
+export LSMexec=${CYCLEDIR}/ufs-land-driver/run/ufsLand.exe
+export EnsForcGenExe=${CYCLEDIR}/stochastic_physics/GenEnsForc.x
+
 export DADIR=${CYCLEDIR}/DA_update/
 export DAscript=${DADIR}/do_landDA.sh
 
@@ -51,7 +53,7 @@ export FREQ=$(( 3600 * $FCSTHR ))
 export RDD=$(( $FCSTHR / 24 )) 
 export RHH=$(( $FCSTHR % 24 )) 
 
-############################
+#############################
 # set up directories
 
 #workdir
@@ -60,57 +62,183 @@ if [[ -e ${WORKDIR} ]]; then
 fi
 mkdir ${WORKDIR}
 
+###############################
+# create dirs and copy in ICS if needed
+
+mem_ens="mem000"  # single member, use ensemble 0
+MEM_WORKDIR=${WORKDIR}/${mem_ens}
+mkdir $MEM_WORKDIR
+
 #outdir for model
-if [[ ! -e ${OUTDIR} ]]; then
+if [[ ! -e ${OUTDIR} ]]; then    
+
     mkdir -p  ${OUTDIR}
-fi 
 
-###############################
-# create dirs and copy in ICS if needed
+    # ensemble outdir (model only)
+    MEM_MODL_OUTDIR=${OUTDIR}/${mem_ens}    
+    if [[ ! -e $MEM_MODL_OUTDIR ]]; then  
+        mkdir -p $MEM_MODL_OUTDIR
+    fi
 
-mem_ens="mem000"  # single member, us ensemble 0
+    # outdir subdirs
+    if [[ ! -e ${MEM_MODL_OUTDIR}/restarts/ ]]; then  
+        mkdir -p ${MEM_MODL_OUTDIR}/restarts/
+        mkdir -p ${MEM_MODL_OUTDIR}/restarts/vector/ 
+        mkdir ${MEM_MODL_OUTDIR}/restarts/tile/
+    fi
 
-MEM_WORKDIR=${WORKDIR}/${mem_ens}
-if [[ ! -e $MEM_WORKDIR ]]; then
-  mkdir $MEM_WORKDIR
+    mkdir -p ${MEM_MODL_OUTDIR}/noahmp/
+    ln -sf ${MEM_MODL_OUTDIR}/noahmp ${MEM_WORKDIR}/noahmp_output 
+
+    # stochy: ensemble forcing perturbation
+    if [[ ! -e ${OUTDIR}/STOCHY/ ]]; then  
+        mkdir -p ${OUTDIR}/STOCHY/        
+        mkdir ${OUTDIR}/STOCHY/RESTART/
+        # mkdir ${OUTDIR}/STOCHY/INPUT/
+    fi
 fi
 
-# ensemble outdir (model only)
-MEM_MODL_OUTDIR=${OUTDIR}/${mem_ens}
-if [[ ! -e $MEM_MODL_OUTDIR ]]; then  #ensemble outdir
-    mkdir -p $MEM_MODL_OUTDIR
-fi 
+if [[ "$ensemble_size" -gt 1  ]]; then  
 
-# outdir subdirs
-if [[ ! -e ${MEM_MODL_OUTDIR}/restarts/ ]]; then  # subdirectories
-    mkdir -p ${MEM_MODL_OUTDIR}/restarts/vector/ 
-    mkdir ${MEM_MODL_OUTDIR}/restarts/tile/
-    mkdir -p ${MEM_MODL_OUTDIR}/noahmp/
+    for ie in $(seq $ensemble_size)     
+    do
+        mem_ens="mem`printf %03i $ie`"        
+        MEM_WORKDIR=${WORKDIR}/${mem_ens}
+        mkdir $MEM_WORKDIR
+
+        MEM_MODL_OUTDIR=${OUTDIR}/${mem_ens}
+        if [[ ! -e $MEM_MODL_OUTDIR ]]; then  #ensemble outdir 
+            mkdir -p $MEM_MODL_OUTDIR
+        fi
+
+        # outdir subdirs
+        if [[ ! -e ${MEM_MODL_OUTDIR}/restarts/ ]]; then  
+            mkdir -p ${MEM_MODL_OUTDIR}/restarts/
+            mkdir -p ${MEM_MODL_OUTDIR}/restarts/vector/ 
+            mkdir ${MEM_MODL_OUTDIR}/restarts/tile/            
+        fi
+
+        #TODO: Do we need this?
+        mkdir -p ${MEM_MODL_OUTDIR}/noahmp/
+        ln -sf ${MEM_MODL_OUTDIR}/noahmp ${MEM_WORKDIR}/noahmp_output 
+    done 
+fi
+
+# Forcing perturbation 
+# Make sure the INPUT and RESTART dirs for stochy are in working dir
+# and input.nml has settings right
+if [[ $do_enkf == "YES" ]]; then     
+
+    stochy_init_found="NO"
+
+    if [[ $stochy_init_exist == "YES" ]]; then
+
+        if [[ -e ${stochy_init_dir} ]]; then
+	        echo "Stochy init files found. Copying..."
+            cp ${stochy_init_dir}/*.nc ${OUTDIR}/STOCHY/RESTART/ 
+            stochy_init_found="YES"     	    
+        else
+            echo "directory for Stochy init files $stochy_init_dir doesn't exist."
+            echo "STOCH_INI_VAL will be set to FALSE -- NOT recommended for cycling experiments"
+        fi
+    else
+        echo "STOCH_INI_VAL will be set to FALSE -- NOT recommended for cycling experiments"
+    fi
+
+    ln -fs ${OUTDIR}/STOCHY/RESTART/ ${WORKDIR}/RESTART  
+
+    if [[ ! -e ${WORKDIR}/INPUT ]]; then
+        mkdir -p ${WORKDIR}/INPUT
+        for it in 1 2 3 4 5 6 
+        do
+            ln -fs ${TPATH}/C${RES}_grid.tile${it}.nc  ${WORKDIR}/INPUT/C${RES}_grid.tile${it}.nc 
+            # ln -fs ${TPATH}/C${RES}_ca_condition.tile${it}.nc  ${WORKDIR}/INPUT/C${RES}_ca_condition.tile${it}.nc 
+        done
+        if [[ -e ${TPATH}/C${RES}_grid_spec.nc ]]; then
+            ln -fs ${TPATH}/C${RES}_grid_spec.nc  ${WORKDIR}/INPUT/C${RES}_grid_spec.nc   
+        elif [[ -e ${TPATH}/C${RES}_mosaic.nc ]]; then
+            ln -fs ${TPATH}/C${RES}_mosaic.nc  ${WORKDIR}/INPUT/C${RES}_grid_spec.nc  
+        else
+            echo "Grid spec file not found at ${TPATH}, exiting"
+            exit 10
+        fi          
+    fi
+
+    cp ${CYCLEDIR}/template.input.nml $WORKDIR/input.nml
+
+    if [[ $stochy_init_found == "YES" ]]; then               # true for cycling with temporal correlation 
+        sed -i -e "s/XXSTOCH_INI_VAL/.TRUE./g" $WORKDIR/input.nml
+    else
+        sed -i -e "s/XXSTOCH_INI_VAL/.FALSE./g" $WORKDIR/input.nml
+    fi
+
+    sed -i -e "s/XXRES/${RES}/g"  $WORKDIR/input.nml
+    sed -i -e "s/XXLX/${LayX}/g"  $WORKDIR/input.nml          # Layout
+    sed -i -e "s/XXLY/${LayY}/g"  $WORKDIR/input.nml
+    sed -i -e "s/XXIOLX/${IOLayX}/g"  $WORKDIR/input.nml      # IO Layout
+    sed -i -e "s/XXIOLY/${IOLayY}/g"  $WORKDIR/input.nml
+
+    RESP1=$((RES+1))
+
+    sed -i -e "s/XXREP/${RESP1}/g"  $WORKDIR/input.nml 
+    sed -i -e "s/XXNTIL/${num_tiles}/g"  $WORKDIR/input.nml       # Number of tiles
+    sed -i -e "s/XXGRT/${grid_type}/g"  $WORKDIR/input.nml        # grid type -1 for FV3
+    sed -i -e "s/XXLSC/${lndp_hscale}/g"  $WORKDIR/input.nml      # Spatial/horizontal correlation length = 120000 m
+    sed -i -e "s/XXTAU/${lndp_tscale}/g"  $WORKDIR/input.nml      # Time correlation scale = 86400 s
+
 fi
-ln -sf ${MEM_MODL_OUTDIR}/noahmp ${MEM_WORKDIR}/noahmp_output 
 
 # copy ICS into restarts, if needed 
-rst_out=${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.nc
+mem_ens="mem000"  # single member/ensemble mean, use ensemble 0
+rst_out=${OUTDIR}/${mem_ens}/restarts/vector/ufs_land_restart_back.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.nc
 rst_in=${ICSDIR}/ufs_land_restart_back.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.nc
 # if restart not in experiment out directory, copy the restarts from the ICSDIR
 if [[ ! -e ${rst_out} ]]; then 
     echo "Looking for ICS: ${rst_in}"
     if [[ -e ${rst_in} ]]; then
-       echo "ICS found, copying" 
-       cp ${rst_in} ${rst_out}
+    echo "ICS found, copying" 
+    cp ${rst_in} ${rst_out}
     else  # check if is in output directory structure
         rst_in=${ICSDIR}/${mem_ens}/restarts/vector/ufs_land_restart.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.nc
         echo "Looking for ICS: ${rst_in}"
         if [[ -e ${rst_in} ]]; then
-           echo "ICS found, copying" 
-           cp ${rst_in} ${rst_out}
+        echo "ICS found, copying" 
+        cp ${rst_in} ${rst_out}
         else  
-           echo "ICS not found. Exiting" 
-           exit 10 
+        echo "ICS not found. Exiting" 
+        exit 10 
         fi
     fi 
 fi 
 
+if [[ "$ensemble_size" -gt 1  ]]; then  
+
+    for ie in $(seq $ensemble_size)     
+    do
+        mem_ens="mem`printf %03i $ie`"
+        rst_out=${OUTDIR}/${mem_ens}/restarts/vector/ufs_land_restart_back.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.nc
+        rst_in=${ICSDIR}/ufs_land_restart_back.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.nc
+        # if restart not in experiment out directory, copy the restarts from the ICSDIR
+        if [[ ! -e ${rst_out} ]]; then 
+            echo "Looking for ICS: ${rst_in}"
+            if [[ -e ${rst_in} ]]; then
+            echo "ICS found, copying" 
+            cp ${rst_in} ${rst_out}
+            else  # check if is in output directory structure
+                rst_in=${ICSDIR}/${mem_ens}/restarts/vector/ufs_land_restart.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.nc
+                echo "Looking for ICS: ${rst_in}"
+                if [[ -e ${rst_in} ]]; then
+                echo "ICS found, copying" 
+                cp ${rst_in} ${rst_out}
+                else  
+                echo "ICS not found. Exiting" 
+                exit 10 
+                fi
+            fi 
+        fi 
+    done
+fi
+#---------------
 # create dates file 
 touch analdates.sh 
 cat << EOF > analdates.sh
@@ -120,5 +248,3 @@ EOF
 
 # submit script 
 sbatch submit_cycle.sh
-
-

do_submit_test.sh

@@ -4,7 +4,7 @@ source settings_cycle_test
 
 rm -rf ${OUTDIR}
 
-do_submit_cycle.sh settings_cycle_test
+./do_submit_cycle.sh settings_cycle_test
 
 
 

ens_da/generate_ens_forc.nml

@@ -0,0 +1,31 @@
+&NAMSNO
+  static_filename  = "/scratch1/NCEPDEV/da/Tseganeh.Gichamo/STATIC/C768/ufs-land_C768_static_fields.nc"
+  fv3_prefix       = "workg_T334_2016x1004.tile"
+  vector_prefix    = "ens_forcing"
+  rand_var         = "stc"
+  PRINTRANK        = 0
+  print_debg_info  = .false.
+  IDIM = 768
+  JDIM = 768
+  NUM_TILES = 6
+  i_layout = 1
+  j_layout = 2
+  IY = 2017
+  IM = 06
+  ID = 30
+  IH = 00
+  FH = 0.0
+  DELTSFC = 6.0
+  horz_len_scale = 120.0
+  ver_len_scale = 800.
+  temp_len_scale = 24.
+  ens_size = 20
+  t_len = 24
+  t_indx = 2  
+  n_surf_vars = 6
+  vector_size = 1067297 
+  forc_inp_path = "/scratch1/NCEPDEV/da/Tseganeh.Gichamo/land-offline_workflow/"
+  forc_inp_file = "C768_GDAS_forcing_2017-06-30.nc"
+!  std_dev_f = 0.3, 0.2, 20.0, 3.0, 0.075, 0.075, 0.2, 0.01
+  std_dev_f = 0.6, 0.3, 30.0, 5.0, 0.1075, 0.1075, 0.4, 0.01
+/

ens_da/input.nml

@@ -0,0 +1,83 @@
+&amip_interp_nml
+       interp_oi_sst = .true.
+       use_ncep_sst = .true.
+       use_ncep_ice = .false.
+       no_anom_sst = .false.
+       data_set = 'reynolds_oi',
+       date_out_of_range = 'climo',
+/
+
+&atmos_model_nml
+       blocksize = 32
+       chksum_debug = .false.
+       dycore_only = .false.
+       fhout = 1
+       fhmax = 800
+       fhmaxhf = -1
+       ccpp_suite = 'FV3_GFS_v15p2_coupled'
+/
+
+&diag_manager_nml
+       prepend_date = .F.
+/
+
+&fms_io_nml
+       checksum_required   = .false.
+       max_files_r = 100,
+       max_files_w = 100,
+/
+
+&fms_nml
+       clock_grain = 'ROUTINE',
+       domains_stack_size = 40000000,
+       print_memory_usage = .false.
+/
+
+&fv_grid_nml
+       grid_file = 'INPUT/C768_grid_spec.nc'
+/
+
+&fv_core_nml
+       layout   = 1,2
+       io_layout = 1,1
+       npx      = 769,
+       npy      = 769,
+       ntiles   = 6,
+       grid_type = -1
+/
+
+&gfs_physics_nml
+       do_sppt        = .false.
+       do_shum        = .false.
+       do_skeb        = .false.
+       lndp_type      = 2
+       n_var_lndp     = 6
+/
+&nam_stochy
+  SKEBNORM=1,
+  SKEB_NPASS=30,
+  SKEB_VDOF=5,
+  SKEB=-999,
+  SKEB_TAU=2.16E4,
+  SKEB_LSCALE=1000.E3,
+  SHUM=-999,
+  SHUM_TAU=21600,
+  SHUM_LSCALE=500000,
+  SPPT=-999
+  SPPT_TAU=21600,86400,216000
+  SPPT_LSCALE=500000,1000000,2000000
+  SPPT_LOGIT=.TRUE.,
+  SPPT_SFCLIMIT=.TRUE.,
+  ISEED_SHUM=1,
+  ISEED_SKEB=2,
+  ISEED_SPPT=3,
+  STOCHINI=.TRUE.
+/
+&nam_sfcperts
+  lndp_type=2
+  lndp_var_list='stc','stc','stc','stc','stc','stc'
+  lndp_prt_list=1.0,1.0,1.0,1.0,1.0,1.0
+  iseed_lndp=0
+  lndp_tau= 86400,
+  lndp_lscale = 120000,
+/