From 7253113024a38c17a882d7cd53b97081f102ee9e Mon Sep 17 00:00:00 2001 From: ICCP Guest Date: Mon, 17 Jul 2023 21:47:46 +0900 Subject: [PATCH 01/18] first shot at getting intel compiler with openmpi to work on aleph --- configs/components/oifs/oifs.env.yaml | 23 ++++-- configs/components/xios/xios.yaml | 2 + configs/machines/aleph.yaml | 110 +++++++++++++++++--------- 3 files changed, 91 insertions(+), 44 deletions(-) diff --git a/configs/components/oifs/oifs.env.yaml b/configs/components/oifs/oifs.env.yaml index 9e64a6b0f..780a26834 100644 --- a/configs/components/oifs/oifs.env.yaml +++ b/configs/components/oifs/oifs.env.yaml @@ -450,17 +450,26 @@ compiletime_environment_changes: OIFS_NETCDFF_INCLUDE: '-I$NETCDF_DIR/include' OIFS_NETCDFF_LIB: '"-L$NETCDF_DIR/lib -lnetcdff"' OIFS_FC: '${computer.fc}' - OIFS_FFLAGS: '"-O2 -emf -hthread1 -hflex_mp=conservative -hfp1 -hadd_paren -hbyteswapio -J./ -hcpu=x86-skylake -U_CRAYFTN"' #TODO: Test: -O3 -hfp3 - #OIFS_FFLAGS: '"-O3 -emf -hthread3 -hflex_mp=conservative -hfp3 -hadd_paren -hbyteswapio -J./ -hcpu=x86-skylake -U_CRAYFTN"' #TODO: Test: -O3 -hfp3 OIFS_FFIXED: '"-s real64"' OIFS_FCDEFS: '"BLAS LITTLE LINUX INTEGER_IS_INT"' - OIFS_LFLAGS: '"-dynamic -hbyteswapio"' OIFS_CC: '${computer.cc}' - OIFS_CFLAGS: '"-emf -O2 -hcpu=x86-skylake"' - #OIFS_CFLAGS: '"-emf -O3 -hcpu=x86-skylake"' OIFS_CCDEFS: '"LINUX LITTLE INTEGER_IS_INT _ABI64 BLAS"' - OIFS_XIOS_LIB_NAME: '${computer.c++_lib}' - + choose_computer.useMPI: + cray_mpich: + add_export_vars: + OIFS_FFLAGS: '"-emf -hthread1 -hflex_mp=conservative -hfp1 -hadd_paren -hbyteswapio -J./ -hcpu=x86-skylake -U_CRAYFTN -g -O0 -eZ"' + #OIFS_FFLAGS: '"-emf -hthread1 -hflex_mp=conservative -hfp1 -hadd_paren -hbyteswapio -J./ -hcpu=x86-skylake -U_CRAYFTN -g -O1 -eZ"' + #OIFS_FFLAGS: '"-O1 -emf -hthread1 -hflex_mp=conservative -hfp1 -hadd_paren -hbyteswapio -J./ -hcpu=x86-skylake -U_CRAYFTN"' #TODO: Test: -O3 -hfp3 + #OIFS_FFLAGS: '"-O2 -emf -hthread1 -hflex_mp=conservative -hfp1 -hadd_paren -hbyteswapio -J./ -hcpu=x86-skylake -U_CRAYFTN"' #TODO: Test: -O3 -hfp3 + #OIFS_FFLAGS: '"-O3 -emf -hthread3 -hflex_mp=conservative -hfp3 -hadd_paren -hbyteswapio -J./ -hcpu=x86-skylake -U_CRAYFTN"' #TODO: Test: -O3 -hfp3 + OIFS_CFLAGS: '"-emf -O2 -hcpu=x86-skylake"' + OIFS_LFLAGS: '"-dynamic -hbyteswapio"' + OIFS_XIOS_LIB_NAME: '${computer.c++_lib}' + intelopenmpi: + add_export_vars: + OIFS_FFLAGS: '"-r8 -fp-model precise -align array32byte -O3 -qopenmp -xCORE_AVX2 -g -traceback -convert big_endian -fpe0"' + OIFS_CFLAGS: '"-fp-model precise -O3 -xCORE_AVX2 -g -traceback -qopt-report=0 -fpe0 -qopenmp"' + OIFS_LFLAGS: '"$OIFS_MPI_LIB -qopenmp"' nesh: compiler_mpi: intel2020_impi2020 add_export_vars: diff --git a/configs/components/xios/xios.yaml b/configs/components/xios/xios.yaml index f10aa2a30..33269bdd0 100644 --- a/configs/components/xios/xios.yaml +++ b/configs/components/xios/xios.yaml @@ -68,6 +68,8 @@ choose_version: archfile: ESMTOOLS_generic_oasis_intel parastationmpi: archfile: ESMTOOLS_generic_oasis_intel + intelopenmpi: + archfile: ESMTOOLS_generic_oasis_intel "*": foo: bar # MA: yet another block, this time for albedo. We cannot fix this now, but in the diff --git a/configs/machines/aleph.yaml b/configs/machines/aleph.yaml index 9970ffe78..a1d13eb60 100644 --- a/configs/machines/aleph.yaml +++ b/configs/machines/aleph.yaml @@ -46,53 +46,89 @@ hyper_flag: "" # # -useMPI: cray_mpich - -fc: ftn -f77: ftn -mpifc: ftn -mpicc: cc -cc: cc -cxx: CC - -module_actions: - - "unload craype" - - "load craype/2.6.2" - - "load PrgEnv-cray/6.0.4" - - "load pbs" - - "load cray-mpich/7.7.3" - - "load craype-x86-skylake" - - "load cmake/3.14.0" - - "load cray-hdf5-parallel/1.10.2.0" - - "load cray-netcdf-hdf5parallel/4.6.1.3" - - "load cdo/1.9.5" - - "load fftw/2.1.5.9" - - "load nco/4.9.4" - - "load proj4/5.1.0" - - "load python/3.9.1" - - "list" +useMPI: intelopenmpi #cray_mpich + +choose_useMPI: + cray_mpich: + fc: ftn + f77: ftn + mpifc: ftn + mpicc: cc + cc: cc + cxx: CC + module_actions: + - "unload craype" + - "load craype/2.6.2" + - "load PrgEnv-cray/6.0.4" + - "load pbs" + - "load cray-mpich/7.7.3" + - "load craype-x86-skylake" + - "load cmake/3.14.0" + - "load cray-hdf5-parallel/1.10.2.0" + - "load cray-netcdf-hdf5parallel/4.6.1.3" + - "load cdo/1.9.5" + - "load fftw/2.1.5.9" + - "load nco/4.9.4" + - "load proj4/5.1.0" + - "load python/3.9.1" + - "list" + add_export_vars: + ECCODESROOT: /proj/awiiccp/software/ecmwf/eccodes_cce_mpich + HDF5ROOT: $HDF5_ROOT + + NETCDFFROOT: $NETCDF_DIR + NETCDFROOT: $NETCDF_DIR + NETCDF_Fortran_INCLUDE_DIRECTORIES: $NETCDFROOT/include + NETCDF_CXX_INCLUDE_DIRECTORIES: $NETCDFROOT/include + NETCDF_CXX_LIBRARIES: $NETCDFROOT/lib + ZLIBROOT: /usr + + intelopenmpi: + fc: mpifort + f77: mpif77 + mpifc: mpifort + mpicc: mpicc + cc: mpicc + cxx: mpicxx + module_actions: + - "purge" + - "load PrgEnv-intel/6.0.5" + - "unload pmi" + - "load pbs" + - "load cmake/3.14.0" + - "load cdo/1.9.5" + - "load fftw/2.1.5.9" + - "load nco/4.9.4" + - "load proj4/5.1.0" + - "load python/3.9.1" + - "list" + add_export_vars: + IO_LIB_ROOT: home/awiiccp2/software/HPC_libraries/intel-18.0.3.222_ompi_20230717 + LD_LIBRARY_PATH: $IO_LIB_ROOT/lib:$LD_LIBRARY_PATH + + SZIPROOT: $IO_LIB_ROOT + HDF5ROOT: $IO_LIB_ROOT + HDF5_ROOT: $HDF5ROOT + NETCDFROOT: $IO_LIB_ROOT + NETCDFFROOT: $IO_LIB_ROOT + ECCODESROOT: $IO_LIB_ROOT + + HDF5_C_INCLUDE_DIRECTORIES: $HDF5_ROOT/include + NETCDF_Fortran_INCLUDE_DIRECTORIES: $NETCDFFROOT/include + NETCDF_C_INCLUDE_DIRECTORIES: $NETCDFROOT/include + NETCDF_CXX_INCLUDE_DIRECTORIES: $NETCDFROOT/include + OASIS3MCT_FC_LIB: '"-L$NETCDFFROOT/lib -lnetcdff"' + PATH: $IO_LIB_ROOT/bin:$PATH export_vars: LC_ALL: en_US.UTF-8 - HDF5ROOT: $HDF5_ROOT - - NETCDFFROOT: $NETCDF_DIR - NETCDFROOT: $NETCDF_DIR - NETCDF_Fortran_INCLUDE_DIRECTORIES: $NETCDFROOT/include - NETCDF_CXX_INCLUDE_DIRECTORIES: $NETCDFROOT/include - NETCDF_CXX_LIBRARIES: $NETCDFROOT/lib - PERL5LIB: /usr/lib64/perl5 - #LAPACK_LIB: '"-lmkl_intel_lp64 -lmkl_core -mkl=sequential -lpthread -lm -ldl"' - #LAPACK_LIB_DEFAULT: '"-L/global/AWIsoft/intel/2018/compilers_and_libraries_2018.5.274/linux/mkl/lib/intel64 -lmkl_intel_lp64 -lmkl_core -lmkl_sequential"' XML2ROOT: /usr - ZLIBROOT: /usr TMPDIR: /tmp # Prepend labels to the stdout/sterr lines PMI_LABEL_ERROUT: 1 - ECCODESROOT: /proj/awiiccp/software/ecmwf/eccodes_cce_mpich # enable full MPI thread support level (MPI_THREAD_MULTIPLE) MPICH_MAX_THREAD_SAFETY: multiple # to also switch to an alternative (probably with faster locking) multi threading implementation of the cray-mpich library, use the compiler flag -craympich-mt From 56bdd108025156d9428b1093a4773ce0b1353196 Mon Sep 17 00:00:00 2001 From: ICCP Guest Date: Tue, 18 Jul 2023 18:35:16 +0900 Subject: [PATCH 02/18] more failed configs --- configs/components/oifs/oifs.env.yaml | 37 +++++++++++-------- configs/components/xios/xios.yaml | 2 + configs/machines/aleph.yaml | 53 +++++++++++++++++++++++---- 3 files changed, 68 insertions(+), 24 deletions(-) diff --git a/configs/components/oifs/oifs.env.yaml b/configs/components/oifs/oifs.env.yaml index 780a26834..dade1a12a 100644 --- a/configs/components/oifs/oifs.env.yaml +++ b/configs/components/oifs/oifs.env.yaml @@ -454,22 +454,27 @@ compiletime_environment_changes: OIFS_FCDEFS: '"BLAS LITTLE LINUX INTEGER_IS_INT"' OIFS_CC: '${computer.cc}' OIFS_CCDEFS: '"LINUX LITTLE INTEGER_IS_INT _ABI64 BLAS"' - choose_computer.useMPI: - cray_mpich: - add_export_vars: - OIFS_FFLAGS: '"-emf -hthread1 -hflex_mp=conservative -hfp1 -hadd_paren -hbyteswapio -J./ -hcpu=x86-skylake -U_CRAYFTN -g -O0 -eZ"' - #OIFS_FFLAGS: '"-emf -hthread1 -hflex_mp=conservative -hfp1 -hadd_paren -hbyteswapio -J./ -hcpu=x86-skylake -U_CRAYFTN -g -O1 -eZ"' - #OIFS_FFLAGS: '"-O1 -emf -hthread1 -hflex_mp=conservative -hfp1 -hadd_paren -hbyteswapio -J./ -hcpu=x86-skylake -U_CRAYFTN"' #TODO: Test: -O3 -hfp3 - #OIFS_FFLAGS: '"-O2 -emf -hthread1 -hflex_mp=conservative -hfp1 -hadd_paren -hbyteswapio -J./ -hcpu=x86-skylake -U_CRAYFTN"' #TODO: Test: -O3 -hfp3 - #OIFS_FFLAGS: '"-O3 -emf -hthread3 -hflex_mp=conservative -hfp3 -hadd_paren -hbyteswapio -J./ -hcpu=x86-skylake -U_CRAYFTN"' #TODO: Test: -O3 -hfp3 - OIFS_CFLAGS: '"-emf -O2 -hcpu=x86-skylake"' - OIFS_LFLAGS: '"-dynamic -hbyteswapio"' - OIFS_XIOS_LIB_NAME: '${computer.c++_lib}' - intelopenmpi: - add_export_vars: - OIFS_FFLAGS: '"-r8 -fp-model precise -align array32byte -O3 -qopenmp -xCORE_AVX2 -g -traceback -convert big_endian -fpe0"' - OIFS_CFLAGS: '"-fp-model precise -O3 -xCORE_AVX2 -g -traceback -qopt-report=0 -fpe0 -qopenmp"' - OIFS_LFLAGS: '"$OIFS_MPI_LIB -qopenmp"' + choose_computer.useMPI: + cray_mpich: + add_export_vars: + OIFS_FFLAGS: '"-emf -hthread1 -hflex_mp=conservative -hfp1 -hadd_paren -hbyteswapio -J./ -hcpu=x86-skylake -U_CRAYFTN -g -O0 -eZ"' + #OIFS_FFLAGS: '"-emf -hthread1 -hflex_mp=conservative -hfp1 -hadd_paren -hbyteswapio -J./ -hcpu=x86-skylake -U_CRAYFTN -g -O1 -eZ"' + #OIFS_FFLAGS: '"-O1 -emf -hthread1 -hflex_mp=conservative -hfp1 -hadd_paren -hbyteswapio -J./ -hcpu=x86-skylake -U_CRAYFTN"' #TODO: Test: -O3 -hfp3 + #OIFS_FFLAGS: '"-O2 -emf -hthread1 -hflex_mp=conservative -hfp1 -hadd_paren -hbyteswapio -J./ -hcpu=x86-skylake -U_CRAYFTN"' #TODO: Test: -O3 -hfp3 + #OIFS_FFLAGS: '"-O3 -emf -hthread3 -hflex_mp=conservative -hfp3 -hadd_paren -hbyteswapio -J./ -hcpu=x86-skylake -U_CRAYFTN"' #TODO: Test: -O3 -hfp3 + OIFS_CFLAGS: '"-emf -O2 -hcpu=x86-skylake"' + OIFS_LFLAGS: '"-dynamic -hbyteswapio"' + OIFS_XIOS_LIB_NAME: '${computer.c++_lib}' + intelopenmpi: + add_export_vars: + OIFS_FFLAGS: '"-r8 -fp-model precise -align array32byte -O3 -qopenmp -xCORE_AVX2 -g -traceback -convert big_endian -fpe0"' + OIFS_CFLAGS: '"-fp-model precise -O3 -xCORE_AVX2 -g -traceback -qopt-report=0 -fpe0 -qopenmp"' + OIFS_LFLAGS: '"$OIFS_MPI_LIB -qopenmp"' + intelmpich: + add_export_vars: + OIFS_FFLAGS: '"-r8 -fp-model precise -align array32byte -O3 -qopenmp -xCORE_AVX2 -g -traceback -convert big_endian -fpe0"' + OIFS_CFLAGS: '"-fp-model precise -O3 -xCORE_AVX2 -g -traceback -qopt-report=0 -fpe0 -qopenmp"' + OIFS_LFLAGS: '"$OIFS_MPI_LIB -qopenmp"' nesh: compiler_mpi: intel2020_impi2020 add_export_vars: diff --git a/configs/components/xios/xios.yaml b/configs/components/xios/xios.yaml index 33269bdd0..4107008b4 100644 --- a/configs/components/xios/xios.yaml +++ b/configs/components/xios/xios.yaml @@ -70,6 +70,8 @@ choose_version: archfile: ESMTOOLS_generic_oasis_intel intelopenmpi: archfile: ESMTOOLS_generic_oasis_intel + intelmpich: + archfile: ESMTOOLS_generic_oasis_intel "*": foo: bar # MA: yet another block, this time for albedo. We cannot fix this now, but in the diff --git a/configs/machines/aleph.yaml b/configs/machines/aleph.yaml index a1d13eb60..841c154da 100644 --- a/configs/machines/aleph.yaml +++ b/configs/machines/aleph.yaml @@ -46,7 +46,7 @@ hyper_flag: "" # # -useMPI: intelopenmpi #cray_mpich +useMPI: cray_mpich #intelopenmpi #cray_mpich choose_useMPI: cray_mpich: @@ -84,16 +84,18 @@ choose_useMPI: ZLIBROOT: /usr intelopenmpi: - fc: mpifort - f77: mpif77 - mpifc: mpifort - mpicc: mpicc - cc: mpicc - cxx: mpicxx + fc: /home/awiiccp2/software/HPC_libraries/intel-19.0.0.117_ompi_20230717/bin/mpifort + f77: /home/awiiccp2/software/HPC_libraries/intel-19.0.0.117_ompi_20230717/bin/mpifort + mpifc: /home/awiiccp2/software/HPC_libraries/intel-19.0.0.117_ompi_20230717/bin/mpifort + mpicc: /home/awiiccp2/software/HPC_libraries/intel-19.0.0.117_ompi_20230717/bin/mpicc + cc: /home/awiiccp2/software/HPC_libraries/intel-19.0.0.117_ompi_20230717/bin/mpicc + cxx: /home/awiiccp2/software/HPC_libraries/intel-19.0.0.117_ompi_20230717/bin/mpicxx module_actions: - "purge" - "load PrgEnv-intel/6.0.5" - "unload pmi" + - "swap intel/19.0.0.117" + - "load gcc/9.3.0" - "load pbs" - "load cmake/3.14.0" - "load cdo/1.9.5" @@ -103,7 +105,9 @@ choose_useMPI: - "load python/3.9.1" - "list" add_export_vars: - IO_LIB_ROOT: home/awiiccp2/software/HPC_libraries/intel-18.0.3.222_ompi_20230717 + CMAKE_C_COMPILER: /home/awiiccp2/software/HPC_libraries/intel-19.0.0.117_ompi_20230717/bin/mpicc + CMAKE_CXX_COMPILER: /home/awiiccp2/software/HPC_libraries/intel-19.0.0.117_ompi_20230717/bin/mpicxx + IO_LIB_ROOT: /home/awiiccp2/software/HPC_libraries/intel-19.0.0.117_ompi_20230717 LD_LIBRARY_PATH: $IO_LIB_ROOT/lib:$LD_LIBRARY_PATH SZIPROOT: $IO_LIB_ROOT @@ -120,6 +124,39 @@ choose_useMPI: OASIS3MCT_FC_LIB: '"-L$NETCDFFROOT/lib -lnetcdff"' PATH: $IO_LIB_ROOT/bin:$PATH + intelmpich: + fc: ftn + f77: ftn + mpifc: ftn + mpicc: cc + cc: cc + cxx: CC + module_actions: + - "purge" + - "load PrgEnv-intel/6.0.5" + - "load gcc/9.3.0" + - "load pbs" + - "load cmake/3.14.0" + - "load cdo/1.9.5" + - "load fftw/2.1.5.9" + - "load nco/4.9.4" + - "load proj4/5.1.0" + - "load python/3.9.1" + - "load cray-netcdf-hdf5parallel/4.6.1.3" + - "load cray-hdf5-parallel/1.10.2.0" + - "load eccodes" + - "list" + add_export_vars: + #LD_LIBRARY_PATH: $IO_LIB_ROOT/lib:$LD_LIBRARY_PATH + ECCODESROOT: $ECCODES_DIR + + HDF5_C_INCLUDE_DIRECTORIES: $HDF5_ROOT/include + NETCDF_Fortran_INCLUDE_DIRECTORIES: $NETCDFFROOT/include + NETCDF_C_INCLUDE_DIRECTORIES: $NETCDFROOT/include + NETCDF_CXX_INCLUDE_DIRECTORIES: $NETCDFROOT/include + OASIS3MCT_FC_LIB: '"-L$NETCDFFROOT/lib -lnetcdff"' + PATH: $IO_LIB_ROOT/bin:$PATH + export_vars: LC_ALL: en_US.UTF-8 From 84407dcb5c0fe0646ce305c4b788937c8aeae2d7 Mon Sep 17 00:00:00 2001 From: ICCP Guest Date: Tue, 18 Jul 2023 23:47:57 +0900 Subject: [PATCH 03/18] cpu specific intel flags for aleph --- configs/components/oifs/oifs.env.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/components/oifs/oifs.env.yaml b/configs/components/oifs/oifs.env.yaml index dade1a12a..3c12ed229 100644 --- a/configs/components/oifs/oifs.env.yaml +++ b/configs/components/oifs/oifs.env.yaml @@ -457,7 +457,7 @@ compiletime_environment_changes: choose_computer.useMPI: cray_mpich: add_export_vars: - OIFS_FFLAGS: '"-emf -hthread1 -hflex_mp=conservative -hfp1 -hadd_paren -hbyteswapio -J./ -hcpu=x86-skylake -U_CRAYFTN -g -O0 -eZ"' + OIFS_FFLAGS: '"-emf -hthread1 -hflex_mp=conservative -hfp1 -hadd_paren -hbyteswapio -J./ -hcpu=x86-skylake -U_CRAYFTN -g -O3 -eZ"' #OIFS_FFLAGS: '"-emf -hthread1 -hflex_mp=conservative -hfp1 -hadd_paren -hbyteswapio -J./ -hcpu=x86-skylake -U_CRAYFTN -g -O1 -eZ"' #OIFS_FFLAGS: '"-O1 -emf -hthread1 -hflex_mp=conservative -hfp1 -hadd_paren -hbyteswapio -J./ -hcpu=x86-skylake -U_CRAYFTN"' #TODO: Test: -O3 -hfp3 #OIFS_FFLAGS: '"-O2 -emf -hthread1 -hflex_mp=conservative -hfp1 -hadd_paren -hbyteswapio -J./ -hcpu=x86-skylake -U_CRAYFTN"' #TODO: Test: -O3 -hfp3 @@ -467,12 +467,12 @@ compiletime_environment_changes: OIFS_XIOS_LIB_NAME: '${computer.c++_lib}' intelopenmpi: add_export_vars: - OIFS_FFLAGS: '"-r8 -fp-model precise -align array32byte -O3 -qopenmp -xCORE_AVX2 -g -traceback -convert big_endian -fpe0"' + OIFS_FFLAGS: '"-r8 -fp-model precise -align array32byte -O3 -qopenmp -xCORE_AVX512 -g -traceback -convert big_endian -fpe0"' OIFS_CFLAGS: '"-fp-model precise -O3 -xCORE_AVX2 -g -traceback -qopt-report=0 -fpe0 -qopenmp"' OIFS_LFLAGS: '"$OIFS_MPI_LIB -qopenmp"' intelmpich: add_export_vars: - OIFS_FFLAGS: '"-r8 -fp-model precise -align array32byte -O3 -qopenmp -xCORE_AVX2 -g -traceback -convert big_endian -fpe0"' + OIFS_FFLAGS: '"-r8 -fp-model precise -align array32byte -O3 -qopenmp -xCORE_AVX512 -g -traceback -convert big_endian -fpe0"' OIFS_CFLAGS: '"-fp-model precise -O3 -xCORE_AVX2 -g -traceback -qopt-report=0 -fpe0 -qopenmp"' OIFS_LFLAGS: '"$OIFS_MPI_LIB -qopenmp"' nesh: From 892b2b82e17d585de165f20a62ec1bfd1c526c27 Mon Sep 17 00:00:00 2001 From: ICCP Guest Date: Wed, 19 Jul 2023 20:22:11 +0900 Subject: [PATCH 04/18] almost there with mpich --- .../components/oasis3mct/oasis3mct.env.yaml | 35 ++++++++++--------- configs/components/oifs/oifs.env.yaml | 10 +++--- configs/components/xios/xios.yaml | 2 +- configs/machines/aleph.yaml | 13 ++++--- 4 files changed, 31 insertions(+), 29 deletions(-) diff --git a/configs/components/oasis3mct/oasis3mct.env.yaml b/configs/components/oasis3mct/oasis3mct.env.yaml index ba742858b..cecf159a9 100644 --- a/configs/components/oasis3mct/oasis3mct.env.yaml +++ b/configs/components/oasis3mct/oasis3mct.env.yaml @@ -1,18 +1,19 @@ add_compiletime_environment_changes: - choose_computer.fc: - ftn: - add_export_vars: - - 'OASIS_FFLAGS=-emf' - choose_computer.name: - levante: - add_export_vars: - OASIS_FFLAGS: '"-march=core-avx2 -mtune=core-avx2"' - albedo: - choose_computer.compiler_suite: - gcc: - add_export_vars: - FFLAGS: "'-fallow-argument-mismatch -w'" - FCFLAGS: "'-fallow-argument-mismatch -w'" - intel-oneapi: - add_export_vars: - OASIS_FFLAGS: '"-march=core-avx2"' + choose_computer.name: + aleph: + choose_computer.useMPI: + cray_mpich: + add_export_vars: + OASIS_FFLAGS: '"-emf"' + levante: + add_export_vars: + OASIS_FFLAGS: '"-march=core-avx2 -mtune=core-avx2"' + albedo: + choose_computer.compiler_suite: + gcc: + add_export_vars: + FFLAGS: "'-fallow-argument-mismatch -w'" + FCFLAGS: "'-fallow-argument-mismatch -w'" + intel-oneapi: + add_export_vars: + OASIS_FFLAGS: '"-march=core-avx2"' diff --git a/configs/components/oifs/oifs.env.yaml b/configs/components/oifs/oifs.env.yaml index 3c12ed229..5aa967447 100644 --- a/configs/components/oifs/oifs.env.yaml +++ b/configs/components/oifs/oifs.env.yaml @@ -450,7 +450,6 @@ compiletime_environment_changes: OIFS_NETCDFF_INCLUDE: '-I$NETCDF_DIR/include' OIFS_NETCDFF_LIB: '"-L$NETCDF_DIR/lib -lnetcdff"' OIFS_FC: '${computer.fc}' - OIFS_FFIXED: '"-s real64"' OIFS_FCDEFS: '"BLAS LITTLE LINUX INTEGER_IS_INT"' OIFS_CC: '${computer.cc}' OIFS_CCDEFS: '"LINUX LITTLE INTEGER_IS_INT _ABI64 BLAS"' @@ -464,17 +463,20 @@ compiletime_environment_changes: #OIFS_FFLAGS: '"-O3 -emf -hthread3 -hflex_mp=conservative -hfp3 -hadd_paren -hbyteswapio -J./ -hcpu=x86-skylake -U_CRAYFTN"' #TODO: Test: -O3 -hfp3 OIFS_CFLAGS: '"-emf -O2 -hcpu=x86-skylake"' OIFS_LFLAGS: '"-dynamic -hbyteswapio"' + OIFS_FFIXED: '"-s real64"' OIFS_XIOS_LIB_NAME: '${computer.c++_lib}' intelopenmpi: add_export_vars: OIFS_FFLAGS: '"-r8 -fp-model precise -align array32byte -O3 -qopenmp -xCORE_AVX512 -g -traceback -convert big_endian -fpe0"' - OIFS_CFLAGS: '"-fp-model precise -O3 -xCORE_AVX2 -g -traceback -qopt-report=0 -fpe0 -qopenmp"' + OIFS_CFLAGS: '"-fp-model precise -O3 -xCORE_AVX512 -g -traceback -qopt-report=0 -fpe0 -qopenmp"' OIFS_LFLAGS: '"$OIFS_MPI_LIB -qopenmp"' - intelmpich: + OIFS_FFIXED: '""' + intel_mpich: add_export_vars: OIFS_FFLAGS: '"-r8 -fp-model precise -align array32byte -O3 -qopenmp -xCORE_AVX512 -g -traceback -convert big_endian -fpe0"' - OIFS_CFLAGS: '"-fp-model precise -O3 -xCORE_AVX2 -g -traceback -qopt-report=0 -fpe0 -qopenmp"' + OIFS_CFLAGS: '"-fp-model precise -O3 -xCORE_AVX512 -g -traceback -qopt-report=0 -fpe0 -qopenmp"' OIFS_LFLAGS: '"$OIFS_MPI_LIB -qopenmp"' + OIFS_FFIXED: '""' nesh: compiler_mpi: intel2020_impi2020 add_export_vars: diff --git a/configs/components/xios/xios.yaml b/configs/components/xios/xios.yaml index 4107008b4..66ce03017 100644 --- a/configs/components/xios/xios.yaml +++ b/configs/components/xios/xios.yaml @@ -70,7 +70,7 @@ choose_version: archfile: ESMTOOLS_generic_oasis_intel intelopenmpi: archfile: ESMTOOLS_generic_oasis_intel - intelmpich: + intel_mpich: archfile: ESMTOOLS_generic_oasis_intel "*": foo: bar diff --git a/configs/machines/aleph.yaml b/configs/machines/aleph.yaml index 841c154da..267db2b37 100644 --- a/configs/machines/aleph.yaml +++ b/configs/machines/aleph.yaml @@ -46,7 +46,7 @@ hyper_flag: "" # # -useMPI: cray_mpich #intelopenmpi #cray_mpich +useMPI: intel_mpich #intelopenmpi #cray_mpich choose_useMPI: cray_mpich: @@ -124,7 +124,7 @@ choose_useMPI: OASIS3MCT_FC_LIB: '"-L$NETCDFFROOT/lib -lnetcdff"' PATH: $IO_LIB_ROOT/bin:$PATH - intelmpich: + intel_mpich: fc: ftn f77: ftn mpifc: ftn @@ -134,7 +134,9 @@ choose_useMPI: module_actions: - "purge" - "load PrgEnv-intel/6.0.5" - - "load gcc/9.3.0" + - "load craype-x86-skylake" + - "load cray-mpich/7.7.3" + - "load gcc/7.3.0" - "load pbs" - "load cmake/3.14.0" - "load cdo/1.9.5" @@ -144,11 +146,9 @@ choose_useMPI: - "load python/3.9.1" - "load cray-netcdf-hdf5parallel/4.6.1.3" - "load cray-hdf5-parallel/1.10.2.0" - - "load eccodes" - "list" add_export_vars: - #LD_LIBRARY_PATH: $IO_LIB_ROOT/lib:$LD_LIBRARY_PATH - ECCODESROOT: $ECCODES_DIR + ECCODESROOT: /home/awiiccp2/software/HPC_libraries/eccodes_intel_mpich/ HDF5_C_INCLUDE_DIRECTORIES: $HDF5_ROOT/include NETCDF_Fortran_INCLUDE_DIRECTORIES: $NETCDFFROOT/include @@ -156,7 +156,6 @@ choose_useMPI: NETCDF_CXX_INCLUDE_DIRECTORIES: $NETCDFROOT/include OASIS3MCT_FC_LIB: '"-L$NETCDFFROOT/lib -lnetcdff"' PATH: $IO_LIB_ROOT/bin:$PATH - export_vars: LC_ALL: en_US.UTF-8 From 660b5e80cd78ea5d3ae2132b52d9764211ec1fb7 Mon Sep 17 00:00:00 2001 From: ICCP Guest Date: Thu, 20 Jul 2023 20:17:40 +0900 Subject: [PATCH 05/18] works with cray mpich in the end --- configs/components/oifs/oifs.env.yaml | 2 +- configs/machines/aleph.yaml | 34 ++++++++++++++------------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/configs/components/oifs/oifs.env.yaml b/configs/components/oifs/oifs.env.yaml index 5aa967447..e0d28ffc2 100644 --- a/configs/components/oifs/oifs.env.yaml +++ b/configs/components/oifs/oifs.env.yaml @@ -473,7 +473,7 @@ compiletime_environment_changes: OIFS_FFIXED: '""' intel_mpich: add_export_vars: - OIFS_FFLAGS: '"-r8 -fp-model precise -align array32byte -O3 -qopenmp -xCORE_AVX512 -g -traceback -convert big_endian -fpe0"' + OIFS_FFLAGS: '"-r8 -fp-model precise -align array32byte -O0 -qopenmp -xCORE_AVX512 -g -traceback -convert big_endian -fpe0"' OIFS_CFLAGS: '"-fp-model precise -O3 -xCORE_AVX512 -g -traceback -qopt-report=0 -fpe0 -qopenmp"' OIFS_LFLAGS: '"$OIFS_MPI_LIB -qopenmp"' OIFS_FFIXED: '""' diff --git a/configs/machines/aleph.yaml b/configs/machines/aleph.yaml index 267db2b37..5eabf25ea 100644 --- a/configs/machines/aleph.yaml +++ b/configs/machines/aleph.yaml @@ -46,7 +46,7 @@ hyper_flag: "" # # -useMPI: intel_mpich #intelopenmpi #cray_mpich +useMPI: cray_mpich #intel_mpich #intelopenmpi #cray_mpich choose_useMPI: cray_mpich: @@ -84,18 +84,18 @@ choose_useMPI: ZLIBROOT: /usr intelopenmpi: - fc: /home/awiiccp2/software/HPC_libraries/intel-19.0.0.117_ompi_20230717/bin/mpifort - f77: /home/awiiccp2/software/HPC_libraries/intel-19.0.0.117_ompi_20230717/bin/mpifort - mpifc: /home/awiiccp2/software/HPC_libraries/intel-19.0.0.117_ompi_20230717/bin/mpifort - mpicc: /home/awiiccp2/software/HPC_libraries/intel-19.0.0.117_ompi_20230717/bin/mpicc - cc: /home/awiiccp2/software/HPC_libraries/intel-19.0.0.117_ompi_20230717/bin/mpicc - cxx: /home/awiiccp2/software/HPC_libraries/intel-19.0.0.117_ompi_20230717/bin/mpicxx + fc: /home/awiiccp2/software/HPC_libraries/intel-18.0.3.222_ompi_20230717/bin/mpifort + f77: /home/awiiccp2/software/HPC_libraries/intel-18.0.3.222_ompi_20230717/bin/mpifort + mpifc: /home/awiiccp2/software/HPC_libraries/intel-18.0.3.222_ompi_20230717/bin/mpifort + mpicc: /home/awiiccp2/software/HPC_libraries/intel-18.0.3.222_ompi_20230717/bin/mpicc + cc: /home/awiiccp2/software/HPC_libraries/intel-18.0.3.222_ompi_20230717/bin/mpicc + cxx: /home/awiiccp2/software/HPC_libraries/intel-18.0.3.222_ompi_20230717/bin/mpicxx module_actions: - - "purge" + - "unload PrgEnv-cray" - "load PrgEnv-intel/6.0.5" - - "unload pmi" - - "swap intel/19.0.0.117" - - "load gcc/9.3.0" + - "load craype-x86-skylake" + - "load cray-mpich/7.7.3" + - "load gcc/7.3.0" - "load pbs" - "load cmake/3.14.0" - "load cdo/1.9.5" @@ -105,9 +105,9 @@ choose_useMPI: - "load python/3.9.1" - "list" add_export_vars: - CMAKE_C_COMPILER: /home/awiiccp2/software/HPC_libraries/intel-19.0.0.117_ompi_20230717/bin/mpicc - CMAKE_CXX_COMPILER: /home/awiiccp2/software/HPC_libraries/intel-19.0.0.117_ompi_20230717/bin/mpicxx - IO_LIB_ROOT: /home/awiiccp2/software/HPC_libraries/intel-19.0.0.117_ompi_20230717 + CMAKE_C_COMPILER: /home/awiiccp2/software/HPC_libraries/intel-18.0.3.222_ompi_20230717/bin/mpicc + CMAKE_CXX_COMPILER: /home/awiiccp2/software/HPC_libraries/intel-18.0.3.222_ompi_20230717/bin/mpicxx + IO_LIB_ROOT: /home/awiiccp2/software/HPC_libraries/intel-18.0.3.222_ompi_20230717 LD_LIBRARY_PATH: $IO_LIB_ROOT/lib:$LD_LIBRARY_PATH SZIPROOT: $IO_LIB_ROOT @@ -123,6 +123,7 @@ choose_useMPI: NETCDF_CXX_INCLUDE_DIRECTORIES: $NETCDFROOT/include OASIS3MCT_FC_LIB: '"-L$NETCDFFROOT/lib -lnetcdff"' PATH: $IO_LIB_ROOT/bin:$PATH + MPIFC: ftn intel_mpich: fc: ftn @@ -132,7 +133,7 @@ choose_useMPI: cc: cc cxx: CC module_actions: - - "purge" + - "unload PrgEnv-cray" - "load PrgEnv-intel/6.0.5" - "load craype-x86-skylake" - "load cray-mpich/7.7.3" @@ -156,6 +157,7 @@ choose_useMPI: NETCDF_CXX_INCLUDE_DIRECTORIES: $NETCDFROOT/include OASIS3MCT_FC_LIB: '"-L$NETCDFFROOT/lib -lnetcdff"' PATH: $IO_LIB_ROOT/bin:$PATH + MPIFC: ftn export_vars: LC_ALL: en_US.UTF-8 @@ -180,7 +182,7 @@ export_vars: # Important for eccodes to work CRAYPE_LINK_TYPE: dynamic -label_format: '"[@MODEL@]%l:"' +label_format: '"[@MODEL@]%r:"' launcher_flags_per_component: "-n @nproc@ -N @nproc_per_node@ -d @cpus_per_proc@ env OMP_NUM_THREADS=@omp_num_threads@ env PMI_LABEL_ERROUT_FORMAT=${label_format}" From 976069e1d5dfa6d05ee9f3200e997e0955bebd0d Mon Sep 17 00:00:00 2001 From: ICCP Guest Date: Thu, 27 Jul 2023 15:39:36 +0900 Subject: [PATCH 06/18] working TCo1279-DART with restart on aleph --- .../frontiers-xios/TCo1279-DART-1D.yaml | 127 ++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 runscripts/awicm3/frontiers-xios/TCo1279-DART-1D.yaml diff --git a/runscripts/awicm3/frontiers-xios/TCo1279-DART-1D.yaml b/runscripts/awicm3/frontiers-xios/TCo1279-DART-1D.yaml new file mode 100644 index 000000000..7d9ce56d5 --- /dev/null +++ b/runscripts/awicm3/frontiers-xios/TCo1279-DART-1D.yaml @@ -0,0 +1,127 @@ +general: + setup_name: "awicm3" + version: "frontiers-xios" + account: "awiiccp" + compute_time: "00:30:00" + initial_date: "1950-01-01" + final_date: "1950-01-03" + base_dir: "/scratch/awiiccp2/" + nday: 1 + nmonth: 0 + nyear: 0 + #use_venv: false # This needs to be deleted as default so that the user gets the dialogue and takes action + runtime_environment_changes: + add_export_vars: + - "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/awiiccp2/software/ecmwf/skylake/lib64/" + +awicm3: + postprocessing: false + model_dir: /home/awiiccp2/model_codes/awicm3-frontiers-xios/ + +fesom: + #setup_dir: "/proj/awiiccp5/awicm3-frontiers-xios/fesom-2.0/" + ini_parent_exp_id: "sp1950d" + ini_parent_date: "1949-12-31" + ini_parent_dir: "/scratch/awiiccp5/ctl1950d/" + resolution: "DART" + mesh_dir: /proj/awi/input/fesom2/dart/ + lresume: true # don't forget to check! + restart_rate: 1 + restart_unit: "d" + restart_first: 1 + choose_general.run_number: + 1: + lasttime: '86160' + restart_in_sources: + par_oce_restart: /scratch/awiiccp5/ctl1950d/run_19490101-19491231/work/fesom.1949.oce.restart/*.nc + par_ice_restart: /scratch/awiiccp5/ctl1950d/run_19490101-19491231/work/fesom.1949.ice.restart/*.nc + "*": + remove_restart_in_files: [par_oce_restart, par_ice_restart] + remove_restart_out_files: [par_oce_restart, par_ice_restart] + add_ignore_sources: + par_oce_restart: fesom.*.oce.restart/* + par_ice_restart: fesom.*.ice.restart/* + namelist_changes: + namelist.config: + machine: + n_levels: 2 + n_part: [10, 72] + restart_log: + raw_restart_length: 1 + raw_restart_length_unit: "m" + namelist_dir: /proj/awiiccp5/esm_tools/namelists/fesom2/2.0/awicm3/DART_niko_kpp/ + add_file_movements: + outdata: + all_directions: move + + time_step: 240 + nproc: 1280 + + +oifs: + resolution: "TCO1279" + levels: "L137" + prepifs_expid: hf05 + input_expid: awi3 + wam: true + lresume: false + time_step: 450 + nproc: 1320 + omp_num_threads: 8 + namelist_changes: + fort.4: + naerad: + NCMIPFIXYR: 1950 + namcumf: + RPRCON: 1.0E-3 + ENTRORG: 1.25E-3 + namgwwms: + GGAUSSB: -0.45 + file_movements: + forcing: + all_directions: "link" + input: + all_directions: "link" + + +oasis3mct: + lresume: true # Set to false to generate the rst files for first leg + time_step: 3600 + coupling_methods: + gauswgt_c: + time_transformation: conserv + remapping: + gauswgt: + search_bin: latitude + nb_of_search_bins: 1 + nb_of_neighbours: 9 + weight: "2" + postprocessing: + conserv: + method: glbpos + algorithm: opt + gauswgt_gss: + time_transformation: conserv + remapping: + gauswgt: + search_bin: latitude + nb_of_search_bins: 1 + nb_of_neighbours: 9 + weight: "2" + postprocessing: + conserv: + method: gsspos + algorithm: opt + gauswgt_i: + time_transformation: instant + remapping: + gauswgt: + search_bin: latitude + nb_of_search_bins: 1 + nb_of_neighbours: 9 + weight: "2" +xios: + with_model: oifs + nproc: 10 + omp_num_threads: 40 + From 053d1874f56b4f05a3796cd4994bd56ca0057838 Mon Sep 17 00:00:00 2001 From: ICCP Guest Date: Mon, 31 Jul 2023 22:25:30 +0900 Subject: [PATCH 07/18] namelist.io as used for TCO1279-DART --- namelists/fesom2/2.0/awicm3/DART/namelist.io | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/namelists/fesom2/2.0/awicm3/DART/namelist.io b/namelists/fesom2/2.0/awicm3/DART/namelist.io index 6c834f1b7..c6dc48691 100644 --- a/namelists/fesom2/2.0/awicm3/DART/namelist.io +++ b/namelists/fesom2/2.0/awicm3/DART/namelist.io @@ -9,6 +9,10 @@ ldiag_DVD =.false. ldiag_forc =.true. / +&nml_output_settings + keep_nth_level = 1 +/ + &nml_listsize io_listsize=100 !number of streams to allocate. shallbe large or equal to the number of streams in &nml_list / @@ -43,9 +47,6 @@ io_list = 'sst ',90,'s', 4, 'u1-31 ',1, 'd', 4, 'v1-31 ',1, 'd', 4, 'w1-31 ',1, 'd', 4, - 'bolus_u ',1, 'y', 4, - 'bolus_v ',1, 'y', 4, - 'bolus_w ',1, 'y', 4, 'snow ',1, 'm', 4, 'subli ',1, 'm', 4, 'thdgrsn ',1, 'm', 4, From f9c591c4596fb21cdfdfe641b556bb434e7a12f1 Mon Sep 17 00:00:00 2001 From: ICCP Guest Date: Tue, 1 Aug 2023 00:19:51 +0900 Subject: [PATCH 08/18] oifs output for high res sims --- .../oifs/43r3/xios/TCO1279_DART/file_def.xml | 3 + .../frontiers-xios/TCo1279-DART-CTRL.yaml | 128 ++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 runscripts/awicm3/frontiers-xios/TCo1279-DART-CTRL.yaml diff --git a/namelists/oifs/43r3/xios/TCO1279_DART/file_def.xml b/namelists/oifs/43r3/xios/TCO1279_DART/file_def.xml index ec82f96fa..e7f7df01e 100644 --- a/namelists/oifs/43r3/xios/TCO1279_DART/file_def.xml +++ b/namelists/oifs/43r3/xios/TCO1279_DART/file_def.xml @@ -23,6 +23,9 @@ + + + diff --git a/runscripts/awicm3/frontiers-xios/TCo1279-DART-CTRL.yaml b/runscripts/awicm3/frontiers-xios/TCo1279-DART-CTRL.yaml new file mode 100644 index 000000000..e3e2c0ece --- /dev/null +++ b/runscripts/awicm3/frontiers-xios/TCo1279-DART-CTRL.yaml @@ -0,0 +1,128 @@ +general: + setup_name: "awicm3" + version: "frontiers-xios" + account: "awiiccp" + compute_time: "00:30:00" + initial_date: "1950-01-01" + final_date: "1960-01-01" + base_dir: "/scratch/awiiccp2/" + nday: 0 + nmonth: 2 + nyear: 0 + #use_venv: false # This needs to be deleted as default so that the user gets the dialogue and takes action + runtime_environment_changes: + add_export_vars: + - "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/awiiccp2/software/ecmwf/skylake/lib64/" + + mail_type: ALL + mail_user: "jan.streffing@awi.de" + +awicm3: + postprocessing: false + model_dir: /home/awiiccp2/model_codes/awicm3-frontiers-xios/ + +fesom: + #setup_dir: "/proj/awiiccp5/awicm3-frontiers-xios/fesom-2.0/" + ini_parent_exp_id: "sp1950d" + ini_parent_date: "1949-12-31" + ini_parent_dir: "/scratch/awiiccp5/ctl1950d/" + resolution: "DART" + mesh_dir: /proj/awi/input/fesom2/dart/ + lresume: true # don't forget to check! + restart_rate: 2 + restart_unit: "m" + restart_first: 2 + choose_general.run_number: + 1: + lasttime: '86160' + restart_in_sources: + par_oce_restart: /scratch/awiiccp5/ctl1950d/run_19490101-19491231/work/fesom.1949.oce.restart/*.nc + par_ice_restart: /scratch/awiiccp5/ctl1950d/run_19490101-19491231/work/fesom.1949.ice.restart/*.nc + "*": + remove_restart_in_files: [par_oce_restart, par_ice_restart] + remove_restart_out_files: [par_oce_restart, par_ice_restart] + add_ignore_sources: + par_oce_restart: fesom.*.oce.restart/* + par_ice_restart: fesom.*.ice.restart/* + namelist_changes: + namelist.config: + machine: + n_levels: 2 + n_part: [10, 72] + restart_log: + raw_restart_length: 1 + raw_restart_length_unit: "m" + namelist.io: + nml_listsize: + filesplit_freq: 'm' + namelist_dir: /proj/awiiccp5/esm_tools/namelists/fesom2/2.0/awicm3/DART_niko_kpp/ + add_file_movements: + outdata: + all_directions: move + + time_step: 240 + nproc: 1280 + + +oifs: + resolution: "TCO1279" + levels: "L137" + prepifs_expid: hf05 + input_expid: awi3 + wam: true + lresume: false + time_step: 450 + nproc: 1320 + omp_num_threads: 8 + namelist_changes: + fort.4: + naerad: + NCMIPFIXYR: 1950 + namcumf: + RPRCON: 1.0E-3 + ENTRORG: 1.25E-3 + namgwwms: + GGAUSSB: -0.45 + + +oasis3mct: + lresume: true # Set to false to generate the rst files for first leg + time_step: 3600 + coupling_methods: + gauswgt_c: + time_transformation: conserv + remapping: + gauswgt: + search_bin: latitude + nb_of_search_bins: 1 + nb_of_neighbours: 9 + weight: "2" + postprocessing: + conserv: + method: glbpos + algorithm: opt + gauswgt_gss: + time_transformation: conserv + remapping: + gauswgt: + search_bin: latitude + nb_of_search_bins: 1 + nb_of_neighbours: 9 + weight: "2" + postprocessing: + conserv: + method: gsspos + algorithm: opt + gauswgt_i: + time_transformation: instant + remapping: + gauswgt: + search_bin: latitude + nb_of_search_bins: 1 + nb_of_neighbours: 9 + weight: "2" +xios: + with_model: oifs + nproc: 10 + omp_num_threads: 40 + From 677235d3a63907a236c2686f13b283bdfe8ba7a2 Mon Sep 17 00:00:00 2001 From: mandresm Date: Mon, 31 Jul 2023 19:13:19 +0200 Subject: [PATCH 09/18] fix issue #1009 --- src/esm_runscripts/filelists.py | 6 +++--- src/esm_runscripts/tidy.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/esm_runscripts/filelists.py b/src/esm_runscripts/filelists.py index 2eb14b84a..fa27f8520 100644 --- a/src/esm_runscripts/filelists.py +++ b/src/esm_runscripts/filelists.py @@ -892,7 +892,7 @@ def check_for_unknown_files(config): return config -def resolve_symlinks(file_source): +def resolve_symlinks(config, file_source): if os.path.islink(file_source): points_to = os.path.realpath(file_source) @@ -905,7 +905,7 @@ def resolve_symlinks(file_source): return file_source # recursively find the file that the link is pointing to - return resolve_symlinks(points_to) + return resolve_symlinks(config, points_to) else: return file_source @@ -956,7 +956,7 @@ def copy_files(config, filetypes, source, target): helpers.print_datetime(config) continue dest_dir = os.path.dirname(file_target) - file_source = resolve_symlinks(file_source) + file_source = resolve_symlinks(config, file_source) if not os.path.isdir(file_source): try: if not os.path.isdir(dest_dir): diff --git a/src/esm_runscripts/tidy.py b/src/esm_runscripts/tidy.py index aeeb5109a..ee2d8857b 100644 --- a/src/esm_runscripts/tidy.py +++ b/src/esm_runscripts/tidy.py @@ -362,12 +362,12 @@ def copy_all_results_to_exp(config): + destination ) else: - linkdest = resolve_symlinks(source) + linkdest = resolve_symlinks(config, source) # newlinkdest = ( # destination.rsplit("/", 1)[0] + "/" + linkdest.rsplit("/", 1)[-1] # ) if os.path.islink(destination): - destdest = resolve_symlinks(source) + destdest = resolve_symlinks(config, source) if linkdest == destdest: # both links are identical, skip continue From 393fa971f58e2bda7aeb306fae9e0800079faa92 Mon Sep 17 00:00:00 2001 From: Jan Streffing Date: Tue, 1 Aug 2023 10:06:29 +0200 Subject: [PATCH 10/18] add warnings --- configs/machines/aleph.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/configs/machines/aleph.yaml b/configs/machines/aleph.yaml index 5eabf25ea..74ac2977c 100644 --- a/configs/machines/aleph.yaml +++ b/configs/machines/aleph.yaml @@ -84,6 +84,10 @@ choose_useMPI: ZLIBROOT: /usr intelopenmpi: + warning: + Configuration does not run: + message: "The intelopenmpi configuration on aleph is experimental. From past experiance it can compile AWI-CM3, but crashes on runtime. Use as your own disgression. Contact jan.streffing@awi.de for further questions" + ask_user_to_continue: True fc: /home/awiiccp2/software/HPC_libraries/intel-18.0.3.222_ompi_20230717/bin/mpifort f77: /home/awiiccp2/software/HPC_libraries/intel-18.0.3.222_ompi_20230717/bin/mpifort mpifc: /home/awiiccp2/software/HPC_libraries/intel-18.0.3.222_ompi_20230717/bin/mpifort @@ -126,6 +130,10 @@ choose_useMPI: MPIFC: ftn intel_mpich: + warning: + Configuration does not run: + message: "The intel_mpich configuration on aleph is experimental. From past experiance it can compile AWI-CM3, but crashes on runtime. Use as your own disgression. Contact jan.streffing@awi.de for further questions" + ask_user_to_continue: True fc: ftn f77: ftn mpifc: ftn From 907e3341ec66ad8f65600891dcd8e8e1461bb937 Mon Sep 17 00:00:00 2001 From: ICCP Guest Date: Wed, 2 Aug 2023 16:49:32 +0900 Subject: [PATCH 11/18] adding CTRL runscript --- .../awicm3/frontiers-xios/TCo1279-DART-CTRL.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/runscripts/awicm3/frontiers-xios/TCo1279-DART-CTRL.yaml b/runscripts/awicm3/frontiers-xios/TCo1279-DART-CTRL.yaml index e3e2c0ece..532c2b7de 100644 --- a/runscripts/awicm3/frontiers-xios/TCo1279-DART-CTRL.yaml +++ b/runscripts/awicm3/frontiers-xios/TCo1279-DART-CTRL.yaml @@ -2,20 +2,17 @@ general: setup_name: "awicm3" version: "frontiers-xios" account: "awiiccp" - compute_time: "00:30:00" + compute_time: "08:00:00" initial_date: "1950-01-01" final_date: "1960-01-01" base_dir: "/scratch/awiiccp2/" nday: 0 nmonth: 2 nyear: 0 - #use_venv: false # This needs to be deleted as default so that the user gets the dialogue and takes action runtime_environment_changes: add_export_vars: - "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/awiiccp2/software/ecmwf/skylake/lib64/" - - mail_type: ALL - mail_user: "jan.streffing@awi.de" + install_esm_tools_branch: feat/aleph_intel_openmpi awicm3: postprocessing: false @@ -83,7 +80,11 @@ oifs: ENTRORG: 1.25E-3 namgwwms: GGAUSSB: -0.45 - + input: + exp_to_run: copy + init_to_exp: link + run_to_work: link + work_to_run: link oasis3mct: lresume: true # Set to false to generate the rst files for first leg From 8aa04ff2f75aef7bb6f0822ce561e00f95b1f8f5 Mon Sep 17 00:00:00 2001 From: ICCP Guest Date: Fri, 4 Aug 2023 06:14:00 +0900 Subject: [PATCH 12/18] splitting monthly --- namelists/oifs/43r3/xios/TCO1279_DART/file_def.xml | 4 ++-- runscripts/awicm3/frontiers-xios/TCo1279-DART-CTRL.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/namelists/oifs/43r3/xios/TCO1279_DART/file_def.xml b/namelists/oifs/43r3/xios/TCO1279_DART/file_def.xml index e7f7df01e..2abfc0fd6 100644 --- a/namelists/oifs/43r3/xios/TCO1279_DART/file_def.xml +++ b/namelists/oifs/43r3/xios/TCO1279_DART/file_def.xml @@ -11,7 +11,7 @@ par_access="collective" name="atm" compression_level="1" - split_freq="1y"> + split_freq="1mo"> + split_freq="1mo"> Date: Tue, 8 Aug 2023 16:12:30 +0900 Subject: [PATCH 13/18] adding potential density to default awicm3 frontiers output --- namelists/fesom2/2.0/awicm3/frontiers/namelist.io | 1 + 1 file changed, 1 insertion(+) diff --git a/namelists/fesom2/2.0/awicm3/frontiers/namelist.io b/namelists/fesom2/2.0/awicm3/frontiers/namelist.io index 2c328e47c..8b0590fcc 100644 --- a/namelists/fesom2/2.0/awicm3/frontiers/namelist.io +++ b/namelists/fesom2/2.0/awicm3/frontiers/namelist.io @@ -37,6 +37,7 @@ io_list = 'sst ',1, 'm', 4, 'ty_sur ',1, 'm', 4, 'temp ',1, 'm', 4, 'salt ',1, 'm', 4, + 'sigma0' ',1, 'm', 4' 'N2 ',1, 'y', 4, 'Kv ',1, 'y', 4, 'u ',1, 'y', 4, From e76ffc627bf77469dd48d92d7d9ccf6e80d59795 Mon Sep 17 00:00:00 2001 From: Jan Streffing Date: Thu, 10 Aug 2023 14:08:34 +0200 Subject: [PATCH 14/18] Update configs/machines/aleph.yaml fixing inlining by @mandresm Co-authored-by: Miguel <63242832+mandresm@users.noreply.github.com> --- configs/machines/aleph.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/machines/aleph.yaml b/configs/machines/aleph.yaml index 74ac2977c..3cb9a27dc 100644 --- a/configs/machines/aleph.yaml +++ b/configs/machines/aleph.yaml @@ -85,9 +85,9 @@ choose_useMPI: intelopenmpi: warning: - Configuration does not run: - message: "The intelopenmpi configuration on aleph is experimental. From past experiance it can compile AWI-CM3, but crashes on runtime. Use as your own disgression. Contact jan.streffing@awi.de for further questions" - ask_user_to_continue: True + "Configuration does not run": + message: "The intelopenmpi configuration on aleph is experimental. From past experiance it can compile AWI-CM3, but crashes on runtime. Use as your own disgression. Contact jan.streffing@awi.de for further questions" + ask_user_to_continue: True fc: /home/awiiccp2/software/HPC_libraries/intel-18.0.3.222_ompi_20230717/bin/mpifort f77: /home/awiiccp2/software/HPC_libraries/intel-18.0.3.222_ompi_20230717/bin/mpifort mpifc: /home/awiiccp2/software/HPC_libraries/intel-18.0.3.222_ompi_20230717/bin/mpifort From b4f21043b04c133c32bba5e07587b33dbcdff6c3 Mon Sep 17 00:00:00 2001 From: ICCP Guest Date: Fri, 25 Aug 2023 21:42:11 +0900 Subject: [PATCH 15/18] fix typo --- namelists/fesom2/2.0/awicm3/frontiers/namelist.io | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/namelists/fesom2/2.0/awicm3/frontiers/namelist.io b/namelists/fesom2/2.0/awicm3/frontiers/namelist.io index 8b0590fcc..45cf58d67 100644 --- a/namelists/fesom2/2.0/awicm3/frontiers/namelist.io +++ b/namelists/fesom2/2.0/awicm3/frontiers/namelist.io @@ -4,7 +4,7 @@ lcurt_stress_surf=.false. ldiag_curl_vel3 =.false. ldiag_energy =.false. ldiag_salt3D =.false. -ldiag_dMOC =.false. +ldiag_dMOC =.true. ldiag_DVD =.false. ldiag_forc =.true. / @@ -37,7 +37,7 @@ io_list = 'sst ',1, 'm', 4, 'ty_sur ',1, 'm', 4, 'temp ',1, 'm', 4, 'salt ',1, 'm', 4, - 'sigma0' ',1, 'm', 4' + 'sigma0' ',1, 'm', 4, 'N2 ',1, 'y', 4, 'Kv ',1, 'y', 4, 'u ',1, 'y', 4, From 3a3a31635137c2c11eb28290cc2dbb453c508181 Mon Sep 17 00:00:00 2001 From: ICCP Guest Date: Mon, 4 Sep 2023 22:04:13 +0900 Subject: [PATCH 16/18] configs for TCO1279 on aleph --- namelists/fesom2/2.0/awicm3/DART/namelist.io | 3 +- .../oifs/43r3/xios/TCO1279_DART/axis_def.xml | 3 +- .../oifs/43r3/xios/TCO1279_DART/grid_def.xml | 4 + .../43r3/xios/TCO95_CORE2/TCo95-CORE2-1Y.yaml | 68 ++++++ .../oifs/43r3/xios/TCO95_CORE2/axis_def.xml | 2 +- .../43r3/xios/TCO95_CORE2/context_ifs.xml | 4 +- .../oifs/43r3/xios/TCO95_CORE2/domain_def.xml | 2 +- .../oifs/43r3/xios/TCO95_CORE2/file_def.xml | 222 ++---------------- .../oifs/43r3/xios/TCO95_CORE2/iodef.xml | 4 +- .../frontiers-xios/TCo1279-DART-1950.yaml | 134 +++++++++++ .../frontiers-xios/TCo1279-DART-1D.yaml | 17 +- .../frontiers-xios/TCo1279-DART-CTRL.yaml | 11 +- .../awicm3/frontiers-xios/TCo95-CORE2-1Y.yaml | 70 ++++++ 13 files changed, 323 insertions(+), 221 deletions(-) create mode 100644 namelists/oifs/43r3/xios/TCO95_CORE2/TCo95-CORE2-1Y.yaml create mode 100644 runscripts/awicm3/frontiers-xios/TCo1279-DART-1950.yaml create mode 100644 runscripts/awicm3/frontiers-xios/TCo95-CORE2-1Y.yaml diff --git a/namelists/fesom2/2.0/awicm3/DART/namelist.io b/namelists/fesom2/2.0/awicm3/DART/namelist.io index c6dc48691..9df96b523 100644 --- a/namelists/fesom2/2.0/awicm3/DART/namelist.io +++ b/namelists/fesom2/2.0/awicm3/DART/namelist.io @@ -4,7 +4,7 @@ lcurt_stress_surf=.false. ldiag_curl_vel3 =.false. ldiag_energy =.true. ldiag_salt3D =.false. -ldiag_dMOC =.false. +ldiag_dMOC =.true. ldiag_DVD =.false. ldiag_forc =.true. / @@ -42,6 +42,7 @@ io_list = 'sst ',90,'s', 4, 'u ',1, 'm', 4, 'v ',1, 'm', 4, 'w ',1, 'm', 4, + 'sigma0 ',1, 'm', 4, 'temp1-31 ',1, 'd', 4, 'salt1-31 ',1, 'd', 4, 'u1-31 ',1, 'd', 4, diff --git a/namelists/oifs/43r3/xios/TCO1279_DART/axis_def.xml b/namelists/oifs/43r3/xios/TCO1279_DART/axis_def.xml index 334b9bb8b..8139fbb3d 100644 --- a/namelists/oifs/43r3/xios/TCO1279_DART/axis_def.xml +++ b/namelists/oifs/43r3/xios/TCO1279_DART/axis_def.xml @@ -9,8 +9,7 @@ - - + diff --git a/namelists/oifs/43r3/xios/TCO1279_DART/grid_def.xml b/namelists/oifs/43r3/xios/TCO1279_DART/grid_def.xml index 4e314a0c8..5c4dc3af6 100644 --- a/namelists/oifs/43r3/xios/TCO1279_DART/grid_def.xml +++ b/namelists/oifs/43r3/xios/TCO1279_DART/grid_def.xml @@ -31,6 +31,10 @@ + + + + diff --git a/namelists/oifs/43r3/xios/TCO95_CORE2/TCo95-CORE2-1Y.yaml b/namelists/oifs/43r3/xios/TCO95_CORE2/TCo95-CORE2-1Y.yaml new file mode 100644 index 000000000..66d015611 --- /dev/null +++ b/namelists/oifs/43r3/xios/TCO95_CORE2/TCo95-CORE2-1Y.yaml @@ -0,0 +1,68 @@ +general: + setup_name: "awicm3" + version: "frontiers-xios" + account: "awiiccp" + compute_time: "24:00:00" + initial_date: "1950-01-01" + final_date: "1951-01-01" + base_dir: "/scratch/awiiccp2/" + nday: 0 + nmonth: 2 + nyear: 0 + runtime_environment_changes: + add_export_vars: + - "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/awiiccp2/software/ecmwf/skylake/lib64/" + +awicm3: + postprocessing: false + model_dir: /home/awiiccp2/model_codes/awicm3-frontiers-xios/ + +fesom: + resolution: "CORE2" + mesh_dir: /proj/awi/input/fesom2/core2/ + lresume: false # don't forget to check! + restart_rate: 2 + restart_unit: "m" + restart_first: 2 + add_file_movements: + outdata: + all_directions: move + add_namelist_changes: + namelist.io: + nml_listsize: + filesplit_freq: 'm' + + time_step: 1800 + nproc: 320 + + +oifs: + resolution: "TCO95" + levels: "L91" + prepifs_expid: aack + input_expid: awi3 + wam: false + lresume: false + time_step: 3600 + nproc: 40 + omp_num_threads: 8 + namelist_changes: + fort.4: + naerad: + NCMIPFIXYR: 1950 + namcumf: + RPRCON: 1.0E-3 + ENTRORG: 1.25E-3 + namgwwms: + GGAUSSB: -0.45 + + +oasis3mct: + lresume: true # Set to false to generate the rst files for first leg + time_step: 3600 + +xios: + with_model: oifs + nproc: 2 + omp_num_threads: 40 + diff --git a/namelists/oifs/43r3/xios/TCO95_CORE2/axis_def.xml b/namelists/oifs/43r3/xios/TCO95_CORE2/axis_def.xml index 498f23766..8139fbb3d 100644 --- a/namelists/oifs/43r3/xios/TCO95_CORE2/axis_def.xml +++ b/namelists/oifs/43r3/xios/TCO95_CORE2/axis_def.xml @@ -9,7 +9,7 @@ - + diff --git a/namelists/oifs/43r3/xios/TCO95_CORE2/context_ifs.xml b/namelists/oifs/43r3/xios/TCO95_CORE2/context_ifs.xml index e3724c1c0..df526436e 100644 --- a/namelists/oifs/43r3/xios/TCO95_CORE2/context_ifs.xml +++ b/namelists/oifs/43r3/xios/TCO95_CORE2/context_ifs.xml @@ -18,8 +18,8 @@ - -6 - -6 + -3 + -3 diff --git a/namelists/oifs/43r3/xios/TCO95_CORE2/domain_def.xml b/namelists/oifs/43r3/xios/TCO95_CORE2/domain_def.xml index 534ded4ee..38d74ca9b 100644 --- a/namelists/oifs/43r3/xios/TCO95_CORE2/domain_def.xml +++ b/namelists/oifs/43r3/xios/TCO95_CORE2/domain_def.xml @@ -4,7 +4,7 @@ - + diff --git a/namelists/oifs/43r3/xios/TCO95_CORE2/file_def.xml b/namelists/oifs/43r3/xios/TCO95_CORE2/file_def.xml index 12e514a58..b923f449a 100644 --- a/namelists/oifs/43r3/xios/TCO95_CORE2/file_def.xml +++ b/namelists/oifs/43r3/xios/TCO95_CORE2/file_def.xml @@ -9,171 +9,18 @@ type="multiple_file" format="netcdf4" par_access="collective" - name="atm" - split_freq="1y"> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + name="atm_reduced" + compression_level="1" + split_freq="1mo"> + timeseries="only" + output_freq="3h" + name_suffix="_3h" + description="ECE4/OIFS 3hourly surface fields"> - + @@ -186,12 +33,12 @@ - + @@ -204,16 +51,12 @@ - - - - - + @@ -223,8 +66,6 @@ - - @@ -271,42 +112,23 @@ - - - - - - - - - - - - - - - + + timeseries="only" + output_freq="3h" + name_suffix="_3h_pl" + description="ECE4/OIFS 3hourly pressure level fields"> - + + + - - diff --git a/namelists/oifs/43r3/xios/TCO95_CORE2/iodef.xml b/namelists/oifs/43r3/xios/TCO95_CORE2/iodef.xml index c3f767033..d34b854b0 100644 --- a/namelists/oifs/43r3/xios/TCO95_CORE2/iodef.xml +++ b/namelists/oifs/43r3/xios/TCO95_CORE2/iodef.xml @@ -6,8 +6,8 @@ true - 50 - 1 + 50 + 1 false 0.5 diff --git a/runscripts/awicm3/frontiers-xios/TCo1279-DART-1950.yaml b/runscripts/awicm3/frontiers-xios/TCo1279-DART-1950.yaml new file mode 100644 index 000000000..2e6115140 --- /dev/null +++ b/runscripts/awicm3/frontiers-xios/TCo1279-DART-1950.yaml @@ -0,0 +1,134 @@ +general: + setup_name: "awicm3" + version: "frontiers-xios" + account: "awiiccp" + compute_time: "12:00:00" + initial_date: "1950-01-01" + final_date: "1960-01-01" + base_dir: "/scratch/awiiccp2/" + nday: 0 + nmonth: 2 + nyear: 0 + runtime_environment_changes: + add_export_vars: + - "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/awiiccp2/software/ecmwf/skylake/lib64/" + install_esm_tools_branch: feat/aleph_intel_openmpi + +computer: + mail_type: abe + mail_user: "jan.streffing@awi.de" + + +awicm3: + postprocessing: false + model_dir: /home/awiiccp2/model_codes/awicm3-frontiers-xios/ + +fesom: + #setup_dir: "/proj/awiiccp5/awicm3-frontiers-xios/fesom-2.0/" + ini_parent_exp_id: "sp1950d" + ini_parent_date: "1949-12-31" + ini_parent_dir: "/scratch/awiiccp5/ctl1950d/" + resolution: "DART" + mesh_dir: /proj/awi/input/fesom2/dart/ + lresume: true # don't forget to check! + restart_rate: 2 + restart_unit: "m" + restart_first: 2 + choose_general.run_number: + 1: + lasttime: '86160' + restart_in_sources: + par_oce_restart: /scratch/awiiccp5/ctl1950d/run_19490101-19491231/work/fesom.1949.oce.restart/*.nc + par_ice_restart: /scratch/awiiccp5/ctl1950d/run_19490101-19491231/work/fesom.1949.ice.restart/*.nc + "*": + remove_restart_in_files: [par_oce_restart, par_ice_restart] + remove_restart_out_files: [par_oce_restart, par_ice_restart] + add_ignore_sources: + par_oce_restart: fesom.*.oce.restart/* + par_ice_restart: fesom.*.ice.restart/* + namelist_changes: + namelist.config: + machine: + n_levels: 2 + n_part: [10, 72] + restart_log: + raw_restart_length: 1 + raw_restart_length_unit: "m" + namelist.io: + nml_listsize: + filesplit_freq: 'm' + namelist_dir: /proj/awiiccp5/esm_tools/namelists/fesom2/2.0/awicm3/DART_niko_kpp/ + add_file_movements: + outdata: + all_directions: move + + time_step: 240 + nproc: 1280 + + +oifs: + resolution: "TCO1279" + levels: "L137" + prepifs_expid: hf05 + input_expid: awi3 + wam: true + lresume: false + time_step: 450 + nproc: 1320 + omp_num_threads: 8 + namelist_changes: + fort.4: + naerad: + NCMIPFIXYR: 1950 + namcumf: + RPRCON: 1.0E-3 + ENTRORG: 1.25E-3 + namgwwms: + GGAUSSB: -0.45 + input: + exp_to_run: copy + init_to_exp: copy + run_to_work: copy + work_to_run: copy + +oasis3mct: + lresume: true # Set to false to generate the rst files for first leg + time_step: 3600 + coupling_methods: + gauswgt_c: + time_transformation: conserv + remapping: + gauswgt: + search_bin: latitude + nb_of_search_bins: 1 + nb_of_neighbours: 9 + weight: "2" + postprocessing: + conserv: + method: glbpos + algorithm: opt + gauswgt_gss: + time_transformation: conserv + remapping: + gauswgt: + search_bin: latitude + nb_of_search_bins: 1 + nb_of_neighbours: 9 + weight: "2" + postprocessing: + conserv: + method: gsspos + algorithm: opt + gauswgt_i: + time_transformation: instant + remapping: + gauswgt: + search_bin: latitude + nb_of_search_bins: 1 + nb_of_neighbours: 9 + weight: "2" +xios: + with_model: oifs + nproc: 10 + omp_num_threads: 40 + diff --git a/runscripts/awicm3/frontiers-xios/TCo1279-DART-1D.yaml b/runscripts/awicm3/frontiers-xios/TCo1279-DART-1D.yaml index 7d9ce56d5..7704dedc8 100644 --- a/runscripts/awicm3/frontiers-xios/TCo1279-DART-1D.yaml +++ b/runscripts/awicm3/frontiers-xios/TCo1279-DART-1D.yaml @@ -2,17 +2,17 @@ general: setup_name: "awicm3" version: "frontiers-xios" account: "awiiccp" - compute_time: "00:30:00" + compute_time: "12:00:00" initial_date: "1950-01-01" - final_date: "1950-01-03" + final_date: "1950-01-02" base_dir: "/scratch/awiiccp2/" nday: 1 nmonth: 0 nyear: 0 - #use_venv: false # This needs to be deleted as default so that the user gets the dialogue and takes action runtime_environment_changes: add_export_vars: - "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/awiiccp2/software/ecmwf/skylake/lib64/" + install_esm_tools_branch: feat/aleph_intel_openmpi awicm3: postprocessing: false @@ -49,6 +49,11 @@ fesom: restart_log: raw_restart_length: 1 raw_restart_length_unit: "m" + namelist.io: + nml_listsize: + filesplit_freq: 'm' + diag_list: + ldiag_dMOC: true namelist_dir: /proj/awiiccp5/esm_tools/namelists/fesom2/2.0/awicm3/DART_niko_kpp/ add_file_movements: outdata: @@ -77,12 +82,6 @@ oifs: ENTRORG: 1.25E-3 namgwwms: GGAUSSB: -0.45 - file_movements: - forcing: - all_directions: "link" - input: - all_directions: "link" - oasis3mct: lresume: true # Set to false to generate the rst files for first leg diff --git a/runscripts/awicm3/frontiers-xios/TCo1279-DART-CTRL.yaml b/runscripts/awicm3/frontiers-xios/TCo1279-DART-CTRL.yaml index 83f97087e..2e6115140 100644 --- a/runscripts/awicm3/frontiers-xios/TCo1279-DART-CTRL.yaml +++ b/runscripts/awicm3/frontiers-xios/TCo1279-DART-CTRL.yaml @@ -14,6 +14,11 @@ general: - "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/awiiccp2/software/ecmwf/skylake/lib64/" install_esm_tools_branch: feat/aleph_intel_openmpi +computer: + mail_type: abe + mail_user: "jan.streffing@awi.de" + + awicm3: postprocessing: false model_dir: /home/awiiccp2/model_codes/awicm3-frontiers-xios/ @@ -82,9 +87,9 @@ oifs: GGAUSSB: -0.45 input: exp_to_run: copy - init_to_exp: link - run_to_work: link - work_to_run: link + init_to_exp: copy + run_to_work: copy + work_to_run: copy oasis3mct: lresume: true # Set to false to generate the rst files for first leg diff --git a/runscripts/awicm3/frontiers-xios/TCo95-CORE2-1Y.yaml b/runscripts/awicm3/frontiers-xios/TCo95-CORE2-1Y.yaml new file mode 100644 index 000000000..0d1480ab5 --- /dev/null +++ b/runscripts/awicm3/frontiers-xios/TCo95-CORE2-1Y.yaml @@ -0,0 +1,70 @@ +general: + setup_name: "awicm3" + version: "frontiers-xios" + account: "awiiccp" + compute_time: "00:00:00" + initial_date: "1950-01-01" + final_date: "1951-01-01" + base_dir: "/scratch/awiiccp2/" + nday: 0 + nmonth: 2 + nyear: 0 + runtime_environment_changes: + add_export_vars: + - "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/awiiccp2/software/ecmwf/skylake/lib64/" + +awicm3: + mail_type: FAIL + mail_user: "jan.streffing@awi.de" + postprocessing: false + model_dir: /home/awiiccp2/model_codes/awicm3-frontiers-xios/ + +fesom: + resolution: "CORE2" + mesh_dir: /proj/awi/input/fesom2/core2/ + lresume: false # don't forget to check! + restart_rate: 2 + restart_unit: "m" + restart_first: 2 + add_file_movements: + outdata: + all_directions: move + add_namelist_changes: + namelist.io: + nml_listsize: + filesplit_freq: 'm' + + time_step: 1800 + nproc: 40 + + +oifs: + resolution: "TCO95" + levels: "L91" + prepifs_expid: aack + input_expid: awi3 + wam: false + lresume: false + time_step: 3600 + nproc: 40 + omp_num_threads: 1 + namelist_changes: + fort.4: + naerad: + NCMIPFIXYR: 1950 + namcumf: + RPRCON: 1.0E-3 + ENTRORG: 1.25E-3 + namgwwms: + GGAUSSB: -0.45 + + +oasis3mct: + lresume: false # Set to false to generate the rst files for first leg + time_step: 3600 + +xios: + with_model: oifs + nproc: 2 + omp_num_threads: 20 + From 972edee1f908de42235d69c29454f57a3a97a2c3 Mon Sep 17 00:00:00 2001 From: ICCP Guest Date: Tue, 17 Oct 2023 17:18:13 +0900 Subject: [PATCH 17/18] update IO to match req --- .../oifs/43r3/xios/TCO1279_DART/file_def.xml | 202 +++--------------- 1 file changed, 25 insertions(+), 177 deletions(-) diff --git a/namelists/oifs/43r3/xios/TCO1279_DART/file_def.xml b/namelists/oifs/43r3/xios/TCO1279_DART/file_def.xml index 2abfc0fd6..af45eb15d 100644 --- a/namelists/oifs/43r3/xios/TCO1279_DART/file_def.xml +++ b/namelists/oifs/43r3/xios/TCO1279_DART/file_def.xml @@ -4,44 +4,47 @@ - - + - - - + + + + + + + - + @@ -49,17 +52,20 @@ + + + - + @@ -121,35 +127,16 @@ - - - - - - - - - - - - - - - + @@ -158,161 +145,22 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + From 30291d80dc689f08ac04e6d793c9b2d63a44ac77 Mon Sep 17 00:00:00 2001 From: ICCP Guest Date: Tue, 17 Oct 2023 17:19:17 +0900 Subject: [PATCH 18/18] netcdf restarts deactivated via binary --- runscripts/awicm3/frontiers-xios/TCo1279-DART-1950.yaml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/runscripts/awicm3/frontiers-xios/TCo1279-DART-1950.yaml b/runscripts/awicm3/frontiers-xios/TCo1279-DART-1950.yaml index 2e6115140..42a1c9567 100644 --- a/runscripts/awicm3/frontiers-xios/TCo1279-DART-1950.yaml +++ b/runscripts/awicm3/frontiers-xios/TCo1279-DART-1950.yaml @@ -40,19 +40,13 @@ fesom: restart_in_sources: par_oce_restart: /scratch/awiiccp5/ctl1950d/run_19490101-19491231/work/fesom.1949.oce.restart/*.nc par_ice_restart: /scratch/awiiccp5/ctl1950d/run_19490101-19491231/work/fesom.1949.ice.restart/*.nc - "*": - remove_restart_in_files: [par_oce_restart, par_ice_restart] - remove_restart_out_files: [par_oce_restart, par_ice_restart] - add_ignore_sources: - par_oce_restart: fesom.*.oce.restart/* - par_ice_restart: fesom.*.ice.restart/* namelist_changes: namelist.config: machine: n_levels: 2 n_part: [10, 72] restart_log: - raw_restart_length: 1 + raw_restart_length: 2 raw_restart_length_unit: "m" namelist.io: nml_listsize: