diff --git a/cmake/FindNETCDF.cmake b/cmake/FindNETCDF.cmake index 03b985d67..cbd823e8e 100644 --- a/cmake/FindNETCDF.cmake +++ b/cmake/FindNETCDF.cmake @@ -13,9 +13,13 @@ if(CMAKE_Fortran_COMPILER_LOADED) if(HAVE_Fortran_NETCDF) set(NETCDF_Fortran_INCLUDE_DIRECTORIES "") set(NETCDF_Fortran_LIBRARIES "") + set(NETCDF_Fortran_FOUND 1) else() - find_path(NETCDF_Fortran_INCLUDE_DIRECTORIES netcdf.inc HINTS $ENV{NETCDF_DIR}/include ENV NETCDF_Fortran_INCLUDE_DIRECTORIES) + find_path(NETCDF_Fortran_INCLUDE_DIRECTORIES netcdf.inc HINTS $ENV{NETCDF_ROOT}/include $ENV{NETCDF_DIR}/include $ENV{NETCDF4_DIR}/include ENV NETCDF_Fortran_INCLUDE_DIRECTORIES) find_library(NETCDF_Fortran_LIBRARIES netcdff HINTS ${NETCDF_Fortran_INCLUDE_DIRECTORIES}/../lib) + if( NETCDF_Fortran_INCLUDE_DIRECTORIES AND NETCDF_Fortran_LIBRARIES ) + set(NETCDF_Fortran_FOUND 1) + endif() endif() endif() @@ -27,14 +31,18 @@ if(CMAKE_C_COMPILER_LOADED OR CMAKE_CXX_COMPILER_LOADED) if(HAVE_C_NETCDF) set(NETCDF_C_INCLUDE_DIRECTORIES "") set(NETCDF_C_LIBRARIES "") + set(NETCDF_C_FOUND 1) else() - find_path(NETCDF_C_INCLUDE_DIRECTORIES netcdf.h HINTS $ENV{NETCDF_DIR}/include ENV NETCDF_C_INCLUDE_DIRECTORIES) + find_path(NETCDF_C_INCLUDE_DIRECTORIES netcdf.h HINTS $ENV{NETCDF_ROOT}/include $ENV{NETCDF_DIR}/include $ENV{NETCDF4_DIR}/include ENV NETCDF_C_INCLUDE_DIRECTORIES) find_library(NETCDF_C_LIBRARIES netcdf HINTS ${NETCDF_C_INCLUDE_DIRECTORIES}/../lib) + if( NETCDF_C_INCLUDE_DIRECTORIES AND NETCDF_C_LIBRARIES ) + set(NETCDF_C_FOUND 1) + endif() endif() endif() if(CMAKE_CXX_COMPILER_LOADED) - find_path(NETCDF_CXX_INCLUDE_DIRECTORIES netcdf HINTS $ENV{NETCDF_DIR}/include ENV NETCDF_CXX_INCLUDE_DIRECTORIES) + find_path(NETCDF_CXX_INCLUDE_DIRECTORIES netcdf HINTS $ENV{NETCDF_ROOT}/include $ENV{NETCDF_DIR}/include $ENV{NETCDF4_DIR} ENV NETCDF_CXX_INCLUDE_DIRECTORIES) # the cray toolchain (e.g. hlrn) disables dynamic linking by default. to enable it at build time do e.g. "CRAYPE_LINK_TYPE=dynamic make". find_library(NETCDF_CXX_LIBRARIES NAMES netcdf_c++4 netcdf-cxx4 HINTS ${NETCDF_CXX_INCLUDE_DIRECTORIES}/../lib) if(NETCDF_CXX_INCLUDE_DIRECTORIES AND NETCDF_C_INCLUDE_DIRECTORIES) @@ -43,4 +51,21 @@ if(CMAKE_CXX_COMPILER_LOADED) if(NETCDF_CXX_LIBRARIES AND NETCDF_C_LIBRARIES) list(APPEND NETCDF_CXX_LIBRARIES ${NETCDF_C_LIBRARIES}) endif() -endif() \ No newline at end of file + if( NETCDF_CXX_INCLUDE_DIRECTORIES AND NETCDF_CXX_LIBRARIES ) + set(NETCDF_CXX_FOUND 1) + endif() +endif() + +if(NOT ${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS) + set(${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS C) +endif() + +unset({CMAKE_FIND_PACKAGE_NAME}_REQUIRED_VARS) +foreach(COMP C CXX Fortran) + if("${COMP}" IN_LIST ${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS) + list(APPEND ${CMAKE_FIND_PACKAGE_NAME}_REQUIRED_VARS NETCDF_${COMP}_FOUND) + endif() +endforeach() +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(${CMAKE_FIND_PACKAGE_NAME} HANDLE_COMPONENTS REQUIRED_VARS ${CMAKE_FIND_PACKAGE_NAME}_REQUIRED_VARS) + diff --git a/env/atosecmwf/shell b/env/atosecmwf/shell index 90f118607..97cd921a7 100644 --- a/env/atosecmwf/shell +++ b/env/atosecmwf/shell @@ -33,7 +33,5 @@ module load cmake/3.20.2 module load ninja/1.10.0 module load fcm/2019.05.0 -export NETCDF_DIR=$NETCDF4_DIR +export FC=ifort CC=icc CXX=icpc -export FC=mpif90 CC=mpicc CXX=mpicxx # MPI wrappers for Fortran, cc and CC similarly -#export FC=mpif90 CC=gcc CXX=mpicxx # MPI wrappers for Fortran, cc and CC similarly diff --git a/env/lumi/shell b/env/lumi/shell index 40a2cb047..aaf0f589c 100644 --- a/env/lumi/shell +++ b/env/lumi/shell @@ -11,10 +11,10 @@ module load cray-netcdf/4.8.1.5 export FC=ftn export CC=cc export CXX=cc -export NETCDF_Fortran_INCLUDE_DIRECTORIES=$CRAY_NETCDF_DIR/include -export NETCDF_C_INCLUDE_DIRECTORIES=$CRAY_NETCDF_DIR/include -export NETCDF_C_LIBRARIES=$CRAY_NETCDF_DIR/lib -export NETCDF_Fortran_LIBRARIES=$CRAY_NETCDF_DIR/lib +#export NETCDF_Fortran_INCLUDE_DIRECTORIES=$CRAY_NETCDF_DIR/include +#export NETCDF_C_INCLUDE_DIRECTORIES=$CRAY_NETCDF_DIR/include +#export NETCDF_C_LIBRARIES=$CRAY_NETCDF_DIR/lib +#export NETCDF_Fortran_LIBRARIES=$CRAY_NETCDF_DIR/lib $CC -v $FC -V $CXX -v diff --git a/lib/parms/src/DDPQ/arms2.c b/lib/parms/src/DDPQ/arms2.c index d7a4ce88d..5dad20138 100755 --- a/lib/parms/src/DDPQ/arms2.c +++ b/lib/parms/src/DDPQ/arms2.c @@ -227,14 +227,15 @@ static int parms_arms_getssize_vcsr(parms_Operator self) } static struct parms_Operator_ops parms_arms_sol_vptr = { - parms_arms_sol_vcsr, - parms_arms_lsol_vcsr, - parms_arms_invs_vcsr, - parms_arms_ascend_vcsr, - parms_arms_getssize_vcsr, - parms_arms_nnz, - arms_free_vcsr, - arms_view_vcsr + parms_arms_sol_vcsr, /* apply */ + parms_arms_lsol_vcsr, /* lsol */ + parms_arms_invs_vcsr, /* invs */ + NULL, /* getu !!! WARNING, UNASSIGNED !!! */ + parms_arms_ascend_vcsr, /* ascend */ + parms_arms_getssize_vcsr, /* getssize */ + parms_arms_nnz, /* getnnz */ + arms_free_vcsr, /* operator_free */ + arms_view_vcsr /* operator_view */ }; diff --git a/lib/parms/src/DDPQ/sets.c b/lib/parms/src/DDPQ/sets.c index 1cfb68b68..1ccf3adb3 100755 --- a/lib/parms/src/DDPQ/sets.c +++ b/lib/parms/src/DDPQ/sets.c @@ -11,7 +11,7 @@ void parms_errexit( char *f_str, ... ) { va_list argp; - char out1[256], out2[256]; + char out1[256], out2[512]; va_start(argp, f_str); vsprintf(out1, f_str, argp); @@ -19,7 +19,7 @@ void parms_errexit( char *f_str, ... ) sprintf(out2, "Error! %s\n", out1); - fprintf(stdout, out2); + fprintf(stdout, "%s", out2); fflush(stdout); exit( -1 ); diff --git a/lib/parms/src/parms_ilu_vcsr.c b/lib/parms/src/parms_ilu_vcsr.c index 116aa51ba..17169e210 100755 --- a/lib/parms/src/parms_ilu_vcsr.c +++ b/lib/parms/src/parms_ilu_vcsr.c @@ -1683,14 +1683,15 @@ int parms_ilu_update(parms_Mat self, parms_FactParam param, void *mat, if(ABS_VALUE(t1) < DBL_EPSILON) continue; if( ii+start < schur_start ){ - for(jj = 1; jj < nnz; jj++) { - iw = jw[rowjj[jj]]; - if(iw != -1) - if(iw < ii+start) - rowm[iw] -= t1*rowmm[jj]; - else - data->U->pa[ii+start][iw-ii-start] -= t1*rowmm[jj]; - } + for(jj = 1; jj < nnz; jj++) { + iw = jw[rowjj[jj]]; + if(iw != -1) { + if(iw < ii+start) + rowm[iw] -= t1*rowmm[jj]; + else + data->U->pa[ii+start][iw-ii-start] -= t1*rowmm[jj]; + } + } } else { for(jj = 1; jj < nnz; jj++){ diff --git a/lib/parms/src/parms_pc_schurras.c b/lib/parms/src/parms_pc_schurras.c index 979acdfe9..505dfc1c2 100755 --- a/lib/parms/src/parms_pc_schurras.c +++ b/lib/parms/src/parms_pc_schurras.c @@ -10,6 +10,11 @@ #endif #endif +/* Forward declarations */ +int parms_OperatorGetU(parms_Operator self, void **mat); +int parms_MatGetOffDiag(parms_Mat self, void **mat); +int parms_CommGetOdvlist(parms_Comm self, int **odvlist); + typedef struct schurras_data { parms_Operator op_out,op_in; diff --git a/src/gen_modules_partitioning.F90 b/src/gen_modules_partitioning.F90 index edf441e73..9c506dac9 100644 --- a/src/gen_modules_partitioning.F90 +++ b/src/gen_modules_partitioning.F90 @@ -93,8 +93,10 @@ subroutine par_ex(COMM, mype, abort) ! finalizes MPI #else !For ECHAM coupled runs we use the old OASIS nameing scheme (prism / prism_proto) use mod_prism -#endif ! oifs/echam -#endif ! oasis +#endif + ! oifs/echam +#endif + ! oasis implicit none integer, intent(in) :: COMM @@ -112,7 +114,8 @@ subroutine par_ex(COMM, mype, abort) ! finalizes MPI call MPI_Barrier(COMM, error) call MPI_Finalize(error) endif -#else ! standalone +#else + ! standalone ! From here on the two coupled options !------------------------------------- @@ -136,8 +139,10 @@ subroutine par_ex(COMM, mype, abort) ! finalizes MPI if (mype==0) print *, 'FESOM calls MPI_Finalize' call MPI_Finalize(error) -#endif ! oifs/echam -#endif ! oasis +#endif + ! oifs/echam +#endif + ! oasis ! Regardless of standalone, OpenIFS oder ECHAM coupling, if we reach to this point ! we should be fine shutting the whole model down diff --git a/src/ice_fct.F90 b/src/ice_fct.F90 index a2ee681ed..8c42cc997 100755 --- a/src/ice_fct.F90 +++ b/src/ice_fct.F90 @@ -136,7 +136,7 @@ subroutine ice_TG_rhs(ice, partit, mesh) rhs_ms(row)=0._WP #if defined (__oifs) || defined (__ifsinterface) rhs_temp(row)=0._WP -#endif /* (__oifs) */ +#endif END DO !$OMP END DO ! Velocities at nodes @@ -174,7 +174,7 @@ subroutine ice_TG_rhs(ice, partit, mesh) rhs_ms(row)=rhs_ms(row)+sum(entries*m_snow(elnodes)) #if defined (__oifs) || defined (__ifsinterface) rhs_temp(row)=rhs_temp(row)+sum(entries*ice_temp(elnodes)) -#endif /* (__oifs) */ +#endif END DO end do !$OMP END DO @@ -210,7 +210,7 @@ subroutine ice_fct_solve(ice, partit, mesh) #if defined (__oifs) || defined (__ifsinterface) call ice_fem_fct(4, ice, partit, mesh) ! ice_temp -#endif /* (__oifs) */ +#endif end subroutine ice_fct_solve ! @@ -297,7 +297,7 @@ subroutine ice_solve_low_order(ice, partit, mesh) m_templ(row)=(rhs_temp(row)+gamma*sum(mass_matrix(clo:clo2)* & ice_temp(location(1:cn))))/area(1,row) + & (1.0_WP-gamma)*ice_temp(row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP !$OMP END PARALLEL DO @@ -305,7 +305,7 @@ subroutine ice_solve_low_order(ice, partit, mesh) call exchange_nod(m_icel,a_icel,m_snowl, partit, luse_g2g = .true.) #if defined (__oifs) || defined (__ifsinterface) call exchange_nod(m_templ, partit, luse_g2g = .true.) -#endif /* (__oifs) */ +#endif !$OMP BARRIER end subroutine ice_solve_low_order @@ -372,7 +372,7 @@ subroutine ice_solve_high_order(ice, partit, mesh) dm_snow(row)=rhs_ms(row)/area(1,row) #if defined (__oifs) || defined (__ifsinterface) dm_temp(row)=rhs_temp(row)/area(1,row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP @@ -380,7 +380,7 @@ subroutine ice_solve_high_order(ice, partit, mesh) call exchange_nod(dm_ice, da_ice, dm_snow, partit, luse_g2g = .true.) #if defined (__oifs) || defined (__ifsinterface) call exchange_nod(dm_temp, partit, luse_g2g = .true.) -#endif /* (__oifs) */ +#endif !$OMP BARRIER !___________________________________________________________________________ !iterate @@ -407,7 +407,7 @@ subroutine ice_solve_high_order(ice, partit, mesh) #if defined (__oifs) || defined (__ifsinterface) rhs_new = rhs_temp(row) - sum(mass_matrix(clo:clo2)*dm_temp(location(1:cn))) m_templ(row)= dm_temp(row)+rhs_new/area(1,row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP @@ -424,7 +424,7 @@ subroutine ice_solve_high_order(ice, partit, mesh) dm_snow(row)=m_snowl(row) #if defined (__oifs) || defined (__ifsinterface) dm_temp(row)=m_templ(row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP @@ -434,7 +434,7 @@ subroutine ice_solve_high_order(ice, partit, mesh) call exchange_nod(dm_ice, da_ice, dm_snow, partit, luse_g2g = .true.) #if defined (__oifs) || defined (__ifsinterface) call exchange_nod(dm_temp, partit, luse_g2g = .true.) -#endif /* (__oifs) */ +#endif !$OMP BARRIER end do end subroutine ice_solve_high_order @@ -569,7 +569,7 @@ subroutine ice_fem_fct(tr_array_id, ice, partit, mesh) dm_temp(elnodes)))*(vol/area(1,elnodes(q)))/12.0_WP end do end if -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP !$OMP END DO @@ -646,7 +646,7 @@ subroutine ice_fem_fct(tr_array_id, ice, partit, mesh) !$ACC END PARALLEL LOOP !$OMP END DO end if -#endif /* (__oifs) */ +#endif !___________________________________________________________________________ ! Sums of positive/negative fluxes to node row @@ -946,12 +946,12 @@ subroutine ice_fem_fct(tr_array_id, ice, partit, mesh) #endif !$OMP END DO end if -#endif /* (__oifs) */ || defined (__ifsinterface) +#endif !$OMP END PARALLEL call exchange_nod(m_ice, a_ice, m_snow, partit, luse_g2g = .true.) #if defined (__oifs) || defined (__ifsinterface) call exchange_nod(ice_temp, partit, luse_g2g = .true.) -#endif /* (__oifs) */ +#endif !$ACC END DATA @@ -1126,13 +1126,13 @@ subroutine ice_TG_rhs_div(ice, partit, mesh) rhs_ms(row)=0.0_WP #if defined (__oifs) || defined (__ifsinterface) rhs_temp(row)=0.0_WP -#endif /* (__oifs) */ +#endif rhs_mdiv(row)=0.0_WP rhs_adiv(row)=0.0_WP rhs_msdiv(row)=0.0_WP #if defined (__oifs) || defined (__ifsinterface) rhs_tempdiv(row)=0.0_WP -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP @@ -1180,7 +1180,7 @@ subroutine ice_TG_rhs_div(ice, partit, mesh) cx3=vol*ice%ice_dt*c4*(sum(m_snow(elnodes))+m_snow(elnodes(n))+sum(entries2*m_snow(elnodes)))/12.0_WP #if defined (__oifs) || defined (__ifsinterface) cx4=vol*ice%ice_dt*c4*(sum(ice_temp(elnodes))+ice_temp(elnodes(n))+sum(entries2*ice_temp(elnodes)))/12.0_WP -#endif /* (__oifs) */ +#endif !___________________________________________________________________ #if defined(_OPENMP) && !defined(__openmp_reproducible) @@ -1212,7 +1212,7 @@ subroutine ice_TG_rhs_div(ice, partit, mesh) !$ACC ATOMIC UPDATE #endif rhs_temp(row)=rhs_temp(row)+tmp_sum+cx4 -#endif /* (__oifs) */ +#endif !___________________________________________________________________ #if !defined(DISABLE_OPENACC_ATOMICS) @@ -1232,7 +1232,7 @@ subroutine ice_TG_rhs_div(ice, partit, mesh) !$ACC ATOMIC UPDATE #endif rhs_tempdiv(row)=rhs_tempdiv(row)-cx4 -#endif /* (__oifs) */ +#endif #if defined(_OPENMP) && !defined(__openmp_reproducible) call omp_unset_lock(partit%plock(row)) #else @@ -1316,7 +1316,7 @@ subroutine ice_update_for_div(ice, partit, mesh) dm_snow(row)=rhs_msdiv(row)/area(1,row) #if defined (__oifs) || defined (__ifsinterface) dm_temp(row)=rhs_tempdiv(row)/area(1,row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP @@ -1326,7 +1326,7 @@ subroutine ice_update_for_div(ice, partit, mesh) call exchange_nod(dm_snow, partit, luse_g2g = .true.) #if defined (__oifs) || defined (__ifsinterface) call exchange_nod(dm_temp, partit, luse_g2g = .true.) -#endif /* (__oifs) */ +#endif !$OMP BARRIER !___________________________________________________________________________ !iterate @@ -1356,7 +1356,7 @@ subroutine ice_update_for_div(ice, partit, mesh) #if defined (__oifs) || defined (__ifsinterface) rhs_new = rhs_tempdiv(row) - sum(mass_matrix(clo:clo2)*dm_temp(location(1:cn))) m_templ(row)= dm_temp(row)+rhs_new/area(1,row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP @@ -1372,7 +1372,7 @@ subroutine ice_update_for_div(ice, partit, mesh) dm_snow(row) = m_snowl(row) #if defined (__oifs) || defined (__ifsinterface) dm_temp(row) = m_templ(row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP @@ -1383,7 +1383,7 @@ subroutine ice_update_for_div(ice, partit, mesh) call exchange_nod(dm_snow, partit, luse_g2g = .true.) #if defined (__oifs) || defined (__ifsinterface) call exchange_nod(dm_temp, partit, luse_g2g = .true.) -#endif /* (__oifs) */ +#endif !$OMP BARRIER end do @@ -1396,7 +1396,7 @@ subroutine ice_update_for_div(ice, partit, mesh) m_snow(row) = m_snow(row)+dm_snow(row) #if defined (__oifs) || defined (__ifsinterface) ice_temp(row)= ice_temp(row)+dm_temp(row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP !$OMP END PARALLEL DO