From 04fe01301e5583aaa31eee7f0f4b4a471836e203 Mon Sep 17 00:00:00 2001 From: Nikolay Koldunov Date: Wed, 11 Oct 2023 14:19:14 +0200 Subject: [PATCH 01/18] update docker image to refactoring --- docs/getting_started/getting_started.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/getting_started/getting_started.rst b/docs/getting_started/getting_started.rst index 24bc94988..f390d6a1f 100644 --- a/docs/getting_started/getting_started.rst +++ b/docs/getting_started/getting_started.rst @@ -461,12 +461,12 @@ The best way to run the model locally is to use Docker container. You obviously - Get the image:: - docker pull koldunovn/fesom2_test:fesom2.1 + docker pull koldunovn/fesom2_test:refactoring2 - Go to the folder with your version of fesom2 folder (NOT inside fesom2 folder, one up, the one you run ``git clone https://github.com/FESOM/fesom2.git`` in). - Run:: - docker run -it -v "$(pwd)"/fesom2:/fesom/fesom2 koldunovn/fesom2_test:fesom2.1 /bin/bash + docker run -it -v "$(pwd)"/fesom2:/fesom/fesom2 koldunovn/fesom2_test:refactoring2 /bin/bash - This should get you inside the container. You now can edit the files in your fesom2 folder (on host system), but run compule and run the model inside the container. - When inside the container, to compile do: From e3d9280de5907e9f2be2c3ce2989160f8044f5db Mon Sep 17 00:00:00 2001 From: Nikolay Koldunov Date: Wed, 11 Oct 2023 14:38:30 +0200 Subject: [PATCH 02/18] update docker image to refactoring again, sorry suvi :) --- docs/getting_started/getting_started.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/getting_started/getting_started.rst b/docs/getting_started/getting_started.rst index f390d6a1f..227fd00de 100644 --- a/docs/getting_started/getting_started.rst +++ b/docs/getting_started/getting_started.rst @@ -474,6 +474,7 @@ The best way to run the model locally is to use Docker container. You obviously :: cd fesom2 + git checkout refactoring bash -l configure.sh ubuntu - To prepare the run (this will do the test with pi mesh):: From fa5d477dc5371d2a7c1709a249c8df8d237a9eae Mon Sep 17 00:00:00 2001 From: Ulf 'Tiggi' Tigerstedt Date: Thu, 19 Oct 2023 13:34:50 +0300 Subject: [PATCH 03/18] Added workflow for sync to lumi and a first version of a gitlab CI workflow --- .github/workflows/lumi-sync.yaml | 26 ++++++++++++++++++++++++++ .gitlab-ci.yml | 13 +++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 .github/workflows/lumi-sync.yaml create mode 100644 .gitlab-ci.yml diff --git a/.github/workflows/lumi-sync.yaml b/.github/workflows/lumi-sync.yaml new file mode 100644 index 000000000..3121338ac --- /dev/null +++ b/.github/workflows/lumi-sync.yaml @@ -0,0 +1,26 @@ +name: Run CICD on Lumi (using .gitlab-ci.yml) + +on: push + branches: + - refactoring + +jobs: + gitlabsync: + runs-on: ubuntu-latest + steps: + - name: "Check out code" + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: "Gitlab mirror and CI" + uses: "tiggi/gitlab-mirror-and-ci-action@tiggi/fixes" + with: + args: "https://gitlab.com/lumi-cicd/fesom2.git" + env: + FOLLOW_TAGS: "true" + FORCE_PUSH: "true" + GITLAB_HOSTNAME: "gitlab.com" + GITLAB_USERNAME: "tiggi" + GITLAB_PASSWORD: ${{ secrets.GITLAB_PASSWORD }} + GITLAB_PROJECT_ID: "51374059" + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..bdd7339e1 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,13 @@ +fesom-job: + stage: build + variables: + SCHEDULER_PARAMETERS: "-N 1 -n 56 --mem=16G -t 00:30:00 -A project_462000376" + tags: + - lumi + artifacts: + paths: + - fesom_build.log + script: + - echo "building fesom branch" + - bash -l configure.sh lumi + - touch fesom_build.log From 33155cc84bfe2bb50fe3e36854997f9ff4735e31 Mon Sep 17 00:00:00 2001 From: Ulf 'Tiggi' Tigerstedt Date: Thu, 19 Oct 2023 13:45:55 +0300 Subject: [PATCH 04/18] Added lumi_gpu_evatali branch to workflow --- .github/workflows/lumi-sync.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lumi-sync.yaml b/.github/workflows/lumi-sync.yaml index 3121338ac..859c0a2eb 100644 --- a/.github/workflows/lumi-sync.yaml +++ b/.github/workflows/lumi-sync.yaml @@ -3,6 +3,7 @@ name: Run CICD on Lumi (using .gitlab-ci.yml) on: push branches: - refactoring + - lumi_gpu_evatali jobs: gitlabsync: From 4a3ea6f97dac6d83003dc2f7dc0b16361aad5517 Mon Sep 17 00:00:00 2001 From: Ulf 'Tiggi' Tigerstedt Date: Thu, 19 Oct 2023 14:03:06 +0300 Subject: [PATCH 05/18] Fixed bad YAML in Lumi CICD workflow --- .github/workflows/lumi-sync.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lumi-sync.yaml b/.github/workflows/lumi-sync.yaml index 859c0a2eb..11ca2531c 100644 --- a/.github/workflows/lumi-sync.yaml +++ b/.github/workflows/lumi-sync.yaml @@ -1,9 +1,10 @@ name: Run CICD on Lumi (using .gitlab-ci.yml) -on: push - branches: - - refactoring - - lumi_gpu_evatali +on: + push: + branches: + - refactoring + - lumi_gpu_evatali jobs: gitlabsync: From 17b396e3019a4ad48b9e75f2333866146fab6675 Mon Sep 17 00:00:00 2001 From: Ulf 'Tiggi' Tigerstedt Date: Thu, 19 Oct 2023 14:16:25 +0300 Subject: [PATCH 06/18] Add a partition for the Lumi CICD job --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bdd7339e1..78c5607bf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,7 @@ fesom-job: stage: build variables: - SCHEDULER_PARAMETERS: "-N 1 -n 56 --mem=16G -t 00:30:00 -A project_462000376" + SCHEDULER_PARAMETERS: "-N 1 -n 56 --mem=32G -p dev-g -t 00:30:00 -A project_462000376" tags: - lumi artifacts: From 62098b4235e049f9a123e8d9b6ab9c7fb7ba20d5 Mon Sep 17 00:00:00 2001 From: Ulf 'Tiggi' Tigerstedt Date: Thu, 19 Oct 2023 14:24:28 +0300 Subject: [PATCH 07/18] Remove the artifact from gitlab-ci, it's better to have it printed out so that github also gets it --- .gitlab-ci.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 78c5607bf..d2dd6cf2c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,10 +4,9 @@ fesom-job: SCHEDULER_PARAMETERS: "-N 1 -n 56 --mem=32G -p dev-g -t 00:30:00 -A project_462000376" tags: - lumi - artifacts: - paths: - - fesom_build.log + # artifacts: + #paths: + # - fesom_build.log script: - echo "building fesom branch" - bash -l configure.sh lumi - - touch fesom_build.log From 473de2a93baf73fb4684861253341d72d21d2f64 Mon Sep 17 00:00:00 2001 From: Ulf 'Tiggi' Tigerstedt Date: Thu, 19 Oct 2023 14:27:11 +0300 Subject: [PATCH 08/18] Remove the artifact from gitlab-ci, it's better to have it printed out so that github also gets it #2 --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d2dd6cf2c..9822c32bf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,9 +4,9 @@ fesom-job: SCHEDULER_PARAMETERS: "-N 1 -n 56 --mem=32G -p dev-g -t 00:30:00 -A project_462000376" tags: - lumi - # artifacts: - #paths: - # - fesom_build.log +# artifacts: +# paths: +# - fesom_build.log script: - echo "building fesom branch" - bash -l configure.sh lumi From 62d3e6a1213e188b682ddb267963a5b94d7bc52d Mon Sep 17 00:00:00 2001 From: Dmitry Sidorenko Date: Thu, 26 Oct 2023 14:36:53 +0200 Subject: [PATCH 09/18] an OMP bux fix in tracer FCT --- src/oce_adv_tra_fct.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/oce_adv_tra_fct.F90 b/src/oce_adv_tra_fct.F90 index 013a64aff..24a43bef2 100644 --- a/src/oce_adv_tra_fct.F90 +++ b/src/oce_adv_tra_fct.F90 @@ -104,7 +104,7 @@ subroutine oce_tra_adv_fct(dt, ttf, lo, adf_h, adf_v, fct_ttf_min, fct_ttf_max, #include "associate_mesh_ass.h" !$OMP PARALLEL DEFAULT(SHARED) PRIVATE(n, nz, k, elem, enodes, num, el, nl1, nl2, nu1, nu2, nl12, nu12, edge, & -!$OMP flux, ae,tvert_max, tvert_min) +!$OMP flux, ae) ! -------------------------------------------------------------------------- ! ttf is the tracer field on step n ! del_ttf is the increment From ef9e91f5d8ddfff34cdfb2e380a1ac169435ffc0 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 22 Aug 2023 10:22:37 +0200 Subject: [PATCH 10/18] Add forward declarations in parms for standard compliance --- lib/parms/src/parms_pc_schurras.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/parms/src/parms_pc_schurras.c b/lib/parms/src/parms_pc_schurras.c index 979acdfe9..505dfc1c2 100755 --- a/lib/parms/src/parms_pc_schurras.c +++ b/lib/parms/src/parms_pc_schurras.c @@ -10,6 +10,11 @@ #endif #endif +/* Forward declarations */ +int parms_OperatorGetU(parms_Operator self, void **mat); +int parms_MatGetOffDiag(parms_Mat self, void **mat); +int parms_CommGetOdvlist(parms_Comm self, int **odvlist); + typedef struct schurras_data { parms_Operator op_out,op_in; From 6298b59867ca8caa1c7808610acc874d56ae78f5 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 10 Oct 2023 12:19:49 +0200 Subject: [PATCH 11/18] Fix warning: format string is not a string literal (potentially insecure) --- lib/parms/src/DDPQ/sets.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/parms/src/DDPQ/sets.c b/lib/parms/src/DDPQ/sets.c index 1cfb68b68..8e3db4e56 100755 --- a/lib/parms/src/DDPQ/sets.c +++ b/lib/parms/src/DDPQ/sets.c @@ -19,7 +19,7 @@ void parms_errexit( char *f_str, ... ) sprintf(out2, "Error! %s\n", out1); - fprintf(stdout, out2); + fprintf(stdout, "%s", out2); fflush(stdout); exit( -1 ); From 66eaaafa8a8d80170de11c328e0a902da5222e31 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 10 Oct 2023 12:24:12 +0200 Subject: [PATCH 12/18] Fix warning: add explicit braces to avoid dangling else --- lib/parms/src/parms_ilu_vcsr.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/parms/src/parms_ilu_vcsr.c b/lib/parms/src/parms_ilu_vcsr.c index 116aa51ba..17169e210 100755 --- a/lib/parms/src/parms_ilu_vcsr.c +++ b/lib/parms/src/parms_ilu_vcsr.c @@ -1683,14 +1683,15 @@ int parms_ilu_update(parms_Mat self, parms_FactParam param, void *mat, if(ABS_VALUE(t1) < DBL_EPSILON) continue; if( ii+start < schur_start ){ - for(jj = 1; jj < nnz; jj++) { - iw = jw[rowjj[jj]]; - if(iw != -1) - if(iw < ii+start) - rowm[iw] -= t1*rowmm[jj]; - else - data->U->pa[ii+start][iw-ii-start] -= t1*rowmm[jj]; - } + for(jj = 1; jj < nnz; jj++) { + iw = jw[rowjj[jj]]; + if(iw != -1) { + if(iw < ii+start) + rowm[iw] -= t1*rowmm[jj]; + else + data->U->pa[ii+start][iw-ii-start] -= t1*rowmm[jj]; + } + } } else { for(jj = 1; jj < nnz; jj++){ From 74f796e555bc21516d6574a495669a68ed6d1342 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 10 Oct 2023 12:43:16 +0200 Subject: [PATCH 13/18] Fix warning: incompatible function pointer types initializing... TODO: Assign 'getu' (#512) --- lib/parms/src/DDPQ/arms2.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/parms/src/DDPQ/arms2.c b/lib/parms/src/DDPQ/arms2.c index d7a4ce88d..5dad20138 100755 --- a/lib/parms/src/DDPQ/arms2.c +++ b/lib/parms/src/DDPQ/arms2.c @@ -227,14 +227,15 @@ static int parms_arms_getssize_vcsr(parms_Operator self) } static struct parms_Operator_ops parms_arms_sol_vptr = { - parms_arms_sol_vcsr, - parms_arms_lsol_vcsr, - parms_arms_invs_vcsr, - parms_arms_ascend_vcsr, - parms_arms_getssize_vcsr, - parms_arms_nnz, - arms_free_vcsr, - arms_view_vcsr + parms_arms_sol_vcsr, /* apply */ + parms_arms_lsol_vcsr, /* lsol */ + parms_arms_invs_vcsr, /* invs */ + NULL, /* getu !!! WARNING, UNASSIGNED !!! */ + parms_arms_ascend_vcsr, /* ascend */ + parms_arms_getssize_vcsr, /* getssize */ + parms_arms_nnz, /* getnnz */ + arms_free_vcsr, /* operator_free */ + arms_view_vcsr /* operator_view */ }; From e5509536c182374ff9a40420c9f9864859990501 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 10 Oct 2023 20:07:46 +0200 Subject: [PATCH 14/18] Fix warning: '%s' directive writing up to 255 bytes into a region of size 249 --- lib/parms/src/DDPQ/sets.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/parms/src/DDPQ/sets.c b/lib/parms/src/DDPQ/sets.c index 8e3db4e56..1ccf3adb3 100755 --- a/lib/parms/src/DDPQ/sets.c +++ b/lib/parms/src/DDPQ/sets.c @@ -11,7 +11,7 @@ void parms_errexit( char *f_str, ... ) { va_list argp; - char out1[256], out2[256]; + char out1[256], out2[512]; va_start(argp, f_str); vsprintf(out1, f_str, argp); From 3cdb8d0657430383c307f8328a820ce479306ba6 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 10 Oct 2023 20:41:02 +0200 Subject: [PATCH 15/18] Fix warning: extra tokens at end of #endif directive --- src/gen_modules_partitioning.F90 | 15 ++++++---- src/ice_fct.F90 | 50 ++++++++++++++++---------------- 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/src/gen_modules_partitioning.F90 b/src/gen_modules_partitioning.F90 index edf441e73..9c506dac9 100644 --- a/src/gen_modules_partitioning.F90 +++ b/src/gen_modules_partitioning.F90 @@ -93,8 +93,10 @@ subroutine par_ex(COMM, mype, abort) ! finalizes MPI #else !For ECHAM coupled runs we use the old OASIS nameing scheme (prism / prism_proto) use mod_prism -#endif ! oifs/echam -#endif ! oasis +#endif + ! oifs/echam +#endif + ! oasis implicit none integer, intent(in) :: COMM @@ -112,7 +114,8 @@ subroutine par_ex(COMM, mype, abort) ! finalizes MPI call MPI_Barrier(COMM, error) call MPI_Finalize(error) endif -#else ! standalone +#else + ! standalone ! From here on the two coupled options !------------------------------------- @@ -136,8 +139,10 @@ subroutine par_ex(COMM, mype, abort) ! finalizes MPI if (mype==0) print *, 'FESOM calls MPI_Finalize' call MPI_Finalize(error) -#endif ! oifs/echam -#endif ! oasis +#endif + ! oifs/echam +#endif + ! oasis ! Regardless of standalone, OpenIFS oder ECHAM coupling, if we reach to this point ! we should be fine shutting the whole model down diff --git a/src/ice_fct.F90 b/src/ice_fct.F90 index a2ee681ed..8c42cc997 100755 --- a/src/ice_fct.F90 +++ b/src/ice_fct.F90 @@ -136,7 +136,7 @@ subroutine ice_TG_rhs(ice, partit, mesh) rhs_ms(row)=0._WP #if defined (__oifs) || defined (__ifsinterface) rhs_temp(row)=0._WP -#endif /* (__oifs) */ +#endif END DO !$OMP END DO ! Velocities at nodes @@ -174,7 +174,7 @@ subroutine ice_TG_rhs(ice, partit, mesh) rhs_ms(row)=rhs_ms(row)+sum(entries*m_snow(elnodes)) #if defined (__oifs) || defined (__ifsinterface) rhs_temp(row)=rhs_temp(row)+sum(entries*ice_temp(elnodes)) -#endif /* (__oifs) */ +#endif END DO end do !$OMP END DO @@ -210,7 +210,7 @@ subroutine ice_fct_solve(ice, partit, mesh) #if defined (__oifs) || defined (__ifsinterface) call ice_fem_fct(4, ice, partit, mesh) ! ice_temp -#endif /* (__oifs) */ +#endif end subroutine ice_fct_solve ! @@ -297,7 +297,7 @@ subroutine ice_solve_low_order(ice, partit, mesh) m_templ(row)=(rhs_temp(row)+gamma*sum(mass_matrix(clo:clo2)* & ice_temp(location(1:cn))))/area(1,row) + & (1.0_WP-gamma)*ice_temp(row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP !$OMP END PARALLEL DO @@ -305,7 +305,7 @@ subroutine ice_solve_low_order(ice, partit, mesh) call exchange_nod(m_icel,a_icel,m_snowl, partit, luse_g2g = .true.) #if defined (__oifs) || defined (__ifsinterface) call exchange_nod(m_templ, partit, luse_g2g = .true.) -#endif /* (__oifs) */ +#endif !$OMP BARRIER end subroutine ice_solve_low_order @@ -372,7 +372,7 @@ subroutine ice_solve_high_order(ice, partit, mesh) dm_snow(row)=rhs_ms(row)/area(1,row) #if defined (__oifs) || defined (__ifsinterface) dm_temp(row)=rhs_temp(row)/area(1,row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP @@ -380,7 +380,7 @@ subroutine ice_solve_high_order(ice, partit, mesh) call exchange_nod(dm_ice, da_ice, dm_snow, partit, luse_g2g = .true.) #if defined (__oifs) || defined (__ifsinterface) call exchange_nod(dm_temp, partit, luse_g2g = .true.) -#endif /* (__oifs) */ +#endif !$OMP BARRIER !___________________________________________________________________________ !iterate @@ -407,7 +407,7 @@ subroutine ice_solve_high_order(ice, partit, mesh) #if defined (__oifs) || defined (__ifsinterface) rhs_new = rhs_temp(row) - sum(mass_matrix(clo:clo2)*dm_temp(location(1:cn))) m_templ(row)= dm_temp(row)+rhs_new/area(1,row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP @@ -424,7 +424,7 @@ subroutine ice_solve_high_order(ice, partit, mesh) dm_snow(row)=m_snowl(row) #if defined (__oifs) || defined (__ifsinterface) dm_temp(row)=m_templ(row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP @@ -434,7 +434,7 @@ subroutine ice_solve_high_order(ice, partit, mesh) call exchange_nod(dm_ice, da_ice, dm_snow, partit, luse_g2g = .true.) #if defined (__oifs) || defined (__ifsinterface) call exchange_nod(dm_temp, partit, luse_g2g = .true.) -#endif /* (__oifs) */ +#endif !$OMP BARRIER end do end subroutine ice_solve_high_order @@ -569,7 +569,7 @@ subroutine ice_fem_fct(tr_array_id, ice, partit, mesh) dm_temp(elnodes)))*(vol/area(1,elnodes(q)))/12.0_WP end do end if -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP !$OMP END DO @@ -646,7 +646,7 @@ subroutine ice_fem_fct(tr_array_id, ice, partit, mesh) !$ACC END PARALLEL LOOP !$OMP END DO end if -#endif /* (__oifs) */ +#endif !___________________________________________________________________________ ! Sums of positive/negative fluxes to node row @@ -946,12 +946,12 @@ subroutine ice_fem_fct(tr_array_id, ice, partit, mesh) #endif !$OMP END DO end if -#endif /* (__oifs) */ || defined (__ifsinterface) +#endif !$OMP END PARALLEL call exchange_nod(m_ice, a_ice, m_snow, partit, luse_g2g = .true.) #if defined (__oifs) || defined (__ifsinterface) call exchange_nod(ice_temp, partit, luse_g2g = .true.) -#endif /* (__oifs) */ +#endif !$ACC END DATA @@ -1126,13 +1126,13 @@ subroutine ice_TG_rhs_div(ice, partit, mesh) rhs_ms(row)=0.0_WP #if defined (__oifs) || defined (__ifsinterface) rhs_temp(row)=0.0_WP -#endif /* (__oifs) */ +#endif rhs_mdiv(row)=0.0_WP rhs_adiv(row)=0.0_WP rhs_msdiv(row)=0.0_WP #if defined (__oifs) || defined (__ifsinterface) rhs_tempdiv(row)=0.0_WP -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP @@ -1180,7 +1180,7 @@ subroutine ice_TG_rhs_div(ice, partit, mesh) cx3=vol*ice%ice_dt*c4*(sum(m_snow(elnodes))+m_snow(elnodes(n))+sum(entries2*m_snow(elnodes)))/12.0_WP #if defined (__oifs) || defined (__ifsinterface) cx4=vol*ice%ice_dt*c4*(sum(ice_temp(elnodes))+ice_temp(elnodes(n))+sum(entries2*ice_temp(elnodes)))/12.0_WP -#endif /* (__oifs) */ +#endif !___________________________________________________________________ #if defined(_OPENMP) && !defined(__openmp_reproducible) @@ -1212,7 +1212,7 @@ subroutine ice_TG_rhs_div(ice, partit, mesh) !$ACC ATOMIC UPDATE #endif rhs_temp(row)=rhs_temp(row)+tmp_sum+cx4 -#endif /* (__oifs) */ +#endif !___________________________________________________________________ #if !defined(DISABLE_OPENACC_ATOMICS) @@ -1232,7 +1232,7 @@ subroutine ice_TG_rhs_div(ice, partit, mesh) !$ACC ATOMIC UPDATE #endif rhs_tempdiv(row)=rhs_tempdiv(row)-cx4 -#endif /* (__oifs) */ +#endif #if defined(_OPENMP) && !defined(__openmp_reproducible) call omp_unset_lock(partit%plock(row)) #else @@ -1316,7 +1316,7 @@ subroutine ice_update_for_div(ice, partit, mesh) dm_snow(row)=rhs_msdiv(row)/area(1,row) #if defined (__oifs) || defined (__ifsinterface) dm_temp(row)=rhs_tempdiv(row)/area(1,row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP @@ -1326,7 +1326,7 @@ subroutine ice_update_for_div(ice, partit, mesh) call exchange_nod(dm_snow, partit, luse_g2g = .true.) #if defined (__oifs) || defined (__ifsinterface) call exchange_nod(dm_temp, partit, luse_g2g = .true.) -#endif /* (__oifs) */ +#endif !$OMP BARRIER !___________________________________________________________________________ !iterate @@ -1356,7 +1356,7 @@ subroutine ice_update_for_div(ice, partit, mesh) #if defined (__oifs) || defined (__ifsinterface) rhs_new = rhs_tempdiv(row) - sum(mass_matrix(clo:clo2)*dm_temp(location(1:cn))) m_templ(row)= dm_temp(row)+rhs_new/area(1,row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP @@ -1372,7 +1372,7 @@ subroutine ice_update_for_div(ice, partit, mesh) dm_snow(row) = m_snowl(row) #if defined (__oifs) || defined (__ifsinterface) dm_temp(row) = m_templ(row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP @@ -1383,7 +1383,7 @@ subroutine ice_update_for_div(ice, partit, mesh) call exchange_nod(dm_snow, partit, luse_g2g = .true.) #if defined (__oifs) || defined (__ifsinterface) call exchange_nod(dm_temp, partit, luse_g2g = .true.) -#endif /* (__oifs) */ +#endif !$OMP BARRIER end do @@ -1396,7 +1396,7 @@ subroutine ice_update_for_div(ice, partit, mesh) m_snow(row) = m_snow(row)+dm_snow(row) #if defined (__oifs) || defined (__ifsinterface) ice_temp(row)= ice_temp(row)+dm_temp(row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP !$OMP END PARALLEL DO From 4ae7703dc810a6299f5841c11667b3b92c110a03 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 10 Oct 2023 23:48:09 +0200 Subject: [PATCH 16/18] Improve FindNETCDF with extra search hints and handling of COMPONENTS --- cmake/FindNETCDF.cmake | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/cmake/FindNETCDF.cmake b/cmake/FindNETCDF.cmake index 03b985d67..cbd823e8e 100644 --- a/cmake/FindNETCDF.cmake +++ b/cmake/FindNETCDF.cmake @@ -13,9 +13,13 @@ if(CMAKE_Fortran_COMPILER_LOADED) if(HAVE_Fortran_NETCDF) set(NETCDF_Fortran_INCLUDE_DIRECTORIES "") set(NETCDF_Fortran_LIBRARIES "") + set(NETCDF_Fortran_FOUND 1) else() - find_path(NETCDF_Fortran_INCLUDE_DIRECTORIES netcdf.inc HINTS $ENV{NETCDF_DIR}/include ENV NETCDF_Fortran_INCLUDE_DIRECTORIES) + find_path(NETCDF_Fortran_INCLUDE_DIRECTORIES netcdf.inc HINTS $ENV{NETCDF_ROOT}/include $ENV{NETCDF_DIR}/include $ENV{NETCDF4_DIR}/include ENV NETCDF_Fortran_INCLUDE_DIRECTORIES) find_library(NETCDF_Fortran_LIBRARIES netcdff HINTS ${NETCDF_Fortran_INCLUDE_DIRECTORIES}/../lib) + if( NETCDF_Fortran_INCLUDE_DIRECTORIES AND NETCDF_Fortran_LIBRARIES ) + set(NETCDF_Fortran_FOUND 1) + endif() endif() endif() @@ -27,14 +31,18 @@ if(CMAKE_C_COMPILER_LOADED OR CMAKE_CXX_COMPILER_LOADED) if(HAVE_C_NETCDF) set(NETCDF_C_INCLUDE_DIRECTORIES "") set(NETCDF_C_LIBRARIES "") + set(NETCDF_C_FOUND 1) else() - find_path(NETCDF_C_INCLUDE_DIRECTORIES netcdf.h HINTS $ENV{NETCDF_DIR}/include ENV NETCDF_C_INCLUDE_DIRECTORIES) + find_path(NETCDF_C_INCLUDE_DIRECTORIES netcdf.h HINTS $ENV{NETCDF_ROOT}/include $ENV{NETCDF_DIR}/include $ENV{NETCDF4_DIR}/include ENV NETCDF_C_INCLUDE_DIRECTORIES) find_library(NETCDF_C_LIBRARIES netcdf HINTS ${NETCDF_C_INCLUDE_DIRECTORIES}/../lib) + if( NETCDF_C_INCLUDE_DIRECTORIES AND NETCDF_C_LIBRARIES ) + set(NETCDF_C_FOUND 1) + endif() endif() endif() if(CMAKE_CXX_COMPILER_LOADED) - find_path(NETCDF_CXX_INCLUDE_DIRECTORIES netcdf HINTS $ENV{NETCDF_DIR}/include ENV NETCDF_CXX_INCLUDE_DIRECTORIES) + find_path(NETCDF_CXX_INCLUDE_DIRECTORIES netcdf HINTS $ENV{NETCDF_ROOT}/include $ENV{NETCDF_DIR}/include $ENV{NETCDF4_DIR} ENV NETCDF_CXX_INCLUDE_DIRECTORIES) # the cray toolchain (e.g. hlrn) disables dynamic linking by default. to enable it at build time do e.g. "CRAYPE_LINK_TYPE=dynamic make". find_library(NETCDF_CXX_LIBRARIES NAMES netcdf_c++4 netcdf-cxx4 HINTS ${NETCDF_CXX_INCLUDE_DIRECTORIES}/../lib) if(NETCDF_CXX_INCLUDE_DIRECTORIES AND NETCDF_C_INCLUDE_DIRECTORIES) @@ -43,4 +51,21 @@ if(CMAKE_CXX_COMPILER_LOADED) if(NETCDF_CXX_LIBRARIES AND NETCDF_C_LIBRARIES) list(APPEND NETCDF_CXX_LIBRARIES ${NETCDF_C_LIBRARIES}) endif() -endif() \ No newline at end of file + if( NETCDF_CXX_INCLUDE_DIRECTORIES AND NETCDF_CXX_LIBRARIES ) + set(NETCDF_CXX_FOUND 1) + endif() +endif() + +if(NOT ${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS) + set(${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS C) +endif() + +unset({CMAKE_FIND_PACKAGE_NAME}_REQUIRED_VARS) +foreach(COMP C CXX Fortran) + if("${COMP}" IN_LIST ${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS) + list(APPEND ${CMAKE_FIND_PACKAGE_NAME}_REQUIRED_VARS NETCDF_${COMP}_FOUND) + endif() +endforeach() +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(${CMAKE_FIND_PACKAGE_NAME} HANDLE_COMPONENTS REQUIRED_VARS ${CMAKE_FIND_PACKAGE_NAME}_REQUIRED_VARS) + From d526ae43632798c227c1539957b2a319d8b7981b Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 11 Oct 2023 02:52:06 +0300 Subject: [PATCH 17/18] Update lumi environment, showing that FindNETCDF works without intervention --- env/lumi/shell | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/env/lumi/shell b/env/lumi/shell index 40a2cb047..aaf0f589c 100644 --- a/env/lumi/shell +++ b/env/lumi/shell @@ -11,10 +11,10 @@ module load cray-netcdf/4.8.1.5 export FC=ftn export CC=cc export CXX=cc -export NETCDF_Fortran_INCLUDE_DIRECTORIES=$CRAY_NETCDF_DIR/include -export NETCDF_C_INCLUDE_DIRECTORIES=$CRAY_NETCDF_DIR/include -export NETCDF_C_LIBRARIES=$CRAY_NETCDF_DIR/lib -export NETCDF_Fortran_LIBRARIES=$CRAY_NETCDF_DIR/lib +#export NETCDF_Fortran_INCLUDE_DIRECTORIES=$CRAY_NETCDF_DIR/include +#export NETCDF_C_INCLUDE_DIRECTORIES=$CRAY_NETCDF_DIR/include +#export NETCDF_C_LIBRARIES=$CRAY_NETCDF_DIR/lib +#export NETCDF_Fortran_LIBRARIES=$CRAY_NETCDF_DIR/lib $CC -v $FC -V $CXX -v From ae6958ef8a429f896802d936cd28af197accc81a Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 10 Oct 2023 23:08:13 +0000 Subject: [PATCH 18/18] Update atosecmwf environment, showing mpi compiler wrappers are no longer needed and FindNETCDF also works --- env/atosecmwf/shell | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/env/atosecmwf/shell b/env/atosecmwf/shell index 90f118607..97cd921a7 100644 --- a/env/atosecmwf/shell +++ b/env/atosecmwf/shell @@ -33,7 +33,5 @@ module load cmake/3.20.2 module load ninja/1.10.0 module load fcm/2019.05.0 -export NETCDF_DIR=$NETCDF4_DIR +export FC=ifort CC=icc CXX=icpc -export FC=mpif90 CC=mpicc CXX=mpicxx # MPI wrappers for Fortran, cc and CC similarly -#export FC=mpif90 CC=gcc CXX=mpicxx # MPI wrappers for Fortran, cc and CC similarly