From 4150608c4dc71932ce1ebe168d9ba8189c75f609 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Sun, 24 Apr 2022 23:05:52 +0100 Subject: [PATCH 01/53] ATLAS-356 Fix NodeColumns remote_index for parallel orca grids --- src/atlas/mesh/actions/BuildParallelFields.cc | 28 ++++++++++++++++--- src/atlas/util/Unique.cc | 6 ++++ src/atlas/util/Unique.h | 2 ++ 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/src/atlas/mesh/actions/BuildParallelFields.cc b/src/atlas/mesh/actions/BuildParallelFields.cc index bb7fc9f96..507ce67b2 100644 --- a/src/atlas/mesh/actions/BuildParallelFields.cc +++ b/src/atlas/mesh/actions/BuildParallelFields.cc @@ -246,18 +246,38 @@ Field& build_nodes_remote_idx(mesh::Nodes& nodes) { idx_t mypart = static_cast(mpi::rank()); idx_t nparts = static_cast(mpi::size()); - UniqueLonLat compute_uid(nodes); std::vector proc(nparts); for (idx_t jpart = 0; jpart < nparts; ++jpart) { proc[jpart] = jpart; } - auto ridx = array::make_indexview(nodes.remote_index()); - auto part = array::make_view(nodes.partition()); + auto ridx = array::make_indexview(nodes.remote_index()); + const auto part = array::make_view(nodes.partition()); + const auto flags = array::make_view(nodes.flags()); + const auto lonlat = array::make_view(nodes.lonlat()); + + const PeriodicTransform transform_periodic_east(-360.); + const PeriodicTransform transform_periodic_west(+360.); + const UniqueLonLat compute_uid_lonlat(nodes); + + auto compute_uid = [&](idx_t jnode) { + constexpr int PERIODIC = util::Topology::PERIODIC; + constexpr int EAST = util::Topology::EAST; + constexpr int WEST = util::Topology::WEST; + const auto flags_view = util::Bitflags::view(flags(jnode)); + if (flags_view.check(PERIODIC | EAST)) { + return compute_uid_lonlat(jnode, transform_periodic_east); + } + if (flags_view.check(PERIODIC | WEST)) { + return compute_uid_lonlat(jnode, transform_periodic_west); + } + return compute_uid_lonlat(jnode); + }; + idx_t nb_nodes = nodes.size(); - idx_t varsize = 2; + constexpr idx_t varsize = 2; std::vector> send_needed(mpi::size()); std::vector> recv_needed(mpi::size()); diff --git a/src/atlas/util/Unique.cc b/src/atlas/util/Unique.cc index 3501edffb..34e810b2d 100644 --- a/src/atlas/util/Unique.cc +++ b/src/atlas/util/Unique.cc @@ -36,6 +36,12 @@ uidx_t UniqueLonLat::operator()(const mesh::Connectivity::Row& elem_nodes, const return unique_lonlat(centroid); } +uidx_t atlas::util::UniqueLonLat::operator()(int node, const PeriodicTransform& transform) const { + std::array _lonlat{lonlat(node, LON), lonlat(node, LAT)}; + transform(_lonlat); + return unique_lonlat(_lonlat[LON], _lonlat[LAT]); +} + } // namespace util } // namespace atlas diff --git a/src/atlas/util/Unique.h b/src/atlas/util/Unique.h index 3ba763185..c33e0c8f0 100644 --- a/src/atlas/util/Unique.h +++ b/src/atlas/util/Unique.h @@ -80,6 +80,8 @@ class UniqueLonLat { /// @return uidx_t Return type depends on ATLAS_BITS_GLOBAL [32/64] bits uidx_t operator()(int node) const; + uidx_t operator()(int node, const PeriodicTransform& transform) const; + /// @brief Compute unique positive index of element defined by node indices. /// The assumption is that the elements exist in a lon-lat domain and don't // degenerate to a line. From 5b5c7abefd49cc7af5fe11f2521069c4b5e3cbed Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Sun, 24 Apr 2022 23:20:17 +0100 Subject: [PATCH 02/53] atlas-meshgen: Only create NodeColumns when required For some mesh generators that are not fully working or buggy it may not be possible to generate NodeColumns. A new option --nodes is introduced to control this. Without --nodes the NodeColumns is only created when halo>0. --- src/apps/atlas-meshgen.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/apps/atlas-meshgen.cc b/src/apps/atlas-meshgen.cc index ccc349f92..3b45a3bcd 100644 --- a/src/apps/atlas-meshgen.cc +++ b/src/apps/atlas-meshgen.cc @@ -124,6 +124,7 @@ class Meshgen2Gmsh : public AtlasTool { private: std::string key; long halo; + bool nodes; bool edges; bool cells; bool brick; @@ -165,7 +166,8 @@ Meshgen2Gmsh::Meshgen2Gmsh(int argc, char** argv): AtlasTool(argc, argv) { add_option(new Separator("Advanced")); add_option(new SimpleOption("halo", "Halo size")); - add_option(new SimpleOption("edges", "Build edge datastructure")); + add_option(new SimpleOption("nodes", "Build nodes datastructure")); + add_option(new SimpleOption("edges", "Build edges datastructure")); add_option(new SimpleOption("cells", "Build cells datastructure")); add_option(new SimpleOption("brick", "Build brick dual mesh")); add_option(new SimpleOption("stats", "Write statistics file")); @@ -203,6 +205,8 @@ int Meshgen2Gmsh::execute(const Args& args) { key = ""; args.get("grid.name", key); + nodes = false; + args.get("nodes", nodes); edges = false; args.get("edges", edges); cells = false; @@ -285,7 +289,7 @@ int Meshgen2Gmsh::execute(const Args& args) { } - if (grid.projection().units() == "degrees") { + if ((grid.projection().units() == "degrees" && halo > 0) || nodes) { functionspace::NodeColumns nodes_fs(mesh, option::halo(halo)); } else { From db7733d1e97841b0d91326912c89cec0adc5b224 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 26 Apr 2022 09:16:53 +0100 Subject: [PATCH 03/53] Add Fortran API for Interpolation::execute_adjoint() --- src/atlas/interpolation/Interpolation.cc | 14 ++++++++++++ src/atlas/interpolation/Interpolation.h | 4 ++++ .../atlas_Interpolation_module.F90 | 22 +++++++++++++++++++ 3 files changed, 40 insertions(+) diff --git a/src/atlas/interpolation/Interpolation.cc b/src/atlas/interpolation/Interpolation.cc index 7a26b95b2..926d8f87d 100644 --- a/src/atlas/interpolation/Interpolation.cc +++ b/src/atlas/interpolation/Interpolation.cc @@ -177,6 +177,20 @@ void atlas__Interpolation__execute_fieldset(Interpolation::Implementation* This, This->execute(FieldSet(source), t); } + +void atlas__Interpolation__execute_adjoint_field(Interpolation::Implementation* This, field::FieldImpl* source, + const field::FieldImpl* target) { + Field s(source); + This->execute_adjoint(s, Field(target)); +} + +void atlas__Interpolation__execute_adjoint_fieldset(Interpolation::Implementation* This, field::FieldSetImpl* source, + const field::FieldSetImpl* target) { + FieldSet s(source); + This->execute_adjoint(s, FieldSet(target)); +} + + } // extern "C" } // namespace atlas diff --git a/src/atlas/interpolation/Interpolation.h b/src/atlas/interpolation/Interpolation.h index 50144b4b9..be7b576ed 100644 --- a/src/atlas/interpolation/Interpolation.h +++ b/src/atlas/interpolation/Interpolation.h @@ -99,6 +99,10 @@ void atlas__Interpolation__execute_field(Interpolation::Implementation* This, co field::FieldImpl* target); void atlas__Interpolation__execute_fieldset(Interpolation::Implementation* This, const field::FieldSetImpl* source, field::FieldSetImpl* target); +void atlas__Interpolation__execute_adjoint_field(Interpolation::Implementation* This, field::FieldImpl* source, + const field::FieldImpl* target); +void atlas__Interpolation__execute_adjoint_fieldset(Interpolation::Implementation* This, field::FieldSetImpl* source, + const field::FieldSetImpl* target); } #endif diff --git a/src/atlas_f/interpolation/atlas_Interpolation_module.F90 b/src/atlas_f/interpolation/atlas_Interpolation_module.F90 index ea3daea32..869284577 100644 --- a/src/atlas_f/interpolation/atlas_Interpolation_module.F90 +++ b/src/atlas_f/interpolation/atlas_Interpolation_module.F90 @@ -38,7 +38,10 @@ module atlas_Interpolation_module contains procedure, private :: execute_field procedure, private :: execute_fieldset + procedure, private :: execute_adjoint_field + procedure, private :: execute_adjoint_fieldset generic, public :: execute => execute_field, execute_fieldset + generic, public :: execute_adjoint => execute_adjoint_field, execute_adjoint_fieldset #if FCKIT_FINAL_NOT_INHERITING final :: atlas_Interpolation__final_auto @@ -124,6 +127,25 @@ subroutine execute_fieldset(this,source,target) call atlas__Interpolation__execute_fieldset(this%CPTR_PGIBUG_A,source%CPTR_PGIBUG_A,target%CPTR_PGIBUG_A) end subroutine +subroutine execute_adjoint_field(this,source,target) + use atlas_Interpolation_c_binding + use atlas_Field_module, only : atlas_Field + class(atlas_Interpolation), intent(in) :: this + class(atlas_Field), intent(inout) :: source + class(atlas_Field), intent(in) :: target + call atlas__Interpolation__execute_adjoint_field(this%CPTR_PGIBUG_A,source%CPTR_PGIBUG_A,target%CPTR_PGIBUG_A) +end subroutine + +subroutine execute_adjoint_fieldset(this,source,target) + use atlas_Interpolation_c_binding + use atlas_FieldSet_module, only : atlas_FieldSet + class(atlas_Interpolation), intent(in) :: this + class(atlas_FieldSet), intent(inout) :: source + class(atlas_FieldSet), intent(in) :: target + call atlas__Interpolation__execute_adjoint_fieldset(this%CPTR_PGIBUG_A,source%CPTR_PGIBUG_A,target%CPTR_PGIBUG_A) +end subroutine + + !------------------------------------------------------------------------------- #if FCKIT_FINAL_NOT_INHERITING From 6c1a2241b12807331b3ab9179bf09060781b84b0 Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Thu, 21 Apr 2022 16:04:25 +0100 Subject: [PATCH 04/53] ATLAS-350 Fix global numbering of cells matching HEALPix grid --- .../detail/HealpixMeshGenerator.cc | 105 ++++++++++-------- 1 file changed, 60 insertions(+), 45 deletions(-) diff --git a/src/atlas/meshgenerator/detail/HealpixMeshGenerator.cc b/src/atlas/meshgenerator/detail/HealpixMeshGenerator.cc index 0e082b2eb..80fb67b0b 100644 --- a/src/atlas/meshgenerator/detail/HealpixMeshGenerator.cc +++ b/src/atlas/meshgenerator/detail/HealpixMeshGenerator.cc @@ -498,7 +498,7 @@ void HealpixMeshGenerator::generate_mesh(const StructuredGrid& grid, const grid: } #if DEBUG_OUTPUT - std::cout << "[" << mypart << "] : nnodes_SB = " << nnodes_SB << "\n"; + Log::info() << "[" << mypart << "] : nnodes_SB = " << nnodes_SB << "\n"; #endif // partitions and local indices in SB @@ -546,24 +546,24 @@ void HealpixMeshGenerator::generate_mesh(const StructuredGrid& grid, const grid: } #if DEBUG_OUTPUT_DETAIL - std::cout << "[" << mypart << "] : " - << "parts_SB = "; + Log::info() << "[" << mypart << "] : " + << "parts_SB = "; for (ii = 0; ii < nnodes_SB; ii++) { - std::cout << parts_SB[ii] << ","; + Log::info() << parts_SB[ii] << ","; } - std::cout << std::endl; - std::cout << "[" << mypart << "] : " - << "local_idx_SB = "; + Log::info() << std::endl; + Log::info() << "[" << mypart << "] : " + << "local_idx_SB = "; for (ii = 0; ii < nnodes_SB; ii++) { - std::cout << local_idx_SB[ii] << ","; + Log::info() << local_idx_SB[ii] << ","; } - std::cout << std::endl; - std::cout << "[" << mypart << "] : " - << "is_ghost_SB = "; + Log::info() << std::endl; + Log::info() << "[" << mypart << "] : " + << "is_ghost_SB = "; for (ii = 0; ii < nnodes_SB; ii++) { - std::cout << is_ghost_SB[ii] << ","; + Log::info() << is_ghost_SB[ii] << ","; } - std::cout << std::endl; + Log::info() << std::endl; #endif // vectors marking nodes that are necessary for this proc's cells @@ -577,9 +577,9 @@ void HealpixMeshGenerator::generate_mesh(const StructuredGrid& grid, const grid: for (iy = iy_min; iy <= iy_max; iy++) { int nx = latPoints(iy); for (ix = 0; ix < nx; ix++) { - int is_cell = (iy == 0 ? ix % 2 : 1) * (iy == ny - 1 ? ix % 2 : 1); + int not_duplicate_cell = (iy == 0 ? ix % 2 : 1) * (iy == ny - 1 ? ix % 2 : 1); - if (!is_ghost_SB[ii] && is_cell) { + if (!is_ghost_SB[ii] && not_duplicate_cell) { // mark this node as being used if (!is_node_SB[ii]) { ++nnodes; @@ -625,16 +625,18 @@ void HealpixMeshGenerator::generate_mesh(const StructuredGrid& grid, const grid: } #if DEBUG_OUTPUT - std::cout << "[" << mypart << "] : " - << "nnodes = " << nnodes << ", ncells = " << ncells << ", parts_sidx = " << parts_sidx << std::endl; + Log::info() << "[" << mypart << "] : " + << "nnodes = " << nnodes << ", ncells = " << ncells << ", parts_sidx = " << parts_sidx << std::endl; + Log::info() << "[" << mypart << "] : " + << "iy_min = " << iy_min << ", iy_max = " << iy_max << std::endl; #endif #if DEBUG_OUTPUT_DETAIL - std::cout << "[" << mypart << "] : " - << "is_node_SB = "; + Log::info() << "[" << mypart << "] : " + << "is_node_SB = "; for (int ii = 0; ii < nnodes_SB; ii++) { - std::cout << is_node_SB[ii] << ","; + Log::info() << is_node_SB[ii] << ","; } - std::cout << std::endl; + Log::info() << std::endl; #endif // define nodes and associated properties @@ -651,9 +653,10 @@ void HealpixMeshGenerator::generate_mesh(const StructuredGrid& grid, const grid: // define cells and associated properties mesh.cells().add(new mesh::temporary::Quadrilateral(), ncells); - int quad_begin = mesh.cells().elements(0).begin(); - auto cells_part = array::make_view(mesh.cells().partition()); - mesh::HybridElements::Connectivity& node_connectivity = mesh.cells().node_connectivity(); + int quad_begin = mesh.cells().elements(0).begin(); + auto cells_part = array::make_view(mesh.cells().partition()); + auto cells_glb_idx = array::make_view(mesh.cells().global_index()); + auto& node_connectivity = mesh.cells().node_connectivity(); idx_t quad_nodes[4]; int jcell = quad_begin; @@ -758,24 +761,25 @@ void HealpixMeshGenerator::generate_mesh(const StructuredGrid& grid, const grid: local_idx_SB[iil] = inode; #if DEBUG_OUTPUT_DETAIL - std::cout << "[" << mypart << "] : " - << "New node \tinode=" << inode << "; iil= " << iil << "; ix=" << ix << "; iy=" << iy - << "; glon=" << lonlat(inode, 0) << "; glat=" << lonlat(inode, 1) - << "; glb_idx=" << glb_idx(inode) << "; loc_idx=" << local_idx_SB[iil] << std::endl; + Log::info() << "[" << mypart << "] : " + << "New node \tinode=" << inode << "; iil= " << iil << "; ix=" << ix << "; iy=" << iy + << "; glon=" << lonlat(inode, 0) << "; glat=" << lonlat(inode, 1) + << "; glb_idx=" << glb_idx(inode) << "; loc_idx=" << local_idx_SB[iil] << std::endl; #endif } ii += (ix != nx - 1 ? 1 : 0); } } - ii = 0; // index inside SB (surrounding belt) + ii = 0; // index inside SB (surrounding belt) + int jcell_offset = 0; for (iy = iy_min; iy <= iy_max; iy++) { int nx = latPoints(iy) + 1; for (ix = 0; ix < nx; ix++) { - int is_cell = (iy == 0 ? ix % 2 : 1) * (iy == ny - 1 ? ix % 2 : 1); - int iil = idx_xy_to_x(ix, iy, ns); + int not_duplicate_cell = (iy == 0 ? ix % 2 : 1) * (iy == ny - 1 ? ix % 2 : 1); + int iil = idx_xy_to_x(ix, iy, ns); iil -= (iil < 12 * ns * ns + 16 ? parts_sidx : -nnodes_SB + iy_max + 12 * ns * ns + 17); - if (!is_ghost_SB[iil] && is_cell) { + if (!is_ghost_SB[iil] && not_duplicate_cell) { // define cell corners (local indices) quad_nodes[0] = local_idx_SB[iil]; @@ -795,11 +799,23 @@ void HealpixMeshGenerator::generate_mesh(const StructuredGrid& grid, const grid: quad_nodes[3] = local_idx_SB[quad_nodes[3]]; node_connectivity.set(jcell, quad_nodes); + if (iy == 0) { + cells_glb_idx(jcell) = 12 * ns * ns + 1 + ix / 2; + jcell_offset++; + } + else if (iy == ny - 1) { + cells_glb_idx(jcell) = 12 * ns * ns + 5 + ix / 2; + jcell_offset++; + } + else { + cells_glb_idx(jcell) = parts_sidx + iil - 3 - (mypart != 0 ? 4 : jcell_offset); + } cells_part(jcell) = mypart; #if DEBUG_OUTPUT_DETAIL - std::cout << "[" << mypart << "] : " - << "New quad " << jcell << ": " << glb_idx(quad_nodes[0]) << "," << glb_idx(quad_nodes[1]) - << "," << glb_idx(quad_nodes[2]) << "," << glb_idx(quad_nodes[3]) << std::endl; + Log::info() << "[" << mypart << "] : " + << "New quad: loc-idx " << jcell << ", glb-idx " << cells_glb_idx(jcell) << ": " + << glb_idx(quad_nodes[0]) << "," << glb_idx(quad_nodes[1]) << "," << glb_idx(quad_nodes[2]) + << "," << glb_idx(quad_nodes[3]) << std::endl; #endif ++jcell; } @@ -810,18 +826,18 @@ void HealpixMeshGenerator::generate_mesh(const StructuredGrid& grid, const grid: #if DEBUG_OUTPUT_DETAIL // list nodes for (inode = 0; inode < nnodes; inode++) { - std::cout << "[" << mypart << "] : " - << " node " << inode << ": ghost = " << ghost(inode) << ", glb_idx = " << glb_idx(inode) - << ", part = " << part(inode) << ", lon = " << lonlat(inode, 0) << ", lat = " << lonlat(inode, 1) - << ", remote_idx = " << remote_idx(inode) << std::endl; + Log::info() << "[" << mypart << "] : " + << " node " << inode << ": ghost = " << ghost(inode) << ", glb_idx = " << glb_idx(inode) + << ", part = " << part(inode) << ", lon = " << lonlat(inode, 0) << ", lat = " << lonlat(inode, 1) + << ", remote_idx = " << remote_idx(inode) << std::endl; } int* cell_nodes; for (jcell = 0; jcell < ncells; jcell++) { - std::cout << "[" << mypart << "] : " - << " cell " << jcell << ": " << glb_idx(node_connectivity(jcell, 0)) << "," - << glb_idx(node_connectivity(jcell, 1)) << "," << glb_idx(node_connectivity(jcell, 2)) << "," - << glb_idx(node_connectivity(jcell, 3)) << std::endl; + Log::info() << "[" << mypart << "] : " + << " cell " << jcell << ", glb-idx " << cells_glb_idx(jcell) << ": " + << glb_idx(node_connectivity(jcell, 0)) << "," << glb_idx(node_connectivity(jcell, 1)) << "," + << glb_idx(node_connectivity(jcell, 2)) << "," << glb_idx(node_connectivity(jcell, 3)) << std::endl; } #endif @@ -832,8 +848,7 @@ void HealpixMeshGenerator::generate_mesh(const StructuredGrid& grid, const grid: nodes.global_index().metadata().set("min", 1); nodes.global_index().metadata().set("max", nvertices + grid.ny() + 2); - - generateGlobalElementNumbering(mesh); + //generateGlobalElementNumbering(mesh); } // generate_mesh From a2f8ccaa2e2c2d27ad1e7b0653b5a5531ec3b92d Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 27 Apr 2022 22:58:08 +0100 Subject: [PATCH 05/53] atlas::vector::capacity() --- src/atlas/util/vector.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/atlas/util/vector.h b/src/atlas/util/vector.h index 5cc4e94c8..77dcc5dbf 100644 --- a/src/atlas/util/vector.h +++ b/src/atlas/util/vector.h @@ -106,6 +106,8 @@ class vector { idx_t size() const { return size_; } + idx_t capacity() const { return capacity_; } + template ::value, int>::type = 0> void assign(Size n, const value_type& value) { resize(n); From 45da29145fa4cfeef4749a0df680016289fbb798 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 27 Apr 2022 23:20:10 +0100 Subject: [PATCH 06/53] ATLAS-357 StructuredColumns functionspace now works for regional grids --- .../detail/StructuredColumns_setup.cc | 23 ++- src/tests/functionspace/CMakeLists.txt | 7 + .../test_structuredcolumns_regional.cc | 191 ++++++++++++++++++ .../test_interpolation_structured2D.cc | 2 +- 4 files changed, 219 insertions(+), 4 deletions(-) create mode 100644 src/tests/functionspace/test_structuredcolumns_regional.cc diff --git a/src/atlas/functionspace/detail/StructuredColumns_setup.cc b/src/atlas/functionspace/detail/StructuredColumns_setup.cc index e095270e3..7fd8b6087 100644 --- a/src/atlas/functionspace/detail/StructuredColumns_setup.cc +++ b/src/atlas/functionspace/detail/StructuredColumns_setup.cc @@ -72,6 +72,7 @@ struct GridPointSet { } idx_t size() const { return static_cast(gp_.size()); } + idx_t capacity() const { return static_cast(gp_.capacity()); } const GridPoint& operator[](idx_t i) const { return gp_[i]; } @@ -94,6 +95,8 @@ void StructuredColumns::setup(const grid::Distribution& distribution, const ecki config.get("periodic_x", periodic_x); config.get("periodic_y", periodic_y); + bool regional = (!periodic_x && !periodic_y && !grid_->domain().global()); + const double eps = 1.e-12; ny_ = grid_->ny(); @@ -225,6 +228,10 @@ void StructuredColumns::setup(const grid::Distribution& distribution, const ecki j_end_halo_ = j_end_ + halo; i_begin_halo_.resize(-halo, grid_->ny() - 1 + halo); i_end_halo_.resize(-halo, grid_->ny() - 1 + halo); + if (regional) { + j_begin_halo_ = std::max(j_begin_halo_, 0); + j_end_halo_ = std::min(j_end_halo_, grid_->ny()); + } auto compute_i = [this](idx_t i, idx_t j) -> idx_t { const idx_t nx = grid_->nx(j); @@ -345,7 +352,7 @@ void StructuredColumns::setup(const grid::Distribution& distribution, const ecki idx_t jmax = -std::numeric_limits::max(); ATLAS_TRACE_SCOPE("Compute bounds halo") { - for (idx_t j = j_begin_halo_; j < j_end_halo_; ++j) { + for (idx_t j = j_begin_ - halo; j < j_end_ + halo; ++j) { i_begin_halo_(j) = imin; i_end_halo_(j) = imax; } @@ -362,7 +369,13 @@ void StructuredColumns::setup(const grid::Distribution& distribution, const ecki double x_next = grid_->x(i + 1, j); double x_prev = grid_->x(i - 1, j); - for (idx_t jj = j - halo; jj <= j + halo; ++jj) { + idx_t jj_min = j - halo; + idx_t jj_max = j + halo; + if (regional) { + jj_min = std::max(jj_min, 0); + jj_max = std::min(jj_max, grid_->nx(j) - 1); + } + for (idx_t jj = jj_min; jj <= jj_max; ++jj) { idx_t jjj = compute_j(jj); idx_t nx_jjj = grid_->nx(jjj); idx_t last = grid_->nx(jjj) - 1; @@ -412,6 +425,11 @@ void StructuredColumns::setup(const grid::Distribution& distribution, const ecki iii = std::min(iii, last); idx_t i_plus_halo = iii + halo; + if (regional) { + i_minus_halo = std::max(i_minus_halo, 0); + i_plus_halo = std::min(i_plus_halo, grid_->nx(jj) - 1); + } + imin = std::min(imin, i_minus_halo); imax = std::max(imax, i_plus_halo); i_begin_halo_(jj) = std::min(i_begin_halo_(jj), i_minus_halo); @@ -433,7 +451,6 @@ void StructuredColumns::setup(const grid::Distribution& distribution, const ecki extra_halo += i_end_halo_(j) - i_begin_halo_(j); } - ATLAS_TRACE_SCOPE("Assemble gridpoints") { gridpoints.reserve(owned + extra_halo); gridpoints.resize(owned); diff --git a/src/tests/functionspace/CMakeLists.txt b/src/tests/functionspace/CMakeLists.txt index 686c362d1..bf7d5d378 100644 --- a/src/tests/functionspace/CMakeLists.txt +++ b/src/tests/functionspace/CMakeLists.txt @@ -51,6 +51,13 @@ ecbuild_add_test( TARGET test_structuredcolumns_biperiodic CONDITION eckit_HAVE_MPI AND MPI_SLOTS GREATER_EQUAL 5 ) +ecbuild_add_test( TARGET test_structuredcolumns_regional + SOURCES test_structuredcolumns_regional.cc + LIBS atlas + ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} +) + + ecbuild_add_test( TARGET atlas_test_structuredcolumns SOURCES test_structuredcolumns.cc LIBS atlas diff --git a/src/tests/functionspace/test_structuredcolumns_regional.cc b/src/tests/functionspace/test_structuredcolumns_regional.cc new file mode 100644 index 000000000..52f8c2117 --- /dev/null +++ b/src/tests/functionspace/test_structuredcolumns_regional.cc @@ -0,0 +1,191 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#include +#include +#include +#include + +#include "atlas/array.h" +#include "atlas/field.h" +#include "atlas/functionspace.h" +#include "atlas/grid.h" +#include "atlas/parallel/mpi/mpi.h" +#include "atlas/util/Config.h" + + +#include +#include +#include +#include + + +#include "tests/AtlasTestEnvironment.h" + +using Config = atlas::util::Config; + +namespace atlas { +namespace test { + +//----------------------------------------------------------------------------- + +CASE("regional lonlat") { + const int Nx = 8, Ny = 8; + const double xmin = +20, xmax = +60, ymin = +20, ymax = +60; + + StructuredGrid::XSpace xspace(Config("type", "linear")("N", Nx)("start", xmin)("end", xmax)); + StructuredGrid::YSpace yspace(Config("type", "linear")("N", Ny)("start", ymin)("end", ymax)); + + StructuredGrid grid(xspace, yspace); + + functionspace::StructuredColumns fs(grid, grid::Partitioner("checkerboard"), Config("halo", 1)); + + ATLAS_TRACE_SCOPE("Output python") { + auto xy = array::make_view(fs.xy()); + auto g = array::make_view(fs.global_index()); + auto p = array::make_view(fs.partition()); + + + eckit::PathName filepath("test_structuredcolumns_regional_p" + std::to_string(mpi::comm().rank()) + ".py"); + + std::ofstream f(filepath.asString().c_str(), std::ios::trunc); + + f << "\n" + "import matplotlib.pyplot as plt" + "\n" + "from matplotlib.path import Path" + "\n" + "import matplotlib.patches as patches" + "\n" + "" + "\n" + "from itertools import cycle" + "\n" + "import matplotlib.cm as cm" + "\n" + "import numpy as np" + "\n" + "" + "\n" + "fig = plt.figure(figsize=(20,10))" + "\n" + "ax = fig.add_subplot(111,aspect='equal')" + "\n" + ""; + + double xmin = std::numeric_limits::max(); + double xmax = -std::numeric_limits::max(); + double ymin = std::numeric_limits::max(); + double ymax = -std::numeric_limits::max(); + f << "\n" + "x = ["; + for (idx_t j = fs.j_begin_halo(); j < fs.j_end_halo(); ++j) { + for (idx_t i = fs.i_begin_halo(j); i < fs.i_end_halo(j); ++i) { + idx_t n = fs.index(i, j); + f << xy(n, XX) << ", "; + xmin = std::min(xmin, xy(n, XX)); + xmax = std::max(xmax, xy(n, XX)); + } + } + f << "]"; + + f << "\n" + "y = ["; + for (idx_t j = fs.j_begin_halo(); j < fs.j_end_halo(); ++j) { + for (idx_t i = fs.i_begin_halo(j); i < fs.i_end_halo(j); ++i) { + idx_t n = fs.index(i, j); + f << xy(n, YY) << ", "; + ymin = std::min(ymin, xy(n, YY)); + ymax = std::max(ymax, xy(n, YY)); + } + } + f << "]"; + + f << "\n" + "g = ["; + for (idx_t j = fs.j_begin_halo(); j < fs.j_end_halo(); ++j) { + for (idx_t i = fs.i_begin_halo(j); i < fs.i_end_halo(j); ++i) { + idx_t n = fs.index(i, j); + f << g(n) << ", "; + } + } + f << "]"; + + f << "\n" + "p = ["; + for (idx_t j = fs.j_begin_halo(); j < fs.j_end_halo(); ++j) { + for (idx_t i = fs.i_begin_halo(j); i < fs.i_end_halo(j); ++i) { + idx_t n = fs.index(i, j); + f << p(n) << ", "; + } + } + f << "]"; + + // f << "\n" + // "r = ["; + // for (idx_t j = fs.j_begin_halo(); j < fs.j_end_halo(); ++j) { + // for (idx_t i = fs.i_begin_halo(j); i < fs.i_end_halo(j); ++i) { + // idx_t n = fs.index(i, j); + // f << r(n) << ", "; + // } + // } + // f << "]"; + + f << "\n" + "" + "\n" + "c = [ cm.Paired( float(pp%13)/12. ) for pp in p ]" + "\n" + "ax.scatter(x, y, color=c, marker='o')" + "\n" + "for i in range(" + << fs.size() + << "):" + "\n" + " ax.annotate(g[i], (x[i],y[i]), fontsize=8)" + "\n" + ""; + f << "\n" + "ax.set_xlim( " + << std::min(0., xmin) << "-5, " << std::max(360., xmax) + << "+5)" + "\n" + "ax.set_ylim( " + << std::min(-90., ymin) << "-5, " << std::max(90., ymax) + << "+5)" + "\n" + "ax.set_xticks([0,45,90,135,180,225,270,315,360])" + "\n" + "ax.set_yticks([-90,-45,0,45,90])" + "\n" + "plt.grid()" + "\n" + "plt.show()" + "\n"; + } + + auto lonlat = array::make_view(fs.lonlat()); + for (idx_t n = 0; n < lonlat.shape(0); ++n) { + // Log::info() << n << "\t" << PointLonLat(lonlat(n, LON), lonlat(n, LAT)) << std::endl; + EXPECT(lonlat(n, LON) >= xmin); + EXPECT(lonlat(n, LON) <= xmax); + EXPECT(lonlat(n, LAT) >= ymin); + EXPECT(lonlat(n, LAT) <= ymax); + } +} + +//----------------------------------------------------------------------------- + +} // namespace test +} // namespace atlas + +int main(int argc, char** argv) { + return atlas::test::run(argc, argv); +} diff --git a/src/tests/interpolation/test_interpolation_structured2D.cc b/src/tests/interpolation/test_interpolation_structured2D.cc index be4c13ad0..c5847f03d 100644 --- a/src/tests/interpolation/test_interpolation_structured2D.cc +++ b/src/tests/interpolation/test_interpolation_structured2D.cc @@ -197,7 +197,7 @@ CASE("test_interpolation_structured using grid API") { ATLAS_TRACE_SCOPE("output") { output::Gmsh gmsh(scheme().getString("name") + "-output-section" + std::to_string(_subsection) + ".msh", Config("coordinates", "xy")); - gmsh.write(MeshGenerator("structured").generate(output_grid)); + gmsh.write(Mesh(output_grid)); gmsh.write(field_target, StructuredColumns(output_grid)); } }; From eb20c37fe1d08e80cfdcca436eef601b0bd9fbf5 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Thu, 28 Apr 2022 13:21:12 +0100 Subject: [PATCH 07/53] Add Fortran API for FieldSet::name() --- src/atlas/field/FieldSet.cc | 5 +++++ src/atlas/field/FieldSet.h | 1 + src/atlas_f/field/atlas_FieldSet_module.F90 | 12 ++++++++++++ src/tests/field/fctest_field.F90 | 3 +++ 4 files changed, 21 insertions(+) diff --git a/src/atlas/field/FieldSet.cc b/src/atlas/field/FieldSet.cc index 9b7a307b5..49a6ba3be 100644 --- a/src/atlas/field/FieldSet.cc +++ b/src/atlas/field/FieldSet.cc @@ -97,6 +97,11 @@ void atlas__FieldSet__delete(FieldSetImpl* This) { delete This; } +const char* atlas__FieldSet__name(FieldSetImpl* This) { + ATLAS_ASSERT(This != nullptr, "Cannot access name of uninitialised atlas_FieldSet"); + return This->name().c_str(); +} + void atlas__FieldSet__add_field(FieldSetImpl* This, FieldImpl* field) { ATLAS_ASSERT(This != nullptr, "Reason: Use of uninitialised atlas_FieldSet"); ATLAS_ASSERT(field != nullptr, "Reason: Use of uninitialised atlas_Field"); diff --git a/src/atlas/field/FieldSet.h b/src/atlas/field/FieldSet.h index b93eb43f2..46f804a1c 100644 --- a/src/atlas/field/FieldSet.h +++ b/src/atlas/field/FieldSet.h @@ -130,6 +130,7 @@ FieldSetImpl* atlas__FieldSet__new(char* name); void atlas__FieldSet__delete(FieldSetImpl* This); void atlas__FieldSet__add_field(FieldSetImpl* This, FieldImpl* field); int atlas__FieldSet__has_field(const FieldSetImpl* This, char* name); +const char* atlas__FieldSet__name(FieldSetImpl* This); idx_t atlas__FieldSet__size(const FieldSetImpl* This); FieldImpl* atlas__FieldSet__field_by_name(FieldSetImpl* This, char* name); FieldImpl* atlas__FieldSet__field_by_idx(FieldSetImpl* This, idx_t idx); diff --git a/src/atlas_f/field/atlas_FieldSet_module.F90 b/src/atlas_f/field/atlas_FieldSet_module.F90 index 3ecaa4749..a8a09457c 100644 --- a/src/atlas_f/field/atlas_FieldSet_module.F90 +++ b/src/atlas_f/field/atlas_FieldSet_module.F90 @@ -41,6 +41,7 @@ module atlas_FieldSet_module !------------------------------------------------------------------------------ contains + procedure, public :: name => FieldSet__name procedure, public :: size => FieldSet__size procedure, public :: has procedure, private :: field_by_name @@ -94,6 +95,17 @@ function atlas_FieldSet__ctor(name) result(fieldset) call fieldset%return() end function +function FieldSet__name(this) result(fieldset_name) + use, intrinsic :: iso_c_binding, only : c_ptr + use fckit_c_interop_module, only : c_ptr_to_string, c_str + use atlas_fieldset_c_binding + class(atlas_FieldSet), intent(in) :: this + character(len=:), allocatable :: fieldset_name + type(c_ptr) :: fieldset_name_c_str + fieldset_name_c_str = atlas__FieldSet__name(this%CPTR_PGIBUG_A) + fieldset_name = c_ptr_to_string(fieldset_name_c_str) +end function FieldSet__name + subroutine add(this,field) use atlas_fieldset_c_binding use atlas_Field_module, only: atlas_Field diff --git a/src/tests/field/fctest_field.F90 b/src/tests/field/fctest_field.F90 index ef915ce6e..a0264f7e4 100644 --- a/src/tests/field/fctest_field.F90 +++ b/src/tests/field/fctest_field.F90 @@ -72,6 +72,9 @@ module fcta_Field_fixture call state%remove("field_test_owners") FCTEST_CHECK_EQUAL( f%owners() , 1 ) fields = atlas_FieldSet("fields") + + FCTEST_CHECK_EQUAL( fields%name() , "fields" ) + call fields%add(f) FCTEST_CHECK_EQUAL( f%owners() , 2 ) From e5e39458db09b20c52e714170500c19014b327a8 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 3 May 2022 10:37:55 +0100 Subject: [PATCH 08/53] Fix clang-tidy warning --- src/atlas/projection/detail/CubedSphereEquiAnglProjection.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/atlas/projection/detail/CubedSphereEquiAnglProjection.cc b/src/atlas/projection/detail/CubedSphereEquiAnglProjection.cc index b7b2a56a8..eddcd9f88 100644 --- a/src/atlas/projection/detail/CubedSphereEquiAnglProjection.cc +++ b/src/atlas/projection/detail/CubedSphereEquiAnglProjection.cc @@ -79,7 +79,7 @@ void CubedSphereEquiAnglProjection::xy2alphabeta(double crd[], idx_t t) const { (greaterEqual(crd[YY], xyCentre[YY] - 45.) && lessEqual(crd[YY], xyCentre[YY] + 45.)); }; if (!inCross(crd)) { - auto sStream = std::stringstream(); + std::stringstream sStream; sStream << "xy coordinate (" << crd[0] << ", " << crd[1] << ") is not in range for tile " << t << "."; ATLAS_THROW_EXCEPTION(sStream.str()); } From f325363e942b657bb378fc0b9ffe5bfc09e433ee Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 3 May 2022 11:31:36 +0100 Subject: [PATCH 09/53] Fix type of indices in test_transgeneral CASE test_2level_adjoint_test_with_powerspectrum_convolution --- src/tests/trans/test_transgeneral.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/tests/trans/test_transgeneral.cc b/src/tests/trans/test_transgeneral.cc index 63b523f27..33072cffa 100644 --- a/src/tests/trans/test_transgeneral.cc +++ b/src/tests/trans/test_transgeneral.cc @@ -1609,8 +1609,6 @@ CASE("test_2level_adjoint_test_with_powerspectrum_convolution") { Log::info() << "transIFS backend" << transIFS.backend() << std::endl; std::vector powerSpectrum(2 * N, 0.0); - float i(1.0); - float t(0.0); for (std::size_t w = 0; w < powerSpectrum.size(); ++w) { powerSpectrum[w] = 1.0 / static_cast(w + 1); @@ -1651,8 +1649,8 @@ CASE("test_2level_adjoint_test_with_powerspectrum_convolution") { auto spfView = atlas::array::make_view(spf); const auto zonal_wavenumbers = specFS.zonal_wavenumbers(); const int nb_zonal_wavenumbers = zonal_wavenumbers.size(); - i = 0; double adj_value(0.0); + int i{0}; for (int jm = 0; jm < nb_zonal_wavenumbers; ++jm) { const std::size_t m1 = zonal_wavenumbers(jm); for (std::size_t n1 = m1; n1 <= static_cast(2 * N - 1); ++n1) { From 9c6471d734bc3175e078f34e60263e21af44be9d Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Sun, 8 May 2022 14:19:24 +0100 Subject: [PATCH 10/53] Fix in atlas-meshgen where --3d option was not properly propagated to mesh generators --- src/apps/atlas-meshgen.cc | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/apps/atlas-meshgen.cc b/src/apps/atlas-meshgen.cc index 3b45a3bcd..f5d6d635c 100644 --- a/src/apps/atlas-meshgen.cc +++ b/src/apps/atlas-meshgen.cc @@ -62,6 +62,10 @@ MeshGenerator make_meshgenerator(const Grid& grid, const AtlasTool::Args& args) config.set("type", args.getString("generator")); } + if (args.has("3d")) { + config.set("3d", true); + } + if (mpi::comm().size() > 1 || args.getBool("edges", false)) { config.set("3d", false); } @@ -139,8 +143,10 @@ class Meshgen2Gmsh : public AtlasTool { //----------------------------------------------------------------------------- Meshgen2Gmsh::Meshgen2Gmsh(int argc, char** argv): AtlasTool(argc, argv) { - add_option(new SimpleOption("lonlat", "Output mesh in lon,lat coordinates")); - add_option(new SimpleOption("ij", "Output mesh in i,j coordinates")); + add_option(new SimpleOption("coordinates", "Output mesh in given coordinates")); + add_option( + new SimpleOption("lonlat", "Output mesh in lon,lat coordinates (shorthand for --coordinates=lonlat)")); + add_option(new SimpleOption("ij", "Output mesh in i,j coordinates (shorthand for --coordinates=ij)")); add_option(new SimpleOption("3d", "Output mesh as sphere, and generate " "mesh connecting East and West in " @@ -319,6 +325,7 @@ int Meshgen2Gmsh::execute(const Args& args) { bool lonlat = args.getBool("lonlat", false); bool ij = args.getBool("ij", false); std::string coordinates = dim_3d ? "xyz" : lonlat ? "lonlat" : ij ? "ij" : "xy"; + args.get("coordinates", coordinates); if (args.getBool("gmsh", true)) { bool torus = false; From 9c99a162026e19067b64e30508c7789df6f89202 Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Mon, 9 May 2022 17:05:32 +0100 Subject: [PATCH 11/53] ATLAS-350 Fix bug in healpix global numbering of periodic cells and nodes (tnx Willem) --- src/atlas/meshgenerator/detail/HealpixMeshGenerator.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/atlas/meshgenerator/detail/HealpixMeshGenerator.cc b/src/atlas/meshgenerator/detail/HealpixMeshGenerator.cc index 80fb67b0b..f9b3d3afa 100644 --- a/src/atlas/meshgenerator/detail/HealpixMeshGenerator.cc +++ b/src/atlas/meshgenerator/detail/HealpixMeshGenerator.cc @@ -846,10 +846,12 @@ void HealpixMeshGenerator::generate_mesh(const StructuredGrid& grid, const grid: nodes.metadata().set("NbVirtualPts", size_t(0)); nodes.global_index().metadata().set("human_readable", true); nodes.global_index().metadata().set("min", 1); - nodes.global_index().metadata().set("max", nvertices + grid.ny() + 2); + nodes.global_index().metadata().set("max", nvertices + 4 * ns + 1); + mesh.cells().global_index().metadata().set("human_readable", true); + mesh.cells().global_index().metadata().set("min", 1); + mesh.cells().global_index().metadata().set("max", grid.size() + 8); //generateGlobalElementNumbering(mesh); - } // generate_mesh namespace { From 3f53fd0be6006b9eace67afd77b4f8e80ba69faf Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Thu, 12 May 2022 10:51:32 +0100 Subject: [PATCH 12/53] Small fixes and diagnostics in sandbox atlas-parallel-interpolation --- src/sandbox/interpolation/PartitionedMesh.cc | 4 +- src/sandbox/interpolation/PartitionedMesh.h | 2 +- .../atlas-parallel-interpolation.cc | 41 +++++++++---------- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/src/sandbox/interpolation/PartitionedMesh.cc b/src/sandbox/interpolation/PartitionedMesh.cc index 65bf8e590..18712651a 100644 --- a/src/sandbox/interpolation/PartitionedMesh.cc +++ b/src/sandbox/interpolation/PartitionedMesh.cc @@ -19,10 +19,10 @@ namespace atlas { namespace interpolation { PartitionedMesh::PartitionedMesh(const std::string& partitioner, const std::string& generator, - bool generatorTriangulate, double generatorAngle): + bool generatorTriangulate, double generatorAngle, bool patchPole): optionPartitioner_(partitioner), optionGenerator_(generator) { generatorParams_.set("three_dimensional", false); - generatorParams_.set("patch_pole", true); + generatorParams_.set("patch_pole", patchPole); generatorParams_.set("include_pole", false); generatorParams_.set("triangulate", generatorTriangulate); generatorParams_.set("angle", generatorAngle); diff --git a/src/sandbox/interpolation/PartitionedMesh.h b/src/sandbox/interpolation/PartitionedMesh.h index 6e79e87ae..6b2a3e569 100644 --- a/src/sandbox/interpolation/PartitionedMesh.h +++ b/src/sandbox/interpolation/PartitionedMesh.h @@ -22,7 +22,7 @@ struct PartitionedMesh { typedef grid::Partitioner Partitioner; PartitionedMesh(const std::string& partitioner, const std::string& generator, bool meshGeneratorTriangulate = false, - double meshGeneratorAngle = 0); + double meshGeneratorAngle = 0, bool patchPole = true); virtual ~PartitionedMesh() {} diff --git a/src/sandbox/interpolation/atlas-parallel-interpolation.cc b/src/sandbox/interpolation/atlas-parallel-interpolation.cc index 5a96dcbd2..05669b875 100644 --- a/src/sandbox/interpolation/atlas-parallel-interpolation.cc +++ b/src/sandbox/interpolation/atlas-parallel-interpolation.cc @@ -90,13 +90,6 @@ int AtlasParallelInterpolation::execute(const AtlasTool::Args& args) { bool log_statistics = false; args.get("log-statistics", log_statistics); - idx_t log_rank = 0; - args.get("log-rank", log_rank); - - if (idx_t(eckit::mpi::comm().rank()) != log_rank) { - Log::reset(); - } - std::string interpolation_method = "finite-element"; args.get("method", interpolation_method); @@ -105,6 +98,9 @@ int AtlasParallelInterpolation::execute(const AtlasTool::Args& args) { linalg::sparse::current_backend(option); } + bool with_backward = false; + args.get("with-backward", with_backward); + // Generate and partition source & target mesh // source mesh is partitioned on its own, the target mesh uses // (pre-partitioned) source mesh @@ -121,19 +117,15 @@ int AtlasParallelInterpolation::execute(const AtlasTool::Args& args) { interpolation::PartitionedMesh src(args.get("source-mesh-partitioner", option) ? option : "default", args.get("source-mesh-generator", option) ? option : "default", args.get("source-mesh-generator-triangulate", trigs) ? trigs : false, - args.get("source-mesh-generator-angle", angle) ? angle : 0.); - - Grid tgt_grid(target_gridname); - - idx_t target_mesh_halo = args.getInt("target-mesh-halo", 0); - - interpolation::PartitionedMesh tgt(args.get("target-mesh-partitioner", option) ? option : "spherical-polygon", - args.get("target-mesh-generator", option) ? option : "default", - args.get("target-mesh-generator-triangulate", trigs) ? trigs : false, - args.get("target-mesh-generator-angle", angle) ? angle : 0.); + args.get("source-mesh-generator-angle", angle) ? angle : 0., true); Log::info() << "Partitioning source grid, halo of " << eckit::Plural(source_mesh_halo, "element") << std::endl; src.partition(src_grid); + + if (eckit::Resource("--output-polygons", false)) { + src.mesh().polygon(0).outputPythonScript("src-polygons.py"); + } + FunctionSpace src_functionspace; bool structured = false; for (auto& is_structured : {"structured-bicubic", "bicubic", "structured-bilinear", "bilinear"}) { @@ -152,15 +144,24 @@ int AtlasParallelInterpolation::execute(const AtlasTool::Args& args) { } src.writeGmsh("src-mesh.msh"); + Grid tgt_grid(target_gridname); + + idx_t target_mesh_halo = args.getInt("target-mesh-halo", 0); + + interpolation::PartitionedMesh tgt(args.get("target-mesh-partitioner", option) ? option : "spherical-polygon", + args.get("target-mesh-generator", option) ? option : "default", + args.get("target-mesh-generator-triangulate", trigs) ? trigs : false, + args.get("target-mesh-generator-angle", angle) ? angle : 0., + with_backward ? true : false); Log::info() << "Partitioning target grid, halo of " << eckit::Plural(target_mesh_halo, "element") << std::endl; tgt.partition(tgt_grid, src); + functionspace::NodeColumns tgt_functionspace(tgt.mesh(), option::halo(target_mesh_halo)); tgt.writeGmsh("tgt-mesh.msh"); - /// For debugging purposes if (eckit::Resource("--output-polygons", false)) { - src.mesh().polygon(0).outputPythonScript("polygons.py"); + tgt.mesh().polygon(0).outputPythonScript("tgt-polygons.py"); } // Setup interpolator relating source & target meshes before setting a source @@ -170,8 +171,6 @@ int AtlasParallelInterpolation::execute(const AtlasTool::Args& args) { Interpolation interpolator_forward(option::type(interpolation_method), src_functionspace, tgt_functionspace); Interpolation interpolator_backward; - bool with_backward = false; - args.get("with-backward", with_backward); if (with_backward) { std::string backward_interpolation_method = "finite-element"; args.get("method", backward_interpolation_method); From 59b3d2070e4f9d815b96adeb5b23f98263e04aaf Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Fri, 13 May 2022 09:31:03 +0100 Subject: [PATCH 13/53] ATLAS-352 HEALPix mesh with '3d' option, corresponding to original healpix numbering --- .../detail/HealpixMeshGenerator.cc | 380 +++++++++++------- .../detail/HealpixMeshGenerator.h | 9 + 2 files changed, 244 insertions(+), 145 deletions(-) diff --git a/src/atlas/meshgenerator/detail/HealpixMeshGenerator.cc b/src/atlas/meshgenerator/detail/HealpixMeshGenerator.cc index f9b3d3afa..c95c0467e 100644 --- a/src/atlas/meshgenerator/detail/HealpixMeshGenerator.cc +++ b/src/atlas/meshgenerator/detail/HealpixMeshGenerator.cc @@ -59,6 +59,11 @@ HealpixMeshGenerator::HealpixMeshGenerator(const eckit::Parametrisation& p) { options.set("part", part); } + bool three_dimensional; + if (p.get("3d", three_dimensional)) { + options.set("3d", three_dimensional); + } + std::string partitioner; if (p.get("partitioner", partitioner)) { if (not grid::Partitioner::exists(partitioner)) { @@ -77,6 +82,9 @@ void HealpixMeshGenerator::configure_defaults() { // This option sets the part that will be generated options.set("part", mpi::rank()); + // This option switches between original HEALPix (=1) or HEALPix with 8 points are the pole (=8) + options.set("3d", false); + // This options sets the default partitioner std::string partitioner; if (grid::Partitioner::exists("equal_regions") && mpi::size() > 1) { @@ -88,66 +96,97 @@ void HealpixMeshGenerator::configure_defaults() { options.set("partitioner", partitioner); } -namespace { -int idx_xy_to_x(const int xidx, const int yidx, const int ns) { + +// match glb_idx of node in (nb_pole_nodes==8)-mesh to glb_idx of nodes in (nb_pole_nodes==1)-mesh +gidx_t HealpixMeshGenerator::match_idx(gidx_t gidx, const int ns) const { + const gidx_t nb_nodes_orig = 12 * ns * ns; + if (nb_pole_nodes_ == 1 or gidx >= nb_nodes_) { + return gidx; + } + if (gidx == 4) { + return 0; + } + if (gidx == nb_nodes_ - 4) { + return nb_nodes_orig + 1; + } + bool at_north_pole = (gidx < 8); + bool at_south_pole = (gidx <= nb_nodes_ and gidx >= nb_nodes_orig + nb_pole_nodes_); + if (at_north_pole) { + return nb_nodes_orig + 2 + gidx - (gidx > 4 ? 1 : 0); + } + if (at_south_pole) { + return gidx - nb_pole_nodes_ + 9 - (gidx > nb_nodes_orig + 12 ? 1 : 0); + } + return gidx - nb_pole_nodes_ + 1; +} + + +// return "global_id - 1" +gidx_t HealpixMeshGenerator::idx_xy_to_x(const int xidx, const int yidx, const int ns) const { ATLAS_ASSERT(yidx < 4 * ns + 1 && yidx >= 0); ATLAS_ASSERT(xidx >= 0); - auto ghostIdx = [ns](int latid) { return 12 * ns * ns + 16 + latid; }; + const gidx_t nb_nodes_orig = 12 * ns * ns; + auto ghostIdx = [ns, this](int latid) { return this->nb_nodes_ + latid; }; + gidx_t ret; if (yidx == 0) { - ATLAS_ASSERT(xidx < 9 && xidx >= 0); - return (xidx != 8 ? xidx : ghostIdx(yidx)); + ATLAS_ASSERT(xidx <= nb_pole_nodes_ && xidx >= 0); + ret = (xidx != nb_pole_nodes_ ? xidx : ghostIdx(yidx)); } else if (yidx < ns) { ATLAS_ASSERT(xidx < 4 * yidx + 1 && xidx >= 0); - return (xidx != 4 * yidx ? 2 * yidx * (yidx - 1) + 8 + xidx : ghostIdx(yidx)); + ret = (xidx != 4 * yidx ? 2 * yidx * (yidx - 1) + nb_pole_nodes_ + xidx : ghostIdx(yidx)); } else if (yidx <= 2 * ns) { ATLAS_ASSERT(xidx < 4 * ns + 1 && xidx >= 0); - return (xidx != 4 * ns ? 2 * ns * (ns - 1) + 4 * ns * (yidx - ns) + 8 + xidx : ghostIdx(yidx)); + ret = (xidx != 4 * ns ? 2 * ns * (ns - 1) + 4 * ns * (yidx - ns) + nb_pole_nodes_ + xidx : ghostIdx(yidx)); } else if (yidx <= 3 * ns) { ATLAS_ASSERT(xidx < 4 * ns + 1 && xidx >= 0); - return (xidx != 4 * ns ? 2 * ns * (3 * ns + 1) + 4 * ns * (yidx - 2 * ns - 1) + 8 + xidx : ghostIdx(yidx)); + ret = (xidx != 4 * ns ? 2 * ns * (3 * ns + 1) + 4 * ns * (yidx - 2 * ns - 1) + nb_pole_nodes_ + xidx + : ghostIdx(yidx)); } else if (yidx == 3 * ns + 1 && ns > 1) { ATLAS_ASSERT(xidx < 4 * (ns - 1) + 1 && xidx >= 0); - return (xidx != 4 * (ns - 1) ? 2 * ns * (5 * ns + 1) + 4 * ns * (yidx - 3 * ns - 1) + 8 + xidx - : ghostIdx(yidx)); + ret = (xidx != 4 * (ns - 1) ? 2 * ns * (5 * ns + 1) + 4 * ns * (yidx - 3 * ns - 1) + nb_pole_nodes_ + xidx + : ghostIdx(yidx)); } else if (yidx < 4 * ns) { ATLAS_ASSERT(xidx < 4 * (ns - (yidx - 3 * ns)) + 1 && xidx >= 0); - return (xidx != 4 * (ns - (yidx - 3 * ns)) ? 2 * ns * (5 * ns + 1) + 4 * ns * (yidx - 3 * ns - 1) - - 2 * (yidx - 3 * ns) * (yidx - 3 * ns - 1) + 8 + xidx - : ghostIdx(yidx)); + ret = + (xidx != 4 * (ns - (yidx - 3 * ns)) ? 2 * ns * (5 * ns + 1) + 4 * ns * (yidx - 3 * ns - 1) - + 2 * (yidx - 3 * ns) * (yidx - 3 * ns - 1) + nb_pole_nodes_ + xidx + : ghostIdx(yidx)); } else { - ATLAS_ASSERT(xidx < 9 && xidx >= 0); - return (xidx != 8 ? 12 * ns * ns + 8 + xidx : ghostIdx(yidx)); + ATLAS_ASSERT(xidx <= nb_pole_nodes_ && xidx >= 0); + ret = (xidx != nb_pole_nodes_ ? nb_nodes_orig + nb_pole_nodes_ + xidx : ghostIdx(yidx)); } + return ret; } -int up_idx(const int xidx, const int yidx, const int ns) { +// return global_id of the node "above" (xidx,yidx) node +gidx_t HealpixMeshGenerator::up_idx(const int xidx, const int yidx, const int ns) const { ATLAS_ASSERT(yidx <= 4 * ns && yidx >= 0); - auto ghostIdx = [ns](int latid) { return 12 * ns * ns + 16 + latid; }; + const gidx_t nb_nodes_orig = 12 * ns * ns; + auto ghostIdx = [ns, this](int latid) { return this->nb_nodes_ + latid; }; int ret; - // global idx if (yidx == 0) { - ATLAS_ASSERT(xidx < 8); - ret = (xidx != 7 ? xidx + 1 : ghostIdx(0)); + ATLAS_ASSERT(xidx < nb_pole_nodes_); + ret = (xidx != nb_pole_nodes_ - 1 ? xidx + 1 : ghostIdx(0)); } else if (yidx == 1) { ATLAS_ASSERT(xidx < 4); - ret = 2 * xidx; + ret = (nb_pole_nodes_ == 8 ? 2 * xidx : 0); } else if (yidx < ns) { ATLAS_ASSERT(xidx < 4 * yidx); if (xidx != 4 * yidx - 1) { - ret = 2 * (yidx - 2) * (yidx - 1) + 8 + xidx - std::floor(xidx / (double)yidx); + ret = 2 * (yidx - 2) * (yidx - 1) + nb_pole_nodes_ + xidx - std::floor(xidx / (double)yidx); } else { ret = ghostIdx(yidx - 1); @@ -156,7 +195,7 @@ int up_idx(const int xidx, const int yidx, const int ns) { else if (yidx == ns && ns < 3) { ATLAS_ASSERT(xidx < 4 * ns); if (xidx != 4 * ns - 1) { - ret = 2 * ns * (ns - 1) + 8 - 4 * (ns - 1) + (xidx + 1) / 2; + ret = 2 * ns * (ns - 1) + nb_pole_nodes_ - 4 * (ns - 1) + (xidx + 1) / 2; } else { ret = ghostIdx(yidx - 1); @@ -165,7 +204,7 @@ int up_idx(const int xidx, const int yidx, const int ns) { else if (yidx == ns) { ATLAS_ASSERT(xidx < 4 * ns); if (xidx != 4 * ns - 1) { - ret = 2 * (ns - 2) * (ns - 1) + 8 + xidx - std::floor(xidx / (double)yidx); + ret = 2 * (ns - 2) * (ns - 1) + nb_pole_nodes_ + xidx - std::floor(xidx / (double)yidx); } else { ret = ghostIdx(yidx - 1); @@ -173,9 +212,9 @@ int up_idx(const int xidx, const int yidx, const int ns) { } else if (yidx <= 3 * ns) { ATLAS_ASSERT(xidx < 4 * ns); - int stg = (yidx - ns) % 2; - if (xidx != 4 * ns - 1 || (xidx == 4 * ns - 1 && stg)) { - ret = 2 * ns * (ns - 1) + 8 + 4 * ns * (yidx - ns - 1) + xidx + 1 - stg; + int staggering = (yidx - ns) % 2; + if (xidx != 4 * ns - 1 || (xidx == 4 * ns - 1 && staggering)) { + ret = 2 * ns * (ns - 1) + nb_pole_nodes_ + 4 * ns * (yidx - ns - 1) + xidx + 1 - staggering; } else { ret = ghostIdx(yidx - 1); @@ -184,70 +223,76 @@ int up_idx(const int xidx, const int yidx, const int ns) { else if (yidx < 4 * ns - 1) { int yidxl = 4 * ns - yidx; ATLAS_ASSERT(xidx < 4 * yidxl); - ret = 12 * ns * ns + 9 - 2 * (yidxl + 2) * (yidxl + 1) + xidx + std::floor(xidx / (double)yidxl); + ret = nb_nodes_orig + nb_pole_nodes_ + 1 - 2 * (yidxl + 2) * (yidxl + 1) + xidx + + std::floor(xidx / (double)yidxl); } else if (yidx == 4 * ns - 1) { ATLAS_ASSERT(xidx < 4); - ret = 12 * ns * ns + 5 - (ns == 1 ? 4 : 8) + 2 * xidx; + ret = nb_nodes_orig + nb_pole_nodes_ - 3 - (ns == 1 ? 4 : 8) + 2 * xidx; } else { - ATLAS_ASSERT(xidx < 8); + ATLAS_ASSERT(xidx < nb_pole_nodes_); if (ns == 1) { - if (xidx != 7) { - ret = 12 * ns * ns + 4 + (xidx % 2 ? -4 + (xidx + 1) / 2 : xidx / 2); + if (xidx != nb_pole_nodes_ - 1) { + ret = nb_nodes_orig + nb_pole_nodes_ - 4 + (xidx % 2 ? -4 + (xidx + 1) / 2 : xidx / 2); } else { - ret = ghostIdx(4 * ns - 2); + ret = (nb_pole_nodes_ == 8 ? ghostIdx(4 * ns - 2) : ghostIdx(4 * ns)); } } else { - ret = 12 * ns * ns + 8 + (xidx % 2 ? xidx - 12 : xidx - 4 - xidx / 2); + ret = nb_nodes_orig + nb_pole_nodes_ + (xidx % 2 ? xidx - 12 : xidx - 4 - xidx / 2); } } return ret; } -int down_idx(const int xidx, const int yidx, const int ns) { +// return global_id of the node "below" (xidx,yidx) node +gidx_t HealpixMeshGenerator::down_idx(const int xidx, const int yidx, const int ns) const { ATLAS_ASSERT(yidx <= 4 * ns); - auto ghostIdx = [ns](int latid) { return 12 * ns * ns + 16 + latid; }; + const gidx_t nb_nodes_orig = 12 * ns * ns; + auto ghostIdx = [ns, this](int latid) { return this->nb_nodes_ + latid; }; int ret; - // global idx if (yidx == 0) { - if (xidx < 8) { - ATLAS_ASSERT(xidx < 8); - } + ATLAS_ASSERT(xidx < nb_pole_nodes_); if (ns == 1) { - if (xidx != 7) { - ret = 8 + (xidx % 2 ? 4 + (xidx + 1) / 2 : xidx / 2); + if (xidx != nb_pole_nodes_ - 1) { + if (nb_pole_nodes_ == 8) { + ret = 8 + (xidx % 2 ? 4 + (xidx + 1) / 2 : xidx / 2); + } + else { + ret = 1 + xidx; + } } else { ret = ghostIdx(2); } } else { - ret = 8 + ((xidx + 1) % 2 ? xidx / 2 : 4 + xidx); + ret = nb_pole_nodes_ + ((xidx + 1) % 2 ? xidx / 2 : 4 + xidx); } } else if (yidx < ns) { ATLAS_ASSERT(xidx < 4 * yidx); - ret = 2 * yidx * (yidx + 1) + 9 + xidx + std::floor(xidx / (double)yidx); + ret = 2 * yidx * (yidx + 1) + nb_pole_nodes_ + 1 + xidx + std::floor(xidx / (double)yidx); } else if (yidx == ns && ns == 1) { ATLAS_ASSERT(xidx < 4); - ret = (xidx != 3 ? 13 + xidx : ghostIdx(2)); + ret = (xidx != 3 ? nb_pole_nodes_ + 5 + xidx : ghostIdx(2)); } else if (yidx == 2 * ns && ns == 1) { ATLAS_ASSERT(xidx < 4); - ret = 16 + xidx; + ret = (nb_pole_nodes_ == 8 ? 16 + xidx : 9 + xidx); } else if (yidx < 3 * ns && ns > 1) { ATLAS_ASSERT(xidx < 4 * ns); - int stg = (yidx - ns) % 2; - if (xidx != 4 * ns - 1 || (xidx == 4 * ns - 1 && stg)) { - ret = 2 * ns * (ns - 1) + 8 + 4 * ns * (yidx - ns + 1) + xidx + (yidx != 3 * ns ? 1 - stg : 0); + int staggering = (yidx - ns) % 2; + if (xidx != 4 * ns - 1 || (xidx == 4 * ns - 1 && staggering)) { + ret = 2 * ns * (ns - 1) + nb_pole_nodes_ + 4 * ns * (yidx - ns + 1) + xidx + + (yidx != 3 * ns ? 1 - staggering : 0); } else { ret = ghostIdx(yidx + 1); @@ -255,17 +300,27 @@ int down_idx(const int xidx, const int yidx, const int ns) { } else if (yidx == 4 * ns - 2) { ATLAS_ASSERT(xidx < 8); - ret = (xidx != 7 ? 12 * ns * ns + 4 + (xidx + 1) / 2 : ghostIdx(4 * ns - 1)); + if (nb_pole_nodes_ == 8) { + ret = (xidx != 7 ? nb_nodes_orig + 4 + (xidx + 1) / 2 : ghostIdx(4 * ns - 1)); + } + else { + ret = (xidx != 7 ? nb_nodes_orig - 3 + (xidx + 1) / 2 : ghostIdx(4 * ns - 1)); + } } else if (yidx == 4 * ns - 1) { ATLAS_ASSERT(xidx < 4); - ret = 12 * ns * ns + 8 + 2 * xidx; + if (nb_pole_nodes_ == 8) { + ret = nb_nodes_orig + nb_pole_nodes_ + 2 * xidx; + } + else { + ret = nb_nodes_orig + 1; + } } else if (yidx < 4 * ns - 1) { int yidxl = yidx - 3 * ns; ATLAS_ASSERT(xidx < 4 * (ns - yidxl)); if (xidx != 4 * (ns - yidxl) - 1) { - ret = 2 * ns * (5 * ns + 1) + 8 + 4 * ns * yidxl - 2 * (yidxl + 1) * yidxl + xidx - + ret = 2 * ns * (5 * ns + 1) + nb_pole_nodes_ + 4 * ns * yidxl - 2 * (yidxl + 1) * yidxl + xidx - std::floor(xidx / (double)(ns - yidxl)); } else { @@ -273,8 +328,8 @@ int down_idx(const int xidx, const int yidx, const int ns) { } } else if (yidx == 4 * ns) { - ATLAS_ASSERT(xidx < 8); - ret = (xidx != 7 ? 12 * ns * ns + 8 + xidx + 1 : ghostIdx(yidx)); + ATLAS_ASSERT(xidx < nb_pole_nodes_); + ret = (xidx != nb_pole_nodes_ - 1 ? nb_nodes_orig + nb_pole_nodes_ + xidx + 1 : ghostIdx(yidx)); } else { throw_AssertionFailed("Invalid value of yidx", Here()); @@ -282,45 +337,54 @@ int down_idx(const int xidx, const int yidx, const int ns) { return ret; } -int right_idx(const int xidx, const int yidx, const int ns) { +// return global_id of the node "to the right of" (xidx,yidx) node +gidx_t HealpixMeshGenerator::right_idx(const int xidx, const int yidx, const int ns) const { ATLAS_ASSERT(yidx <= 4 * ns); - auto ghostIdx = [ns](int latid) { return 12 * ns * ns + 16 + latid; }; - int ret = -1; + const gidx_t nb_nodes_orig = 12 * ns * ns; + auto ghostIdx = [ns, this](int latid) { return this->nb_nodes_ + latid; }; + int ret = -1; if (yidx == 0) { - if (xidx < 8) { - ATLAS_ASSERT(xidx < 8); - } + ATLAS_ASSERT(xidx < nb_pole_nodes_); if (ns == 1) { - ret = (xidx != 7 ? (xidx % 2 ? 8 + (xidx + 1) / 2 : 13 + xidx / 2) : ghostIdx(1)); + ret = (xidx != nb_pole_nodes_ - 1 + ? (xidx % 2 ? nb_pole_nodes_ + (xidx + 1) / 2 : nb_pole_nodes_ + 5 + xidx / 2) + : ghostIdx(1)); } else { - ret = (xidx < 7 ? (xidx % 2 ? 8 + (xidx + 1) / 2 : 13 + xidx) : ghostIdx(1)); + ret = (xidx != nb_pole_nodes_ - 1 ? (xidx % 2 ? nb_pole_nodes_ + (xidx + 1) / 2 : nb_pole_nodes_ + 5 + xidx) + : ghostIdx(1)); } } else if (yidx == 1) { ATLAS_ASSERT(xidx < 4); - ret = (xidx < 7 ? 1 + 2 * xidx : ghostIdx(0)); + if (nb_pole_nodes_ == 8) { + ret = 1 + 2 * xidx; + } + else { + ret = (xidx != 3 ? xidx + 2 : ghostIdx(1)); + } } else if (yidx < ns) { ATLAS_ASSERT(xidx < 4 * yidx); - ret = (xidx != 4 * yidx - 1 ? 2 * yidx * (yidx - 1) + 9 + xidx : ghostIdx(yidx)); + ret = (xidx != 4 * yidx - 1 ? 2 * yidx * (yidx - 1) + nb_pole_nodes_ + 1 + xidx : ghostIdx(yidx)); } else if (yidx == 3 && ns == 1) { ATLAS_ASSERT(xidx < 4); - ret = 21 + 2 * xidx; + ret = (nb_pole_nodes_ == 8 ? 21 + 2 * xidx : (xidx != 3 ? 10 + xidx : ghostIdx(yidx))); } else if (yidx <= 3 * ns) { ATLAS_ASSERT(xidx < 4 * ns + 1); - ret = (xidx != 4 * ns - 1 ? 2 * ns * (ns - 1) + 4 * ns * (yidx - ns) + 9 + xidx : ghostIdx(yidx)); + ret = (xidx != 4 * ns - 1 ? 2 * ns * (ns - 1) + 4 * ns * (yidx - ns) + nb_pole_nodes_ + 1 + xidx + : ghostIdx(yidx)); } else if (yidx < 4 * ns - 1 && ns > 1) { int yidxl = yidx - 3 * ns; ATLAS_ASSERT(xidx < 4 * (ns - yidxl)); if (xidx != 4 * (ns - yidxl) - 1) { - ret = 2 * ns * (5 * ns + 1) + 4 * ns * (yidx - 3 * ns - 1) - 2 * (yidx - 3 * ns) * (yidx - 3 * ns - 1) + 9 + - xidx; + ret = 2 * ns * (5 * ns + 1) + 4 * ns * (yidx - 3 * ns - 1) - 2 * (yidx - 3 * ns) * (yidx - 3 * ns - 1) + + nb_pole_nodes_ + 1 + xidx; } else { ret = ghostIdx(yidx); @@ -328,12 +392,17 @@ int right_idx(const int xidx, const int yidx, const int ns) { } else if (yidx == 4 * ns - 1) { ATLAS_ASSERT(xidx < 4); - ret = 12 * ns * ns + 9 + 2 * xidx; + if (nb_pole_nodes_ == 8) { + ret = nb_nodes_orig + nb_pole_nodes_ + 1 + 2 * xidx; + } + else { + ret = (xidx != 3 ? nb_nodes_orig - 2 + xidx : ghostIdx(yidx)); + } } else if (yidx == 4 * ns) { - ATLAS_ASSERT(xidx < 8); - if (xidx != 7) { - ret = (xidx % 2 ? 12 * ns * ns + 4 + (xidx + 1) / 2 : 12 * ns * ns + 4 - (ns == 1 ? 3 : 7) + xidx); + ATLAS_ASSERT(xidx < nb_pole_nodes_); + if (xidx != nb_pole_nodes_ - 1) { + ret = (xidx % 2 ? nb_nodes_orig + 4 + (xidx + 1) / 2 : nb_nodes_orig + 4 - (ns == 1 ? 3 : 7) + xidx); } else { ret = ghostIdx(yidx - 1); @@ -341,7 +410,6 @@ int right_idx(const int xidx, const int yidx, const int ns) { } return ret; } -} // namespace void HealpixMeshGenerator::generate(const Grid& grid, Mesh& mesh) const { ATLAS_ASSERT(HealpixGrid(grid), "Grid could not be cast to a HealpixGrid"); @@ -422,14 +490,23 @@ void HealpixMeshGenerator::generate_mesh(const StructuredGrid& grid, const grid: ATLAS_ASSERT(HealpixGrid(grid)); - const int mypart = options.get("part"); - const int nparts = options.get("nb_parts"); - const int ny = grid.ny() + 2; - const int ns = (ny - 1) / 4; - const int nvertices = 12 * ns * ns + 16; + const int mypart = options.get("part"); + const int nparts = options.get("nb_parts"); + const bool three_dimensional = options.get("3d"); + const int nb_pole_nodes = three_dimensional ? 1 : 8; + const int ny = grid.ny() + 2; + const int ns = (ny - 1) / 4; + const int nvertices = 12 * ns * ns + 2 * nb_pole_nodes; + + + nb_pole_nodes_ = nb_pole_nodes; + nb_points_ = 12 * ns * ns + (nb_pole_nodes == 8 ? 8 : 0); + nb_nodes_ = nvertices; int inode; - auto latPoints = [ny, &grid](int latid) { return (latid == 0 ? 8 : (latid == ny - 1 ? 8 : grid.nx()[latid - 1])); }; + auto nb_lat_nodes = [ny, nb_pole_nodes, &grid](int latid) { + return ((latid == 0) or (latid == ny - 1) ? nb_pole_nodes : grid.nx()[latid - 1]); + }; int ii, ix, iy, ii_ghost, ii_glb; int iy_min, iy_max; // a belt (iy_min:iy_max) surrounding the nodes on this processor @@ -439,22 +516,40 @@ void HealpixMeshGenerator::generate_mesh(const StructuredGrid& grid, const grid: std::vector local_idx(nvertices, -1); std::vector current_idx(nparts, 0); // index counter for each proc + auto compute_part = [&](int iy, gidx_t ii_glb) -> int { + // nodes at the pole belong to proc_0 (north) and proc_maxRank (south) + // a node gets its proc rank from the element for which this node would be its west vertex + return (iy == 0 ? 0 : (iy == ny - 1 ? mpi::comm().size() - 1 : distribution.partition(ii_glb - nb_pole_nodes))); + }; + +#if DEBUG_OUTPUT_DETAIL + for (iy = 0; iy < ny; iy++) { + int nx = nb_lat_nodes(iy); + for (ix = 0; ix < nx; ix++) { + Log::info() << "iy, ix, glb_idx, up_idx, down_idx, right_idx : " << iy << ", " << ix << ", " + << idx_xy_to_x(ix, iy, ns) + 1 << ", " << up_idx(ix, iy, ns) + 1 << ", " + << down_idx(ix, iy, ns) + 1 << ", " << right_idx(ix, iy, ns) + 1 << std::endl; + } + Log::info() << std::endl; + } +#endif + // loop over all points to determine local indices and surrounding rectangle - ii_glb = 0; + ii_glb = 0; // global index starting from 0 iy_min = ny + 1; iy_max = 0; nnodes_nonghost = 0; for (iy = 0; iy < ny; iy++) { - int nx = latPoints(iy); + int nx = nb_lat_nodes(iy); for (ix = 0; ix < nx; ix++) { - int proc_id = (iy == 0 ? 0 : (iy == ny - 1 ? mpi::comm().size() - 1 : distribution.partition(ii_glb - 8))); + int proc_id = compute_part(iy, ii_glb); local_idx[ii_glb] = current_idx[proc_id]++; if (proc_id == mypart) { ++nnodes_nonghost; iy_min = std::min(iy_min, iy); iy_max = std::max(iy_max, iy); } - ++ii_glb; // global index + ++ii_glb; } } @@ -462,7 +557,7 @@ void HealpixMeshGenerator::generate_mesh(const StructuredGrid& grid, const grid: inode = 0; Log::info() << "local_idx : " << std::endl; for (size_t ilat = 0; ilat < ny; ilat++) { - for (size_t ilon = 0; ilon < latPoints(ilat); ilon++) { + for (size_t ilon = 0; ilon < nb_lat_nodes(ilat); ilon++) { Log::info() << std::setw(4) << local_idx[inode]; inode++; } @@ -471,7 +566,7 @@ void HealpixMeshGenerator::generate_mesh(const StructuredGrid& grid, const grid: inode = 0; Log::info() << "global_idx : " << std::endl; for (size_t ilat = 0; ilat < ny; ilat++) { - for (size_t ilon = 0; ilon < latPoints(ilat); ilon++) { + for (size_t ilon = 0; ilon < nb_lat_nodes(ilat); ilon++) { Log::info() << std::setw(4) << inode; inode++; } @@ -494,7 +589,8 @@ void HealpixMeshGenerator::generate_mesh(const StructuredGrid& grid, const grid: ++iy_max; } for (int iy = iy_min; iy <= iy_max; iy++) { - nnodes_SB += latPoints(iy) + 1; + // (east) periodic point adds +1 here + nnodes_SB += nb_lat_nodes(iy) + 1; } #if DEBUG_OUTPUT @@ -506,38 +602,23 @@ void HealpixMeshGenerator::generate_mesh(const StructuredGrid& grid, const grid: std::vector local_idx_SB(nnodes_SB, -1); std::vector is_ghost_SB(nnodes_SB, true); - // global starting node index for the partition + // starting from index 0, first global node-index for this partition int parts_sidx = idx_xy_to_x(0, iy_min, ns); - auto compute_part = [&](int ix, int iy, gidx_t ii_glb) -> int { - if (ii_glb < 8) { - // HACK! expects equal_regions partitioner. Better would be partition of attached element of which this node would be the North-West point. - return 0; - } - if (ii_glb > nvertices - 9) { - // HACK! expects equal_regions partitioner. Better would be partition of attached element of which this node would be the South-West point. - // Also, we should not have mpi here. - return mpi::comm().size() - 1; - } - return distribution.partition(idx_xy_to_x(ix, iy, ns) - 8); - }; - - ii = 0; // index inside SB - ii_ghost = nnodes_SB - (iy_max - iy_min + 1); + ii = 0; // index inside SB + ii_ghost = nnodes_SB - (iy_max - iy_min + 1); // first local ghost idx for (iy = iy_min; iy <= iy_max; iy++) { - int nx = latPoints(iy) + 1; - ii_glb = ii + parts_sidx; - int part0 = compute_part(0, iy, ii_glb); + int nx = nb_lat_nodes(iy) + 1; for (ix = 0; ix < nx; ix++) { if (ix != nx - 1) { ii_glb = ii + parts_sidx; - parts_SB[ii] = compute_part(ix, iy, ii_glb); + parts_SB[ii] = compute_part(iy, ii_glb); local_idx_SB[ii] = ii; - is_ghost_SB[ii] = !((parts_SB[ii] == mypart)); + is_ghost_SB[ii] = !(parts_SB[ii] == mypart); ++ii; } else { - parts_SB[ii_ghost] = part0; + parts_SB[ii_ghost] = compute_part(iy, ii_glb); local_idx_SB[ii_ghost] = ii_ghost; is_ghost_SB[ii_ghost] = true; ++ii_ghost; @@ -575,38 +656,41 @@ void HealpixMeshGenerator::generate_mesh(const StructuredGrid& grid, const grid: ii = 0; int iil; for (iy = iy_min; iy <= iy_max; iy++) { - int nx = latPoints(iy); + int nx = nb_lat_nodes(iy); for (ix = 0; ix < nx; ix++) { int not_duplicate_cell = (iy == 0 ? ix % 2 : 1) * (iy == ny - 1 ? ix % 2 : 1); + if ((iy == 0 or iy == ny - 1) and nb_pole_nodes == 1) { + not_duplicate_cell = false; + } - if (!is_ghost_SB[ii] && not_duplicate_cell) { + if (not is_ghost_SB[ii] && not_duplicate_cell) { // mark this node as being used - if (!is_node_SB[ii]) { + if (not is_node_SB[ii]) { ++nnodes; is_node_SB[ii] = true; } ++ncells; - const int glb2loc_ghost_offset = -nnodes_SB + iy_max + 12 * ns * ns + 17; + const int glb2loc_ghost_offset = -nnodes_SB + iy_max + nb_nodes_ + 1; // mark upper corner iil = up_idx(ix, iy, ns); - iil -= (iil < 12 * ns * ns + 16 ? parts_sidx : glb2loc_ghost_offset); - if (!is_node_SB[iil]) { + iil -= (iil < nb_nodes_ ? parts_sidx : glb2loc_ghost_offset); + if (not is_node_SB[iil]) { ++nnodes; is_node_SB[iil] = true; } // mark lower corner iil = down_idx(ix, iy, ns); - iil -= (iil < 12 * ns * ns + 16 ? parts_sidx : glb2loc_ghost_offset); - if (!is_node_SB[iil]) { + iil -= (iil < nb_nodes_ ? parts_sidx : glb2loc_ghost_offset); + if (not is_node_SB[iil]) { ++nnodes; is_node_SB[iil] = true; } // mark right corner iil = right_idx(ix, iy, ns); - iil -= (iil < 12 * ns * ns + 16 ? parts_sidx : glb2loc_ghost_offset); - if (!is_node_SB[iil]) { + iil -= (iil < nb_nodes_ ? parts_sidx : glb2loc_ghost_offset); + if (not is_node_SB[iil]) { ++nnodes; is_node_SB[iil] = true; } @@ -668,10 +752,10 @@ void HealpixMeshGenerator::generate_mesh(const StructuredGrid& grid, const grid: inode_ghost = nnodes_nonghost; // ghost nodes start counting after nonghost nodes ii = 0; // index inside SB for (iy = iy_min; iy <= iy_max; iy++) { - int nx = latPoints(iy) + 1; + int nx = nb_lat_nodes(iy) + 1; for (ix = 0; ix < nx; ix++) { int iil = idx_xy_to_x(ix, iy, ns); - iil -= (iil < 12 * ns * ns + 16 ? parts_sidx : -nnodes_SB + iy_max + 12 * ns * ns + 17); + iil -= (iil < nb_nodes_ ? parts_sidx : -nnodes_SB + iy_max + nb_nodes_ + 1); if (is_node_SB[iil]) { // set node counter if (is_ghost_SB[iil]) { @@ -684,18 +768,18 @@ void HealpixMeshGenerator::generate_mesh(const StructuredGrid& grid, const grid: // flags Topology::reset(flags(inode)); - glb_idx(inode) = idx_xy_to_x(ix, iy, ns) + 1; + glb_idx(inode) = 1 + match_idx(idx_xy_to_x(ix, iy, ns), ns); // grid coordinates double _xy[2]; double xy1[2], xy2[2]; if (iy == 0) { - _xy[0] = 45. * ix; + _xy[0] = (nb_pole_nodes == 8 ? 45. * ix : 180.); _xy[1] = 90.; Topology::set(flags(inode), Topology::BC); } else if (iy == ny - 1) { - _xy[0] = 45. * ix; + _xy[0] = (nb_pole_nodes == 8 ? 45. * ix : 180.); _xy[1] = -90.; Topology::set(flags(inode), Topology::BC); } @@ -772,43 +856,49 @@ void HealpixMeshGenerator::generate_mesh(const StructuredGrid& grid, const grid: } ii = 0; // index inside SB (surrounding belt) - int jcell_offset = 0; + int jcell_offset = 0; // global index offset due to extra points at the north pole for (iy = iy_min; iy <= iy_max; iy++) { - int nx = latPoints(iy) + 1; + int nx = nb_lat_nodes(iy) + 1; for (ix = 0; ix < nx; ix++) { int not_duplicate_cell = (iy == 0 ? ix % 2 : 1) * (iy == ny - 1 ? ix % 2 : 1); int iil = idx_xy_to_x(ix, iy, ns); - iil -= (iil < 12 * ns * ns + 16 ? parts_sidx : -nnodes_SB + iy_max + 12 * ns * ns + 17); + iil -= (iil < nb_nodes_ ? parts_sidx : -nnodes_SB + iy_max + nb_nodes_ + 1); if (!is_ghost_SB[iil] && not_duplicate_cell) { // define cell corners (local indices) quad_nodes[0] = local_idx_SB[iil]; - quad_nodes[1] = down_idx(ix, iy, ns); // point to the right - quad_nodes[1] -= - (quad_nodes[1] < 12 * ns * ns + 16 ? parts_sidx : -nnodes_SB + iy_max + 12 * ns * ns + 17); - quad_nodes[1] = local_idx_SB[quad_nodes[1]]; + quad_nodes[1] = down_idx(ix, iy, ns); // point to the right, global idx + quad_nodes[1] -= (quad_nodes[1] < nb_nodes_ ? parts_sidx : -nnodes_SB + iy_max + nb_nodes_ + 1); + quad_nodes[1] = local_idx_SB[quad_nodes[1]]; // now, local idx - quad_nodes[2] = right_idx(ix, iy, ns); // point above right - quad_nodes[2] -= - (quad_nodes[2] < 12 * ns * ns + 16 ? parts_sidx : -nnodes_SB + iy_max + 12 * ns * ns + 17); - quad_nodes[2] = local_idx_SB[quad_nodes[2]]; + quad_nodes[2] = right_idx(ix, iy, ns); // point above right, global idx + quad_nodes[2] -= (quad_nodes[2] < nb_nodes_ ? parts_sidx : -nnodes_SB + iy_max + nb_nodes_ + 1); + quad_nodes[2] = local_idx_SB[quad_nodes[2]]; // now, local idx - quad_nodes[3] = up_idx(ix, iy, ns); // point above - quad_nodes[3] -= - (quad_nodes[3] < 12 * ns * ns + 16 ? parts_sidx : -nnodes_SB + iy_max + 12 * ns * ns + 17); - quad_nodes[3] = local_idx_SB[quad_nodes[3]]; + quad_nodes[3] = up_idx(ix, iy, ns); // point above, global idx + quad_nodes[3] -= (quad_nodes[3] < nb_nodes_ ? parts_sidx : -nnodes_SB + iy_max + nb_nodes_ + 1); + quad_nodes[3] = local_idx_SB[quad_nodes[3]]; // now, local idx node_connectivity.set(jcell, quad_nodes); if (iy == 0) { - cells_glb_idx(jcell) = 12 * ns * ns + 1 + ix / 2; - jcell_offset++; + if (nb_pole_nodes == 8) { + cells_glb_idx(jcell) = 12 * ns * ns + 1 + ix / 2; + jcell_offset++; + } } else if (iy == ny - 1) { - cells_glb_idx(jcell) = 12 * ns * ns + 5 + ix / 2; - jcell_offset++; + if (nb_pole_nodes == 8) { + cells_glb_idx(jcell) = 12 * ns * ns + 5 + ix / 2; + jcell_offset++; + } } else { - cells_glb_idx(jcell) = parts_sidx + iil - 3 - (mypart != 0 ? 4 : jcell_offset); + if (nb_pole_nodes == 8) { + cells_glb_idx(jcell) = parts_sidx + iil - 3 - (mypart != 0 ? 4 : jcell_offset); + } + else { + cells_glb_idx(jcell) = parts_sidx + iil; + } } cells_part(jcell) = mypart; #if DEBUG_OUTPUT_DETAIL @@ -846,10 +936,10 @@ void HealpixMeshGenerator::generate_mesh(const StructuredGrid& grid, const grid: nodes.metadata().set("NbVirtualPts", size_t(0)); nodes.global_index().metadata().set("human_readable", true); nodes.global_index().metadata().set("min", 1); - nodes.global_index().metadata().set("max", nvertices + 4 * ns + 1); + nodes.global_index().metadata().set("max", nb_nodes_ + grid.ny() + 2); mesh.cells().global_index().metadata().set("human_readable", true); mesh.cells().global_index().metadata().set("min", 1); - mesh.cells().global_index().metadata().set("max", grid.size() + 8); + mesh.cells().global_index().metadata().set("max", nb_points_); //generateGlobalElementNumbering(mesh); } // generate_mesh diff --git a/src/atlas/meshgenerator/detail/HealpixMeshGenerator.h b/src/atlas/meshgenerator/detail/HealpixMeshGenerator.h index 28df7872b..a8feeab19 100644 --- a/src/atlas/meshgenerator/detail/HealpixMeshGenerator.h +++ b/src/atlas/meshgenerator/detail/HealpixMeshGenerator.h @@ -58,8 +58,17 @@ class HealpixMeshGenerator : public MeshGenerator::Implementation { void generate_mesh(const StructuredGrid&, const grid::Distribution&, Mesh& m) const; + gidx_t match_idx(gidx_t gidx, const int ns) const; + gidx_t idx_xy_to_x(const int xidx, const int yidx, const int ns) const; + gidx_t up_idx(const int xidx, const int yidx, const int ns) const; + gidx_t down_idx(const int xidx, const int yidx, const int ns) const; + gidx_t right_idx(const int xidx, const int yidx, const int ns) const; + private: util::Metadata options; + mutable gidx_t nb_points_; + mutable gidx_t nb_nodes_; + mutable int nb_pole_nodes_; }; //---------------------------------------------------------------------------------------------------------------------- From 0f8712e4fa5017293c76330b86d3ce064d4c9822 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Sun, 8 May 2022 18:31:48 +0200 Subject: [PATCH 14/53] ATLAS-358 Pentagon element type --- src/atlas/mesh/ElementType.h | 32 ++++++++ src/atlas/output/detail/GmshIO.cc | 19 +++-- src/tests/mesh/CMakeLists.txt | 5 ++ src/tests/mesh/test_pentagon_element.cc | 102 ++++++++++++++++++++++++ 4 files changed, 152 insertions(+), 6 deletions(-) create mode 100644 src/tests/mesh/test_pentagon_element.cc diff --git a/src/atlas/mesh/ElementType.h b/src/atlas/mesh/ElementType.h index 88a3447c1..c96464d3c 100644 --- a/src/atlas/mesh/ElementType.h +++ b/src/atlas/mesh/ElementType.h @@ -200,6 +200,38 @@ class Line : public Edge { return s; } }; + +class Pentagon : public Face { +public: + enum + { + EDGES = 5 + }; + enum + { + VERTICES = 5 + }; + enum + { + FACETS = EDGES + }; + enum + { + RIDGES = VERTICES + }; + virtual ~Pentagon() {} + virtual bool parametric() const { return false; } + virtual idx_t nb_vertices() const { return VERTICES; } + virtual idx_t nb_edges() const { return EDGES; } + virtual idx_t nb_nodes() const { return VERTICES; } + virtual idx_t nb_facets() const { return FACETS; } + virtual idx_t nb_ridges() const { return RIDGES; } + virtual const std::string& name() const { + static std::string s("Pentagon"); + return s; + } +}; + } // namespace temporary extern "C" { diff --git a/src/atlas/output/detail/GmshIO.cc b/src/atlas/output/detail/GmshIO.cc index 83e91b482..4c4d84d65 100644 --- a/src/atlas/output/detail/GmshIO.cc +++ b/src/atlas/output/detail/GmshIO.cc @@ -1024,8 +1024,11 @@ void GmshIO::write(const Mesh& mesh, const PathName& file_path) const { for (const mesh::HybridElements* hybrid : grouped_elements) { for (idx_t etype = 0; etype < hybrid->nb_types(); ++etype) { - const mesh::Elements& elements = hybrid->elements(etype); - const mesh::ElementType& element_type = elements.element_type(); + const mesh::Elements& elements = hybrid->elements(etype); + const mesh::ElementType& element_type = elements.element_type(); + const mesh::BlockConnectivity& node_connectivity = elements.node_connectivity(); + size_t nb_nodes = node_connectivity.cols(); + int gmsh_elem_type; if (element_type.name() == "Line") { gmsh_elem_type = 1; @@ -1036,11 +1039,15 @@ void GmshIO::write(const Mesh& mesh, const PathName& file_path) const { else if (element_type.name() == "Quadrilateral") { gmsh_elem_type = 3; } + else if (element_type.name() == "Pentagon") { + // Hack: treat as quadrilateral and ignore 5th point + gmsh_elem_type = 3; + nb_nodes = 4; + } else { ATLAS_NOTIMPLEMENTED; } - const mesh::BlockConnectivity& node_connectivity = elements.node_connectivity(); auto elems_glb_idx = elements.view(elements.global_index()); auto elems_partition = elements.view(elements.partition()); @@ -1092,12 +1099,12 @@ void GmshIO::write(const Mesh& mesh, const PathName& file_path) const { data[1] = 1; data[2] = 1; data[3] = 1; - size_t datasize = sizeof(int) * (5 + node_connectivity.cols()); + size_t datasize = sizeof(int) * (5 + nb_nodes); for (idx_t elem = 0; elem < nb_elems; ++elem) { if (include_ghost || !elems_halo(elem)) { data[0] = elems_glb_idx(elem); data[4] = elems_partition(elem); - for (idx_t n = 0; n < node_connectivity.cols(); ++n) { + for (idx_t n = 0; n < nb_nodes; ++n) { data[5 + n] = glb_idx(node_connectivity(elem, n)); } file.write(reinterpret_cast(&data), datasize); @@ -1111,7 +1118,7 @@ void GmshIO::write(const Mesh& mesh, const PathName& file_path) const { for (idx_t elem = 0; elem < elements.size(); ++elem) { if (include(elem)) { file << elems_glb_idx(elem) << elem_info << elems_partition(elem); - for (idx_t n = 0; n < node_connectivity.cols(); ++n) { + for (idx_t n = 0; n < nb_nodes; ++n) { file << " " << glb_idx(node_connectivity(elem, n)); } file << "\n"; diff --git a/src/tests/mesh/CMakeLists.txt b/src/tests/mesh/CMakeLists.txt index b25d98b6a..7dd3a7007 100644 --- a/src/tests/mesh/CMakeLists.txt +++ b/src/tests/mesh/CMakeLists.txt @@ -140,3 +140,8 @@ atlas_add_cuda_test( ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ) +ecbuild_add_test( TARGET atlas_test_pentagon_element + SOURCES test_pentagon_element.cc + LIBS atlas + ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} +) diff --git a/src/tests/mesh/test_pentagon_element.cc b/src/tests/mesh/test_pentagon_element.cc new file mode 100644 index 000000000..59e3e84f0 --- /dev/null +++ b/src/tests/mesh/test_pentagon_element.cc @@ -0,0 +1,102 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#include "atlas/mesh.h" +#include "atlas/output/Gmsh.h" +#include "atlas/output/Output.h" + +#include "tests/AtlasTestEnvironment.h" +#include "tests/TestMeshes.h" + +namespace atlas { +namespace test { + +//----------------------------------------------------------------------------- + +CASE("test_pentagon_like_healpix") { + Mesh mesh; + + // 1 2 3 4 5 + // +------+------+------+------+ + // | 0 | 1 | 2 | 3 | + // +6 +7 +8 +9 +10 + // \ / \ / \ / \ / + // + + + + + // 11 12 13 14 + + auto points = std::vector{ + {0., 90.}, // 1 + {90, 90.}, // 2 + {180., 90.}, // 3 + {270., 90.}, // 4 + {360., 90.}, // 5 + {0., 80.}, // 6 + {90, 80.}, // 7 + {180., 80.}, // 8 + {270., 80.}, // 9 + {360., 80.}, // 10 + {45., 70.}, // 11 + {135., 70.}, // 12 + {225., 70.}, // 13 + {315., 70.}, // 14 + }; + + + // nodes + { + mesh.nodes().resize(14); + auto xy = array::make_view(mesh.nodes().xy()); + auto lonlat = array::make_view(mesh.nodes().lonlat()); + auto nodes_glb_idx = array::make_view(mesh.nodes().global_index()); + for (idx_t i = 0; i < mesh.nodes().size(); ++i) { + lonlat(i, LON) = points[i][LON]; + lonlat(i, LAT) = points[i][LAT]; + nodes_glb_idx(i) = i + 1; + } + xy.assign(lonlat); + } + + // elements + { + mesh.cells().add(new mesh::temporary::Pentagon(), 4); + auto connect = [&](idx_t cell, std::array nodes) { + mesh.cells().node_connectivity().block(0).set(cell, nodes.data()); + }; + connect(0, {0, 5, 10, 6, 1}); + connect(1, {1, 6, 11, 7, 2}); + connect(2, {2, 7, 12, 8, 3}); + connect(3, {3, 8, 13, 9, 4}); + + auto cell_glb_idx = array::make_view(mesh.cells().global_index()); + for (idx_t i = 0; i < mesh.cells().size(); ++i) { + cell_glb_idx(i) = i + 1; + } + } + + { + output::Gmsh gmsh("test_pentagon_xyz.msh", util::Config("coordinates", "xyz")); + gmsh.write(mesh); + } + + { + output::Gmsh gmsh("test_pentagon_lonlat.msh", util::Config("coordinates", "lonlat")); + gmsh.write(mesh); + } +} + + +//----------------------------------------------------------------------------- + +} // namespace test +} // namespace atlas + +int main(int argc, char** argv) { + return atlas::test::run(argc, argv); +} From 2273808b95f39aac2e0b90d99aa8805756e11267 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Fri, 13 May 2022 13:51:45 +0100 Subject: [PATCH 15/53] ATLAS-358 Support degenerate pentagon in FiniteElement interpolation --- .../method/unstructured/FiniteElement.cc | 38 +++++++++++-------- src/atlas/util/Point.h | 8 ++++ src/tests/mesh/test_pentagon_element.cc | 27 +++++++++++++ 3 files changed, 58 insertions(+), 15 deletions(-) diff --git a/src/atlas/interpolation/method/unstructured/FiniteElement.cc b/src/atlas/interpolation/method/unstructured/FiniteElement.cc index 2ec6ad994..c949e3858 100644 --- a/src/atlas/interpolation/method/unstructured/FiniteElement.cc +++ b/src/atlas/interpolation/method/unstructured/FiniteElement.cc @@ -337,7 +337,22 @@ Method::Triplets FiniteElement::projectPointToElements(size_t ip, const ElemInde const idx_t elem_id = idx_t((*itc).value().payload()); ATLAS_ASSERT(elem_id < connectivity_->rows()); - const idx_t nb_cols = connectivity_->cols(elem_id); + const idx_t nb_cols = [&]() { + int nb_cols = connectivity_->cols(elem_id); + if (nb_cols == 5) { + // Check if pentagon degenerates to quad. Otherwise abort. + // For now only check if first and last point coincide. + auto i1 = (*connectivity_)(elem_id, 0); + auto iN = (*connectivity_)(elem_id, nb_cols - 1); + auto first = PointXYZ{(*icoords_)(i1, XX), (*icoords_)(i1, YY), (*icoords_)(i1, ZZ)}; + auto last = PointXYZ{(*icoords_)(iN, XX), (*icoords_)(iN, YY), (*icoords_)(iN, ZZ)}; + if (first == last) { + return 4; + } + } + return nb_cols; + }(); + ATLAS_ASSERT(nb_cols == 3 || nb_cols == 4); for (idx_t i = 0; i < nb_cols; ++i) { @@ -437,12 +452,9 @@ Method::Triplets FiniteElement::projectPointToElements(size_t ip, const ElemInde if (nb_cols == 3) { /* triangle */ - element::Triag3D triag(PointXYZ{(*icoords_)(idx[0], size_t(0)), (*icoords_)(idx[0], size_t(1)), - (*icoords_)(idx[0], size_t(2))}, - PointXYZ{(*icoords_)(idx[1], size_t(0)), (*icoords_)(idx[1], size_t(1)), - (*icoords_)(idx[1], size_t(2))}, - PointXYZ{(*icoords_)(idx[2], size_t(0)), (*icoords_)(idx[2], size_t(1)), - (*icoords_)(idx[2], size_t(2))}); + element::Triag3D triag(PointXYZ{(*icoords_)(idx[0], XX), (*icoords_)(idx[0], YY), (*icoords_)(idx[0], ZZ)}, + PointXYZ{(*icoords_)(idx[1], XX), (*icoords_)(idx[1], YY), (*icoords_)(idx[1], ZZ)}, + PointXYZ{(*icoords_)(idx[2], XX), (*icoords_)(idx[2], YY), (*icoords_)(idx[2], ZZ)}); // pick an epsilon based on a characteristic length (sqrt(area)) // (this scales linearly so it better compares with linear weights u,v,w) @@ -483,14 +495,10 @@ Method::Triplets FiniteElement::projectPointToElements(size_t ip, const ElemInde } else { /* quadrilateral */ - element::Quad3D quad(PointXYZ{(*icoords_)(idx[0], (size_t)0), (*icoords_)(idx[0], (size_t)1), - (*icoords_)(idx[0], (size_t)2)}, - PointXYZ{(*icoords_)(idx[1], (size_t)0), (*icoords_)(idx[1], (size_t)1), - (*icoords_)(idx[1], (size_t)2)}, - PointXYZ{(*icoords_)(idx[2], (size_t)0), (*icoords_)(idx[2], (size_t)1), - (*icoords_)(idx[2], (size_t)2)}, - PointXYZ{(*icoords_)(idx[3], (size_t)0), (*icoords_)(idx[3], (size_t)1), - (*icoords_)(idx[3], (size_t)2)}); + element::Quad3D quad(PointXYZ{(*icoords_)(idx[0], XX), (*icoords_)(idx[0], YY), (*icoords_)(idx[0], ZZ)}, + PointXYZ{(*icoords_)(idx[1], XX), (*icoords_)(idx[1], YY), (*icoords_)(idx[1], ZZ)}, + PointXYZ{(*icoords_)(idx[2], XX), (*icoords_)(idx[2], YY), (*icoords_)(idx[2], ZZ)}, + PointXYZ{(*icoords_)(idx[3], XX), (*icoords_)(idx[3], YY), (*icoords_)(idx[3], ZZ)}); // pick an epsilon based on a characteristic length (sqrt(area)) // (this scales linearly so it better compares with linear weights u,v,w) diff --git a/src/atlas/util/Point.h b/src/atlas/util/Point.h index 91994a221..aadb1fcf3 100644 --- a/src/atlas/util/Point.h +++ b/src/atlas/util/Point.h @@ -36,6 +36,14 @@ inline bool operator!=(const Point2& p1, const Point2& p2) { return !eckit::geometry::points_equal(p1, p2); } +inline bool operator==(const Point3& p1, const Point3& p2) { + return eckit::geometry::points_equal(p1, p2); +} +inline bool operator!=(const Point3& p1, const Point3& p2) { + return !eckit::geometry::points_equal(p1, p2); +} + + /// @brief Point in arbitrary XY-coordinate system class PointXY : public eckit::geometry::Point2 { using array_t = std::array; diff --git a/src/tests/mesh/test_pentagon_element.cc b/src/tests/mesh/test_pentagon_element.cc index 59e3e84f0..454aaab0d 100644 --- a/src/tests/mesh/test_pentagon_element.cc +++ b/src/tests/mesh/test_pentagon_element.cc @@ -8,7 +8,10 @@ * nor does it submit to any jurisdiction. */ +#include "atlas/functionspace.h" +#include "atlas/interpolation.h" #include "atlas/mesh.h" +#include "atlas/option.h" #include "atlas/output/Gmsh.h" #include "atlas/output/Output.h" @@ -89,6 +92,30 @@ CASE("test_pentagon_like_healpix") { output::Gmsh gmsh("test_pentagon_lonlat.msh", util::Config("coordinates", "lonlat")); gmsh.write(mesh); } + + auto src_fs = functionspace::NodeColumns(mesh); + auto tgt_fs = functionspace::PointCloud({{45, 80}, {0, 89}, {180, 89}, {170, 89}}); + + Interpolation interpolation(option::type("finite-element"), src_fs, tgt_fs); + + auto src_field = src_fs.createField(); + auto tgt_field = tgt_fs.createField(); + + auto src_lonlat = array::make_view(src_fs.lonlat()); + auto src = array::make_view(src_field); + for (auto i = 0; i < src_fs.size(); ++i) { + src(i) = std::cos(src_lonlat(i, LAT)) * std::sin(src_lonlat(i, LON)); + } + + interpolation.execute(src_field, tgt_field); + + auto tgt_lonlat = array::make_view(tgt_fs.lonlat()); + auto tgt = array::make_view(tgt_field); + for (auto i = 0; i < tgt_fs.size(); ++i) { + double analytical = std::cos(tgt_lonlat(i, LAT)) * std::sin(tgt_lonlat(i, LON)); + Log::info() << i << " " << PointLonLat{tgt_lonlat(i, LON), tgt_lonlat(i, LAT)} << " : interpolated=" << tgt(i) + << "; expected=" << analytical << std::endl; + } } From 168f3d82dacfd43f0e305b1813e6f75fed4ecca6 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 17 May 2022 01:55:10 +0100 Subject: [PATCH 16/53] ATLAS-361 Add SphericalHarmonic function This can be used to initialise a source field for interpolation or other algorithms. --- src/atlas/CMakeLists.txt | 2 + src/atlas/util/function/SphericalHarmonic.cc | 142 +++++++++++++++++++ src/atlas/util/function/SphericalHarmonic.h | 53 +++++++ 3 files changed, 197 insertions(+) create mode 100644 src/atlas/util/function/SphericalHarmonic.cc create mode 100644 src/atlas/util/function/SphericalHarmonic.h diff --git a/src/atlas/CMakeLists.txt b/src/atlas/CMakeLists.txt index 9ff575ddb..5b3e7fbf8 100644 --- a/src/atlas/CMakeLists.txt +++ b/src/atlas/CMakeLists.txt @@ -749,6 +749,8 @@ util/detail/BlackMagic.h util/detail/Cache.h util/detail/Debug.h util/detail/KDTree.h +util/function/SphericalHarmonic.h +util/function/SphericalHarmonic.cc util/function/VortexRollup.h util/function/VortexRollup.cc ) diff --git a/src/atlas/util/function/SphericalHarmonic.cc b/src/atlas/util/function/SphericalHarmonic.cc new file mode 100644 index 000000000..f85b98e7e --- /dev/null +++ b/src/atlas/util/function/SphericalHarmonic.cc @@ -0,0 +1,142 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#include +#include + +#include "atlas/runtime/Exception.h" +#include "atlas/util/Constants.h" +#include "atlas/util/function/SphericalHarmonic.h" + +namespace atlas { + +namespace util { + +namespace function { + + +namespace { +static double factorial(double v) { + if (v == 0) { + return 1; + } + double result = v; + while (--v > 0) { + result *= v; + } + return result; +} + +static double double_factorial(double x) { + if (x == 0 || x == -1) { + return 1; + } + + double result = x; + while ((x -= 2) > 0) { + result *= x; + } + return result; +} + +// Associated Legendre Polynomial +static double P(const int n, const int m, const double x) { + // No recursive calculation needed + if (n == m) { + return (std::pow(-1.0, m) * double_factorial(2 * m - 1) * std::pow(std::sqrt(1. - x * x), m)); + } + + if (n == m + 1) { + return x * (2 * m + 1) * P(m, m, x); + } + + // Formula 1 + return (x * (2 * n - 1) * P(n - 1, m, x) - (n + m - 1) * P(n - 2, m, x)) / (n - m); +} + +static double K(const int n, const int m) { + //When m is less than 0, multiply - 1 to pass in + return std::sqrt(((2 * n + 1) * factorial(n - m)) / (4 * M_PI * factorial(n + m))); +} +} // namespace + +double spherical_harmonic(int n, int m, double lon, double lat) { + const int abs_m = std::abs(m); + + ATLAS_ASSERT(n >= abs_m); + + double colat = (90. - lat) * Constants::degreesToRadians(); + lon *= Constants::degreesToRadians(); + + if (m == 0) { + return (K(n, 0) * P(n, 0, std::cos(colat))); + } + + if (m > 0) { + return (M_SQRT2 * K(n, m) * std::cos(m * lon) * P(n, m, std::cos(colat))); + } + + // When m is less than 0, multiply - 1 in advance and send it to K + return (M_SQRT2 * K(n, abs_m) * std::sin(abs_m * lon) * P(n, abs_m, std::cos(colat))); +} + +SphericalHarmonic::SphericalHarmonic(int n, int m, bool caching) { + const int abs_m = std::abs(m); + + ATLAS_ASSERT(n >= abs_m); + + const double Knm = K(n, abs_m); + std::function Pnm = [n, abs_m](double x) { return P(n, abs_m, x); }; + + if (caching) { + // Computation of associated legendre polynomials is + // very expensive. With this trick (off by default), + // we can cache repeated values, useful for structured + // grids with constant latitude. + Pnm = [Pnm](double x) { + static std::map memo; + + auto it = memo.find(x); + if (it != memo.end()) { + return it->second; + } + auto result = Pnm(x); + memo[x] = result; + return result; + }; + } + + if (m == 0) { + Y_ = [Knm, Pnm](double lon, double colat) { return Knm * Pnm(std::cos(colat)); }; + } + else if (m > 0) { + Y_ = [n, m, Knm, Pnm](double lon, double colat) { + return M_SQRT2 * Knm * std::cos(m * lon) * Pnm(std::cos(colat)); + }; + } + else { + Y_ = [m, Knm, Pnm](double lon, double colat) { + return M_SQRT2 * Knm * std::sin(-m * lon) * Pnm(std::cos(colat)); + }; + } +} + +double SphericalHarmonic::operator()(double lon, double lat) const { + double colat = (90. - lat) * Constants::degreesToRadians(); + lon *= Constants::degreesToRadians(); + return Y_(lon, colat); +} + + +} // namespace function + +} // namespace util + +} // namespace atlas diff --git a/src/atlas/util/function/SphericalHarmonic.h b/src/atlas/util/function/SphericalHarmonic.h new file mode 100644 index 000000000..1edf1a2d3 --- /dev/null +++ b/src/atlas/util/function/SphericalHarmonic.h @@ -0,0 +1,53 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + + +#pragma once + +#include + +namespace atlas { + +namespace util { + +namespace function { + +/// \brief An analytic function that provides a spherical harmonic on a 2D sphere +/// +/// \param n Total wave number +/// \param m Zonal wave number +/// \param lon Longitude in degrees +/// \param lat Latitude in degrees +/// \return spherical harmonic +double spherical_harmonic(int n, int m, double lon, double lat); + +/// \brief SphericalHarmonic operator that provides a spherical harmonic on a 2D sphere +class SphericalHarmonic { +public: + /// \brief Constructor + /// + /// \param n Total wave number + /// \param m Zonal wave number + /// \param caching When true, internally cache the results of Associated Legendre Polynomials + /// in a map. Warning: this is not thread-safe + SphericalHarmonic(int n, int m, bool caching = false); + + /// \brief Evaluate the spherical harmonic function for given longitude and latitude + double operator()(double lon, double lat) const; + +private: + std::function Y_; +}; + +} // namespace function + +} // namespace util + +} // namespace atlas From c790259272e158dac2b2093d818ba31e39410bb1 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 17 May 2022 01:55:43 +0100 Subject: [PATCH 17/53] Make Grid::type() public --- src/atlas/grid/Grid.cc | 4 ++++ src/atlas/grid/Grid.h | 1 + 2 files changed, 5 insertions(+) diff --git a/src/atlas/grid/Grid.cc b/src/atlas/grid/Grid.cc index 609aa1307..5d28ac81e 100644 --- a/src/atlas/grid/Grid.cc +++ b/src/atlas/grid/Grid.cc @@ -85,6 +85,10 @@ std::string Grid::name() const { return get()->name(); } +std::string Grid::type() const { + return get()->type(); +} + std::string Grid::uid() const { return get()->uid(); } diff --git a/src/atlas/grid/Grid.h b/src/atlas/grid/Grid.h index 32a957250..9873d5550 100644 --- a/src/atlas/grid/Grid.h +++ b/src/atlas/grid/Grid.h @@ -92,6 +92,7 @@ class Grid : DOXYGEN_HIDE(public util::ObjectHandle) { const Domain& domain() const; RectangularLonLatDomain lonlatBoundingBox() const; std::string name() const; + std::string type() const; std::string uid() const; /// Adds to the hash the information that makes this Grid unique From ba931aa300993359fc2f4fa57c0d3793756ad5f7 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 18 May 2022 16:24:10 +0100 Subject: [PATCH 18/53] Add output operator for Interpolation --- src/atlas/interpolation/Interpolation.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/atlas/interpolation/Interpolation.h b/src/atlas/interpolation/Interpolation.h index be7b576ed..2bb9643cf 100644 --- a/src/atlas/interpolation/Interpolation.h +++ b/src/atlas/interpolation/Interpolation.h @@ -69,6 +69,11 @@ class Interpolation : DOXYGEN_HIDE(public util::ObjectHandle Date: Wed, 18 May 2022 16:55:02 +0100 Subject: [PATCH 19/53] Fix more ConvexSphericalPolygon intersection edge cases --- src/atlas/util/ConvexSphericalPolygon.cc | 8 ++- src/tests/util/test_convexsphericalpolygon.cc | 52 +++++++++++++++++-- 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/src/atlas/util/ConvexSphericalPolygon.cc b/src/atlas/util/ConvexSphericalPolygon.cc index b81f65540..fe266f87f 100644 --- a/src/atlas/util/ConvexSphericalPolygon.cc +++ b/src/atlas/util/ConvexSphericalPolygon.cc @@ -297,6 +297,9 @@ bool ConvexSphericalPolygon::validate() { } bool ConvexSphericalPolygon::equals(const ConvexSphericalPolygon& plg, const double deg_prec) const { + if (size_ == 0 and plg.size_ == 0) { + return true; + } if ((not plg.valid_) || (not valid_) || size_ != plg.size()) { Log::info() << " ConvexSphericalPolygon::equals == not compatible\n"; return false; @@ -439,7 +442,7 @@ void ConvexSphericalPolygon::clip(const GreatCircleSegment& great_circle) { StackVector vertex_in(size_); int num_vertices_in = 0; for (int i = 0; i < size_; i++) { - vertex_in[i] = great_circle.inLeftHemisphere(sph_coords_[i], -EPS); + vertex_in[i] = great_circle.inLeftHemisphere(sph_coords_[i], -10. * TOL); num_vertices_in += vertex_in[i] ? 1 : 0; } @@ -526,6 +529,9 @@ ConvexSphericalPolygon ConvexSphericalPolygon::intersect(const ConvexSphericalPo } } intersection.simplify(); + intersection.computed_area_ = false; + intersection.computed_radius_ = false; + intersection.computed_centroid_ = false; return intersection; } diff --git a/src/tests/util/test_convexsphericalpolygon.cc b/src/tests/util/test_convexsphericalpolygon.cc index 56bdf5663..f4e158855 100644 --- a/src/tests/util/test_convexsphericalpolygon.cc +++ b/src/tests/util/test_convexsphericalpolygon.cc @@ -250,14 +250,58 @@ CASE("source_covered") { CASE("edge cases") { Log::info().precision(20); + SECTION("CS-LFR-256 -> H1280 problem polygon intersection") { + const auto plg0 = make_polygon({{-23.55468749999994, -41.11286269132660}, + {-23.20312500000000, -41.18816845938357}, + {-23.20312500000000, -40.83947225425061}, + {-23.55468749999994, -40.76429594967151}}); + const auto plg1 = make_polygon({{-23.30859375000000, -40.81704944888558}, + {-23.27343750000000, -40.85649237345376}, + {-23.23828125000000, -40.81704944888558}, + {-23.27343750000000, -40.77762996221442}}); + auto iplg = plg0.intersect(plg1); + auto jplg = plg1.intersect(plg0); + EXPECT_APPROX_EQ(iplg.area(), jplg.area(), 2e-12); // can not take 1e-15 + EXPECT_EQ(iplg.size(), 3); + EXPECT_EQ(jplg.size(), 3); + EXPECT(iplg.equals(jplg, 5.e-7)); + Log::info() << "Intersection area difference: " << std::abs(iplg.area() - jplg.area()) << "\n"; + } + SECTION("CS-LFR-16 -> O32 problem polygon intersection") { + const auto plg0 = make_polygon({{174.3750000000001, -16.79832945594544}, + {174.3750000000001, -11.19720014633353}, + {168.7500000000000, -11.03919441545243}, + {168.7500000000000, -16.56868711281520}}); + const auto plg1 = make_polygon({{174.3750000000000, -12.55775611523100}, + {174.1935483870968, -15.34836475949100}, + {177.0967741935484, -15.34836475949100}}); + auto iplg = plg0.intersect(plg1); + auto jplg = plg1.intersect(plg0); + EXPECT_APPROX_EQ(iplg.area(), jplg.area(), 1e-13); // can not take 1e-15 + EXPECT_EQ(iplg.size(), 3); + EXPECT_EQ(jplg.size(), 3); + EXPECT(iplg.equals(jplg, 1.e-11)); + Log::info() << "Intersection area difference: " << std::abs(iplg.area() - jplg.area()) << "\n"; + } + SECTION("CS-LFR-2 -> O48 problem polygon intersection") { + const auto plg0 = make_polygon({{180, -45}, {180, 0}, {135, 0}, {135, -35.26438968}}); + const auto plg1 = make_polygon({{180, -23.31573073}, {180, -25.18098558}, {-177.75, -23.31573073}}); + auto iplg = plg0.intersect(plg1); + auto jplg = plg1.intersect(plg0); + EXPECT_APPROX_EQ(iplg.area(), jplg.area(), 1e-15); + EXPECT_EQ(iplg.size(), 0); + EXPECT_EQ(jplg.size(), 0); + EXPECT(iplg.equals(jplg, 1.e-12)); + Log::info() << "Intersection area difference: " << std::abs(iplg.area() - jplg.area()) << "\n"; + } SECTION("H128->H256 problem polygon intersection") { const auto plg0 = make_polygon({{42.45283019, 50.48004426}, {43.33333333, 49.70239033}, - {44.1509434, 50.48004426}, - {43.26923077, 51.2558069}}); + {44.15094340, 50.48004426}, + {43.26923077, 51.25580690}}); const auto plg1 = make_polygon({{43.30188679, 50.48004426}, - {43.73831776, 50.0914568}, - {44.1509434, 50.48004426}, + {43.73831776, 50.09145680}, + {44.15094340, 50.48004426}, {43.71428571, 50.86815893}}); auto iplg = plg0.intersect(plg1); auto jplg = plg1.intersect(plg0); From 89229b42d6127dc18703a869767a2ee0d49e9b8a Mon Sep 17 00:00:00 2001 From: odlomax Date: Wed, 18 May 2022 17:04:54 +0100 Subject: [PATCH 20/53] Fix CubedSphereInterpolation bug and CubedSphereProjection bug (#104) Squashed commit of the following: commit 1598a2e728fae7019741fa907af56ed3c1185b99 Author: Willem Deconinck Date: Wed May 18 17:03:40 2022 +0100 Apply clang-format commit 28b19c440e566a4d51ed4f61c1357c5cccf85616 Merge: 8ea4e2271 2273808b9 Author: odlomax Date: Tue May 17 19:15:17 2022 +0100 Merge branch 'feature/ecmwf-develop' of https://github.com/JCSDA-internal/atlas into feature/ecmwf-develop commit 8ea4e2271a8778448f7db510b3d96aed6710f152 Author: odlomax Date: Tue May 17 19:14:08 2022 +0100 Corrected interpolation and projection bugs. --- .../method/cubedsphere/CubedSphereBilinear.cc | 8 ++++---- src/atlas/projection/detail/CubedSphereProjectionBase.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/atlas/interpolation/method/cubedsphere/CubedSphereBilinear.cc b/src/atlas/interpolation/method/cubedsphere/CubedSphereBilinear.cc index da39e1ecf..0fb1a7793 100644 --- a/src/atlas/interpolation/method/cubedsphere/CubedSphereBilinear.cc +++ b/src/atlas/interpolation/method/cubedsphere/CubedSphereBilinear.cc @@ -83,11 +83,11 @@ void CubedSphereBilinear::do_setup(const FunctionSpace& source, const FunctionSp } } } - - // fill sparse matrix and return. - Matrix A(target_.size(), source_.size(), weights); - setMatrix(A); } + + // fill sparse matrix and return. + Matrix A(target_.size(), source_.size(), weights); + setMatrix(A); } void CubedSphereBilinear::print(std::ostream&) const { diff --git a/src/atlas/projection/detail/CubedSphereProjectionBase.h b/src/atlas/projection/detail/CubedSphereProjectionBase.h index cc4ff37e2..b71154853 100644 --- a/src/atlas/projection/detail/CubedSphereProjectionBase.h +++ b/src/atlas/projection/detail/CubedSphereProjectionBase.h @@ -31,7 +31,7 @@ class CubedSphereProjectionBase : public ProjectionImpl { void hash(eckit::Hash&) const; - atlas::grid::CubedSphereTiles getCubedSphereTiles() const { return tiles_; }; + const atlas::grid::CubedSphereTiles& getCubedSphereTiles() const { return tiles_; }; /// @brief Convert (x, y) coordinate to (alpha, beta) on tile t. /// From 7cab4572cbabfd822509b049f7eed53b737c4828 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Thu, 19 May 2022 09:32:04 +0100 Subject: [PATCH 21/53] Only compile Fortran sandbox programs when HAVE_FORTRAN --- src/sandbox/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sandbox/CMakeLists.txt b/src/sandbox/CMakeLists.txt index 02b7c9e54..427f997af 100644 --- a/src/sandbox/CMakeLists.txt +++ b/src/sandbox/CMakeLists.txt @@ -11,10 +11,12 @@ if( ECKIT_INCLUDE_DIRS ) # eckit not yet ported to CMake3 endif() add_subdirectory( apps ) +if( HAVE_FORTRAN ) add_subdirectory( fortran_submodule ) add_subdirectory( fortran_object ) add_subdirectory( example_fortran ) add_subdirectory( fortran_acc_fields ) +endif() add_subdirectory( interpolation ) add_subdirectory( interpolation-fortran ) add_subdirectory( grid_distribution ) From b68da09fce5f360329374c1efe1f37c39650a0c7 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Thu, 19 May 2022 10:18:42 +0100 Subject: [PATCH 22/53] Fix AddressSanitizer issues --- src/atlas/grid/detail/grid/Structured.cc | 1 + src/tests/grid/test_largegrid.cc | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/atlas/grid/detail/grid/Structured.cc b/src/atlas/grid/detail/grid/Structured.cc index 1414d80b2..e79d0b56b 100644 --- a/src/atlas/grid/detail/grid/Structured.cc +++ b/src/atlas/grid/detail/grid/Structured.cc @@ -90,6 +90,7 @@ Structured::Structured(const std::string& name, XSpace xspace, YSpace yspace, Pr "Recompile atlas with idx_t 64bits, or wait for pending development where grid index to become gidx_t"); } crop(domain); + ny = nx_.size(); computeTruePeriodicity(); diff --git a/src/tests/grid/test_largegrid.cc b/src/tests/grid/test_largegrid.cc index 2d8fde0d2..c273fafb0 100644 --- a/src/tests/grid/test_largegrid.cc +++ b/src/tests/grid/test_largegrid.cc @@ -42,20 +42,20 @@ struct Trace { } }; -constexpr size_t Mbytes = 1000000; +constexpr size_t MB = 1024 * 1024; CASE("test resources for cropping large grids") { std::vector gridnames{"L40000x20000", "N8000", "O8000"}; for (auto& gridname : gridnames) { - EXPECT(Trace::peakMemory() < 100 * Mbytes); + EXPECT(Trace::peakMemory() < 100 * MB); SECTION(std::string("section") + gridname) { Trace trace(Here(), "Grid{" + gridname + ", GlobalDomain{-180}}"); auto grid = Grid{gridname, GlobalDomain{-180}}; trace.stopAndReport(); EXPECT(trace.elapsed() < 10.); - EXPECT(trace.peakMemory() < 100 * Mbytes); + EXPECT(trace.peakMemory() < 100 * MB); } } } From 124438c3376fb3ff2181c4e5744b61ed86aa4fd5 Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Thu, 19 May 2022 11:18:06 +0100 Subject: [PATCH 23/53] ATLAS-362 New Interpolation: ConservativeSphericalPolygonInterpolation Capable to interpolate from any grid to any grid Squashed commit of the following: commit 2bbc9fa5f05ab5fe19792aaf8454c0688d18fe2d Author: Willem Deconinck Date: Thu May 19 10:48:43 2022 +0100 Relax tolerances, for nvhpc with Github Actions commit f3b4c9ad04d0738d71e1f001b0831c8cb50d1ec8 Author: Willem Deconinck Date: Wed May 18 16:59:00 2022 +0100 Cosmetic fixes commit bce3d39ef6d9e9860de3e9011e07f8dc9cc87b34 Author: Willem Deconinck Date: Tue May 17 02:07:46 2022 +0100 Revamp atlas-conservative-interpolation sandbox commit 7daaaf69c30fe414b72c803f9374160d2bf79b08 Author: Willem Deconinck Date: Tue May 17 21:41:51 2022 +0100 Improve ConservativeSphericalPolygonInterpolation metadata and configuration commit 7e136ed7f110103e0c23429e4cafef3eda70a7bb Author: Willem Deconinck Date: Tue May 17 02:06:25 2022 +0100 Fix ConservativeSphericalPolygonInterpolation with functionspace API commit 144d6cc25b2c0bd0f950fe2061a96aac7d3e62c7 Author: Willem Deconinck Date: Thu May 19 09:32:28 2022 +0100 Fix memory problem with accuracy computation commit 4e4396dad21523d37025c4d6d330ea3250f2fa9f Merge: 03646d314 b68da09fc Author: Willem Deconinck Date: Thu May 19 10:18:54 2022 +0100 Merge branch 'develop' into features/conservative_remap * develop: (58 commits) Fix AddressSanitizer issues Only compile Fortran sandbox programs when HAVE_FORTRAN Fix CubedSphereInterpolation bug and CubedSphereProjection bug (#104) Fix more ConvexSphericalPolygon intersection edge cases Add output operator for Interpolation Make Grid::type() public ATLAS-361 Add SphericalHarmonic function ATLAS-358 Support degenerate pentagon in FiniteElement interpolation ATLAS-358 Pentagon element type ATLAS-352 HEALPix mesh with '3d' option, corresponding to original healpix numbering Small fixes and diagnostics in sandbox atlas-parallel-interpolation ATLAS-350 Fix bug in healpix global numbering of periodic cells and nodes (tnx Willem) Fix in atlas-meshgen where --3d option was not properly propagated to mesh generators Fix type of indices in test_transgeneral CASE test_2level_adjoint_test_with_powerspectrum_convolution Fix clang-tidy warning Add Fortran API for FieldSet::name() ATLAS-357 StructuredColumns functionspace now works for regional grids atlas::vector::capacity() ATLAS-350 Fix global numbering of cells matching HEALPix grid Add Fortran API for Interpolation::execute_adjoint() ... commit 03646d314e56675cf0f8abe7f316b83d5708a25a Author: Willem Deconinck Date: Wed Apr 6 17:00:13 2022 +0100 Remove unnecessary code commit 74eb0b475e2424abd3efab35d134b217a1f27c82 Author: Willem Deconinck Date: Thu Mar 24 09:50:22 2022 +0000 Use MatrixCache with a (probably too simple) uid to distinguish first and second order matrix in ConservativeSphericalPolygonInterpolation commit 9e4f8f2d8cf28508553158351ae6fc68ee08607f Merge: a805e3730 2c8ba8332 Author: Willem Deconinck Date: Thu Mar 24 10:33:18 2022 +0000 Merge branch 'develop' into features/conservative_remap [DOES NOT COMPILE] * develop: Make interpolation::Method::matrix_ members private and add protected accessor functions Add uid to interpolation::MatrixCache File reorganisation: fe/FiniteElement -> unstructured/FiniteElement Rename BilinearRemapping to UnstructuredBilinearLonLat commit a805e37303229138403db9d0c209b7809cbdea77 Author: Willem Deconinck Date: Wed Mar 23 17:23:35 2022 +0000 Review part 9: Cleanups and Rename ConservativeMethod to ConservativeSphericalPolygonInterpolation commit 23a4215d7400bc4095e51576757954599d8b0801 Author: Willem Deconinck Date: Wed Mar 23 12:58:06 2022 +0000 Review part 8: Simple method to fix bug in duplicated triplets at cost of some performance commit 330a982beda272e30a1ecf7070f51422aadead07 Author: Willem Deconinck Date: Wed Mar 23 11:49:03 2022 +0000 Review part 7: Remove set_order() and add printing of interpolation commit 7596bba0ac4cd9676e24d5215e958970d19fde80 Author: Pedro Maciel Date: Wed Mar 23 09:57:57 2022 +0000 ConservativeMethod simpler sort (same thing) commit c0f04a13c01bd8c0465fc6feaba3a6b7fb07284b Author: Willem Deconinck Date: Wed Mar 23 09:52:11 2022 +0000 Minor cleanup commit c74632dbc9f29a755f5f51f36a64373542062fe6 Author: Willem Deconinck Date: Tue Mar 22 19:37:33 2022 +0000 Review part 6: Remove RemapStat::compute and add some contribution to do_execute commit e79aac53ca1ad44529a5d30abd046a7a9d1e0863 Author: Willem Deconinck Date: Tue Mar 22 15:54:33 2022 +0000 Review part 5: Don't require call to setup_stat commit 7b7889e9ab27b096b8883ab6c9661fa9c76fe15e Author: Willem Deconinck Date: Tue Mar 22 15:45:17 2022 +0000 Review part 4: Use Metadata to pass RemapStat commit d48d686b256da42c034ff3cea46d701f42280b67 Merge: 5d07c600b c1e77945d Author: Willem Deconinck Date: Tue Mar 22 14:09:32 2022 +0000 Merge branch 'develop' into features/conservative_remap * develop: Add Metadata return value to Interpolation::execute() Add interpolation::Cache constructor with filter, and apply it to interpolation::MatrixCache and interpolation::IndexKDTreeCache commit 5d07c600bbfefd9b0fc27d405a6b7b0f40cbf109 Author: Willem Deconinck Date: Tue Mar 22 13:11:41 2022 +0000 Review part 3: Also cache function spaces and add more tests commit 051ec3d1dc6499fa12f93ca55d50467ea92a7336 Author: Willem Deconinck Date: Mon Mar 21 20:13:20 2022 +0000 Review part 2: Reuse cachable data, and correct virtual functions override commit f50451b19b089e7fb36d9baa993fcc07a506fdb2 Author: Willem Deconinck Date: Mon Mar 21 12:22:39 2022 +0000 Review part 1: Introduce cachable data. It cannot be reused yet. commit c5da7070e580de2260ffb1bcc58d166572795d74 Author: Willem Deconinck Date: Fri Mar 18 11:57:40 2022 +0000 apply clang-format commit fefc05daff68c6841f47ba023f0349d3a509e5cb Merge: 41f387ce9 2eed5368b Author: Willem Deconinck Date: Fri Mar 18 11:56:58 2022 +0000 Merge branch 'develop' into features/conservative_remap * develop: Fix bug where using ectrans was not enabling adjoint of invtrans Version 0.28.1 Fix ambiguous call to std::min Fix ambiguous call to std::min Fix bilinear remapping (#85): Catch periodicity issues using 3D kdtree (#96) Add extended cubed-sphere dual halo with a correct alpha-beta projection (#95) Move Projection::Jacobian to namespace Apply clang-format Removed rank restriction in redistribution. commit 41f387ce9046be6a871bf53c9dc52e46fa9ad9c9 Author: Slavko Brdar Date: Tue Mar 15 17:29:23 2022 +0000 try fixing other compilers in unit test for convexsphericalpolygon commit 62ea32c49935430ad41f9e627bd5e23d192b132b Author: Slavko Brdar Date: Tue Mar 15 16:34:31 2022 +0000 fix problematic polygon intersection in CS-LFR-256-->H1280 commit 44025be7f71ce889697ff5772c11e49550ab015e Author: Slavko Brdar Date: Tue Mar 15 16:27:27 2022 +0000 add unit test for polygon intersections for problematic CS-LFR-256-->H1280 remapping commit b2c4d4cf951237efc13afa2acec506c8a54e7234 Author: Slavko Brdar Date: Tue Mar 15 15:34:28 2022 +0000 fix polygon intersection problem with CS-LFR-16->O32 and add a unit test for it commit 1a541e74a3119e051c732ac5fb63ef6138bc5dc9 Author: Slavko Brdar Date: Tue Mar 15 13:50:51 2022 +0000 add unit test for polygon intersection problem from CS-LFR-16 -> O32 commit 46011f8fbecbbd4342f7f188bb4d07b9baa74c37 Author: Slavko Brdar Date: Tue Mar 15 13:40:20 2022 +0000 warn about target and source cell intersections not summing up to their host source and target cells, tolerance for now is at 0.1 % mismatch commit cde3c31df4340750ab066177933e2999f0616729 Author: Slavko Brdar Date: Tue Mar 15 01:20:07 2022 +0000 fix in analysis output of conservative remapping commit f1cda6c1419047b615b64c67112eae557e2887da Author: Slavko Brdar Date: Tue Mar 15 01:02:41 2022 +0000 fix debuggin in conservative remapping: 1) use PLG_DEBUG instead of NDEBUG; 2) fix a bug in computing tgt_cell coverage commit fa9d82cd1afe47b1e4436968d726b46e1e355492 Merge: cfcc3ec28 5189664eb Author: Slavko Brdar Date: Mon Mar 14 18:49:05 2022 +0000 Merge branch 'features/conservative_remap' of ssh://git.ecmwf.int/atlas/atlas into features/conservative_remap commit cfcc3ec28a9bc1bbfcd00b82c9a571c6e2bbba23 Author: Slavko Brdar Date: Mon Mar 14 18:48:59 2022 +0000 add analysis for detection of target cells not fully covered its intersections commit 9c74896fd7d3e0a65760ba5b4910366b891ba813 Author: Slavko Brdar Date: Mon Mar 14 17:28:03 2022 +0000 improve debugging in polygon intersection commit 5189664ebf6e22a15eae3d461f55d568628dc3a7 Author: Willem Deconinck Date: Mon Mar 14 11:01:41 2022 +0000 Fix "8af432f18 - fix bug in CS-LFR-2->O48 remapping" differently commit b134080ea5e42a1ee4bdafeb2e84ba26199c3078 Author: Slavko Brdar Date: Fri Mar 11 12:34:24 2022 +0000 try fixing conservative remap unit test for nvhpc compiler commit 8543d05d97450c3213f8b6b97f8488485eb2f8f6 Author: Slavko Brdar Date: Fri Mar 11 12:16:33 2022 +0000 remove unnecessary variable commit 36323202e6e9ebd3da8b46ee969a4fa1ce4a0459 Author: Slavko Brdar Date: Thu Mar 10 17:18:58 2022 +0000 code readability: replace weights->src_weights, sweights->tgt_weights commit bc8f5efcf9ca3df064f7c86640074b4531365e1b Author: Slavko Brdar Date: Thu Mar 10 17:08:05 2022 +0000 add area check on intersection polygon commit 8af432f186881a966e755122ea662f00ba3c2f33 Author: Slavko Brdar Date: Thu Mar 10 16:25:25 2022 +0000 fix bug in CS-LFR-2->O48 remapping commit 690ba8d48916af29a8cccf03369e111189329444 Merge: 5d57ba9de c2789a992 Author: Slavko Brdar Date: Thu Mar 10 14:49:14 2022 +0000 Merge branch 'features/conservative_remap' of ssh://git.ecmwf.int/atlas/atlas into features/conservative_remap commit 5d57ba9de4e63882c6a040a938cc777ce3cdb07c Author: Slavko Brdar Date: Thu Mar 10 14:46:41 2022 +0000 in polygon intersection, empty polygons are equal commit 9e6c828e4c9dbd402d191e15e2963dfa281f5df5 Author: Slavko Brdar Date: Thu Mar 10 13:59:20 2022 +0000 add problematic CS-LFR-2 -> O48 remapping polygons to unit tests commit c2789a9922dd21f10dc3c882cb1cd25f1ae9f795 Author: Willem Deconinck Date: Thu Mar 10 09:30:34 2022 +0000 Fix compiler warning commit c200d2b64342b7ced830478859789e60d1f469b5 Author: Slavko Brdar Date: Wed Mar 9 16:50:01 2022 +0000 fix for parallel boundary problem, fix for CubedSphere issues (together with Willem) commit bf60601fd7c5345a24d2e96fb8ede69933020763 Author: Slavko Brdar Date: Tue Mar 8 11:01:14 2022 +0000 correct setting of the halo size for CubedSphere meshes commit 305916ccf5298778bfc070ac3742f2f912018261 Merge: e52804934 3901c057c Author: Slavko Brdar Date: Wed Mar 2 16:42:59 2022 +0000 Merge branch 'develop' into features/conservative_remap commit e5280493444dfc7b2c21e02674cab69425a6f22f Author: Slavko Brdar Date: Wed Mar 2 16:40:15 2022 +0000 remove building of obsolete unit test -spherical_geo- commit ed1c3e0934ba04e42db15f506d1bb8d131c84922 Author: Slavko Brdar Date: Wed Mar 2 14:09:24 2022 +0000 -spherical_geo- unit test has been replaced with -convexsphericalpolygon- commit 11d9634817189ea68ecbc674facd42b7a0575b07 Merge: 7bd9fa7c2 bfced3fcc Author: Slavko Brdar Date: Mon Feb 28 18:10:51 2022 +0000 Merge branch 'develop' into features/conservative_remap, and add inside_vertices for debugging polygon intersections commit 7bd9fa7c2abee2cc865d2dd187b46e5a35534060 Author: Slavko Brdar Date: Mon Feb 28 13:49:25 2022 +0000 bugfix for tgt-cell-data=0, index -inode- overshoots commit a0a5701b17229093082ab8f34ddb39e89496ba2b Author: Slavko Brdar Date: Mon Feb 28 13:47:09 2022 +0000 bugfix in remap_stat commit 7782362248c591fe508b6d54621b8618f144379c Author: Slavko Brdar Date: Mon Feb 28 13:46:04 2022 +0000 also fix the new intersection unit test w.r.t. constexpr compilation problem commit 2a897b5132815763feb7239be6e3cf25edf8f4f1 Author: Slavko Brdar Date: Mon Feb 28 13:45:00 2022 +0000 do not use deprecated function; 2) fix compilation issue with static constexpr (tnx Willem) commit 31e8bce911e2f846dbc6bd17b94a9c046d0ebfa1 Author: Slavko Brdar Date: Wed Feb 23 16:57:07 2022 +0000 improve debbuging by using does_intersect method to check for bad polygon intersections commit 43ec7ab6cf240c1c00650ae16625f74beda5ea6f Author: Slavko Brdar Date: Wed Feb 23 16:20:43 2022 +0000 change intersection unit test for ConvexSphericalPolygon to work with with 3d point and away from spherical coordinates commit 30b832b1b0ae7a196cb04ac19783e557213cfd93 Author: Slavko Brdar Date: Wed Feb 23 16:20:30 2022 +0000 use constant EPS and TOL in ConvexSphericalPolygon instead of individually set thresholds commit 5d0d59727508d8203289db911f65585886071161 Author: Slavko Brdar Date: Fri Feb 18 17:20:43 2022 +0000 fix unit test for cray, nvhpc, pgi commit 1332749dccd4af7f3ba546db53a68733ef01c0de Author: Slavko Brdar Date: Fri Feb 18 16:17:01 2022 +0000 fix bug in NodeColumn indexing of subpolygons and revamp the code of subpolygon creation commit a63b4d73280ff65b079619d0c2ea46c489029624 Author: Slavko Brdar Date: Fri Feb 18 15:05:26 2022 +0000 code cleanup commit 3cc41110a7882e9a88dd9532b38b661a28288149 Author: Slavko Brdar Date: Fri Feb 18 13:18:28 2022 +0000 simplify initialisation of src_field in unit test commit 3857246d0eeae5aff3f060da78aefcb67d18d7ee Author: Slavko Brdar Date: Fri Feb 18 11:22:49 2022 +0000 improve output from unit-test commit a6b85c6ef9f3223bbea35aa8481495e93f11aa50 Author: Slavko Brdar Date: Fri Feb 18 11:03:19 2022 +0000 adjust tolerances to pass unit-test on Cray & co commit c7c0ef7be6b6f8ba041b16dd00c06b5578e42ac2 Merge: 7ac1929ab a2a53f02c Author: Slavko Brdar Date: Fri Feb 18 00:53:23 2022 +0000 Merge branch 'develop' into features/conservative_remap commit 7ac1929ab1a5a24e3d8d12b126d530988ebe8707 Author: Slavko Brdar Date: Fri Feb 18 00:50:20 2022 +0000 fix errors in std output commit f02d8bfdc1464c06d2545f94a1cb31ed1e60dac4 Author: Slavko Brdar Date: Fri Feb 18 00:04:30 2022 +0000 encapsulate remapping statistics in RemapStat commit d715ccb0afc633f5988811c7eaed69d123ce0e42 Author: Slavko Brdar Date: Thu Feb 17 17:13:46 2022 +0000 make unit test of cons remap pass commit 19a68fb5800126caba8f53b1694b1fbfd034c29d Author: Slavko Brdar Date: Thu Feb 17 16:41:13 2022 +0000 improve unit test for conservative remapping commit 7d76021e189343e3e70f59eb63f674c34281912f Author: Slavko Brdar Date: Thu Feb 17 14:53:56 2022 +0000 add RemapStat subclass of ConservativeMethod to collect statistics of the last remapping commit cd866d9e176e0fb0ab06798423b59034abe0d89d Author: Slavko Brdar Date: Thu Feb 17 13:16:24 2022 +0000 move member definition to .cc commit 4f1c2cda77e0e7dc9a914c57eb451f038c6f397a Author: Slavko Brdar Date: Wed Feb 16 15:08:36 2022 +0000 clean up unit test for conservative remapping - atlas excepts still missing commit a8a9281ff92d9a7ca38c608933a764f8a0b055af Author: Slavko Brdar Date: Wed Feb 16 13:42:20 2022 +0000 add conservative interpolation tests in sandbox commit a008d41c6719171271b3b23b091de7516dac4522 Author: Willem Deconinck Date: Wed Feb 16 11:57:22 2022 +0000 nb_points are retrieved from functionspace rather than from mesh commit 820268ddab5b44f680ae9dc34c7e244e2e5caa95 Merge: dc0dee0b4 1f59e67c2 Author: Willem Deconinck Date: Wed Feb 16 11:39:26 2022 +0000 Merge branch 'develop' into features/conservative_remap * develop: ATLAS-346 Fix CellColumns::haloExchange for multiple element types Add += and -= operators to IndexView elements commit dc0dee0b4f6cd3e5d51d09f915b161171611d343 Author: Slavko Brdar Date: Mon Feb 14 23:44:13 2022 +0000 add unit test to check node-to-cell connectivities of Atlas grids - Healpix grid fails vim test_mesh_node2cell.cc commit 962271cc82db6e74a3710413ffdee5e3385faedb Author: Slavko Brdar Date: Mon Feb 14 16:27:08 2022 +0000 skip neighbourhood check for the periodic points for now - potentially a problem commit c7e3181116c073f20e4be30d73edd5688d389ec8 Author: Slavko Brdar Date: Mon Feb 14 14:58:48 2022 +0000 fix an issue in NodeColumns with nodes which have no subpolygon associated with commit 3bdb0ae5afde75c17278155fa3eba4e62d047366 Author: Slavko Brdar Date: Mon Feb 14 14:13:23 2022 +0000 make compile again commit 736d3bf038a55025cd39b1318d66da2694f94a66 Merge: b30b86de6 7afd801ac Author: Slavko Brdar Date: Mon Feb 14 14:05:35 2022 +0000 Merge branch 'develop' into features/conservative_remap commit b30b86de6fadd6402193377bd9f8834b6ac458a4 Author: Slavko Brdar Date: Mon Feb 14 11:51:06 2022 +0000 two assertion in unit test commit cac25e92d6cc07ec62792ba9ba3b15f017a0bc4d Author: Slavko Brdar Date: Fri Feb 11 17:58:57 2022 +0000 enable matrix-free conservative remapping again commit ab982af441839e2ec670ac97409f418192bfddae Author: Slavko Brdar Date: Fri Feb 11 17:31:57 2022 +0000 output more info for remap commit 80697d567625ad07196125b3fca2dceb551f84d0 Author: Slavko Brdar Date: Fri Feb 11 17:30:51 2022 +0000 use next_index and prev_index commit ff9784d80431fa69416b6da1d1a43067bf085cdb Author: Slavko Brdar Date: Fri Feb 11 14:15:57 2022 +0000 fix issue with pole multiplicity when contructing subpolygons in NodeColumn -> {Cell,Node}Column remappings commit f3b301b4de041c5c133bfe212435d9ccd2a9093a Author: Slavko Brdar Date: Thu Feb 10 16:00:54 2022 +0000 remove unnecessary -if- condition commit c7cf03537536848caccfe11a54a282cba1b4eb0d Author: Slavko Brdar Date: Tue Feb 8 16:52:53 2022 +0000 small fixes commit 1966ec61b8f0beef3882e57f6a8705c38e385d09 Author: Slavko Brdar Date: Tue Feb 8 13:39:07 2022 +0000 readable coding commit cb19bfe2cf63452fd87ab09903673500e6b537fb Author: Slavko Brdar Date: Tue Feb 8 09:54:55 2022 +0000 remove unused variable commit 3ec5fe2b0968645b2123e9793ad4162e8401b71d Author: Slavko Brdar Date: Mon Feb 7 13:03:43 2022 +0000 build node-to-cell connectivity only when absolutely needed commit ae682e1a2706e30c5cf7f8ddbc9ca484176e3af1 Author: Slavko Brdar Date: Mon Feb 7 11:22:02 2022 +0000 construct nodes-to-cell connectivity only if needed commit 8f029e7ba08fcd0f0073ff162c2633101d1b461b Author: Slavko Brdar Date: Mon Feb 7 11:09:51 2022 +0000 use lonlat as default coordinates in test_interpolation_conservative commit 82a6f7f44407d2d2382d256ea4e4b12042b40a0d Author: Slavko Brdar Date: Mon Feb 7 10:50:14 2022 +0000 fix error in std output for mpi > 1 commit e23056d6eb84087427c3b28f6a409a7cae506569 Author: Slavko Brdar Date: Mon Feb 7 09:47:11 2022 +0000 fix std output commit 6cac0bef051c1f52862ad76d28666ea356b726c6 Author: Slavko Brdar Date: Sun Feb 6 01:38:07 2022 +0000 fixing remap statistics for mpi > 1 commit 0522abe97d45f514a4c1c24650743f4be1d778ad Author: Slavko Brdar Date: Sat Feb 5 01:15:24 2022 +0000 fix remap statistics for mpi>1 commit afff032d7a83bf05e50da77730e8e5451315e856 Author: Slavko Brdar Date: Fri Feb 4 22:34:58 2022 +0000 avoid duplicate creation of functionspaces in setup of ConservativeMethod commit c53b0b49c1f9ceb693119a78dbf1fdf4fca6a1c7 Author: Slavko Brdar Date: Fri Feb 4 17:56:52 2022 +0000 cleanup requirements on function spaces in ConsRemap commit bcecd07d2535ba60498574c5221d86b6ebf30704 Author: Slavko Brdar Date: Fri Feb 4 17:47:08 2022 +0000 more statistic on remap with NodeColumns commit d8ec712c8af0e5236206b14cb188621bedaeb6a3 Author: Slavko Brdar Date: Fri Feb 4 16:37:46 2022 +0000 check if 0 before dividing commit 8a1b28798d4d9b2331425e60de75d62837190714 Author: Slavko Brdar Date: Fri Feb 4 16:13:37 2022 +0000 cleanup, remove edge connectivities commit 5d68220de7c0380bca4ddaa54522fc86921a0495 Author: Slavko Brdar Date: Fri Feb 4 16:00:13 2022 +0000 cleanup, no more edge connectivity commit 33c0606e3deae1a0c0ea74b6de41c6fad0d5fec1 Author: Slavko Brdar Date: Fri Feb 4 14:55:57 2022 +0000 add command line argumets for test_interplation_conservative commit 3a55a6b2e34d7e461acee3e27847f5f2a7b82f97 Author: Slavko Brdar Date: Fri Feb 4 14:05:10 2022 +0000 more info on polygon inteersection in ConsRemap commit 7bd25ba220af47fb3678e3bd32cd8ddc765721ff Author: Slavko Brdar Date: Fri Feb 4 13:18:14 2022 +0000 use same function signatures as in branch feature/spherical_polygons commit 2bf4aec16268f17b7143308994f780a41ed14265 Author: Slavko Brdar Date: Thu Feb 3 15:59:36 2022 +0000 remove all edge connectivities in ConservativeMethod commit 20321cbe3a640a6f12ced723a865b9028c555f27 Merge: c9efaf685 86737ea22 Author: Slavko Brdar Date: Tue Feb 1 00:16:04 2022 +0000 Merge branch 'features/conservative_remap' of ssh://git.ecmwf.int/atlas/atlas into features/conservative_remap commit c9efaf68532267d63c21772f0e7d98fbc9554625 Author: Slavko Brdar Date: Tue Feb 1 00:15:51 2022 +0000 remove edge connectivity in creation of subpolygons for cons remap on NodeColumns commit 86737ea22818685610464625fc21119a69e5cf16 Merge: 088adc99d f548684e8 Author: Willem Deconinck Date: Fri Jan 28 11:53:12 2022 +0000 Merge branch 'develop' into features/conservative_remap * develop: Add *= and /= operators to PointXYZ with scalar ATLAS-345 Use new eckit::linalg::LinearAlgebraSparse and eckit::linalg::LinearAlgebraDense API (see ECKIT-585) Prevent optimisation with NVHPC compiler in VariableResolutionProjection Adapt atlas_test_field_missingvalue as there is no standard on comparisons with NaN Workaround for Cray 8.7 compiler bug resulting in FE_DIVBYZERO in atlas_test_jacobian in Release build ATLAS-327 Workaround for Cray 8.7 compiler bug with CXX flags "-O2 -hfp1 -g -DNDEBUG" commit 6ac9ecbe8671b99be5eb76469be387679fd3ac44 Author: Slavko Brdar Date: Thu Jan 27 11:15:51 2022 +0000 fix in USE_EDGE_CONNECTIVITY commit 088adc99da33f8bc4d7697a30cea7004a1071e3b Author: Slavko Brdar Date: Wed Jan 26 14:20:21 2022 +0000 rewrite get_node_neighbours without edge_connectivity commit 3472ca1601bc7b1b80c6b0b4644c4d29a19f1930 Author: Slavko Brdar Date: Thu Jan 20 10:45:41 2022 +0000 cleanup commit 0a5869063e329d8f96b579e2ed632af05ca34e72 Author: Willem Deconinck Date: Tue Jan 18 22:31:46 2022 +0000 Add some debugging assertions and command-line options commit 8ea624bcf4a50e3a39ed533879949cf3c7c50e9e Merge: 9b51fbb12 298eb2e62 Author: Willem Deconinck Date: Tue Jan 18 22:23:46 2022 +0000 Merge branch 'develop' into features/conservative_remap * develop: Fixup previous commit commit 9b51fbb12e7b9c1e7d052692e3b171ff6445432e Merge: e35aa47f3 4349a3ee3 Author: Willem Deconinck Date: Mon Jan 17 09:07:56 2022 +0000 Merge branch 'develop' into features/conservative_remap * develop: (44 commits) Allow use of build_cells_remote_index() when halo was already built Apply clang-format Addressed reviewer comments. Permitted variable sized halos on functionspaces. Added ghost field to CubedSphereCellColumns functionspace. ATLAS-344 Fix atlas_test_healpixmeshgen CI with nvhpc-21.9 ATLAS-344 Fix MatchingMeshPartitionerLonLatPolygon to work with non-matching domains ATLAS-344 PolygonXY with Point2 member variables ATLAS-344 Printing of util::PolygonCoordinates ATLAS-344 Initialise halo field for healpix mesh Attempt at workaround of internal compiler error with OpenSuseTumbleweed Docker build ATLAS-343 [Github #82] Add experimental VariableResolutionProjection for UKV grid support Apply clang-format ATLAS-342 Patches to allow support for the Fujitsu compiler ATLAS-341 [Github #84] FiniteElement Interpolation with configurable max_fraction_elems_to_try ATLAS-340 [Github #81] Add CubedSphereDualMeshGenerator ATLAS-339 ArrayView assign from another ArrayView Fix clang-tidy warnings Update Changelog clang-format 13.0.0 -- WARNING, all files touched ... commit e35aa47f344aab5d4c9e4e95e20b3e5bc1422a57 Author: Slavko Brdar Date: Thu Jan 13 14:08:42 2022 +0000 remove git merge conflict markers commit 0b8f49dda2db0ab3bed3c20f443ed7ba6927b712 Merge: 894f6c8df fb23e2c82 Author: Slavko Brdar Date: Thu Jan 13 11:55:42 2022 +0000 Merge branch 'features/conservative_remap' of ssh://git.ecmwf.int/atlas/atlas into features/conservative_remap commit 894f6c8df864a9be24a2a7b617cd90383d2d7e94 Author: Slavko Brdar Date: Thu Jan 13 11:43:12 2022 +0000 a fix in remap statistics commit fb23e2c823448a33ad3952c3cef426bf7ad2535f Author: Slavko Brdar Date: Wed Dec 15 10:54:42 2021 +0000 make conservative remapping support function spaces as input commit 45ee4b0a301fa118097fbba5bbd839d1d25e63d8 Author: Slavko Brdar Date: Tue Dec 14 15:19:49 2021 +0000 use eckit::parametrisation instead util::Config commit 8d62fdd0ac02966574252d7183fe6a16319cf918 Author: Slavko Brdar Date: Wed Nov 24 18:06:18 2021 +0000 fix periodic boundary in cons remap for CellColumns on source mesh commit ee790e0db66803bb8bb69ffad37bc89000295d12 Author: Slavko Brdar Date: Mon Nov 22 16:35:57 2021 +0000 make remapping work in parallel for tgt_halo_size > 0 (periodic bnd issue still) commit 99e8f1427c202619dd08baeb93de9a60732da0c8 Author: Slavko Brdar Date: Tue Nov 9 15:43:29 2021 +0000 fix a bug in ConservativeMethod::tgt_points commit 236c306354c59653178fc3898505304ce09a7013 Author: Slavko Brdar Date: Tue Nov 9 11:33:07 2021 +0000 cleanup commit 1ac747f2b342393418dca013ccc0a16b1e82112a Author: Slavko Brdar Date: Mon Nov 8 17:18:09 2021 +0000 in ConservativeMethod, src_points(ID) and tgt_points(ID) has barycentre of the cell and/or dual cell of the node with index ID commit 06ed30f1c8c874e15295aa9f5a27d8b218266ea0 Author: Slavko Brdar Date: Mon Nov 8 16:11:02 2021 +0000 working parallel version of 1/2 order cons remap, periodic boundary still a problem commit f89924a30af6513b1c482d9f633a3a374c83d736 Author: Slavko Brdar Date: Fri Nov 5 17:57:31 2021 +0000 create src_mesh decomposition from tgt_mesh; problem when src is NodeColumns commit 00571340dec0334d27c8cd587f6c10f0177b6ecc Author: Slavko Brdar Date: Thu Nov 4 16:51:38 2021 +0000 an almost correct running version of parallel 1st and 2nd order remapping, not working for NodeColumns-to-any commit 35f6f7801e3dc9a58e28514f15712d8947ba67bf Author: Slavko Brdar Date: Wed Nov 3 15:02:22 2021 +0000 cleanup commit 45595b3c4d29092a374e928a3a51dc88cb99c6f0 Author: Slavko Brdar Date: Mon Nov 1 18:04:56 2021 +0000 use std::tuple instead of std::pair in ConservativeMethod commit 78eb1e7c83a9db303c17675395b4f8ee232e5ab6 Author: Slavko Brdar Date: Fri Oct 29 15:04:04 2021 +0100 allow mpi to be used, program crashes for mpi>1 commit 3203569a8e94ad37a61f3c7d2e5036cffc59e2d6 Author: Slavko Brdar Date: Fri Oct 22 14:33:01 2021 +0100 fix matrix_free for halo>0. matrix_free works only for cell-to-cell remapping commit c68929b6e98f5929caf0ece36529edc116e079f7 Author: Slavko Brdar Date: Fri Oct 22 10:29:47 2021 +0100 code readability commit ffb8a8bb57d2436cf332cf20332aaf5e0675ba5b Author: Slavko Brdar Date: Thu Oct 21 18:05:35 2021 +0100 do not multiply weights in Gmsh error-fields commit f95ca78bcdc88c6859e715a2cb12486458772d3b Author: Slavko Brdar Date: Thu Oct 21 17:40:39 2021 +0100 fix remapping output to Gmsh error-fields commit f5acb0415d64f88fa6b0be4c5debf3d00d844a06 Author: Slavko Brdar Date: Thu Oct 21 15:49:20 2021 +0100 fix global conservation for 2nd order vertex-to-vertex remap commit d09f38a614ed9e5bb1c1d04bb1b33124d28947f4 Author: Slavko Brdar Date: Thu Oct 21 12:02:36 2021 +0100 small fix readability commit 411198aa28814f131464bcea6d006f058254fa09 Author: Slavko Brdar Date: Tue Oct 19 18:32:00 2021 +0100 use Field instead of std::vector commit 53ebc6eb5d2ec3923ee93bd144f298f17d19196c Author: Slavko Brdar Date: Tue Oct 19 16:23:07 2021 +0100 fix computation of remap errors for halo > 0 commit 8b9d655b1d5f6b3fd2c281af3bef9e42a1fa8c7b Author: Slavko Brdar Date: Thu Oct 14 16:34:25 2021 +0100 move remap error computation to ConservativeMethod commit cac9399fa34331376e19e9bc69a9d21a8674fb2c Author: Slavko Brdar Date: Wed Oct 13 14:18:23 2021 +0100 clean up in ConservativeMethod commit 89fea6441ccedcd3f9a565d5ff3d3a2834571675 Author: Slavko Brdar Date: Wed Oct 13 00:03:23 2021 +0100 correct use of ghost and halo in conservative remapping (tnx Willem) commit 26c789f61adb0ffa6d6a1cd2f8864c59224d9d77 Author: Slavko Brdar Date: Thu Oct 7 22:50:29 2021 +0100 do not recompute gradient-sum in vertex 2nd order conservative remap commit 9c4f4e4661246190df9694c863cca2680610f785 Author: Slavko Brdar Date: Thu Oct 7 17:38:00 2021 +0100 simplify for vertex and centroid versions commit c8f411d53b9006a57016ddd1b7e8f3e7492a77b6 Author: Slavko Brdar Date: Thu Oct 7 16:19:53 2021 +0100 renaming variables for code consistency commit 59c7b8985df7657e25745fd54e7d24ecb887812c Author: Slavko Brdar Date: Thu Oct 7 15:54:08 2021 +0100 cleanup commit f01a8615f8cde4de03ad602e88caefa2e17d444d Author: Slavko Brdar Date: Thu Oct 7 15:46:03 2021 +0100 avoid recomputation of node-gradient for every subcell commit 23dc725624a4accea5678b43bc32ba4c4413f781 Author: Slavko Brdar Date: Thu Oct 7 12:14:39 2021 +0100 bug fix in sorting edges (tnx Willem); vertex-to-vertex runs in 2nd order but no advantage over 1st order commit dff44b56e0996b93197e6e17955d1a7a9c654714 Author: Slavko Brdar Date: Tue Oct 5 17:24:11 2021 +0100 build edges of both src and tgt meshes early in setup commit bfdc9418f6c55e3326f7970a79f72c00fa622c45 Author: Slavko Brdar Date: Tue Oct 5 17:14:31 2021 +0100 assembled matrix for 2nd order from NodeColumn to NodeColumn/CellColumn - seg error still commit eb0dd9bce2f336b4765b7f0c6159beaa2b50cee3 Author: Slavko Brdar Date: Fri Oct 1 23:10:39 2021 +0100 make 2nd order remap work for cell-to-vertex data commit df9ec9f8dece91c5320c3ea60232aa36f471bd8c Author: Slavko Brdar Date: Fri Oct 1 18:06:02 2021 +0100 always compute subcell-to-node and node-to-subcell mapping in NodeColumns commit 1cf1093f01811ba7c3f8456192c6daaf3d518749 Author: Slavko Brdar Date: Fri Oct 1 17:20:02 2021 +0100 finished 1st order remap for vertex-to-vertex, vertex-to-cell, cell-to-vertex, cell-to-cell data commit 564c23b328fd056c380f6fdfe328199d7f880bc4 Author: Slavko Brdar Date: Fri Oct 1 15:15:58 2021 +0100 first vertex-to-cell remapping for 1st order working commit f1c07866ffdab26c1d34c15d4d11c8ad60462450 Author: Slavko Brdar Date: Thu Sep 30 15:58:08 2021 +0100 correct computation of errors in polygon intersection for node data commit e2d064b14533f11ecb418c89feed25ceb69c6740 Author: Slavko Brdar Date: Tue Sep 21 23:59:55 2021 +0100 compute area around node based on constructed subcells of mesh commit 00214f6f1def7231376ca79668e76646f4e8518b Author: Slavko Brdar Date: Tue Sep 21 23:24:28 2021 +0100 cleaning -intersect_polygon- method commit 708657f0f67b8e4800a4184c79ee3503f2d31b9c Author: Slavko Brdar Date: Tue Sep 21 23:10:21 2021 +0100 rename -centroid- to -point- to indicate where data points are store, consistent for node2node remapping commit eb9172d71152c1fdeb3f61a27b85f168d2efc7b1 Merge: 60e66ded7 96a976eb3 Author: Slavko Brdar Date: Tue Sep 21 14:47:12 2021 +0100 Merge branch 'features/conservative_remap' of ssh://git.ecmwf.int/atlas/atlas into features/conservative_remap commit 60e66ded7a397377ea40b69ac8951a155a4f868d Author: Slavko Brdar Date: Tue Sep 21 14:42:57 2021 +0100 rebasing with develop branch: resolve conflicts commit b7ca7c135bbd204225f529a9307042bfb7fd3547 Author: Slavko Brdar Date: Tue Sep 21 00:24:49 2021 +0100 compute node2subcell and subcell2node in node2node remapping commit 9018aaa6f6a4cb47f3ac10a7aed4bbdf87c5c349 Author: Slavko Brdar Date: Mon Sep 20 23:13:35 2021 +0100 further restructuring for node values commit 5510e166fa370cf7e44c421e968d1b83b9c87066 Author: Slavko Brdar Date: Mon Sep 20 18:17:56 2021 +0100 more restructuring for vertex2vertex remapping commit afacba67d8ef69b7bd7f90d35b8b22e6c361a518 Author: Slavko Brdar Date: Mon Sep 20 17:14:26 2021 +0100 add src_cell_data and tgt_cell_data to ConservativeMethod to allow for Cell- and NodeColumns in remapping commit 7a6d00d47761a8183f69d2e5601189e79906e8f0 Author: Slavko Brdar Date: Mon Sep 20 15:21:16 2021 +0100 new method for circular ordering of edges and neighbouring cells commit 0f7f1dabf1b02876d0e0f2d4f3e3e1c20f4e8d20 Author: Slavko Brdar Date: Thu Sep 16 17:04:55 2021 +0100 fix problem with non existing neighbour (tnx Willem) commit 76b0d332ca5d72f6c2d74930157ae1d6253d7d61 Author: Slavko Brdar Date: Wed Sep 15 15:02:21 2021 +0100 clean up commit 04d42261565adf88e07c89712ddf3bf1df5bbd57 Author: Slavko Brdar Date: Mon Sep 13 18:29:09 2021 +0100 consistent naming commit dbcdd158ced7518f9bfb9b6ad7bcb1f83265b227 Author: Slavko Brdar Date: Mon Sep 13 18:15:55 2021 +0100 creation of sub-elements for vertex-vertex remapping commit b7ff02fa25b1428e26f1f69d148326f9ebe8c676 Author: Slavko Brdar Date: Tue Jun 22 16:33:15 2021 +0100 bug fix in computing the global conservation error commit bf948f4098cc7e876958fae98655f75a457373a9 Author: Slavko Brdar Date: Wed Jun 16 15:28:55 2021 +0100 apply clang tool commit 39f7f99bcf6a4465af1fce0ee4ba24906db32347 Author: Slavko Brdar Date: Wed Jun 16 15:28:38 2021 +0100 revisit computation of errors in conservative remapping commit 3c1f5c4c12f5ffd92406073f3d5e5101222dcbcf Author: Slavko Brdar Date: Tue Apr 20 15:49:36 2021 +0100 setup of cached matrix for cons-2 can work without pre-sorting of triples commit 7a93b85e0182b3fca2cb2795c8924bcdb9263eeb Author: Slavko Brdar Date: Tue Apr 20 15:45:11 2021 +0100 simplification in computing cached matrix for cons-2 commit 9c5d226f72e111cb572d874e4fb746d5fb9d50c0 Author: Slavko Brdar Date: Tue Apr 20 15:40:42 2021 +0100 generate sparse matrix for Method::execute for cons-2 commit aa00c3309cd1257c489fdd6c8a803fceaad62c90 Author: Slavko Brdar Date: Wed Apr 14 21:23:43 2021 +0100 set target_field to zero only when necessary, i.e. not for Method::do_execute commit cd48eff63dcfd0d3758d6934f71bf67e9ecc18b1 Author: Slavko Brdar Date: Wed Apr 14 18:17:54 2021 +0100 make ConservativeMethod inherit from Method and assemble Method::Matrix for cons-1 commit ca0269a8d2bf9b428800230e80b7e8ab4826d18d Author: Slavko Brdar Date: Mon Apr 12 18:35:52 2021 +0100 code clean up commit 9deeed6b5a36655d2358e8fbdcb3a3dcb160358e Author: Slavko Brdar Date: Mon Apr 12 18:28:29 2021 +0100 final 2x speed up of cons-2 in matrix-nonfree version / conservative remapping commit 6ae3bd549d7d6aec64581910b1fc6fac2f318f9d Author: Slavko Brdar Date: Fri Apr 9 17:40:08 2021 +0100 compute barycentre for 2nd order remap after all, not to lose conservation commit 7a7d82f8226187799b56a684f3f16d770c66b502 Author: Slavko Brdar Date: Thu Apr 8 16:39:40 2021 +0100 use matrix_free=false by default and correct const return on non-reference types (tnx Willem) commit 59f30e87a417fcaf3188d04918acdcdd52e40abc Author: Slavko Brdar Date: Thu Apr 8 16:27:59 2021 +0100 speed up 20x for cons-2 conservative remapping commit 1d553a80c732ec73e6d41a5fd0ba3057048e711a Author: Slavko Brdar Date: Wed Apr 7 17:05:19 2021 +0100 no need to recompute barycentres in ConservativeMethod commit 75f2a43769b85b53291328a1d4c6d67e01d47b7b Author: Slavko Brdar Date: Wed Mar 31 16:39:32 2021 +0100 apply clang commit fee80963c092de14da6df9be83aedd8389df64c7 Author: Slavko Brdar Date: Wed Mar 31 16:30:37 2021 +0100 add matrix_free option to ConservativeMethod commit b07e8d2f396ade4f080d27d9db382093fd6ef5e2 Author: Slavko Brdar Date: Wed Mar 31 15:43:17 2021 +0100 1) add get_neighbours for consecutive number of neighbour cells; 2) preparing for v2v, c2v etc commit 8253e0e97417a34bb0b5afd745a3089fa4ea361c Author: Slavko Brdar Date: Tue Mar 30 18:26:04 2021 +0100 use member function get_polygon to select ConvexSphericalPolygons from Mesh commit 6b9eba58984bf316234df8e7061a95c24690208c Author: Slavko Brdar Date: Tue Mar 30 16:06:36 2021 +0100 add dump_intersection in ConvexSphericalPolygon commit 5a8f1c58ba160344055642b42bb9fbfffe96d61a Author: Slavko Brdar Date: Tue Mar 30 15:44:49 2021 +0100 make 1st cons remapping 5x faster commit 60dde3fa97b6096fff76bc1a4941803972f14b05 Author: Slavko Brdar Date: Mon Mar 29 22:20:00 2021 +0100 more debug in case of large intersection accuracy error commit e042a363817bfabea38a8fa37ce5d4b1e3dd3d48 Author: Slavko Brdar Date: Thu Mar 25 15:48:20 2021 +0000 cleanup commit 1ca6100114c7ebdb4373c1a115cb6ccfacb48135 Author: Slavko Brdar Date: Thu Mar 25 14:39:21 2021 +0000 remove warning commit 08057b924b01a4051c9ee9afa6e8a1ac28808825 Author: Slavko Brdar Date: Thu Mar 18 16:23:04 2021 +0000 fix accuracy bug in 2nd order cons remap commit 20180a5f7f99fec7b266305982dbd71b9aa8801e Author: Slavko Brdar Date: Thu Mar 18 14:14:54 2021 +0000 apply clang commit 5ffad33483f902bb1ea6a9b1c3a527318a8ef474 Author: Slavko Brdar Date: Thu Mar 18 12:03:03 2021 +0000 better variant of polygon intersection commit b177870c74809b668c7c51302d46491bd5a0a636 Author: Slavko Brdar Date: Thu Mar 18 09:06:45 2021 +0000 change in polygon intersections commit c2ec2a4a26df4cf026f0dff47711675af18613b3 Author: Slavko Brdar Date: Mon Mar 15 15:12:30 2021 +0000 compute polygon area by triangulating first commit ae54a85b05b05809e993c3a191409e3dbfb04c69 Author: Slavko Brdar Date: Mon Mar 15 09:17:05 2021 +0000 forgot file in the pervious commit commit f647cb09466f013f8b86638752ba5b4e59f4d529 Author: Slavko Brdar Date: Mon Mar 15 09:16:39 2021 +0000 specify polygon-intersection tolerances each time explicitly commit 3983ba5c786777f7af9566ae5d6ab508f3c77c47 Author: Slavko Brdar Date: Thu Mar 11 17:42:09 2021 +0000 fix in CSPolygon unit tests commit e7f72010598fed7b906eb8ef284a70e706f9a4a2 Author: Slavko Brdar Date: Thu Mar 11 17:39:05 2021 +0000 cons remappign for every grid with ever grid; apply clang commit b355cb519f9a9451847fe78c355a76b46218c6d6 Author: Slavko Brdar Date: Wed Mar 10 10:43:40 2021 +0000 check for global cons error between mapping fields commit 2013e086caad73e406e3c5619beaa04ad8463c17 Author: Slavko Brdar Date: Tue Mar 9 14:36:26 2021 +0000 explicitly specify tolerance in ConvexSphericalPolygon intersections commit 6ef6a38400f2c87c787e242dd6e8f0f9a3407eee Author: Slavko Brdar Date: Mon Mar 8 18:29:14 2021 +0000 cleanup and normalisation of remap to remedy polygon-intersection issues commit b524e688ae4d70e2b1a544e98ce2a94d40da4c7c Author: Slavko Brdar Date: Thu Mar 4 11:57:56 2021 +0000 Sutherland-Hodgman polygon intersection on sphere - not robust yet commit 8905f3359cb1eece45accd985a8f917d80e5a8cf Author: Slavko Brdar Date: Fri Feb 12 18:12:55 2021 +0000 debug output for ConvexSphericalPolygon commit 32fbbe7a32512394e17634b983a817e5720c70ce Author: Slavko Brdar Date: Fri Feb 12 17:59:33 2021 +0000 more debug for ConvexSphericalPolygon commit cdda1ef7ed1174b2abf86bab862e2482a1972074 Author: Slavko Brdar Date: Fri Feb 12 17:05:41 2021 +0000 prepare debug in case of bad ConvexSphericalPolygon intersections commit 998c889700cdaee5dc146245ff0c3164f6af3f6d Author: Slavko Brdar Date: Sat Feb 6 00:55:59 2021 +0000 remove unneccessary function commit 0306c2a2bb0fa12a3238f04e31cfae1731af0849 Author: Slavko Brdar Date: Thu Feb 4 18:36:42 2021 +0000 make empty polygons equal commit 7f621acb49d8760c6fa3fe259290c1a3bf30a32c Author: Slavko Brdar Date: Thu Feb 4 14:50:24 2021 +0000 back to not-so-bad polygon intersecting (not ideal yet) commit 47fec9cacafd14efa5edcf44772dea6b859c1fdd Author: Slavko Brdar Date: Wed Feb 3 19:37:42 2021 +0000 fix bug in polygon intersection algorithm commit 165c0f77c98622a82c6a623c7d824caa4cfeeeec Author: Slavko Brdar Date: Tue Feb 2 11:51:16 2021 +0000 use kdtree to search for candidate intersector polygons (significant speed up) commit 85c4e397eb09f42d5c8d738306a5ac59b46b2194 Author: Slavko Brdar Date: Tue Feb 2 11:50:26 2021 +0000 one more scpecific polygon intersecting in unit test commit 075de5f62ef1cf241d1bd2bc9d0c20b1a15571c4 Author: Slavko Brdar Date: Tue Feb 2 01:23:58 2021 +0000 bug fix commit ceab090e5813c676277e234f3b3f005c3c6cadfc Author: Slavko Brdar Date: Tue Feb 2 01:18:13 2021 +0000 warn about non covered source mesh area in remapping commit 43b217bf7dea61c6a397d53d45db8afc132b837a Author: Slavko Brdar Date: Tue Feb 2 01:14:49 2021 +0000 another fix for remap H16 -> H256 commit 6a01e925b8f216e3b3e51dfa82ab9edda662614b Author: Slavko Brdar Date: Mon Feb 1 18:15:10 2021 +0000 cleanup and one more unit test for polygon intersections commit cb1502608861e3e04ce94f74b1d114f88a79180b Author: Slavko Brdar Date: Mon Feb 1 18:12:09 2021 +0000 fix crashes in intersecting polygons of higher resolution Atlas grids commit 2cd239a6852b68fe90e3778e32af32ea07a1b8cb Author: Slavko Brdar Date: Sat Jan 30 00:56:49 2021 +0000 fix 2nd order accuracy in conservative remap commit 31950f2bcef3bf0960ba731fe64d5253a2ab4cd0 Author: Slavko Brdar Date: Thu Jan 28 18:13:31 2021 +0000 output test diagnostic in separate files for each CASE of cons.remap. unit test commit 8eb08f1b9821c7e473aba88e32971b10e33e286a Author: Slavko Brdar Date: Thu Jan 28 18:12:44 2021 +0000 ask 1e-14 precision for vertex2vertrex equality of polygons commit 3947afe9e7b0ac4f3196d8579c47ce1a063a2b80 Author: Slavko Brdar Date: Thu Jan 28 18:11:10 2021 +0000 add specific problematic polygons for intersections commit 377f035752469620e9a970d7084b07a791128b28 Author: Slavko Brdar Date: Thu Jan 28 16:43:09 2021 +0000 bug fixes and more diagnostic output commit 9c8308989135017114e9565362338ca5d1dc0aca Author: Slavko Brdar Date: Tue Jan 19 18:18:16 2021 +0000 write positive error thru gmsh and data in lonlat space commit 9a0dd5940ff710b7d0064503559cabaa491db294 Author: Slavko Brdar Date: Tue Jan 19 14:02:32 2021 +0000 cleanup commit 6046b881f93d5d18b4cea2f40cb73450776e615b Author: Slavko Brdar Date: Tue Jan 19 12:07:48 2021 +0000 more fixed in computing errors in cons. remap commit c5ea17b7fded09d8a9ccf22e28776eab8356c4e1 Author: Slavko Brdar Date: Mon Jan 18 19:37:16 2021 +0000 a not so good example O32->N32 commit d5361984a8b831c944bda1d27ec1c0c1bbf04903 Author: Slavko Brdar Date: Mon Jan 18 19:13:21 2021 +0000 cleanup commit 6045d04b0b2d8aa83c5ff2530b10fd9e15aa1bb4 Author: Slavko Brdar Date: Mon Jan 18 19:11:10 2021 +0000 cleanup commit 60f1a279fc595fe72699a1b98d218a286d050876 Author: Slavko Brdar Date: Mon Jan 18 18:17:46 2021 +0000 cleanup commit 4b83a11391b08bc946357fb4be304ce6cb848bc7 Author: Slavko Brdar Date: Mon Jan 18 16:45:33 2021 +0000 minor corrections commit 5a5ab5f0ed27f63ee3c94955bc66350452365645 Author: Slavko Brdar Date: Fri Jan 15 14:06:03 2021 +0000 improve unit test for cons remaps commit 7fa29ef103fff85e2b139c957614e37329654909 Author: Slavko Brdar Date: Fri Jan 15 14:03:09 2021 +0000 debugging 2nd order cons commit 65d23ec1f9ef5a146f67a4b10b6ceb0d9befaeaa Author: Slavko Brdar Date: Thu Jan 14 17:49:36 2021 +0000 bug fix commit d282655a14e7304344f395933c92959a381640b5 Author: Slavko Brdar Date: Thu Jan 14 15:30:40 2021 +0000 another fix for higher resolution: increase search radius by 1.3x commit a14f0f9852788a1e30c0feaa86b41925b3d738f5 Author: Slavko Brdar Date: Wed Jan 13 18:58:37 2021 +0000 more comprehensive test for soncervative remapping commit f71a228d0f3ced50201018d5a420daafbd46cf35 Author: Slavko Brdar Date: Wed Jan 13 18:52:54 2021 +0000 not too strict on polygon area commit e59671f75f672a636bc030f6c846181f58d7e095 Author: Slavko Brdar Date: Wed Jan 13 16:53:06 2021 +0000 8x speed up of ConservativeMethod::do_setup commit 228d362d0e5dc8835624799d5947783ea266f088 Author: Slavko Brdar Date: Wed Jan 13 16:15:41 2021 +0000 allow zero area polygons commit fc56644db06de69d2073e417148d5886c7753bdb Author: Slavko Brdar Date: Tue Jan 12 18:50:20 2021 +0000 fix bugs in onSegment for high-res atlas grids (still checking for more bugs) commit 5dd8d8840ec90b5866101cd61481acacdc30f515 Author: Slavko Brdar Date: Mon Jan 11 18:53:40 2021 +0000 expand unit test for conservative remap commit 4cd98e308ab680aed86dde347f958bdd4edb39ef Author: Slavko Brdar Date: Mon Jan 11 18:50:06 2021 +0000 improve polygon equality commit c808383070427e4cc4a2fa237d6cd4681897deb6 Author: Slavko Brdar Date: Sun Jan 10 00:39:39 2021 +0000 compute errors as in Kritsikis et al. commit 833c01d4f0fbc72e8fbe0023c86336251f963405 Author: Slavko Brdar Date: Thu Jan 7 19:14:02 2021 +0000 compute global and local conservation errors commit c5e9ff09a10be24efa5cc193fd39488f946dcc92 Author: Slavko Brdar Date: Thu Jan 7 18:14:08 2021 +0000 when intersecting quads, max number of vertices of intersection polygon can be 8 commit 50a47049fcd791ef1484e3d884f1c681b0c201a3 Author: Slavko Brdar Date: Wed Jan 6 15:41:33 2021 +0000 make 1st and 2nd order convervative run with healpix as well (tnx Willem) commit c0584dded115340db1bf099647acccc370f89ce2 Author: Slavko Brdar Date: Wed Jan 6 15:06:56 2021 +0000 make cons. remap work for Healpix meshes (2nd order cons. does not work git diff ../interpolation/method/knn/ConservativeMethod.cc ConvexSphericalPolygon.cc ../../tests/interpolation/test_interpolation_conservative.cc) commit 0240547e435ce23666ea7a9b29e4da9291f24da4 Author: Slavko Brdar Date: Tue Jan 5 18:06:03 2021 +0000 add source mesh as member of ConservativeMethod commit 30e7819474ba2a9a33a3509035b95780e0c6df87 Author: Slavko Brdar Date: Tue Jan 5 17:53:58 2021 +0000 add ProgressTimer in brute force polygon intersections commit 2b84b6dc82123bf31e4eb8d06ec99d44719c4e9f Author: Slavko Brdar Date: Tue Jan 5 17:41:04 2021 +0000 a better test for 1st order cons remap commit 43ae480ce7289e20799d703612b3eb7aec10768b Author: Slavko Brdar Date: Tue Jan 5 17:36:29 2021 +0000 a working 1st order conservative remap on O8 commit 7489e7e05c5ff10b0dbf6f8a6893f18a2235e9a0 Author: Slavko Brdar Date: Tue Jan 5 17:15:53 2021 +0000 correct counter-clockwise orientation for vertices of a grid cell based on structured (regular meshgen has the oposite) commit f5e36b2f10f2e9f4abde4da9ac44e0a1a033d372 Author: Slavko Brdar Date: Tue Jan 5 16:00:04 2021 +0000 apply clang-format commit 7267abbe3084793ec0ffdab6386e5d9c22822c29 Author: Slavko Brdar Date: Tue Jan 5 15:59:11 2021 +0000 move cons. interpolation to atlas/interpolation/method/knn/ConservativeMethod.h,cc commit 706752be4870a08553ccc1fd772fc986a9c25c88 Author: Slavko Brdar Date: Mon Jan 4 19:06:47 2021 +0000 apply clang-format commit 37b458d2144a3846dbe02b5a9e47db6910f1ff15 Author: Slavko Brdar Date: Mon Jan 4 19:04:08 2021 +0000 ConvexSPhericalPolygon does not need vertex orientation commit 82014a0637a2163baadef79754f4391c35701a76 Author: Slavko Brdar Date: Mon Jan 4 15:57:54 2021 +0000 make area of ConvexSphericalPolygon with two distinct point equal 0 commit 3a4691c8768e4f69786db39fafa2f816b42329d9 Author: Slavko Brdar Date: Mon Jan 4 15:18:14 2021 +0000 use coordinate-based distance rather that vector norm commit 6905957a40a77d3ba1aae7b0484acbf392861495 Author: Slavko Brdar Date: Mon Jan 4 12:10:50 2021 +0000 print lonlat instead of xyz coords in debug prints of ConvexSphericalPolygon; apply clang-format commit 4737bd47aab4c65ef3549e97a9566011034b9cfc Author: Slavko Brdar Date: Sun Jan 3 23:57:59 2021 +0000 cache area for ConvexSphericalPolygon commit abf9dfb8c613c618836b39be592fbb16c168fd3a Author: Slavko Brdar Date: Sun Jan 3 01:12:25 2021 +0000 fix gradient of 2nd order cons as in Kritsikis et al. commit d46f5a0a62d191f627d2b054623c64da87135d09 Author: Slavko Brdar Date: Sun Jan 3 00:29:32 2021 +0000 fix gradient in 2nd order cons, print error on target grid commit fa21d9cc05d41060975dcbf9993c63eab62314a9 Author: Slavko Brdar Date: Thu Dec 31 17:57:13 2020 +0000 a non-constant test working correct commit 27d1bbf18f11e1de1a6f041c0bcdfc2e53494b45 Author: Slavko Brdar Date: Thu Dec 31 17:31:00 2020 +0000 running correctly for constant function commit 1026abc7896f9b36fc35dcd66b156b468ee86aaf Author: Slavko Brdar Date: Wed Dec 30 19:10:52 2020 +0000 first running version of 2nd order conservative remap example (albeit buggy) commit bd3f86fc03e48b2b78daf3483572d8638f79b614 Author: Slavko Brdar Date: Tue Dec 29 18:13:57 2020 +0000 test case for conservative remapping (not done yet) commit f6e0e4c14dd39397c4ae1ff70fd9bdf0a31250bf Author: Slavko Brdar Date: Tue Dec 29 18:13:07 2020 +0000 centroid of ConvexSphericalPolygon should be on sphere commit 9d04bd72665ad67ace8f928e587548bb535c03c3 Author: Slavko Brdar Date: Thu Dec 17 23:22:17 2020 +0000 unit test example: given two grids compute remapping weights, centroids for conservative remapping commit 9116c91e9dad6524539099f673ff9f0d1837c08a Author: Slavko Brdar Date: Thu Dec 17 23:21:01 2020 +0000 bug fix in ConvexSphericalPolygon initialization commit 7e29a3d1cd4e6a07f4544850ee6fdfcd5fb5f30d Author: Slavko Brdar Date: Thu Dec 17 16:24:52 2020 +0000 reverse ordering of vertices from node_connectivity for ConvexSphericalPolygon commit b3fc0dc29d9844902532e5af4c77e7ea28cda979 Author: Slavko Brdar Date: Thu Dec 17 15:26:59 2020 +0000 a buggy test in orientation of element vertices commit 2360a2b890c2197d3ddcb44d4ba865dbb9f1b39c Author: Slavko Brdar Date: Thu Dec 17 15:26:23 2020 +0000 empty polygon should have area = 0 commit 57da76d8e4bc6ebbd8ce4d21fc982fd2f54d6492 Author: Slavko Brdar Date: Thu Dec 17 15:15:08 2020 +0000 use RegularMeshGenerator for regional grids, still get strange elements commit fdf038d1928ce0f40af2c651c5d0b441ef90ea9e Author: Slavko Brdar Date: Thu Dec 17 15:05:22 2020 +0000 unit test for intersecting elements of two regional grids commit 05e64ef8140abbfba8a118a17b904c9038d34163 Author: Slavko Brdar Date: Thu Dec 17 14:59:18 2020 +0000 print of Polygon should be in more user friendly lonlat coordinates commit d78592915ff4c73b28eabeb51ce6d8a29851c25a Author: Slavko Brdar Date: Wed Dec 16 17:53:43 2020 +0000 fix memory leaks (tnx Willem) commit 62b3642b4a916d481f09165dcb6bd7982c4e35c3 Author: Slavko Brdar Date: Wed Dec 16 16:33:34 2020 +0000 remove the last repetition point in ConvexSphericalPolygon commit 7b80f98bbc066eaaf31a7bfa07e73a20f5c80ec1 Author: Willem Deconinck Date: Wed Dec 16 14:07:53 2020 +0000 ATLAS-322 ConvexSphericalPolygon lives on stack entirely, with support of MAX_SIZE=8 points commit ee2721108a7cbeba950892baae9b3a630340e29b Author: Willem Deconinck Date: Wed Dec 16 13:34:15 2020 +0000 ATLAS-322 clang-format commit 585031d9a26f674500ec9b9222eb51efb7e12633 Author: Willem Deconinck Date: Wed Dec 16 13:32:27 2020 +0000 ATLAS-322 No more pointers (and memory leaks) commit 7d88a3938b6c67289e7defd2f9df7b54e8cc7e79 Author: Willem Deconinck Date: Wed Dec 16 13:03:04 2020 +0000 ATLAS-322 ConvexSphericalPolygon operator<< commit 9ae25ea9e3173337244eca1fc1668a4112597177 Author: Willem Deconinck Date: Wed Dec 16 12:59:15 2020 +0000 ATLAS-322 No more pointers in ConvexSphericalPolygon commit 87fc2d095d54fc9eec2570f373487ead5a3d0557 Author: Willem Deconinck Date: Wed Dec 16 12:47:54 2020 +0000 ATLAS-322 ConvexSphericalPolygon default constructor commit 2359bc004bf86ca56f1b56968d22021e8ae16c3e Author: Willem Deconinck Date: Wed Dec 16 12:40:25 2020 +0000 ATLAS-322 Do not inherit ConvexSphericalPolygon from PolygonCoordinates commit 330b58fe971b774e15a8c8803336cf4fb09fc9ed Author: Slavko Brdar Date: Tue Dec 15 17:01:18 2020 +0000 add centroid of spherical polygon commit bb12955c9ea5e427fdafd1b53169cabaa5fff393 Author: Slavko Brdar Date: Tue Dec 15 16:33:23 2020 +0000 area of spherical polygons commit 6a311d80b8dbdb6194eef88b8c1a5108e0468e00 Author: Slavko Brdar Date: Mon Dec 14 17:58:51 2020 +0000 more comprehensive unit test for spherical polygon intersections commit f7f583305540d6559d9ad1c0364f9b4f5fe0b3a1 Author: Slavko Brdar Date: Thu Dec 10 16:40:10 2020 +0000 34 example of polygon intersections working fine, note sensitive value for eps commit 89459670f951f02c86db8bd4c6a195db4ae7d255 Author: Slavko Brdar Date: Thu Dec 10 15:52:57 2020 +0000 improve in unit test output, show what two polygons intersect, and the resulting polygon commit 6b12251ba4ed03953e1f5479514791b4a7276f6a Author: Slavko Brdar Date: Thu Dec 10 15:26:49 2020 +0000 another bug fix, more compact unit test commit 4745ffbc821bd7b98b0343a116f8d01afbd72aac Author: Slavko Brdar Date: Thu Dec 10 14:06:38 2020 +0000 replace fabs with abs (thx Willem) commit 5185990acd1bb37d8c52b70cd701d2eb6e5eb45b Author: Slavko Brdar Date: Thu Dec 10 12:42:40 2020 +0000 fix bugs, plus more test cases commit 414e37064bdf0e575ee8b9f37546817a52406e4d Author: Slavko Brdar Date: Wed Dec 9 18:54:40 2020 +0000 first working version of convex polygon intersections on sphere commit d0a310e8ef106c51b255b792018c277de810586e Author: Slavko Brdar Date: Wed Dec 9 11:30:23 2020 +0000 almost working spherical polygon intersection commit f607b052c71f4794b9d2baa3ae7d342acb00c656 Author: Slavko Brdar Date: Tue Dec 1 17:22:47 2020 +0000 recursive search of intersection polygon, not done yet commit 8577c38d96c742038570c67b1b060b51ad9ec7c9 Author: Slavko Brdar Date: Mon Nov 23 18:14:55 2020 +0000 correct intersection of spherical segments commit 06e7292a04bfae4dc85e5bc2aa8e5bfbbe2b1094 Author: Slavko Brdar Date: Fri Nov 20 17:09:43 2020 +0000 add ConvexSphericalPolygon and test_spherical_geo commit 96a976eb366897496239f97035ceacd84767f15a Author: Slavko Brdar Date: Tue Sep 21 00:24:49 2021 +0100 compute node2subcell and subcell2node in node2node remapping commit aa1b79229d8777cd4cc09e0e9aafe7a419e1e2c0 Author: Slavko Brdar Date: Mon Sep 20 23:13:35 2021 +0100 further restructuring for node values commit b6f5410a06bec11f7244c9da8a09f2fbd45de68e Author: Slavko Brdar Date: Mon Sep 20 18:17:56 2021 +0100 more restructuring for vertex2vertex remapping commit bf43285bf26a7b8aa10c45c3bc95205f38e3b657 Author: Slavko Brdar Date: Mon Sep 20 17:14:26 2021 +0100 add src_cell_data and tgt_cell_data to ConservativeMethod to allow for Cell- and NodeColumns in remapping commit a47a959f9bb4cefc33c0b4b45bb46645fb173edf Author: Slavko Brdar Date: Mon Sep 20 15:21:16 2021 +0100 new method for circular ordering of edges and neighbouring cells commit 196f0a705391d7c87777974b5734680bf54fac69 Author: Slavko Brdar Date: Thu Sep 16 17:04:55 2021 +0100 fix problem with non existing neighbour (tnx Willem) commit 5c266a312bf58d56538b2cb153bd39a5d4b93b56 Author: Slavko Brdar Date: Wed Sep 15 15:02:21 2021 +0100 clean up commit eb5e2789d2434c87b07213a1897ac7ebfbdd3a8f Author: Slavko Brdar Date: Mon Sep 13 18:29:09 2021 +0100 consistent naming commit 096374097aceb5aee8c3546efdc17f751eec9792 Author: Slavko Brdar Date: Mon Sep 13 18:15:55 2021 +0100 creation of sub-elements for vertex-vertex remapping commit 8e9d9dac0b7e0e331dde76a5d9bdc0ab09c6fabf Author: Slavko Brdar Date: Tue Jun 22 16:33:15 2021 +0100 bug fix in computing the global conservation error commit 22cb2961af2a9d74f8835c2c298d16ed10de6b32 Author: Slavko Brdar Date: Wed Jun 16 15:28:55 2021 +0100 apply clang tool commit c8d33dce988a44329d541ce3f8ce117bc9e9f2ca Author: Slavko Brdar Date: Wed Jun 16 15:28:38 2021 +0100 revisit computation of errors in conservative remapping commit 6eb377042db67e74efaa98764696d54082f8cf1e Merge: a2ca551bb 99fedb2c6 Author: Slavko Brdar Date: Mon Jun 7 14:59:28 2021 +0100 Merge remote-tracking branch 'ec-atlas/develop' into features/conservative_remap commit a2ca551bb5f65709c65879270c3694fdaec7f10d Author: Slavko Brdar Date: Tue Apr 20 15:49:36 2021 +0100 setup of cached matrix for cons-2 can work without pre-sorting of triples commit efc85767ed8523fef8d2e49f3dbc15298a53cba9 Author: Slavko Brdar Date: Tue Apr 20 15:45:11 2021 +0100 simplification in computing cached matrix for cons-2 commit a6a8fe5257b13508ca9e5c8db348946901e6af0b Author: Slavko Brdar Date: Tue Apr 20 15:40:42 2021 +0100 generate sparse matrix for Method::execute for cons-2 commit 4105c59e21453f6a5eb1b5cc73987c717c64134e Author: Slavko Brdar Date: Wed Apr 14 21:23:43 2021 +0100 set target_field to zero only when necessary, i.e. not for Method::do_execute commit 82c060070e0d7bb9d85d5455d7ebdd4ba876d476 Author: Slavko Brdar Date: Wed Apr 14 18:17:54 2021 +0100 make ConservativeMethod inherit from Method and assemble Method::Matrix for cons-1 commit deb5761c749d43739da1cb0b09f5cb78dccc606d Author: Slavko Brdar Date: Mon Apr 12 18:35:52 2021 +0100 code clean up commit 67c7053f05b3cef5352f73815eccdfe4ca8acc4c Author: Slavko Brdar Date: Mon Apr 12 18:28:29 2021 +0100 final 2x speed up of cons-2 in matrix-nonfree version / conservative remapping commit b13be195c662ceaf6184da7a491980669659ea8b Merge: 15550b590 5cba4fbae Author: Slavko Brdar Date: Fri Apr 9 19:37:31 2021 +0100 Merge branch 'features/conservative_remap_fix_conservation' into features/conservative_remap commit 5cba4fbae1d63f5c54f6a97400770d1d9652d4d7 Author: Slavko Brdar Date: Fri Apr 9 17:40:08 2021 +0100 compute barycentre for 2nd order remap after all, not to lose conservation commit 15550b5909b6a0ecdb4b02b3bee60045b8089923 Author: Slavko Brdar Date: Thu Apr 8 16:39:40 2021 +0100 use matrix_free=false by default and correct const return on non-reference types (tnx Willem) commit 07740d8f27d008b2bb87827b0340f6325c99565a Author: Slavko Brdar Date: Thu Apr 8 16:27:59 2021 +0100 speed up 20x for cons-2 conservative remapping commit 0a28ec1e83021774edc47bcf905790b1515d905d Author: Slavko Brdar Date: Wed Apr 7 17:05:19 2021 +0100 no need to recompute barycentres in ConservativeMethod commit 7e2ac67da6128bb49607f759b61c20f0d6561e5a Author: Slavko Brdar Date: Wed Mar 31 16:39:32 2021 +0100 apply clang commit afda5fe50deea464dffb0758a4cca50e44b02ab5 Author: Slavko Brdar Date: Wed Mar 31 16:30:37 2021 +0100 add matrix_free option to ConservativeMethod commit e55b411d21b7f3e05d6525de613cd7c36c9f3b3b Author: Slavko Brdar Date: Wed Mar 31 15:43:17 2021 +0100 1) add get_neighbours for consecutive number of neighbour cells; 2) preparing for v2v, c2v etc commit 4f7a7dd84b7d707a7c9e103aed49e0ec4ce75c87 Author: Slavko Brdar Date: Tue Mar 30 18:26:04 2021 +0100 use member function get_polygon to select ConvexSphericalPolygons from Mesh commit faced0e3b6d6b3c1691cc957f5956cf06dfe1565 Author: Slavko Brdar Date: Tue Mar 30 16:06:36 2021 +0100 add dump_intersection in ConvexSphericalPolygon commit 99a7c056b5815fb8efd840572c9cc12ccfa8b09f Author: Slavko Brdar Date: Tue Mar 30 15:44:49 2021 +0100 make 1st cons remapping 5x faster commit ab0bd248be2a2a4eee3d947ff6ea9da48f0f9c11 Author: Slavko Brdar Date: Mon Mar 29 22:20:00 2021 +0100 more debug in case of large intersection accuracy error commit 69baeade93ad710e162d5ec80c5597f760cabfaa Author: Slavko Brdar Date: Thu Mar 25 15:48:20 2021 +0000 cleanup commit 06b72842644123cc89fb34f9f1199bb8ebe3add9 Author: Slavko Brdar Date: Thu Mar 25 14:39:21 2021 +0000 remove warning commit c896a9a30979a94aeba9e5329b468df034b00889 Merge: d96b81603 071bbb18c Author: Slavko Brdar Date: Fri Mar 19 14:22:03 2021 +0000 Merge remote-tracking branch 'ec-atlas/master' into features/conservative_remap commit d96b8160333628de16298a073ee882eac55b3008 Author: Slavko Brdar Date: Thu Mar 18 16:23:04 2021 +0000 fix accuracy bug in 2nd order cons remap commit a2c77f98756884b590dc7957b5fb90cb4962bbfd Author: Slavko Brdar Date: Thu Mar 18 14:14:54 2021 +0000 apply clang commit b0d137e417ce37a3d854de726c674bca6b7514f7 Author: Slavko Brdar Date: Thu Mar 18 12:03:03 2021 +0000 better variant of polygon intersection commit 5fe50c95286c5db643a30ff719b6e2405ed7a69a Author: Slavko Brdar Date: Thu Mar 18 09:06:45 2021 +0000 change in polygon intersections commit cbcd82939c3b1289c15f009fac7400f064aa0139 Author: Slavko Brdar Date: Mon Mar 15 15:12:30 2021 +0000 compute polygon area by triangulating first commit feb9af50836910c646848e8cfe188eb06bf66324 Author: Slavko Brdar Date: Mon Mar 15 09:17:05 2021 +0000 forgot file in the pervious commit commit fe55468e8c1ccf01d6733259e069ebd27fb22814 Author: Slavko Brdar Date: Mon Mar 15 09:16:39 2021 +0000 specify polygon-intersection tolerances each time explicitly commit d1d26816ac6619ac6084c14039891b79d9ea4f0a Author: Slavko Brdar Date: Thu Mar 11 17:42:09 2021 +0000 fix in CSPolygon unit tests commit eec657c678f9516460573258ba794bec4c59a532 Author: Slavko Brdar Date: Thu Mar 11 17:39:05 2021 +0000 cons remappign for every grid with ever grid; apply clang commit 86f814ef35053cf70796c31a258423878c387eae Author: Slavko Brdar Date: Wed Mar 10 10:43:40 2021 +0000 check for global cons error between mapping fields commit e3af4af9df1c419cbd5ad73bece4a036c34db1c4 Author: Slavko Brdar Date: Tue Mar 9 14:36:26 2021 +0000 explicitly specify tolerance in ConvexSphericalPolygon intersections commit 1e3316b7dfb92538c9f6c9eafbc20df4e540f623 Author: Slavko Brdar Date: Mon Mar 8 18:29:14 2021 +0000 cleanup and normalisation of remap to remedy polygon-intersection issues commit 0e26ac14e8fe25c40bf1b2931b7e06c1b085ea8e Author: Slavko Brdar Date: Thu Mar 4 11:57:56 2021 +0000 Sutherland-Hodgman polygon intersection on sphere - not robust yet commit ce5ef2d2af9d863f06ed1d5d4c9921a6741c0b4e Author: Slavko Brdar Date: Fri Feb 12 18:12:55 2021 +0000 debug output for ConvexSphericalPolygon commit 9b98fc7ef8b9b607d1c74c5df27e23c8cb72ce21 Author: Slavko Brdar Date: Fri Feb 12 17:59:33 2021 +0000 more debug for ConvexSphericalPolygon commit 2f26bb4b9775b801612ee104d1d7ff59c65cd12e Author: Slavko Brdar Date: Fri Feb 12 17:05:41 2021 +0000 prepare debug in case of bad ConvexSphericalPolygon intersections commit 10369c5c7ba3548b030bdac1a432984b022e4cd7 Author: Slavko Brdar Date: Sat Feb 6 00:55:59 2021 +0000 remove unneccessary function commit 4f36855193d207ce138c549952026184ed77fb9c Author: Slavko Brdar Date: Thu Feb 4 18:36:42 2021 +0000 make empty polygons equal commit 2b68ef7d2850bdca934013883c3e07bcf466252f Author: Slavko Brdar Date: Thu Feb 4 14:50:24 2021 +0000 back to not-so-bad polygon intersecting (not ideal yet) commit 449638efa3bc5dccd178b26b056c162a6323eca2 Author: Slavko Brdar Date: Wed Feb 3 19:37:42 2021 +0000 fix bug in polygon intersection algorithm commit 2f88fcabce9294332f900811d463d6537189de6e Author: Slavko Brdar Date: Tue Feb 2 11:51:16 2021 +0000 use kdtree to search for candidate intersector polygons (significant speed up) commit ae1d804eb1428e0af57b675f256a8e9154d79db7 Author: Slavko Brdar Date: Tue Feb 2 11:50:26 2021 +0000 one more scpecific polygon intersecting in unit test commit 5dace1981d657bfa16e4fc7193fc4e7bc3384cb4 Author: Slavko Brdar Date: Tue Feb 2 01:23:58 2021 +0000 bug fix commit cfedea0c6a68505a734bc25503f2420326e7973e Author: Slavko Brdar Date: Tue Feb 2 01:18:13 2021 +0000 warn about non covered source mesh area in remapping commit f98e9fcf7e2aa5a0965b62a7d4f38586bcb3a0c1 Author: Slavko Brdar Date: Tue Feb 2 01:14:49 2021 +0000 another fix for remap H16 -> H256 commit 562ecd98cb1ef893f31a8c588a25d04cc86ca5a8 Author: Slavko Brdar Date: Mon Feb 1 18:15:10 2021 +0000 cleanup and one more unit test for polygon intersections commit ae956765af4d5fd7fe19e810cb6855d15abeb809 Author: Slavko Brdar Date: Mon Feb 1 18:12:09 2021 +0000 fix crashes in intersecting polygons of higher resolution Atlas grids commit f055727cb2e4465cf050243b7d95044f30254b5e Author: Slavko Brdar Date: Sat Jan 30 00:56:49 2021 +0000 fix 2nd order accuracy in conservative remap commit 5bfa6aa7315b293c1d4af28d6962300eb621d8d5 Author: Slavko Brdar Date: Thu Jan 28 18:13:31 2021 +0000 output test diagnostic in separate files for each CASE of cons.remap. unit test commit bcf97419abae0f370b51a0e0a106611a1e279e9e Author: Slavko Brdar Date: Thu Jan 28 18:12:44 2021 +0000 ask 1e-14 precision for vertex2vertrex equality of polygons commit 77d06381ff0390bff19fdc4c39129fe4f63088ed Author: Slavko Brdar Date: Thu Jan 28 18:11:10 2021 +0000 add specific problematic polygons for intersections commit 9c5529dab553e55ac44d6519e25cebea0e937c59 Author: Slavko Brdar Date: Thu Jan 28 16:43:09 2021 +0000 bug fixes and more diagnostic output commit 01bc63b30d08cdaeffa035498c488be1c47f58f3 Author: Slavko Brdar Date: Tue Jan 19 18:18:16 2021 +0000 write positive error thru gmsh and data in lonlat space commit ccdd15900b67d38ec861533a5ab86fcb112a8035 Author: Slavko Brdar Date: Tue Jan 19 14:02:32 2021 +0000 cleanup commit c3947a857f2c485ff774e793b1d7e84ab0941c64 Author: Slavko Brdar Date: Tue Jan 19 12:07:48 2021 +0000 more fixed in computing errors in cons. remap commit a13b277d7c4568f5780a59af8c13dcfdb64888fc Author: Slavko Brdar Date: Mon Jan 18 19:37:16 2021 +0000 a not so good example O32->N32 commit f61a4874abcf3cd76c1053e15d539dd46f57b359 Merge: 4011de89a ab7318a76 Author: Slavko Brdar Date: Mon Jan 18 19:20:00 2021 +0000 Merge branch 'features/conservative_remap' of ssh://git.ecmwf.int/atlas/atlas into features/conservative_remap commit 4011de89a12b1d8e2567b58af42ff6b4c8a94b07 Author: Slavko Brdar Date: Mon Jan 18 19:13:21 2021 +0000 cleanup commit c85810bb9c720f03a5ee477b4d6e5ef52e694c37 Author: Slavko Brdar Date: Mon Jan 18 19:11:10 2021 +0000 cleanup commit 3186c055c9f90e183d5b7c2a339d68bdfdcafea6 Author: Slavko Brdar Date: Mon Jan 18 18:17:46 2021 +0000 cleanup commit dffab1cd654f0448c4c2ecdba5285e97ae46a742 Author: Slavko Brdar Date: Mon Jan 18 16:45:33 2021 +0000 minor corrections commit ab7318a76af9c0c38460514e59a665c9202c3af6 Author: Slavko Brdar Date: Fri Jan 15 14:06:03 2021 +0000 improve unit test for cons remaps commit 569a302ad21796887af6bdeb3dc8dc25a160b2c0 Author: Slavko Brdar Date: Fri Jan 15 14:03:09 2021 +0000 debugging 2nd order cons commit c86d71475ca1111e1bf64409e4df29693f99cad3 Author: Slavko Brdar Date: Thu Jan 14 17:49:36 2021 +0000 bug fix commit e6f8cc27e8acbd11d60ed4b697630f6d50e98aef Author: Slavko Brdar Date: Thu Jan 14 15:30:40 2021 +0000 another fix for higher resolution: increase search radius by 1.3x commit 669590e5ca0110669aa1a62b30fc161b3f6d0202 Author: Slavko Brdar Date: Wed Jan 13 18:58:37 2021 +0000 more comprehensive test for soncervative remapping commit 17c0a669b02831a8598f07d94aad170282095490 Author: Slavko Brdar Date: Wed Jan 13 18:52:54 2021 +0000 not too strict on polygon area commit f2f8a39a0eb6040c7d5b3d53fc5d874587ff9b1f Author: Slavko Brdar Date: Wed Jan 13 16:53:06 2021 +0000 8x speed up of ConservativeMethod::do_setup commit 7cd6761b2c45931c74f918d56a7fa39563284fe7 Author: Slavko Brdar Date: Wed Jan 13 16:15:41 2021 +0000 allow zero area polygons commit 2d15bab63e801eb8ec73051f8ed27563ab07482d Author: Slavko Brdar Date: Tue Jan 12 18:50:20 2021 +0000 fix bugs in onSegment for high-res atlas grids (still checking for more bugs) commit 642b51ddd1ed5acd9d6a61d73b3e9b8c61a43e7a Author: Slavko Brdar Date: Mon Jan 11 18:53:40 2021 +0000 expand unit test for conservative remap commit cb289b31cfc6f434993d7067f0cd264b1ba6cbb6 Author: Slavko Brdar Date: Mon Jan 11 18:50:06 2021 +0000 improve polygon equality commit 9418930dc5d842a467935b4c4c5e963465ab4315 Author: Slavko Brdar Date: Sun Jan 10 00:39:39 2021 +0000 compute errors as in Kritsikis et al. commit f5ca3004018c0d07dc6460ac4ccc7ac07b210bbe Author: Slavko Brdar Date: Thu Jan 7 19:14:02 2021 +0000 compute global and local conservation errors commit b4a494f3d3c2a2e2fae4ad1a4495187045dcaf67 Author: Slavko Brdar Date: Thu Jan 7 18:14:08 2021 +0000 when intersecting quads, max number of vertices of intersection polygon can be 8 commit 78af0a8fc2d7018f890be99ae49ddb31de6e00b0 Author: Slavko Brdar Date: Wed Jan 6 15:41:33 2021 +0000 make 1st and 2nd order convervative run with healpix as well (tnx Willem) commit 91abeb3bab2cd7404f4640b0864530840c57fce9 Author: Slavko Brdar Date: Wed Jan 6 15:06:56 2021 +0000 make cons. remap work for Healpix meshes (2nd order cons. does not work git diff ../interpolation/method/knn/ConservativeMethod.cc ConvexSphericalPolygon.cc ../../tests/interpolation/test_interpolation_conservative.cc) commit a3ad33d355ab8f23b230338c3168e2f3393e66e8 Author: Slavko Brdar Date: Tue Jan 5 18:06:03 2021 +0000 add source mesh as member of ConservativeMethod commit ab77088d2c1e21c0f8c7401f574682b7c1f7ccc5 Author: Slavko Brdar Date: Tue Jan 5 17:53:58 2021 +0000 add ProgressTimer in brute force polygon intersections commit fa4812fc16d88b2bca6d1e0a05097739a4b7c32e Author: Slavko Brdar Date: Tue Jan 5 17:41:04 2021 +0000 a better test for 1st order cons remap commit f33f45006ca6d2fe0d4adfe585a7d2142aa12cf6 Author: Slavko Brdar Date: Tue Jan 5 17:36:29 2021 +0000 a working 1st order conservative remap on O8 commit 11304b584511a3f34cf31ae88dbd58d6454f8f04 Author: Slavko Brdar Date: Tue Jan 5 17:15:53 2021 +0000 correct counter-clockwise orientation for vertices of a grid cell based on structured (regular meshgen has the oposite) commit 8bfe6ba672dc88d63e09ed5e29da35d4932860ee Author: Slavko Brdar Date: Tue Jan 5 16:00:04 2021 +0000 apply clang-format commit 093f5bb4c436b504b0366e25c3e3d0f88eec5dce Author: Slavko Brdar Date: Tue Jan 5 15:59:11 2021 +0000 move cons. interpolation to atlas/interpolation/method/knn/ConservativeMethod.h,cc commit 87a2db3d2ff1091b2a90f8a13b2d54ccd2dc0180 Author: Slavko Brdar Date: Mon Jan 4 19:06:47 2021 +0000 apply clang-format commit f9968b3d199127fc882049fadd58b2ededaf7167 Author: Slavko Brdar Date: Mon Jan 4 19:04:08 2021 +0000 ConvexSPhericalPolygon does not need vertex orientation commit c598faf71e2cebb5529d924c55765b4baa9e58af Author: Slavko Brdar Date: Mon Jan 4 15:57:54 2021 +0000 make area of ConvexSphericalPolygon with two distinct point equal 0 commit 3d3526158ef924bee218fde0a4c2a47ed38a3bda Author: Slavko Brdar Date: Mon Jan 4 15:18:14 2021 +0000 use coordinate-based distance rather that vector norm commit 5d0e68604c34ed03562f426f52fc7beb5f1b3e59 Author: Slavko Brdar Date: Mon Jan 4 12:10:50 2021 +0000 print lonlat instead of xyz coords in debug prints of ConvexSphericalPolygon; apply clang-format commit 795b1cf649efb6ed7f15f510b4e5219e1db32aab Author: Slavko Brdar Date: Sun Jan 3 23:57:59 2021 +0000 cache area for ConvexSphericalPolygon commit ab96fa3a4ebb8503ff7ac9b5e70f15f4451ed636 Author: Slavko Brdar Date: Sun Jan 3 01:12:25 2021 +0000 fix gradient of 2nd order cons as in Kritsikis et al. commit 4366b1273913a10153e49682aa3701785f66111f Author: Slavko Brdar Date: Sun Jan 3 00:29:32 2021 +0000 fix gradient in 2nd order cons, print error on target grid commit 8990cd1c22cb042d689bb95f29d3a156bccaec81 Author: Slavko Brdar Date: Thu Dec 31 17:57:13 2020 +0000 a non-constant test working correct commit 5a5f3e5a47088ee49e31aa2793daecdd2a3736ab Author: Slavko Brdar Date: Thu Dec 31 17:31:00 2020 +0000 running correctly for constant function commit 6c58aae34829493f3290de3cd7b82030291550a4 Author: Slavko Brdar Date: Wed Dec 30 19:10:52 2020 +0000 first running version of 2nd order conservative remap example (albeit buggy) commit 4a5093511ed6b8629533fae21a58a323e0dfdb5f Author: Slavko Brdar Date: Tue Dec 29 18:13:57 2020 +0000 test case for conservative remapping (not done yet) commit b1854f48c1dca70f1beae660f82824eedada0844 Author: Slavko Brdar Date: Tue Dec 29 18:13:07 2020 +0000 centroid of ConvexSphericalPolygon should be on sphere commit 4bac63559f52c2b4fd3414e20b1b8b61c97ea55b Author: Slavko Brdar Date: Thu Dec 17 23:22:17 2020 +0000 unit test example: given two grids compute remapping weights, centroids for conservative remapping commit 84ded57b86baf385beb823232f048b55b0f1cac7 Author: Slavko Brdar Date: Thu Dec 17 23:21:01 2020 +0000 bug fix in ConvexSphericalPolygon initialization commit f3eb614f3d0f1459689eb2788c03887a77d9de0d Author: Slavko Brdar Date: Thu Dec 17 16:24:52 2020 +0000 reverse ordering of vertices from node_connectivity for ConvexSphericalPolygon commit e780f6e3b652db6fd0ff0cc62b61d576862b257f Author: Slavko Brdar Date: Thu Dec 17 15:26:59 2020 +0000 a buggy test in orientation of element vertices commit 9b5b8a4b5500044523c4fffd926c48d93930b214 Author: Slavko Brdar Date: Thu Dec 17 15:26:23 2020 +0000 empty polygon should have area = 0 commit af97401611899b7bb12756edd084bec1b3c2f79c Author: Slavko Brdar Date: Thu Dec 17 15:15:08 2020 +0000 use RegularMeshGenerator for regional grids, still get strange elements commit a94d029b3f6aab72939865066fb8af2c12dd9332 Author: Slavko Brdar Date: Thu Dec 17 15:05:22 2020 +0000 unit test for intersecting elements of two regional grids commit 28dfe24d7945402d57b639afea3adad293986abc Author: Slavko Brdar Date: Thu Dec 17 14:59:18 2020 +0000 print of Polygon should be in more user friendly lonlat coordinates commit d65804e96a7805506c67126216fa955734aff655 Author: Slavko Brdar Date: Wed Dec 16 17:53:43 2020 +0000 fix memory leaks (tnx Willem) commit da2cfdd15b6846c23b4f0a3204a5a9ee20d7c42b Merge: d817ed9c4 d38dd7bc2 Author: Slavko Brdar Date: Wed Dec 16 16:45:48 2020 +0000 merge with ATLAS-322 for the missing d38dd7bc206 commit d817ed9c4741787849bd92a7b16a3110d2a15b40 Author: Slavko Brdar Date: Wed Dec 16 16:33:34 2020 +0000 remove the last repetition point in ConvexSphericalPolygon commit 0131af6c49f1a5a012dbda573c56ad28aa84904c Merge: 0a99832ff 4bdbba99d Author: Slavko Brdar Date: Wed Dec 16 15:24:17 2020 +0000 merge with ATLAS-322 and add centroid commit d38dd7bc20668018f6c26a18e3f5d5e339dbfdcc Author: Willem Deconinck Date: Wed Dec 16 14:07:53 2020 +0000 ATLAS-322 ConvexSphericalPolygon lives on stack entirely, with support of MAX_SIZE=8 points commit 4bdbba99d016845ad7af0dcafc8abeb3180a589e Author: Willem Deconinck Date: Wed Dec 16 13:34:15 2020 +0000 ATLAS-322 clang-format commit a8bee0a46f3bbf95fae37f5e8bc4185230b2fdcf Author: Willem Deconinck Date: Wed Dec 16 13:32:27 2020 +0000 ATLAS-322 No more pointers (and memory leaks) commit a10cc79525ab041a6af2c0c2b3d642cfc682af5e Author: Willem Deconinck Date: Wed Dec 16 13:03:04 2020 +0000 ATLAS-322 ConvexSphericalPolygon operator<< commit 40d38550b92c234cd16e7e0b47624f2ddf880e54 Author: Willem Deconinck Date: Wed Dec 16 12:59:15 2020 +0000 ATLAS-322 No more pointers in ConvexSphericalPolygon commit 84116c3a6ce6c77f4be2477028503b8eb27827ab Author: Willem Deconinck Date: Wed Dec 16 12:47:54 2020 +0000 ATLAS-322 ConvexSphericalPolygon default constructor commit 0769e1babe1e32a8a3e876aa9aa78b5c213da92b Author: Willem Deconinck Date: Wed Dec 16 12:40:25 2020 +0000 ATLAS-322 Do not inherit ConvexSphericalPolygon from PolygonCoordinates commit 0a99832ff89346e59626d2ed0a7e2c99e95b56e8 Author: Slavko Brdar Date: Tue Dec 15 17:01:18 2020 +0000 add centroid of spherical polygon commit 81702f7b10dbc808bb53f8c7d8d38338534bd3f2 Author: Slavko Brdar Date: Tue Dec 15 16:33:23 2020 +0000 area of spherical polygons commit 9c08b61b1eb02d719c8dbace175346abbe829015 Author: Slavko Brdar Date: Mon Dec 14 17:58:51 2020 +0000 more comprehensive unit test for spherical polygon intersections commit 55bfbe57700e703e4ca417f51567a0961bcd509b Author: Slavko Brdar Date: Thu Dec 10 16:40:10 2020 +0000 34 example of polygon intersections working fine, note sensitive value for eps commit dbc82781f551a7bf04d23469cf3fa62be6485432 Author: Slavko Brdar Date: Thu Dec 10 15:52:57 2020 +0000 improve in unit test output, show what two polygons intersect, and the resulting polygon commit 86fea5cc60fcd8c1dc2f4180c3f7e90caefe948d Author: Slavko Brdar Date: Thu Dec 10 15:26:49 2020 +0000 another bug fix, more compact unit test commit c4241718b8ea8aa274cc4c68f4a2ba6763742c71 Author: Slavko Brdar Date: Thu Dec 10 14:06:38 2020 +0000 replace fabs with abs (thx Willem) commit 58aededf1347200bc5070ad3331c4f7d2981f2dc Author: Slavko Brdar Date: Thu Dec 10 12:42:40 2020 +0000 fix bugs, plus more test cases commit 1750edc65ceb35f365dc2ea74511ab2525dad643 Author: Slavko Brdar Date: Wed Dec 9 18:54:40 2020 +0000 first working version of convex polygon intersections on sphere commit 0dd3a47d43f3a17059f192613133b31b6ffb00ba Author: Slavko Brdar Date: Wed Dec 9 11:30:23 2020 +0000 almost working spherical polygon intersection commit 52a3c7a25cb8202dcdbc7adea8bcf2eef7238189 Author: Slavko Brdar Date: Tue Dec 1 17:22:47 2020 +0000 recursive search of intersection polygon, not done yet commit 19710830a1b90e489a610d1ceb5eaef1881f04e0 Author: Slavko Brdar Date: Mon Nov 23 18:14:55 2020 +0000 correct intersection of spherical segments commit 287940e73e0ffaa1e7337682750fe791a75df52d Author: Slavko Brdar Date: Fri Nov 20 17:09:43 2020 +0000 add ConvexSphericalPolygon and test_spherical_geo Co-authored-by: Willem Deconinck --- src/atlas/CMakeLists.txt | 2 + ...nservativeSphericalPolygonInterpolation.cc | 1799 +++++++++++++++++ ...onservativeSphericalPolygonInterpolation.h | 211 ++ src/sandbox/interpolation/CMakeLists.txt | 6 + .../atlas-conservative-interpolation.cc | 306 +++ src/tests/interpolation/CMakeLists.txt | 6 + .../test_interpolation_conservative.cc | 205 ++ src/tests/mesh/CMakeLists.txt | 2 - src/tests/mesh/test_mesh_node2cell.cc | 15 + 9 files changed, 2550 insertions(+), 2 deletions(-) create mode 100644 src/atlas/interpolation/method/unstructured/ConservativeSphericalPolygonInterpolation.cc create mode 100644 src/atlas/interpolation/method/unstructured/ConservativeSphericalPolygonInterpolation.h create mode 100644 src/sandbox/interpolation/atlas-conservative-interpolation.cc create mode 100644 src/tests/interpolation/test_interpolation_conservative.cc diff --git a/src/atlas/CMakeLists.txt b/src/atlas/CMakeLists.txt index 5b3e7fbf8..9d2201fa0 100644 --- a/src/atlas/CMakeLists.txt +++ b/src/atlas/CMakeLists.txt @@ -601,6 +601,8 @@ interpolation/method/structured/kernels/LinearVerticalKernel.h interpolation/method/structured/kernels/QuasiCubic3DKernel.cc interpolation/method/structured/kernels/QuasiCubic3DKernel.h interpolation/method/structured/kernels/QuasiCubicHorizontalKernel.h +interpolation/method/unstructured/ConservativeSphericalPolygonInterpolation.cc +interpolation/method/unstructured/ConservativeSphericalPolygonInterpolation.h interpolation/method/unstructured/FiniteElement.cc interpolation/method/unstructured/FiniteElement.h interpolation/method/unstructured/UnstructuredBilinearLonLat.cc diff --git a/src/atlas/interpolation/method/unstructured/ConservativeSphericalPolygonInterpolation.cc b/src/atlas/interpolation/method/unstructured/ConservativeSphericalPolygonInterpolation.cc new file mode 100644 index 000000000..b87c17987 --- /dev/null +++ b/src/atlas/interpolation/method/unstructured/ConservativeSphericalPolygonInterpolation.cc @@ -0,0 +1,1799 @@ +/* + * (C) Copyright 2021- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#include +#include + +#include "ConservativeSphericalPolygonInterpolation.h" + +#include "eckit/log/ProgressTimer.h" + +#include "atlas/grid.h" +#include "atlas/interpolation/Interpolation.h" +#include "atlas/interpolation/method/MethodFactory.h" +#include "atlas/mesh/actions/BuildHalo.h" +#include "atlas/mesh/actions/BuildNode2CellConnectivity.h" +#include "atlas/meshgenerator.h" +#include "atlas/parallel/mpi/mpi.h" +#include "atlas/runtime/Exception.h" +#include "atlas/runtime/Log.h" +#include "atlas/runtime/Trace.h" +#include "atlas/util/ConvexSphericalPolygon.h" +#include "atlas/util/KDTree.h" +#include "atlas/util/Topology.h" + +#include "eckit/log/Bytes.h" + +namespace atlas { +namespace interpolation { +namespace method { + +using runtime::trace::StopWatch; +using util::ConvexSphericalPolygon; + +namespace { + +MethodBuilder __builder("conservative-spherical-polygon"); + +constexpr double unit_sphere_area() { + // 4*pi*r^2 with r=1 + return 4. * M_PI; +} + +template +size_t memory_of(const std::vector& vector) { + return sizeof(T) * vector.size(); +} +template +size_t memory_of(const std::vector>& vector_of_vector) { + size_t mem = 0; + for (const auto& vector : vector_of_vector) { + mem += memory_of(vector); + } + return mem; +} + +size_t memory_of( + const std::vector& vector_of_params) { + size_t mem = 0; + for (const auto& params : vector_of_params) { + mem += memory_of(params.cell_idx); + mem += memory_of(params.centroids); + mem += memory_of(params.src_weights); + mem += memory_of(params.tgt_weights); + } + return mem; +} + +Mesh extract_mesh(FunctionSpace fs) { + if (functionspace::CellColumns(fs)) { + return functionspace::CellColumns(fs).mesh(); + } + else if (functionspace::NodeColumns(fs)) { + return functionspace::NodeColumns(fs).mesh(); + } + else { + ATLAS_THROW_EXCEPTION("Cannot extract mesh from FunctionSpace" << fs.type()); + } +} + +void sort_and_accumulate_triplets(std::vector& triplets) { + ATLAS_TRACE(); + std::map, double> triplet_map; + ATLAS_TRACE_SCOPE("accumulate in map") + for (auto& triplet : triplets) { + auto loc = std::make_pair(triplet.row(), triplet.col()); + auto entry = triplet_map.find(loc); + if (entry == triplet_map.end()) { + triplet_map[loc] = triplet.value(); + } + else { + entry->second += triplet.value(); + } + } + triplets.clear(); + ATLAS_TRACE_SCOPE("recontruct sorted vector from map") + for (auto& triplet : triplet_map) { + auto& row = triplet.first.first; + auto& col = triplet.first.second; + auto& val = triplet.second; + triplets.emplace_back(row, col, val); + } +} + + +} // namespace + +int inside_vertices(const ConvexSphericalPolygon& plg1, const ConvexSphericalPolygon& plg2, int& pout) { + int points_in = 0; + pout = 0; + for (int j = 0; j < plg2.size(); j++) { + int i = 0; + for (; i < plg1.size(); i++) { + int in = (i != plg1.size() - 1) ? i + 1 : 0; + auto gss = ConvexSphericalPolygon::GreatCircleSegment(plg1[i], plg1[in]); + if (not gss.inLeftHemisphere(plg2[j], -std::numeric_limits::epsilon())) { + pout++; + break; + }; + } + if (i == plg1.size()) { + points_in++; + } + } + ATLAS_ASSERT(points_in + pout == plg2.size()); + return points_in; +} + +ConservativeSphericalPolygonInterpolation::ConservativeSphericalPolygonInterpolation(const Config& config): + Method(config) { + config.get("validate", validate_ = false); + config.get("order", order_ = 1); + config.get("normalise_intersections", normalise_intersections_ = 0); + config.get("matrix_free", matrix_free_ = false); + config.get("src_cell_data", src_cell_data_ = true); + config.get("tgt_cell_data", tgt_cell_data_ = true); + + + config.get("statistics.intersection", statistics_intersection_ = false); + config.get("statistics.conservation", statistics_conservation_ = false); + + sharable_data_ = std::make_shared(); + cache_ = Cache(sharable_data_); + data_ = sharable_data_.get(); + + ATLAS_ASSERT(sharable_data_.use_count() == 2); +} + +int ConservativeSphericalPolygonInterpolation::next_index(int current_index, int size, int offset) const { + ATLAS_ASSERT(current_index >= 0 && current_index < size); + ATLAS_ASSERT(offset >= 0 && offset <= size); + return (current_index < size - offset) ? current_index + offset : current_index + offset - size; +} +int ConservativeSphericalPolygonInterpolation::prev_index(int current_index, int size, int offset) const { + ATLAS_ASSERT(current_index >= 0 && current_index < size); + ATLAS_ASSERT(offset >= 0 && offset <= size); + return (current_index >= offset) ? current_index - offset : current_index - offset + size; +} + +// get counter-clockwise sorted neighbours of a cell +std::vector ConservativeSphericalPolygonInterpolation::get_cell_neighbours(Mesh& mesh, idx_t cell) const { + const auto& cell2node = mesh.cells().node_connectivity(); + const auto& nodes_ll = array::make_view(mesh.nodes().lonlat()); + const idx_t n_nodes = cell2node.cols(cell); + std::vector nbr_cells; + nbr_cells.reserve(n_nodes); + if (mesh.nodes().cell_connectivity().rows() == 0) { + mesh::actions::build_node_to_cell_connectivity(mesh); + } + const auto& node2cell = mesh.nodes().cell_connectivity(); + const auto n2c_missval = node2cell.missing_value(); + + for (idx_t inode = 0; inode < n_nodes; ++inode) { + idx_t node0 = cell2node(cell, inode); + idx_t node1 = cell2node(cell, next_index(inode, n_nodes)); + const PointLonLat p0_ll = PointLonLat{nodes_ll(node0, 0), nodes_ll(node0, 1)}; + const PointLonLat p1_ll = PointLonLat{nodes_ll(node1, 0), nodes_ll(node1, 1)}; + PointXYZ p0, p1; + eckit::geometry::Sphere::convertSphericalToCartesian(1., p0_ll, p0); + eckit::geometry::Sphere::convertSphericalToCartesian(1., p1_ll, p1); + if (PointXYZ::norm(p0 - p1) < 1e-14) { + continue; // edge = point + } + bool still_search = true; // still search the cell having vertices node0 & node1, not having index "cell" + int n_cells0 = node2cell.cols(node0); + int n_cells1 = node2cell.cols(node1); + for (int icell0 = 0; still_search && icell0 < n_cells0; icell0++) { + int cell0 = node2cell(node0, icell0); + if (cell0 == cell) { + continue; + } + for (int icell1 = 0; still_search && icell1 < n_cells1; icell1++) { + int cell1 = node2cell(node1, icell1); + if (cell0 == cell1 && cell0 != n2c_missval) { + nbr_cells.emplace_back(cell0); + still_search = false; + } + } + } + } + return nbr_cells; +} + +// get cyclically sorted node neighbours without using edge connectivity +std::vector ConservativeSphericalPolygonInterpolation::get_node_neighbours(Mesh& mesh, idx_t node_id) const { + const auto& cell2node = mesh.cells().node_connectivity(); + if (mesh.nodes().cell_connectivity().rows() == 0) { + mesh::actions::build_node_to_cell_connectivity(mesh); + } + const auto& node2cell = mesh.nodes().cell_connectivity(); + std::vector nbr_nodes; + std::vector nbr_nodes_od; + const int ncells = node2cell.cols(node_id); + ATLAS_ASSERT(ncells > 0, "There is a node which does not connect to any cell"); + idx_t cnodes[ncells][2]; + nbr_nodes.reserve(ncells + 1); + nbr_nodes_od.reserve(ncells + 1); + for (idx_t icell = 0; icell < ncells; ++icell) { + const idx_t cell = node2cell(node_id, icell); + const int nnodes = cell2node.cols(cell); + idx_t cnode = 0; + for (; cnode < nnodes; ++cnode) { + if (node_id == cell2node(cell, cnode)) { + break; + } + } + cnodes[icell][0] = cell2node(cell, prev_index(cnode, nnodes)); + cnodes[icell][1] = cell2node(cell, next_index(cnode, nnodes)); + } + if (ncells == 1) { + nbr_nodes.emplace_back(cnodes[0][0]); + nbr_nodes.emplace_back(cnodes[0][1]); + return nbr_nodes; + } + // cycle one direction + idx_t find = cnodes[0][1]; + idx_t prev = cnodes[0][0]; + nbr_nodes.emplace_back(prev); + nbr_nodes.emplace_back(find); + for (idx_t icycle = 0; nbr_nodes[0] != find;) { + idx_t jcell = 0; + for (; jcell < ncells; ++jcell) { + idx_t ocell = (icycle + jcell + 1) % ncells; + idx_t cand0 = cnodes[ocell][0]; + idx_t cand1 = cnodes[ocell][1]; + if (find == cand0 && prev != cand1) { + if (cand1 == nbr_nodes[0]) { + return nbr_nodes; + } + nbr_nodes.emplace_back(cand1); + prev = find; + find = cand1; + break; + } + if (find == cand1 && prev != cand0) { + if (cand0 == nbr_nodes[0]) { + return nbr_nodes; + } + nbr_nodes.emplace_back(cand0); + prev = find; + find = cand0; + break; + } + } + if (jcell == ncells) { // not found + if (nbr_nodes[0] != find && find != nbr_nodes[nbr_nodes.size() - 1]) { + nbr_nodes.emplace_back(find); + } + break; + } + else { + icycle++; + } + } + if (nbr_nodes[0] == find) { + return nbr_nodes; + } + // cycle the oposite direction + find = cnodes[0][0]; + prev = cnodes[0][1]; + nbr_nodes_od.emplace_back(prev); + nbr_nodes_od.emplace_back(find); + for (idx_t icycle = 0; nbr_nodes_od[0] != find;) { + idx_t jcell = 0; + for (; jcell < ncells; ++jcell) { + idx_t ocell = (icycle + jcell + 1) % ncells; + if (find == cnodes[ocell][0] && prev != cnodes[ocell][1]) { + nbr_nodes_od.emplace_back(cnodes[ocell][1]); + prev = find; + find = cnodes[ocell][1]; + break; + } + if (find == cnodes[ocell][1] && prev != cnodes[ocell][0]) { + nbr_nodes_od.emplace_back(cnodes[ocell][0]); + prev = find; + find = cnodes[ocell][0]; + break; + } + } + if (jcell == ncells) { + if (find != nbr_nodes_od[nbr_nodes_od.size() - 1]) { + nbr_nodes_od.emplace_back(find); + } + break; + } + icycle++; + } + // put together + int ow_size = nbr_nodes_od.size(); + for (int i = 0; i < ow_size - 2; i++) { + nbr_nodes.emplace_back(nbr_nodes_od[ow_size - 1 - i]); + } + return nbr_nodes; +} + +// Create polygons for cell-centred data. Here, the polygons are mesh cells +ConservativeSphericalPolygonInterpolation::CSPolygonArray +ConservativeSphericalPolygonInterpolation::get_polygons_celldata(Mesh& mesh) const { + CSPolygonArray cspolygons; + const idx_t n_cells = mesh.cells().size(); + cspolygons.resize(n_cells); + const auto& cell2node = mesh.cells().node_connectivity(); + const auto lonlat = array::make_view(mesh.nodes().lonlat()); + const auto cell_halo = array::make_view(mesh.cells().halo()); + const auto& cell_flags = array::make_view(mesh.cells().flags()); + const auto& cell_part = array::make_view(mesh.cells().partition()); + std::vector pts_ll; + for (idx_t cell = 0; cell < n_cells; ++cell) { + int halo_type = cell_halo(cell); + const idx_t n_nodes = cell2node.cols(cell); + pts_ll.clear(); + pts_ll.resize(n_nodes); + for (idx_t jnode = 0; jnode < n_nodes; ++jnode) { + idx_t inode = cell2node(cell, jnode); + pts_ll[jnode] = PointLonLat{lonlat(inode, 0), lonlat(inode, 1)}; + } + const auto& bitflag = util::Bitflags::view(cell_flags(cell)); + if (bitflag.check(util::Topology::PERIODIC) and mpi::rank() == cell_part(cell)) { + halo_type = -1; + } + std::get<0>(cspolygons[cell]) = ConvexSphericalPolygon(pts_ll); + std::get<1>(cspolygons[cell]) = halo_type; + } + return cspolygons; +} + +// Create polygons for cell-vertex data. Here, the polygons are subcells of mesh cells created as +// (cell_centre, edge_centre, cell_vertex, edge_centre) +// additionally, subcell-to-node and node-to-subcells mapping are computed +ConservativeSphericalPolygonInterpolation::CSPolygonArray +ConservativeSphericalPolygonInterpolation::get_polygons_nodedata(Mesh& mesh, std::vector& csp2node, + std::vector>& node2csp, + std::array& errors) const { + CSPolygonArray cspolygons; + csp2node.clear(); + node2csp.clear(); + node2csp.resize(mesh.nodes().size()); + const auto nodes_ll = array::make_view(mesh.nodes().lonlat()); + const auto& cell2node = mesh.cells().node_connectivity(); + const auto cell_halo = array::make_view(mesh.cells().halo()); + const auto cell_flags = array::make_view(mesh.cells().flags()); + const auto cell_part = array::make_view(mesh.cells().partition()); + auto xyz2ll = [](const atlas::PointXYZ& p_xyz) { + PointLonLat p_ll; + eckit::geometry::Sphere::convertCartesianToSpherical(1., p_xyz, p_ll); + return p_ll; + }; + auto ll2xyz = [](const atlas::PointLonLat& p_ll) { + PointXYZ p_xyz; + eckit::geometry::Sphere::convertSphericalToCartesian(1., p_ll, p_xyz); + return p_xyz; + }; + idx_t cspol_id = 0; // subpolygon enumeration + errors = {0., 0.}; // over/undershoots in creation of subpolygons + for (idx_t cell = 0; cell < mesh.cells().size(); ++cell) { + ATLAS_ASSERT(cell < cell2node.rows()); + const idx_t n_nodes = cell2node.cols(cell); + ATLAS_ASSERT(n_nodes > 2); + PointXYZ cell_mid(0., 0., 0.); // cell centre + std::vector pts_xyz; + std::vector pts_ll; + std::vector pts_idx; + pts_xyz.reserve(n_nodes); + pts_ll.reserve(n_nodes); + pts_idx.reserve(n_nodes); + for (idx_t inode = 0; inode < n_nodes; ++inode) { + idx_t node0 = cell2node(cell, inode); + idx_t node1 = cell2node(cell, next_index(inode, n_nodes)); + const PointLonLat p0_ll = PointLonLat{nodes_ll(node0, 0), nodes_ll(node0, 1)}; + const PointLonLat p1_ll = PointLonLat{nodes_ll(node1, 0), nodes_ll(node1, 1)}; + PointXYZ p0 = ll2xyz(p0_ll); + PointXYZ p1 = ll2xyz(p1_ll); + if (PointXYZ::norm(p0 - p1) < 1e-14) { + continue; // skip this edge = a pole point + } + pts_xyz.emplace_back(p0); + pts_ll.emplace_back(p0_ll); + pts_idx.emplace_back(inode); + cell_mid = cell_mid + p0; + cell_mid = cell_mid + p1; + } + cell_mid = PointXYZ::div(cell_mid, PointXYZ::norm(cell_mid)); + PointLonLat cell_ll = xyz2ll(cell_mid); + double loc_csp_area_shoot = ConvexSphericalPolygon(pts_ll).area(); + // get ConvexSphericalPolygon for each valid edge + for (int inode = 0; inode < pts_idx.size(); inode++) { + int inode_n = next_index(inode, pts_idx.size()); + idx_t node = cell2node(cell, inode); + idx_t node_n = cell2node(cell, inode_n); + PointXYZ iedge_mid = pts_xyz[inode] + pts_xyz[inode_n]; + iedge_mid = PointXYZ::div(iedge_mid, PointXYZ::norm(iedge_mid)); + csp2node.emplace_back(node_n); + node2csp[node_n].emplace_back(cspol_id); + int inode_nn = next_index(inode_n, pts_idx.size()); + if (PointXYZ::norm(pts_xyz[inode_nn] - pts_xyz[inode_n]) < 1e-14) { + ATLAS_THROW_EXCEPTION("Three cell vertices on a same great arc!"); + } + PointXYZ jedge_mid; + jedge_mid = pts_xyz[inode_nn] + pts_xyz[inode_n]; + jedge_mid = PointXYZ::div(jedge_mid, PointXYZ::norm(jedge_mid)); + std::vector subpol_pts_ll(4); + subpol_pts_ll[0] = cell_ll; + subpol_pts_ll[1] = xyz2ll(iedge_mid); + subpol_pts_ll[2] = pts_ll[inode_n]; + subpol_pts_ll[3] = xyz2ll(jedge_mid); + int halo_type = cell_halo(cell); + if (util::Bitflags::view(cell_flags(cell)).check(util::Topology::PERIODIC) and + cell_part(cell) == mpi::rank()) { + halo_type = -1; + } + ConvexSphericalPolygon cspi(subpol_pts_ll); + loc_csp_area_shoot -= cspi.area(); + cspolygons.emplace_back(cspi, halo_type); + cspol_id++; + } + errors[0] += std::abs(loc_csp_area_shoot); + errors[1] = std::max(std::abs(loc_csp_area_shoot), errors[1]); + } + ATLAS_TRACE_MPI(ALLREDUCE) { + mpi::comm().allReduceInPlace(&errors[0], 1, eckit::mpi::sum()); + mpi::comm().allReduceInPlace(&errors[1], 1, eckit::mpi::max()); + } + return cspolygons; +} + +void ConservativeSphericalPolygonInterpolation::do_setup_impl(const Grid& src_grid, const Grid& tgt_grid) { + ATLAS_TRACE("ConservativeMethod::do_setup( Grid, Grid )"); + ATLAS_ASSERT(src_grid); + ATLAS_ASSERT(tgt_grid); + + tgt_fs_ = data_->tgt_fs_; + src_fs_ = data_->src_fs_; + + if (not tgt_fs_) { + auto tgt_mesh_config = tgt_grid.meshgenerator() | option::halo(0); + ATLAS_TRACE_SCOPE("Generate target mesh") { tgt_mesh_ = MeshGenerator(tgt_mesh_config).generate(tgt_grid); } + ATLAS_TRACE_SCOPE("Create target functionspace") { + if (tgt_cell_data_) { + tgt_fs_ = functionspace::CellColumns(tgt_mesh_, option::halo(0)); + } + else { + tgt_fs_ = functionspace::NodeColumns(tgt_mesh_, option::halo(0)); + } + } + sharable_data_->tgt_fs_ = tgt_fs_; + ATLAS_ASSERT(data_->tgt_fs_); + } + + if (not src_fs_) { + auto src_mesh_config = src_grid.meshgenerator() | option::halo(2); + ATLAS_TRACE_SCOPE("Generate source mesh") { + if (mpi::size() > 1) { + src_mesh_ = MeshGenerator(src_mesh_config).generate(src_grid, grid::MatchingPartitioner(tgt_mesh_)); + } + else { + src_mesh_ = MeshGenerator(src_mesh_config).generate(src_grid); + } + } + ATLAS_TRACE_SCOPE("Create source functionspace") { + if (src_cell_data_) { + src_fs_ = functionspace::CellColumns(src_mesh_, option::halo(2)); + } + else { + src_fs_ = functionspace::NodeColumns(src_mesh_, option::halo(2)); + } + } + sharable_data_->src_fs_ = src_fs_; + ATLAS_ASSERT(data_->tgt_fs_); + } + + do_setup(src_fs_, tgt_fs_); +} + + +void ConservativeSphericalPolygonInterpolation::do_setup(const Grid& src_grid, const Grid& tgt_grid, + const interpolation::Cache& cache) { + ATLAS_TRACE("ConservativeSphericalPolygonInterpolation::do_setup(Grid, Grid, Cache)"); + + if (Cache(cache)) { + Log::debug() << "Interpolation data found in cache -> no polygon intersections required" << std::endl; + cache_ = Cache(cache); + data_ = cache_.get(); + sharable_data_.reset(); + + src_fs_ = data_->src_fs_; + tgt_fs_ = data_->tgt_fs_; + + src_cell_data_ = functionspace::CellColumns(src_fs_); + tgt_cell_data_ = functionspace::CellColumns(tgt_fs_); + + src_mesh_ = extract_mesh(src_fs_); + tgt_mesh_ = extract_mesh(tgt_fs_); + + if (order_ == 1 && matrix_free_) { + // We don't need to continue with setups required for first order matrix-free + // such as mesh generation and functionspace creation. + return; + } + } + + if (not matrix_free_) { + auto matrix_cache = interpolation::MatrixCache(cache); + if (matrix_cache) { + if (matrix_cache.uid() == std::to_string(order_) || matrix_cache.uid().empty()) { + Log::debug() << "Matrix found in cache -> no setup required at all" << std::endl; + setMatrix(matrix_cache); + return; + } + } + } + + do_setup_impl(src_grid, tgt_grid); +} + +void ConservativeSphericalPolygonInterpolation::do_setup(const FunctionSpace& src_fs, const FunctionSpace& tgt_fs) { + ATLAS_TRACE("ConservativeMethod::do_setup( FunctionSpace, FunctionSpace )"); + ATLAS_ASSERT(src_fs); + ATLAS_ASSERT(tgt_fs); + + bool compute_cache = data_->src_points_.empty(); + + if (not data_->tgt_fs_) { + tgt_fs_ = tgt_fs; + sharable_data_->tgt_fs_ = tgt_fs_; + } + if (not data_->src_fs_) { + src_fs_ = src_fs; + sharable_data_->src_fs_ = src_fs_; + } + + src_cell_data_ = functionspace::CellColumns(src_fs_); + tgt_cell_data_ = functionspace::CellColumns(tgt_fs_); + + src_mesh_ = extract_mesh(src_fs_); + tgt_mesh_ = extract_mesh(tgt_fs_); + + { + // we need src_halo_size >= 2, whereas tgt_halo_size >= 0 is enough + int src_halo_size = 0; + src_mesh_.metadata().get("halo", src_halo_size); + ATLAS_ASSERT(src_halo_size > 1); + } + CSPolygonArray src_csp; + CSPolygonArray tgt_csp; + std::array errors = {0., 0.}; + if (compute_cache) { + ATLAS_TRACE("Get source polygons"); + StopWatch stopwatch; + stopwatch.start(); + if (src_cell_data_) { + src_csp = get_polygons_celldata(src_mesh_); + } + else { + src_csp = + get_polygons_nodedata(src_mesh_, sharable_data_->src_csp2node_, sharable_data_->src_node2csp_, errors); + } + stopwatch.stop(); + sharable_data_->timings.source_polygons_assembly = stopwatch.elapsed(); + } + remap_stat_.errors[Statistics::Errors::SRC_PLG_L1] = errors[0]; + remap_stat_.errors[Statistics::Errors::SRC_PLG_LINF] = errors[1]; + if (compute_cache) { + ATLAS_TRACE("Get target polygons"); + StopWatch stopwatch; + stopwatch.start(); + if (tgt_cell_data_) { + tgt_csp = get_polygons_celldata(tgt_mesh_); + } + else { + tgt_csp = + get_polygons_nodedata(tgt_mesh_, sharable_data_->tgt_csp2node_, sharable_data_->tgt_node2csp_, errors); + } + stopwatch.stop(); + sharable_data_->timings.target_polygons_assembly = stopwatch.elapsed(); + } + remap_stat_.counts[Statistics::Counts::SRC_PLG] = src_csp.size(); + remap_stat_.counts[Statistics::Counts::TGT_PLG] = tgt_csp.size(); + remap_stat_.errors[Statistics::Errors::TGT_PLG_L1] = errors[0]; + remap_stat_.errors[Statistics::Errors::TGT_PLG_LINF] = errors[1]; + + n_spoints_ = src_fs_.size(); + n_tpoints_ = tgt_fs_.size(); + + if (compute_cache) { + intersect_polygons(src_csp, tgt_csp); + + auto& src_points_ = sharable_data_->src_points_; + auto& tgt_points_ = sharable_data_->tgt_points_; + src_points_.resize(n_spoints_); + tgt_points_.resize(n_tpoints_); + sharable_data_->src_areas_.resize(n_spoints_); + auto& src_areas_v = sharable_data_->src_areas_; + if (src_cell_data_) { + for (idx_t spt = 0; spt < n_spoints_; ++spt) { + const auto& s_csp = std::get<0>(src_csp[spt]); + src_points_[spt] = s_csp.centroid(); + src_areas_v[spt] = s_csp.area(); + } + } + else { + auto& src_node2csp_ = sharable_data_->src_node2csp_; + const auto lonlat = array::make_view(src_mesh_.nodes().lonlat()); + for (idx_t spt = 0; spt < n_spoints_; ++spt) { + if (src_node2csp_[spt].size() == 0) { + // this is a node to which no subpolygon is associated + // maximal twice per mesh we end here, and that is only when mesh has nodes on poles + auto p = PointLonLat{lonlat(spt, 0), lonlat(spt, 1)}; + eckit::geometry::Sphere::convertSphericalToCartesian(1., p, src_points_[spt]); + } + else { + // .. in the other case, start computing the barycentre + src_points_[spt] = PointXYZ{0., 0., 0.}; + } + src_areas_v[spt] = 0.; + for (idx_t isubcell = 0; isubcell < src_node2csp_[spt].size(); ++isubcell) { + idx_t subcell = src_node2csp_[spt][isubcell]; + const auto& s_csp = std::get<0>(src_csp[subcell]); + src_areas_v[spt] += s_csp.area(); + src_points_[spt] = src_points_[spt] + PointXYZ::mul(s_csp.centroid(), s_csp.area()); + } + double src_point_norm = PointXYZ::norm(src_points_[spt]); + ATLAS_ASSERT(src_point_norm > 0.); + src_points_[spt] = PointXYZ::div(src_points_[spt], src_point_norm); + } + } + sharable_data_->tgt_areas_.resize(n_tpoints_); + auto& tgt_areas_v = sharable_data_->tgt_areas_; + if (tgt_cell_data_) { + for (idx_t tpt = 0; tpt < n_tpoints_; ++tpt) { + const auto& t_csp = std::get<0>(tgt_csp[tpt]); + tgt_points_[tpt] = t_csp.centroid(); + tgt_areas_v[tpt] = t_csp.area(); + } + } + else { + auto& tgt_node2csp_ = sharable_data_->tgt_node2csp_; + const auto lonlat = array::make_view(tgt_mesh_.nodes().lonlat()); + for (idx_t tpt = 0; tpt < n_tpoints_; ++tpt) { + if (tgt_node2csp_[tpt].size() == 0) { + // this is a node to which no subpolygon is associated + // maximal twice per mesh we end here, and that is only when mesh has nodes on poles + auto p = PointLonLat{lonlat(tpt, 0), lonlat(tpt, 1)}; + eckit::geometry::Sphere::convertSphericalToCartesian(1., p, tgt_points_[tpt]); + } + else { + // .. in the other case, start computing the barycentre + tgt_points_[tpt] = PointXYZ{0., 0., 0.}; + } + tgt_areas_v[tpt] = 0.; + for (idx_t isubcell = 0; isubcell < tgt_node2csp_[tpt].size(); ++isubcell) { + idx_t subcell = tgt_node2csp_[tpt][isubcell]; + const auto& t_csp = std::get<0>(tgt_csp[subcell]); + tgt_areas_v[tpt] += t_csp.area(); + tgt_points_[tpt] = tgt_points_[tpt] + PointXYZ::mul(t_csp.centroid(), t_csp.area()); + } + double tgt_point_norm = PointXYZ::norm(tgt_points_[tpt]); + ATLAS_ASSERT(tgt_point_norm > 0.); + tgt_points_[tpt] = PointXYZ::div(tgt_points_[tpt], tgt_point_norm); + } + } + } + + + if (not matrix_free_) { + StopWatch stopwatch; + stopwatch.start(); + switch (order_) { + case 1: { + auto M = compute_1st_order_matrix(); + setMatrix(M, "1"); + break; + } + case 2: { + auto M = compute_2nd_order_matrix(); + setMatrix(M, "2"); + break; + } + default: { + ATLAS_NOTIMPLEMENTED; + } + } + stopwatch.stop(); + if (compute_cache) { + sharable_data_->timings.matrix_assembly = stopwatch.elapsed(); + } + } + + data_->print(Log::debug()); + + if (statistics_intersection_) { + setup_stat(); + } +} + +namespace { +// needed for intersect_polygons only, merely for detecting duplicate points +struct ComparePointXYZ { + bool operator()(const PointXYZ& f, const PointXYZ& s) const { + // eps = ConvexSphericalPolygon::EPS which is the threshold when two points are "same" + double eps = 1e4 * std::numeric_limits::epsilon(); + if (f[0] < s[0] - eps) { + return true; + } + else if (std::abs(f[0] - s[0]) < eps) { + if (f[1] < s[1] - eps) { + return true; + } + else if (std::abs(f[1] - s[1]) < eps) { + if (f[2] < s[2] - eps) { + return true; + } + } + } + return false; + } +}; +} // namespace + +void ConservativeSphericalPolygonInterpolation::intersect_polygons(const CSPolygonArray& src_csp, + const CSPolygonArray& tgt_csp) { + ATLAS_TRACE(); + auto& timings = sharable_data_->timings; + StopWatch stopwatch; + stopwatch.start(); + util::KDTree kdt_search; + kdt_search.reserve(tgt_csp.size()); + double max_tgtcell_rad = 0.; + for (idx_t jcell = 0; jcell < tgt_csp.size(); ++jcell) { + if (std::get<1>(tgt_csp[jcell]) == 0) { + const auto& t_csp = std::get<0>(tgt_csp[jcell]); + kdt_search.insert(t_csp.centroid(), jcell); + max_tgtcell_rad = std::max(max_tgtcell_rad, t_csp.radius()); + } + } + kdt_search.build(); + stopwatch.stop(); + timings.target_kdtree_assembly = stopwatch.elapsed(); + + StopWatch stopwatch_src_already_in; + StopWatch stopwatch_kdtree_search; + StopWatch stopwatch_polygon_intersections; + + stopwatch_src_already_in.start(); + std::set src_cent; + auto polygon_point = [](const ConvexSphericalPolygon& pol) { + PointXYZ p{0., 0., 0.}; + for (int i = 0; i < pol.size(); i++) { + p = p + pol[i]; + } + p /= pol.size(); + return p; + }; + auto src_already_in = [&](const PointXYZ& halo_cent) { + if (src_cent.find(halo_cent) == src_cent.end()) { + src_cent.insert(halo_cent); + return false; + } + return true; + }; + stopwatch_src_already_in.stop(); + + enum MeshSizeId + { + SRC, + TGT, + SRC_TGT_INTERSECT, + SRC_NONINTERSECT + }; + std::array num_pol{0, 0, 0, 0}; + enum AreaCoverageId + { + TOTAL_SRC, + MAX_SRC + }; + std::array area_coverage{0., 0.}; + auto& src_iparam_ = sharable_data_->src_iparam_; + src_iparam_.resize(src_csp.size()); + + std::vector tgt_iparam; // only used for debugging + if (validate_) { + tgt_iparam.resize(tgt_csp.size()); + } + + eckit::Channel blackhole; + eckit::ProgressTimer progress("Intersecting polygons ", src_csp.size(), " cell", double(10), + src_csp.size() > 50 ? Log::info() : blackhole); + for (idx_t scell = 0; scell < src_csp.size(); ++scell, ++progress) { + stopwatch_src_already_in.start(); + if (src_already_in(polygon_point(std::get<0>(src_csp[scell])))) { + stopwatch_src_already_in.stop(); + continue; + } + stopwatch_src_already_in.stop(); + + const auto& s_csp = std::get<0>(src_csp[scell]); + const double s_csp_area = s_csp.area(); + double src_cover_area = 0.; + + stopwatch_kdtree_search.start(); + auto tgt_cells = kdt_search.closestPointsWithinRadius(s_csp.centroid(), s_csp.radius() + max_tgtcell_rad); + stopwatch_kdtree_search.stop(); + for (idx_t ttcell = 0; ttcell < tgt_cells.size(); ++ttcell) { + auto tcell = tgt_cells[ttcell].payload(); + const auto& t_csp = std::get<0>(tgt_csp[tcell]); + stopwatch_polygon_intersections.start(); + ConvexSphericalPolygon csp_i = s_csp.intersect(t_csp); + double csp_i_area = csp_i.area(); + stopwatch_polygon_intersections.stop(); + if (validate_) { + // check zero area intersections with inside_vertices + int pout; + if (inside_vertices(s_csp, t_csp, pout) > 2 && csp_i.area() < 3e-16) { + dump_intersection(s_csp, tgt_csp, tgt_cells); + } + } + if (csp_i_area > 0.) { + if (validate_) { + tgt_iparam[tcell].cell_idx.emplace_back(scell); + tgt_iparam[tcell].tgt_weights.emplace_back(csp_i_area); + } + src_iparam_[scell].cell_idx.emplace_back(tcell); + src_iparam_[scell].src_weights.emplace_back(csp_i_area); + double target_weight = csp_i_area / t_csp.area(); + src_iparam_[scell].tgt_weights.emplace_back(target_weight); + src_iparam_[scell].centroids.emplace_back(csp_i.centroid()); + src_cover_area += csp_i_area; + ATLAS_ASSERT(target_weight < 1.1); + ATLAS_ASSERT(csp_i_area / s_csp_area < 1.1); + } + } + const double src_cover_err = std::abs(s_csp_area - src_cover_area); + const double src_cover_err_percent = 100. * src_cover_err / s_csp_area; + if (src_cover_err_percent > 0.1 and std::get<1>(src_csp[scell]) == 0) { + // HACK: source cell at process boundary will not be covered by target cells, skip them + // TODO: mark these source cells beforehand and compute error in them among the processes + + if (validate_) { + if (mpi::size() == 1) { + Log::info() << "WARNING src cell covering error : " << src_cover_err_percent << "%\n"; + dump_intersection(s_csp, tgt_csp, tgt_cells); + } + } + area_coverage[TOTAL_SRC] += src_cover_err; + area_coverage[MAX_SRC] = std::max(area_coverage[MAX_SRC], src_cover_err); + } + if (src_iparam_[scell].cell_idx.size() == 0) { + num_pol[SRC_NONINTERSECT]++; + } + if (normalise_intersections_ && src_cover_err_percent < 1.) { + double wfactor = s_csp.area() / (src_cover_area > 0. ? src_cover_area : 1.); + for (idx_t i = 0; i < src_iparam_[scell].src_weights.size(); i++) { + src_iparam_[scell].src_weights[i] *= wfactor; + src_iparam_[scell].tgt_weights[i] *= wfactor; + } + } + num_pol[SRC_TGT_INTERSECT] += src_iparam_[scell].src_weights.size(); + } + timings.polygon_intersections = stopwatch_polygon_intersections.elapsed(); + timings.target_kdtree_search = stopwatch_kdtree_search.elapsed(); + timings.source_polygons_filter = stopwatch_src_already_in.elapsed(); + num_pol[SRC] = src_csp.size(); + num_pol[TGT] = tgt_csp.size(); + ATLAS_TRACE_MPI(ALLREDUCE) { + mpi::comm().allReduceInPlace(num_pol.data(), num_pol.size(), eckit::mpi::sum()); + mpi::comm().allReduceInPlace(area_coverage.data(), area_coverage.size(), eckit::mpi::max()); + } + remap_stat_.counts[Statistics::Counts::INT_PLG] = num_pol[SRC_TGT_INTERSECT]; + remap_stat_.counts[Statistics::Counts::UNCVR_SRC] = num_pol[SRC_NONINTERSECT]; + remap_stat_.errors[Statistics::Errors::GEO_L1] = area_coverage[TOTAL_SRC]; + remap_stat_.errors[Statistics::Errors::GEO_LINF] = area_coverage[MAX_SRC]; + + double geo_err_l1 = 0.; + double geo_err_linf = 0.; + for (idx_t scell = 0; scell < src_csp.size(); ++scell) { + const int cell_flag = std::get<1>(src_csp[scell]); + if (cell_flag == -1 or cell_flag > 0) { + // skip periodic & halo cells + continue; + } + double diff_cell = std::get<0>(src_csp[scell]).area(); + for (idx_t icell = 0; icell < src_iparam_[scell].src_weights.size(); ++icell) { + diff_cell -= src_iparam_[scell].src_weights[icell]; + } + geo_err_l1 += std::abs(diff_cell); + geo_err_linf = std::max(geo_err_linf, std::abs(diff_cell)); + } + ATLAS_TRACE_MPI(ALLREDUCE) { + mpi::comm().allReduceInPlace(geo_err_l1, eckit::mpi::sum()); + mpi::comm().allReduceInPlace(geo_err_linf, eckit::mpi::max()); + } + remap_stat_.errors[Statistics::Errors::GEO_L1] = geo_err_l1 / unit_sphere_area(); + remap_stat_.errors[Statistics::Errors::GEO_LINF] = geo_err_linf; + + if (validate_) { + for (idx_t tcell = 0; tcell < tgt_csp.size(); ++tcell) { + const auto& t_csp = std::get<0>(tgt_csp[tcell]); + double tgt_cover_area = 0.; + const auto& tiparam = tgt_iparam[tcell]; + for (idx_t icell = 0; icell < tiparam.cell_idx.size(); ++icell) { + tgt_cover_area += tiparam.tgt_weights[icell]; + } + const double tgt_cover_err_percent = 100. * std::abs(t_csp.area() - tgt_cover_area) / t_csp.area(); + if (tgt_cover_err_percent > 0.1 and std::get<1>(tgt_csp[tcell]) == 0) { + Log::info() << "WARNING tgt cell covering error : " << tgt_cover_err_percent << " %\n"; + dump_intersection(t_csp, src_csp, tiparam.cell_idx); + } + } + } +} + +eckit::linalg::SparseMatrix ConservativeSphericalPolygonInterpolation::compute_1st_order_matrix() { + ATLAS_TRACE("ConservativeMethod::setup: build cons-1 interpolant matrix"); + ATLAS_ASSERT(not matrix_free_); + Triplets triplets; + size_t triplets_size = 0; + const auto& src_iparam_ = data_->src_iparam_; + // determine the size of array of triplets used to define the sparse matrix + if (src_cell_data_) { + for (idx_t scell = 0; scell < n_spoints_; ++scell) { + triplets_size += src_iparam_[scell].centroids.size(); + } + } + else { + auto& src_node2csp_ = data_->src_node2csp_; + for (idx_t snode = 0; snode < n_spoints_; ++snode) { + for (idx_t isubcell = 0; isubcell < src_node2csp_[snode].size(); ++isubcell) { + idx_t subcell = src_node2csp_[snode][isubcell]; + triplets_size += src_iparam_[subcell].tgt_weights.size(); + } + } + } + triplets.reserve(triplets_size); + // assemble triplets to define the sparse matrix + const auto& src_areas_v = data_->src_areas_; + const auto& tgt_areas_v = data_->tgt_areas_; + if (src_cell_data_ && tgt_cell_data_) { + for (idx_t scell = 0; scell < n_spoints_; ++scell) { + const auto& iparam = src_iparam_[scell]; + for (idx_t icell = 0; icell < iparam.centroids.size(); ++icell) { + idx_t tcell = iparam.cell_idx[icell]; + triplets.emplace_back(tcell, scell, iparam.tgt_weights[icell]); + } + } + } + else if (not src_cell_data_ && tgt_cell_data_) { + auto& src_node2csp_ = data_->src_node2csp_; + for (idx_t snode = 0; snode < n_spoints_; ++snode) { + for (idx_t isubcell = 0; isubcell < src_node2csp_[snode].size(); ++isubcell) { + const idx_t subcell = src_node2csp_[snode][isubcell]; + const auto& iparam = src_iparam_[subcell]; + for (idx_t icell = 0; icell < iparam.centroids.size(); ++icell) { + idx_t tcell = iparam.cell_idx[icell]; + triplets.emplace_back(tcell, snode, iparam.tgt_weights[icell]); + } + } + } + } + else if (src_cell_data_ && not tgt_cell_data_) { + auto& tgt_csp2node_ = data_->tgt_csp2node_; + for (idx_t scell = 0; scell < n_spoints_; ++scell) { + const auto& iparam = src_iparam_[scell]; + for (idx_t icell = 0; icell < iparam.centroids.size(); ++icell) { + idx_t tcell = iparam.cell_idx[icell]; + idx_t tnode = tgt_csp2node_[tcell]; + double inv_node_weight = (tgt_areas_v[tnode] > 0. ? 1. / tgt_areas_v[tnode] : 0.); + triplets.emplace_back(tnode, scell, iparam.src_weights[icell] * inv_node_weight); + } + } + } + else if (not src_cell_data_ && not tgt_cell_data_) { + auto& src_node2csp_ = data_->src_node2csp_; + auto& tgt_csp2node_ = data_->tgt_csp2node_; + for (idx_t snode = 0; snode < n_spoints_; ++snode) { + for (idx_t isubcell = 0; isubcell < src_node2csp_[snode].size(); ++isubcell) { + const idx_t subcell = src_node2csp_[snode][isubcell]; + const auto& iparam = src_iparam_[subcell]; + for (idx_t icell = 0; icell < iparam.centroids.size(); ++icell) { + idx_t tcell = iparam.cell_idx[icell]; + idx_t tnode = tgt_csp2node_[tcell]; + double inv_node_weight = (tgt_areas_v[tnode] > 0. ? 1. / tgt_areas_v[tnode] : 0.); + triplets.emplace_back(tnode, snode, iparam.src_weights[icell] * inv_node_weight); + } + } + } + } + sort_and_accumulate_triplets(triplets); // Very expensive!!! (90% of this routine). We need to avoid it + return Matrix(n_tpoints_, n_spoints_, triplets); +} + +eckit::linalg::SparseMatrix ConservativeSphericalPolygonInterpolation::compute_2nd_order_matrix() { + ATLAS_TRACE("ConservativeMethod::setup: build cons-2 interpolant matrix"); + ATLAS_ASSERT(not matrix_free_); + const auto& src_points_ = data_->src_points_; + const auto& src_iparam_ = data_->src_iparam_; + + Triplets triplets; + size_t triplets_size = 0; + const auto& tgt_areas_v = data_->tgt_areas_; + if (src_cell_data_) { + const auto src_halo = array::make_view(src_mesh_.cells().halo()); + for (idx_t scell = 0; scell < n_spoints_; ++scell) { + const auto nb_cells = get_cell_neighbours(src_mesh_, scell); + triplets_size += (2 * nb_cells.size() + 1) * src_iparam_[scell].centroids.size(); + } + triplets.reserve(triplets_size); + for (idx_t scell = 0; scell < n_spoints_; ++scell) { + const auto nb_cells = get_cell_neighbours(src_mesh_, scell); + const auto& iparam = src_iparam_[scell]; + if (iparam.centroids.size() == 0 && not src_halo(scell)) { + continue; + } + /* // better conservation after Kritsikis et al. (2017) + PointXYZ Cs = {0., 0., 0.}; + for ( idx_t icell = 0; icell < iparam.centroids.size(); ++icell ) { + Cs = Cs + PointXYZ::mul( iparam.centroids[icell], iparam.src_weights[icell] ); + } + const double Cs_norm = PointXYZ::norm( Cs ); + ATLAS_ASSERT( Cs_norm > 0. ); + Cs = PointXYZ::div( Cs, Cs_norm ); + */ + const PointXYZ& Cs = src_points_[scell]; + // compute gradient from cells + double dual_area_inv = 0.; + std::vector Rsj; + Rsj.resize(nb_cells.size()); + for (idx_t j = 0; j < nb_cells.size(); ++j) { + idx_t nj = next_index(j, nb_cells.size()); + idx_t sj = nb_cells[j]; + idx_t nsj = nb_cells[nj]; + const auto& Csj = src_points_[sj]; + const auto& Cnsj = src_points_[nsj]; + if (ConvexSphericalPolygon::GreatCircleSegment(Cs, Csj).inLeftHemisphere(Cnsj, -1e-16)) { + Rsj[j] = PointXYZ::cross(Cnsj, Csj); + dual_area_inv += ConvexSphericalPolygon({Cs, Csj, Cnsj}).area(); + } + else { + Rsj[j] = PointXYZ::cross(Csj, Cnsj); + dual_area_inv += ConvexSphericalPolygon({Cs, Cnsj, Csj}).area(); + } + } + dual_area_inv = (dual_area_inv > 0.) ? 1. / dual_area_inv : 0.; + PointXYZ Rs = {0., 0., 0.}; + for (idx_t j = 0; j < nb_cells.size(); ++j) { + Rs = Rs + Rsj[j]; + } + // assemble the matrix + std::vector Aik; + Aik.resize(iparam.centroids.size()); + for (idx_t icell = 0; icell < iparam.centroids.size(); ++icell) { + const PointXYZ& Csk = iparam.centroids[icell]; + const PointXYZ Csk_Cs = Csk - Cs; + Aik[icell] = Csk_Cs - PointXYZ::mul(Cs, PointXYZ::dot(Cs, Csk_Cs)); + Aik[icell] = PointXYZ::mul(Aik[icell], iparam.tgt_weights[icell] * dual_area_inv); + } + if (tgt_cell_data_) { + for (idx_t icell = 0; icell < iparam.centroids.size(); ++icell) { + const idx_t tcell = iparam.cell_idx[icell]; + for (idx_t j = 0; j < nb_cells.size(); ++j) { + idx_t nj = next_index(j, nb_cells.size()); + idx_t sj = nb_cells[j]; + idx_t nsj = nb_cells[nj]; + triplets.emplace_back(tcell, sj, 0.5 * PointXYZ::dot(Rsj[j], Aik[icell])); + triplets.emplace_back(tcell, nsj, 0.5 * PointXYZ::dot(Rsj[j], Aik[icell])); + } + triplets.emplace_back(tcell, scell, iparam.tgt_weights[icell] - PointXYZ::dot(Rs, Aik[icell])); + } + } + else { + auto& tgt_csp2node_ = data_->tgt_csp2node_; + for (idx_t icell = 0; icell < iparam.centroids.size(); ++icell) { + idx_t tcell = iparam.cell_idx[icell]; + idx_t tnode = tgt_csp2node_[tcell]; + double inv_node_weight = (tgt_areas_v[tnode] > 0.) ? 1. / tgt_areas_v[tnode] : 0.; + double csp2node_coef = iparam.src_weights[icell] / iparam.tgt_weights[icell] * inv_node_weight; + for (idx_t j = 0; j < nb_cells.size(); ++j) { + idx_t nj = next_index(j, nb_cells.size()); + idx_t sj = nb_cells[j]; + idx_t nsj = nb_cells[nj]; + triplets.emplace_back(tnode, sj, (0.5 * PointXYZ::dot(Rsj[j], Aik[icell])) * csp2node_coef); + triplets.emplace_back(tnode, nsj, (0.5 * PointXYZ::dot(Rsj[j], Aik[icell])) * csp2node_coef); + } + triplets.emplace_back(tnode, scell, + (iparam.tgt_weights[icell] - PointXYZ::dot(Rs, Aik[icell])) * csp2node_coef); + } + } + } + } + else { // if ( not src_cell_data_ ) + auto& src_node2csp_ = data_->src_node2csp_; + const auto src_halo = array::make_view(src_mesh_.nodes().halo()); + for (idx_t snode = 0; snode < n_spoints_; ++snode) { + const auto nb_nodes = get_node_neighbours(src_mesh_, snode); + for (idx_t isubcell = 0; isubcell < src_node2csp_[snode].size(); ++isubcell) { + idx_t subcell = src_node2csp_[snode][isubcell]; + triplets_size += (2 * nb_nodes.size() + 1) * src_iparam_[subcell].centroids.size(); + } + } + triplets.reserve(triplets_size); + for (idx_t snode = 0; snode < n_spoints_; ++snode) { + const auto nb_nodes = get_node_neighbours(src_mesh_, snode); + // get the barycentre of the dual cell + /* // better conservation + PointXYZ Cs = {0., 0., 0.}; + for ( idx_t isubcell = 0; isubcell < src_node2csp_[snode].size(); ++isubcell ) { + idx_t subcell = src_node2csp_[snode][isubcell]; + const auto& iparam = src_iparam_[subcell]; + for ( idx_t icell = 0; icell < iparam.centroids.size(); ++icell ) { + Cs = Cs + PointXYZ::mul( iparam.centroids[icell], iparam.src_weights[icell] ); + } + } + const double Cs_norm = PointXYZ::norm( Cs ); + ATLAS_ASSERT( Cs_norm > 0. ); + Cs = PointXYZ::div( Cs, Cs_norm ); +*/ + const PointXYZ& Cs = src_points_[snode]; + // compute gradient from nodes + double dual_area_inv = 0.; + std::vector Rsj; + Rsj.resize(nb_nodes.size()); + const auto& Ns = src_points_[snode]; + for (idx_t j = 0; j < nb_nodes.size(); ++j) { + idx_t nj = next_index(j, nb_nodes.size()); + idx_t sj = nb_nodes[j]; + idx_t snj = nb_nodes[nj]; + const auto& Nsj = src_points_[sj]; + const auto& Nsnj = src_points_[snj]; + if (ConvexSphericalPolygon::GreatCircleSegment(Ns, Nsj).inLeftHemisphere(Nsnj, -1e-16)) { + Rsj[j] = PointXYZ::cross(Nsnj, Nsj); + dual_area_inv += ConvexSphericalPolygon({Ns, Nsj, Nsnj}).area(); + } + else { + Rsj[j] = PointXYZ::cross(Nsj, Nsnj); + dual_area_inv += ConvexSphericalPolygon({Ns, Nsnj, Nsj}).area(); + } + } + dual_area_inv = (dual_area_inv > 0.) ? 1. / dual_area_inv : 0.; + PointXYZ Rs = {0., 0., 0.}; + for (idx_t j = 0; j < nb_nodes.size(); ++j) { + Rs = Rs + Rsj[j]; + } + // assemble the matrix + for (idx_t isubcell = 0; isubcell < src_node2csp_[snode].size(); ++isubcell) { + idx_t subcell = src_node2csp_[snode][isubcell]; + const auto& iparam = src_iparam_[subcell]; + if (iparam.centroids.size() == 0) { + continue; + } + std::vector Aik; + Aik.resize(iparam.centroids.size()); + for (idx_t icell = 0; icell < iparam.centroids.size(); ++icell) { + const PointXYZ& Csk = iparam.centroids[icell]; + const PointXYZ Csk_Cs = Csk - Cs; + Aik[icell] = Csk_Cs - PointXYZ::mul(Cs, PointXYZ::dot(Cs, Csk_Cs)); + Aik[icell] = PointXYZ::mul(Aik[icell], iparam.tgt_weights[icell] * dual_area_inv); + } + if (tgt_cell_data_) { + for (idx_t icell = 0; icell < iparam.centroids.size(); ++icell) { + const idx_t tcell = iparam.cell_idx[icell]; + for (idx_t j = 0; j < nb_nodes.size(); ++j) { + idx_t nj = next_index(j, nb_nodes.size()); + idx_t sj = nb_nodes[j]; + idx_t snj = nb_nodes[nj]; + triplets.emplace_back(tcell, sj, 0.5 * PointXYZ::dot(Rsj[j], Aik[icell])); + triplets.emplace_back(tcell, snj, 0.5 * PointXYZ::dot(Rsj[j], Aik[icell])); + } + triplets.emplace_back(tcell, snode, iparam.tgt_weights[icell] - PointXYZ::dot(Rs, Aik[icell])); + } + } + else { + auto& tgt_csp2node_ = data_->tgt_csp2node_; + for (idx_t icell = 0; icell < iparam.centroids.size(); ++icell) { + idx_t tcell = iparam.cell_idx[icell]; + idx_t tnode = tgt_csp2node_[tcell]; + double inv_node_weight = (tgt_areas_v[tnode] > 1e-15) ? 1. / tgt_areas_v[tnode] : 0.; + double csp2node_coef = iparam.src_weights[icell] / iparam.tgt_weights[icell] * inv_node_weight; + for (idx_t j = 0; j < nb_nodes.size(); ++j) { + idx_t nj = next_index(j, nb_nodes.size()); + idx_t sj = nb_nodes[j]; + idx_t snj = nb_nodes[nj]; + triplets.emplace_back(tnode, sj, (0.5 * PointXYZ::dot(Rsj[j], Aik[icell])) * csp2node_coef); + triplets.emplace_back(tnode, snj, + (0.5 * PointXYZ::dot(Rsj[j], Aik[icell])) * csp2node_coef); + } + triplets.emplace_back( + tnode, snode, (iparam.tgt_weights[icell] - PointXYZ::dot(Rs, Aik[icell])) * csp2node_coef); + } + } + } + } + } + sort_and_accumulate_triplets(triplets); // Very expensive!!! (90% of this routine). We need to avoid it + return Matrix(n_tpoints_, n_spoints_, triplets); +} + +void ConservativeSphericalPolygonInterpolation::do_execute(const Field& src_field, Field& tgt_field, + Metadata& metadata) const { + ATLAS_TRACE("ConservativeMethod::do_execute()"); + { + if (src_field.dirty()) { + ATLAS_TRACE("halo exchange source"); + src_field.haloExchange(); + } + } + StopWatch stopwatch; + stopwatch.start(); + if (order_ == 1) { + if (matrix_free_) { + ATLAS_TRACE("matrix_free_order_1"); + const auto& src_iparam_ = data_->src_iparam_; + const auto& tgt_areas_v = data_->tgt_areas_; + + if (not src_cell_data_ or not tgt_cell_data_) { + ATLAS_NOTIMPLEMENTED; + } + const auto src_vals = array::make_view(src_field); + auto tgt_vals = array::make_view(tgt_field); + for (idx_t tcell = 0; tcell < tgt_vals.size(); ++tcell) { + tgt_vals(tcell) = 0.; + } + for (idx_t scell = 0; scell < src_vals.size(); ++scell) { + const auto& iparam = src_iparam_[scell]; + for (idx_t icell = 0; icell < iparam.centroids.size(); ++icell) { + tgt_vals(iparam.cell_idx[icell]) += iparam.src_weights[icell] * src_vals(scell); + } + } + for (idx_t tcell = 0; tcell < tgt_vals.size(); ++tcell) { + tgt_vals[tcell] /= tgt_areas_v[tcell]; + } + } + else { + ATLAS_TRACE("matrix_order_1"); + Method::do_execute(src_field, tgt_field, metadata); + } + } + else if (order_ == 2) { + if (matrix_free_) { + ATLAS_TRACE("matrix_free_order_2"); + const auto& src_iparam_ = data_->src_iparam_; + const auto& tgt_areas_v = data_->tgt_areas_; + + if (not src_cell_data_ or not tgt_cell_data_) { + ATLAS_NOTIMPLEMENTED; + } + + auto& src_points_ = data_->src_points_; + + const auto src_vals = array::make_view(src_field); + auto tgt_vals = array::make_view(tgt_field); + const auto halo = array::make_view(src_mesh_.cells().halo()); + for (idx_t tcell = 0; tcell < tgt_vals.size(); ++tcell) { + tgt_vals(tcell) = 0.; + } + for (idx_t scell = 0; scell < src_vals.size(); ++scell) { + if (halo(scell)) { + continue; + } + const auto& iparam = src_iparam_[scell]; + const PointXYZ& P = src_points_[scell]; + PointXYZ grad = {0., 0., 0.}; + PointXYZ src_barycenter = {0., 0., 0.}; + auto src_neighbour_cells = get_cell_neighbours(src_mesh_, scell); + double dual_area = 0.; + for (idx_t nb_id = 0; nb_id < src_neighbour_cells.size(); ++nb_id) { + idx_t nnb_id = next_index(nb_id, src_neighbour_cells.size()); + idx_t ncell = src_neighbour_cells[nb_id]; + idx_t nncell = src_neighbour_cells[nnb_id]; + const auto& Pn = src_points_[ncell]; + const auto& Pnn = src_points_[nncell]; + if (ncell != scell && nncell != scell) { + double val = 0.5 * (src_vals(ncell) + src_vals(nncell)) - src_vals(scell); + auto csp = ConvexSphericalPolygon({Pn, Pnn, P}); + if (csp.area() < std::numeric_limits::epsilon()) { + csp = ConvexSphericalPolygon({Pn, P, Pnn}); + } + auto NsNsj = ConvexSphericalPolygon::GreatCircleSegment(P, Pn); + val *= (NsNsj.inLeftHemisphere(Pnn, -1e-16) ? -1 : 1); + dual_area += std::abs(csp.area()); + grad = grad + PointXYZ::mul(PointXYZ::cross(Pn, Pnn), val); + } + else if (ncell != scell) { + ATLAS_NOTIMPLEMENTED; + //double val = 0.5 * ( src_vals( ncell ) - src_vals( scell ) ); + //grad = grad + PointXYZ::mul( PointXYZ::cross( Pn, P ), val ); + } + else if (nncell != scell) { + ATLAS_NOTIMPLEMENTED; + //double val = 0.5 * ( src_vals( nncell ) - src_vals( scell ) ); + //grad = grad + PointXYZ::mul( PointXYZ::cross( P, Pnn ), val ); + } + } + if (dual_area > std::numeric_limits::epsilon()) { + grad = PointXYZ::div(grad, dual_area); + } + for (idx_t icell = 0; icell < iparam.centroids.size(); ++icell) { + src_barycenter = src_barycenter + PointXYZ::mul(iparam.centroids[icell], iparam.src_weights[icell]); + } + src_barycenter = PointXYZ::div(src_barycenter, PointXYZ::norm(src_barycenter)); + grad = grad - PointXYZ::mul(src_barycenter, PointXYZ::dot(grad, src_barycenter)); + ATLAS_ASSERT(std::abs(PointXYZ::dot(grad, src_barycenter)) < 1e-14); + for (idx_t icell = 0; icell < iparam.centroids.size(); ++icell) { + tgt_vals(iparam.cell_idx[icell]) += + iparam.src_weights[icell] * + (src_vals(scell) + PointXYZ::dot(grad, iparam.centroids[icell] - src_barycenter)); + } + } + for (idx_t tcell = 0; tcell < tgt_vals.size(); ++tcell) { + tgt_vals[tcell] /= tgt_areas_v[tcell]; + } + } + else { + ATLAS_TRACE("matrix_order_2"); + Method::do_execute(src_field, tgt_field, metadata); + } + } + + stopwatch.stop(); + + auto remap_stat = remap_stat_; + if (statistics_conservation_) { + const auto src_cell_halo = array::make_view(src_mesh_.cells().halo()); + const auto src_node_ghost = array::make_view(src_mesh_.nodes().ghost()); + const auto src_node_halo = array::make_view(src_mesh_.nodes().halo()); + const auto tgt_cell_halo = array::make_view(tgt_mesh_.cells().halo()); + const auto tgt_node_ghost = array::make_view(tgt_mesh_.nodes().ghost()); + const auto tgt_node_halo = array::make_view(tgt_mesh_.nodes().halo()); + const auto& src_areas_v = data_->src_areas_; + const auto& tgt_areas_v = data_->tgt_areas_; + + const auto src_vals = array::make_view(src_field); + const auto tgt_vals = array::make_view(tgt_field); + + double err_remap_cons = 0.; + const auto& tgt_csp2node_ = data_->tgt_csp2node_; + const auto& src_iparam_ = data_->src_iparam_; + if (src_cell_data_) { + for (idx_t spt = 0; spt < src_vals.size(); ++spt) { + if (src_cell_halo(spt)) { + continue; + } + err_remap_cons += src_vals(spt) * src_areas_v[spt]; + } + } + else { + auto& src_node2csp_ = data_->src_node2csp_; + for (idx_t spt = 0; spt < src_vals.size(); ++spt) { + if (src_node_ghost(spt) or src_areas_v[spt] < 1e-14) { + continue; + } + err_remap_cons += src_vals(spt) * src_areas_v[spt]; + } + } + auto& tgt_points_ = data_->tgt_points_; + if (tgt_cell_data_) { + for (idx_t tpt = 0; tpt < tgt_vals.size(); ++tpt) { + if (tgt_cell_halo(tpt)) { + continue; + } + err_remap_cons -= tgt_vals(tpt) * tgt_areas_v[tpt]; + } + } + else { + for (idx_t tpt = 0; tpt < tgt_vals.size(); ++tpt) { + if (tgt_node_ghost(tpt)) { + continue; + } + err_remap_cons -= tgt_vals(tpt) * tgt_areas_v[tpt]; + } + } + ATLAS_TRACE_MPI(ALLREDUCE) { mpi::comm().allReduceInPlace(&err_remap_cons, 1, eckit::mpi::sum()); } + remap_stat.errors[Statistics::Errors::REMAP_CONS] = std::sqrt(std::abs(err_remap_cons) / unit_sphere_area()); + + metadata.set("conservation_error", remap_stat.errors[Statistics::Errors::REMAP_CONS]); + } + if (statistics_intersection_) { + metadata.set("polygons.source", remap_stat.counts[Statistics::SRC_PLG]); + metadata.set("polygons.target", remap_stat.counts[Statistics::TGT_PLG]); + metadata.set("polygons.intersections", remap_stat.counts[Statistics::INT_PLG]); + metadata.set("polygons.uncovered_source", remap_stat.counts[Statistics::UNCVR_SRC]); + metadata.set("source_area_error.L1", remap_stat.errors[Statistics::Errors::GEO_L1]); + metadata.set("source_area_error.Linf", remap_stat.errors[Statistics::Errors::GEO_LINF]); + } + + if (statistics_intersection_ || statistics_conservation_) { + remap_stat.fillMetadata(metadata); + } + + auto& timings = data_->timings; + metadata.set("timings.source_polygons_assembly", timings.source_polygons_assembly); + metadata.set("timings.target_polygons_assembly", timings.target_polygons_assembly); + metadata.set("timings.target_kdtree_assembly", timings.target_kdtree_assembly); + metadata.set("timings.target_kdtree_search", timings.target_kdtree_search); + metadata.set("timings.source_polygons_filter", timings.source_polygons_filter); + metadata.set("timings.polygon_intersections", timings.polygon_intersections); + metadata.set("timings.matrix_assembly", timings.matrix_assembly); + metadata.set("timings.interpolation", stopwatch.elapsed()); + + metadata.set("memory.matrix", matrix_free_ ? 0 : matrix().footprint()); + metadata.set("memory.src_points", memory_of(data_->src_points_)); + metadata.set("memory.tgt_points", memory_of(data_->tgt_points_)); + metadata.set("memory.src_areas", memory_of(data_->src_points_)); + metadata.set("memory.tgt_areas", memory_of(data_->tgt_areas_)); + metadata.set("memory.src_csp2node", memory_of(data_->src_csp2node_)); + metadata.set("memory.tgt_csp2node", memory_of(data_->tgt_csp2node_)); + metadata.set("memory.src_node2csp", memory_of(data_->src_node2csp_)); + metadata.set("memory.tgt_node2csp", memory_of(data_->tgt_node2csp_)); + metadata.set("memory.src_iparam", memory_of(data_->src_iparam_)); + + tgt_field.set_dirty(); +} + +void ConservativeSphericalPolygonInterpolation::print(std::ostream& out) const { + out << "ConservativeMethod{"; + out << "order:" << order_; + out << ", source:" << (src_cell_data_ ? "cells" : "nodes"); + out << ", target:" << (tgt_cell_data_ ? "cells" : "nodes"); + out << ", normalise_intersections:" << normalise_intersections_; + out << ", matrix_free:" << matrix_free_; + out << ", statistics.intersection:" << statistics_intersection_; + out << ", statistics.conservation:" << statistics_conservation_; + out << ", cached_matrix:" << not(matrixAllocated() || matrix_free_); + out << ", cached_data:" << bool(sharable_data_.use_count() == 0); + size_t footprint{}; + if (not matrix_free_) { + footprint += matrix().footprint(); + } + footprint += data_->footprint(); + out << ", footprint:" << eckit::Bytes(footprint); + out << "}"; +} + +Cache ConservativeSphericalPolygonInterpolation::createCache() const { + interpolation::Cache cache; + if (not matrix_free_) { + cache.add(Method::createCache()); + } + cache.add(cache_); + return cache; +} + +void ConservativeSphericalPolygonInterpolation::setup_stat() const { + const auto src_cell_halo = array::make_view(src_mesh_.cells().halo()); + const auto src_node_ghost = array::make_view(src_mesh_.nodes().ghost()); + const auto& src_areas_v = data_->src_areas_; + const auto& tgt_areas_v = data_->tgt_areas_; + double geo_create_err = 0.; + double src_tgt_sums[2] = {0., 0.}; + if (src_cell_data_) { + for (idx_t spt = 0; spt < src_areas_v.size(); ++spt) { + if (not src_cell_halo(spt)) { + src_tgt_sums[0] += src_areas_v[spt]; + } + } + } + else { + for (idx_t src = 0; src < src_areas_v.size(); ++src) { + if (not src_node_ghost(src)) { + src_tgt_sums[0] += src_areas_v[src]; + } + } + } + const auto& tgt_cell_halo = array::make_view(tgt_mesh_.cells().halo()); + const auto& tgt_node_ghost = array::make_view(tgt_mesh_.nodes().ghost()); + if (tgt_cell_data_) { + for (idx_t tpt = 0; tpt < tgt_areas_v.size(); ++tpt) { + if (not tgt_cell_halo(tpt)) { + src_tgt_sums[1] += tgt_areas_v[tpt]; + } + } + } + else { + for (idx_t tpt = 0; tpt < tgt_areas_v.size(); ++tpt) { + if (not tgt_node_ghost(tpt)) { + src_tgt_sums[1] += tgt_areas_v[tpt]; + } + } + } + ATLAS_TRACE_MPI(ALLREDUCE) { mpi::comm().allReduceInPlace(src_tgt_sums, 2, eckit::mpi::sum()); } + + remap_stat_.src_area_sum = src_tgt_sums[0]; + remap_stat_.tgt_area_sum = src_tgt_sums[1]; + + geo_create_err = std::abs(src_tgt_sums[0] - src_tgt_sums[1]) / unit_sphere_area(); + remap_stat_.errors[Statistics::Errors::GEO_DIFF] = geo_create_err; +} + +Field ConservativeSphericalPolygonInterpolation::Statistics::diff(const Interpolation& interpolation, + const Field source, const Field target) { + Field diff = interpolation.source().createField(source, option::name("diff")); + auto diff_vals = array::make_view(diff); + + const auto src_vals = array::make_view(source); + const auto tgt_vals = array::make_view(target); + + auto cachable_data_ = ConservativeSphericalPolygonInterpolation::Cache(interpolation).get(); + const auto& src_areas_v = cachable_data_->src_areas_; + const auto& tgt_areas_v = cachable_data_->tgt_areas_; + const auto& tgt_csp2node_ = cachable_data_->tgt_csp2node_; + const auto& src_node2csp_ = cachable_data_->src_node2csp_; + const auto& src_iparam_ = cachable_data_->src_iparam_; + const auto& src_mesh_ = extract_mesh(cachable_data_->src_fs_); + const auto& tgt_mesh_ = extract_mesh(cachable_data_->tgt_fs_); + const auto src_cell_data_ = bool(functionspace::CellColumns(interpolation.source())); + const auto tgt_cell_data_ = bool(functionspace::CellColumns(interpolation.target())); + const auto src_cell_halo = array::make_view(src_mesh_.cells().halo()); + const auto src_node_ghost = array::make_view(src_mesh_.nodes().ghost()); + const auto src_node_halo = array::make_view(src_mesh_.nodes().halo()); + const auto tgt_cell_halo = array::make_view(tgt_mesh_.cells().halo()); + const auto tgt_node_ghost = array::make_view(tgt_mesh_.nodes().ghost()); + const auto tgt_node_halo = array::make_view(tgt_mesh_.nodes().halo()); + double err_remap_l2 = 0.; + double err_remap_linf = 0.; + if (src_cell_data_) { + for (idx_t spt = 0; spt < src_vals.size(); ++spt) { + if (src_cell_halo(spt)) { + continue; + } + double diff = src_vals(spt) * src_areas_v[spt]; + const auto& iparam = src_iparam_[spt]; + if (tgt_cell_data_) { + for (idx_t icell = 0; icell < iparam.src_weights.size(); ++icell) { + idx_t tcell = iparam.cell_idx[icell]; + if (tgt_cell_halo(tcell) < 1) { + diff -= tgt_vals(iparam.cell_idx[icell]) * iparam.src_weights[icell]; + } + } + } + else { + for (idx_t icell = 0; icell < iparam.src_weights.size(); ++icell) { + idx_t tcell = iparam.cell_idx[icell]; + idx_t tnode = tgt_csp2node_[tcell]; + if (tgt_node_halo(tnode) < 1) { + diff -= tgt_vals(tnode) * iparam.src_weights[icell]; + } + } + } + diff_vals(spt) = std::abs(diff) / src_areas_v[spt]; + } + } + else { + for (idx_t spt = 0; spt < src_vals.size(); ++spt) { + if (src_node_ghost(spt) or src_areas_v[spt] < 1e-14) { + diff_vals(spt) = 0.; + continue; + } + double diff = src_vals(spt) * src_areas_v[spt]; + const auto& node2csp = src_node2csp_[spt]; + for (idx_t subcell = 0; subcell < node2csp.size(); ++subcell) { + const auto& iparam = src_iparam_[node2csp[subcell]]; + if (tgt_cell_data_) { + for (idx_t icell = 0; icell < iparam.src_weights.size(); ++icell) { + diff -= tgt_vals(iparam.cell_idx[icell]) * iparam.src_weights[icell]; + } + } + else { + for (idx_t icell = 0; icell < iparam.src_weights.size(); ++icell) { + idx_t tcell = iparam.cell_idx[icell]; + idx_t tnode = tgt_csp2node_[tcell]; + diff -= tgt_vals(tnode) * iparam.src_weights[icell]; + } + } + } + diff_vals(spt) = std::abs(diff) / src_areas_v[spt]; + } + } + return diff; +} + + +void ConservativeSphericalPolygonInterpolation::Statistics::accuracy(const Interpolation& interpolation, + const Field target, + std::function func) { + auto tgt_vals = array::make_view(target); + auto cachable_data_ = ConservativeSphericalPolygonInterpolation::Cache(interpolation).get(); + auto tgt_mesh_ = extract_mesh(cachable_data_->src_fs_); + auto tgt_cell_data_ = extract_mesh(cachable_data_->tgt_fs_); + const auto tgt_cell_halo = array::make_view(tgt_mesh_.cells().halo()); + const auto tgt_node_ghost = array::make_view(tgt_mesh_.nodes().ghost()); + const auto& tgt_areas_v = cachable_data_->tgt_areas_; + double err_remap_l2 = 0.; + double err_remap_linf = 0.; + auto& tgt_points_ = cachable_data_->tgt_points_; + if (tgt_cell_data_) { + size_t ncells = std::min(tgt_vals.size(), tgt_mesh_.cells().size()); + for (idx_t tpt = 0; tpt < ncells; ++tpt) { + ATLAS_ASSERT(tpt < tgt_cell_halo.size()); + if (tgt_cell_halo(tpt)) { + continue; + } + auto p = tgt_points_[tpt]; + PointLonLat pll; + eckit::geometry::Sphere::convertCartesianToSpherical(1., p, pll); + double err_l = std::abs(tgt_vals(tpt) - func(pll)); + err_remap_l2 += err_l * err_l * tgt_areas_v[tpt]; + err_remap_linf = std::max(err_remap_linf, err_l); + } + } + else { + size_t nnodes = std::min(tgt_vals.size(), tgt_mesh_.nodes().size()); + for (idx_t tpt = 0; tpt < nnodes; ++tpt) { + if (tgt_node_ghost(tpt)) { + continue; + } + auto p = tgt_points_[tpt]; + PointLonLat pll; + eckit::geometry::Sphere::convertCartesianToSpherical(1., p, pll); + double err_l = std::abs(tgt_vals(tpt) - func(pll)); + err_remap_l2 += err_l * err_l * tgt_areas_v[tpt]; + err_remap_linf = std::max(err_remap_linf, err_l); + } + } + ATLAS_TRACE_MPI(ALLREDUCE) { + mpi::comm().allReduceInPlace(&err_remap_l2, 1, eckit::mpi::sum()); + mpi::comm().allReduceInPlace(&err_remap_linf, 1, eckit::mpi::max()); + } + this->errors[Statistics::Errors::REMAP_L2] = std::sqrt(err_remap_l2 / unit_sphere_area()); + this->errors[Statistics::Errors::REMAP_LINF] = err_remap_linf; +} + +auto debug_intersection = [](const ConvexSphericalPolygon& plg_1, const ConvexSphericalPolygon& plg_2, + const ConvexSphericalPolygon& iplg, const ConvexSphericalPolygon& jplg) { + const double intersection_comm_err = std::abs(iplg.area() - jplg.area()) / (plg_1.area() > 0 ? plg_1.area() : 1.); + Log::info().indent(); + if (intersection_comm_err > 1e-6) { + Log::info() << "PLG_1 : " << std::setprecision(10) << plg_1 << "\n"; + Log::info() << "area(PLG_1) : " << plg_1.area() << "\n"; + Log::info() << "PLG_2 :" << plg_2 << "\n"; + Log::info() << "PLG_12 : " << iplg << "\n"; + Log::info() << "PLG_21 : " << jplg << "\n"; + Log::info() << "area(PLG_12 - PLG_21) : " << intersection_comm_err << "\n"; + Log::info() << "area(PLG_21) : " << jplg.area() << "\n"; + //ATLAS_ASSERT( false, "SRC.intersect.TGT =/= TGT.intersect.SRC."); + } + int pout; + int pin = inside_vertices(plg_1, plg_2, pout); + if (pin > 2 && iplg.area() < 3e-16) { + Log::info() << " pin : " << pin << ", pout :" << pout << ", total vertices : " << plg_2.size() << "\n"; + Log::info() << "PLG_2 :" << plg_2 << "\n"; + Log::info() << "PLG_12 : " << iplg << "\n"; + Log::info() << "area(PLG_12) : " << iplg.area() << "\n\n"; + //ATLAS_ASSERT( false, "SRC must intersect TGT." ); + } + Log::info().unindent(); +}; + +void ConservativeSphericalPolygonInterpolation::dump_intersection(const ConvexSphericalPolygon& plg_1, + const CSPolygonArray& plg_2_array, + const std::vector& plg_2_idx_array) const { + double plg_1_coverage = 0.; + for (int i = 0; i < plg_2_idx_array.size(); ++i) { + const auto plg_2_idx = plg_2_idx_array[i]; + const auto& plg_2 = std::get<0>(plg_2_array[plg_2_idx]); + auto iplg = plg_1.intersect(plg_2); + auto jplg = plg_2.intersect(plg_1); + debug_intersection(plg_1, plg_2, iplg, jplg); + plg_1_coverage += iplg.area(); + } + Log::info().indent(); + if (std::abs(plg_1.area() - plg_1_coverage) > 0.01 * plg_1.area()) { + Log::info() << "Polygon coverage incomplete. Printing polygons." << std::endl; + Log::info() << "Polygon 1 : "; + plg_1.print(Log::info()); + Log::info() << std::endl << "Printing " << plg_2_idx_array.size() << " covering polygons -->" << std::endl; + Log::info().indent(); + for (int i = 0; i < plg_2_idx_array.size(); ++i) { + const auto plg_2_idx = plg_2_idx_array[i]; + const auto& plg_2 = std::get<0>(plg_2_array[plg_2_idx]); + Log::info() << "Polygon " << i + 1 << " : "; + plg_2.print(Log::info()); + Log::info() << std::endl; + } + Log::info().unindent(); + } + Log::info().unindent(); +} + +template +void ConservativeSphericalPolygonInterpolation::dump_intersection(const ConvexSphericalPolygon& plg_1, + const CSPolygonArray& plg_2_array, + const TargetCellsIDs& plg_2_idx_array) const { + std::vector idx_array; + idx_array.resize(plg_2_idx_array.size()); + for (int i = 0; i < plg_2_idx_array.size(); ++i) { + idx_array[i] = plg_2_idx_array[i].payload(); + } + dump_intersection(plg_1, plg_2_array, idx_array); +} + +ConservativeSphericalPolygonInterpolation::Cache::Cache(std::shared_ptr entry): + interpolation::Cache(entry), entry_(dynamic_cast(entry.get())) {} + +ConservativeSphericalPolygonInterpolation::Cache::Cache(const interpolation::Cache& c): + interpolation::Cache(c, Data::static_type()), entry_{dynamic_cast(c.get(Data::static_type()))} {} + +ConservativeSphericalPolygonInterpolation::Cache::Cache(const Interpolation& interpolation): + Cache(interpolation::Cache(interpolation)) {} + +size_t ConservativeSphericalPolygonInterpolation::Data::footprint() const { + size_t mem_total{0}; + mem_total += memory_of(src_points_); + mem_total += memory_of(tgt_points_); + mem_total += memory_of(src_areas_); + mem_total += memory_of(tgt_areas_); + mem_total += memory_of(src_csp2node_); + mem_total += memory_of(tgt_csp2node_); + mem_total += memory_of(src_node2csp_); + mem_total += memory_of(tgt_node2csp_); + mem_total += memory_of(src_iparam_); + return mem_total; +} + + +void ConservativeSphericalPolygonInterpolation::Data::print(std::ostream& out) const { + out << "Memory usage of ConservativeMethod: " << eckit::Bytes(footprint()) << "\n"; + out << "- src_points_ \t" << eckit::Bytes(memory_of(src_points_)) << "\n"; + out << "- tgt_points_ \t" << eckit::Bytes(memory_of(tgt_points_)) << "\n"; + out << "- src_areas_ \t" << eckit::Bytes(memory_of(src_areas_)) << "\n"; + out << "- tgt_areas_ \t" << eckit::Bytes(memory_of(tgt_areas_)) << "\n"; + out << "- src_csp2node_ \t" << eckit::Bytes(memory_of(src_csp2node_)) << "\n"; + out << "- tgt_csp2node_ \t" << eckit::Bytes(memory_of(tgt_csp2node_)) << "\n"; + out << "- src_node2csp_ \t" << eckit::Bytes(memory_of(src_node2csp_)) << "\n"; + out << "- tgt_node2csp_ \t" << eckit::Bytes(memory_of(tgt_node2csp_)) << "\n"; + out << "- src_iparam_ \t" << eckit::Bytes(memory_of(src_iparam_)) << "\n"; +} + +void ConservativeSphericalPolygonInterpolation::Statistics::fillMetadata(Metadata& metadata) { + // counts + metadata.set("counts.SRC_PLG", counts[SRC_PLG]); + metadata.set("counts.TGT_PLG", counts[TGT_PLG]); + metadata.set("counts.INT_PLG", counts[INT_PLG]); + metadata.set("counts.UNCVR_SRC", counts[UNCVR_SRC]); + + // errors + metadata.set("errors.SRC_PLG_L1", errors[SRC_PLG_L1]); + metadata.set("errors.SRC_PLG_LINF", errors[SRC_PLG_LINF]); + metadata.set("errors.TGT_PLG_L1", errors[TGT_PLG_L1]); + metadata.set("errors.TGT_PLG_LINF", errors[TGT_PLG_LINF]); + metadata.set("errors.GEO_L1", errors[GEO_L1]); + metadata.set("errors.GEO_LINF", errors[GEO_LINF]); + metadata.set("errors.GEO_DIFF", errors[GEO_DIFF]); + metadata.set("errors.REMAP_CONS", errors[REMAP_CONS]); + metadata.set("errors.REMAP_L2", errors[REMAP_L2]); + metadata.set("errors.REMAP_LINF", errors[REMAP_LINF]); +} + +ConservativeSphericalPolygonInterpolation::Statistics::Statistics() { + std::fill(std::begin(counts), std::end(counts), 0); + std::fill(std::begin(errors), std::end(errors), 0.); +} + +ConservativeSphericalPolygonInterpolation::Statistics::Statistics(const Metadata& metadata): Statistics() { + // counts + metadata.get("counts.SRC_PLG", counts[SRC_PLG]); + metadata.get("counts.TGT_PLG", counts[TGT_PLG]); + metadata.get("counts.INT_PLG", counts[INT_PLG]); + metadata.get("counts.UNCVR_SRC", counts[UNCVR_SRC]); + + // errors + metadata.get("errors.SRC_PLG_L1", errors[SRC_PLG_L1]); + metadata.get("errors.SRC_PLG_LINF", errors[SRC_PLG_LINF]); + metadata.get("errors.TGT_PLG_L1", errors[TGT_PLG_L1]); + metadata.get("errors.TGT_PLG_LINF", errors[TGT_PLG_LINF]); + metadata.get("errors.GEO_L1", errors[GEO_L1]); + metadata.get("errors.GEO_LINF", errors[GEO_LINF]); + metadata.get("errors.GEO_DIFF", errors[GEO_DIFF]); + metadata.get("errors.REMAP_CONS", errors[REMAP_CONS]); + metadata.get("errors.REMAP_L2", errors[REMAP_L2]); + metadata.get("errors.REMAP_LINF", errors[REMAP_LINF]); +} + + +} // namespace method +} // namespace interpolation +} // namespace atlas diff --git a/src/atlas/interpolation/method/unstructured/ConservativeSphericalPolygonInterpolation.h b/src/atlas/interpolation/method/unstructured/ConservativeSphericalPolygonInterpolation.h new file mode 100644 index 000000000..bcab9fdba --- /dev/null +++ b/src/atlas/interpolation/method/unstructured/ConservativeSphericalPolygonInterpolation.h @@ -0,0 +1,211 @@ +/* + * (C) Copyright 2021- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + + +#pragma once + +#include "atlas/functionspace.h" +#include "atlas/interpolation/method/Method.h" +#include "atlas/util/ConvexSphericalPolygon.h" + +namespace atlas { +namespace interpolation { +namespace method { + + +class ConservativeSphericalPolygonInterpolation : public Method { +public: + struct InterpolationParameters { // one polygon intersection + std::vector cell_idx; // target cells used for intersection + std::vector centroids; // intersection cell centroids + std::vector src_weights; // intersection cell areas + + // TODO: tgt_weights can be computed on the fly + std::vector tgt_weights; // (intersection cell areas) / (target cell area) + }; + +private: + class Data : public InterpolationCacheEntry { + public: + ~Data() override = default; + size_t footprint() const override; + static std::string static_type() { return "ConservativeSphericalPolygonInterpolation"; } + std::string type() const override { return static_type(); } + void print(std::ostream& out) const; + + private: + friend class ConservativeSphericalPolygonInterpolation; + + // position and effective area of data points + std::vector src_points_; + std::vector tgt_points_; + std::vector src_areas_; + std::vector tgt_areas_; + + // indexing of subpolygons + std::vector src_csp2node_; + std::vector tgt_csp2node_; + std::vector> src_node2csp_; + std::vector> tgt_node2csp_; + + + // Timings + struct Timings { + double source_polygons_assembly{0}; + double target_polygons_assembly{0}; + double target_kdtree_assembly{0}; + double target_kdtree_search{0}; + double source_polygons_filter{0}; + double polygon_intersections{0}; + double matrix_assembly{0}; + double interpolation{0}; + } timings; + + std::vector src_iparam_; // TODO: remove after setup? + + // Reconstructible if need be + FunctionSpace src_fs_; + FunctionSpace tgt_fs_; + }; + +public: + class Cache final : public interpolation::Cache { + public: + Cache() = default; + Cache(const interpolation::Cache& c); + Cache(const Interpolation&); + + operator bool() const { return entry_; } + const Data* get() const { return entry_; } + + private: + friend class ConservativeSphericalPolygonInterpolation; + Cache(std::shared_ptr entry); + const Data* entry_{nullptr}; + }; + + struct Statistics { + enum Counts + { + SRC_PLG = 0, // index, number of source polygons + TGT_PLG, // index, number of target polygons + INT_PLG, // index, number of intersection polygons + UNCVR_SRC // index, number of uncovered source polygons + }; + std::array counts; + enum Errors + { + SRC_PLG_L1 = 0, // index, over/undershoot in source subpolygon creation + SRC_PLG_LINF, + TGT_PLG_L1, // index, over/untershoot in target subpolygon creation + TGT_PLG_LINF, + GEO_L1, // index, cumulative area mismatch in polygon intersections + GEO_LINF, // index, like GEO_L1 but in L_infinity norm + GEO_DIFF, // index, difference in earth area coverages + REMAP_CONS, // index, error in mass conservation + REMAP_L2, // index, error accuracy for given analytical function + REMAP_LINF // index, like REMAP_L2 but in L_infinity norm + }; + std::array errors; + + double tgt_area_sum; + double src_area_sum; + + void fillMetadata(Metadata&); + + Statistics(); + Statistics(const Metadata&); + + void accuracy(const Interpolation& interpolation, const Field target, + std::function func); + + + // compute difference field of source and target mass + Field diff(const Interpolation&, const Field source, const Field target); + }; + + +public: + ConservativeSphericalPolygonInterpolation(const Config& = util::NoConfig()); + + using Method::do_setup; + void do_setup(const FunctionSpace& src_fs, const FunctionSpace& tgt_fs) override; + void do_setup(const Grid& src_grid, const Grid& tgt_grid, const interpolation::Cache&) override; + void do_execute(const Field& src_field, Field& tgt_field, Metadata&) const override; + + void print(std::ostream& out) const override; + + const FunctionSpace& source() const override { return data_->src_fs_; } + const FunctionSpace& target() const override { return data_->tgt_fs_; } + + inline const PointXYZ& src_points(size_t id) const { return data_->src_points_[id]; } + inline const PointXYZ& tgt_points(size_t id) const { return data_->tgt_points_[id]; } + + interpolation::Cache createCache() const override; + +private: + using ConvexSphericalPolygon = util::ConvexSphericalPolygon; + using PolygonArray = std::vector>; + using CSPolygonArray = std::vector>; + + void do_setup_impl(const Grid& src_grid, const Grid& tgt_grid); + + void intersect_polygons(const CSPolygonArray& src_csp, const CSPolygonArray& tgt_scp); + Matrix compute_1st_order_matrix(); + Matrix compute_2nd_order_matrix(); + void dump_intersection(const ConvexSphericalPolygon& plg_1, const CSPolygonArray& plg_2_array, + const std::vector& plg_2_idx_array) const; + template + void dump_intersection(const ConvexSphericalPolygon& plg_1, const CSPolygonArray& plg_2_array, + const TargetCellsIDs& plg_2_idx_array) const; + std::vector sort_cell_edges(Mesh& mesh, idx_t cell_id) const; + std::vector sort_node_edges(Mesh& mesh, idx_t cell_id) const; + std::vector get_cell_neighbours(Mesh& mesh, idx_t jcell) const; + std::vector get_node_neighbours(Mesh& mesh, idx_t jcell) const; + CSPolygonArray get_polygons_celldata(Mesh& mesh) const; + CSPolygonArray get_polygons_nodedata(Mesh& mesh, std::vector& csp2node, + std::vector>& node2csp, + std::array& errors) const; + + int next_index(int current_index, int size, int offset = 1) const; + int prev_index(int current_index, int size, int offset = 1) const; + + + void setup_stat() const; + +private: + bool validate_; + bool src_cell_data_; + bool tgt_cell_data_; + FunctionSpace src_fs_; + FunctionSpace tgt_fs_; + mutable Mesh src_mesh_; + mutable Mesh tgt_mesh_; + int normalise_intersections_; + int order_; + bool matrix_free_; + bool statistics_intersection_; + bool statistics_conservation_; + + mutable Statistics remap_stat_; + + Cache cache_; // Storage of cache if any was passed to constructor + std::shared_ptr sharable_data_; // Storage of new data_, only allocated if cache is empty + const Data* data_; // Read-only access to data, pointing either to cache_ or sharable_data_ + + // position and effective area of data points + idx_t n_spoints_; + idx_t n_tpoints_; +}; + + +} // namespace method +} // namespace interpolation +} // namespace atlas diff --git a/src/sandbox/interpolation/CMakeLists.txt b/src/sandbox/interpolation/CMakeLists.txt index b2417c2d3..806562089 100644 --- a/src/sandbox/interpolation/CMakeLists.txt +++ b/src/sandbox/interpolation/CMakeLists.txt @@ -22,3 +22,9 @@ ecbuild_add_executable( NOINSTALL ) +ecbuild_add_executable( + TARGET atlas-conservative-interpolation + SOURCES atlas-conservative-interpolation.cc + LIBS atlas + NOINSTALL +) diff --git a/src/sandbox/interpolation/atlas-conservative-interpolation.cc b/src/sandbox/interpolation/atlas-conservative-interpolation.cc new file mode 100644 index 000000000..5242e7321 --- /dev/null +++ b/src/sandbox/interpolation/atlas-conservative-interpolation.cc @@ -0,0 +1,306 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + + +// TODO: +// ----- +// * Fix abort encountered with +// mpirun -np 4 atlas-conservative-interpolation --source.grid=O20 --target.grid=H8 --order=2 +// +// QUESTIONS: +// ---------- +// * Why sqrt in ConservativeSphericalPolygon in line +// remap_stat.errors[Statistics::Errors::REMAP_CONS] = std::sqrt(std::abs(err_remap_cons) / unit_sphere_area()); +// used to compute conservation_error + + +#include +#include +#include +#include + +#include "eckit/geometry/Sphere.h" +#include "eckit/log/Bytes.h" +#include "eckit/log/JSON.h" +#include "eckit/types/FloatCompare.h" + +#include "atlas/array.h" +#include "atlas/array/MakeView.h" +#include "atlas/field.h" +#include "atlas/grid.h" +#include "atlas/interpolation/Interpolation.h" +#include "atlas/interpolation/method/unstructured/ConservativeSphericalPolygonInterpolation.h" +#include "atlas/mesh.h" +#include "atlas/mesh/Mesh.h" +#include "atlas/mesh/actions/Build2DCellCentres.h" +#include "atlas/meshgenerator.h" +#include "atlas/option.h" +#include "atlas/output/Gmsh.h" +#include "atlas/runtime/AtlasTool.h" +#include "atlas/util/Config.h" +#include "atlas/util/function/SphericalHarmonic.h" +#include "atlas/util/function/VortexRollup.h" + +#include "tests/AtlasTestEnvironment.h" + + +namespace atlas { + + +class AtlasParallelInterpolation : public AtlasTool { + int execute(const AtlasTool::Args& args) override; + std::string briefDescription() override { return "Demonstration of parallel interpolation"; } + std::string usage() override { + return name() + + " [--source.grid=gridname] " + "[--target.grid=gridname] [OPTION]... [--help]"; + } + + int numberOfPositionalArguments() override { return -1; } + int minimumPositionalArguments() override { return 0; } + +public: + AtlasParallelInterpolation(int argc, char* argv[]): AtlasTool(argc, argv) { + add_option(new eckit::option::Separator("Source/Target options")); + + add_option(new SimpleOption("source.grid", "source gridname")); + add_option(new SimpleOption("target.grid", "target gridname")); + add_option(new SimpleOption("source.functionspace", + "source functionspace, to override source grid default")); + add_option(new SimpleOption("target.functionspace", + "target functionspace, to override target grid default")); + add_option(new SimpleOption("source.halo", "default=2")); + add_option(new SimpleOption("target.halo", "default=0")); + + add_option(new eckit::option::Separator("Interpolation options")); + add_option(new SimpleOption("order", "Interpolation order. Supported: 1, 2 (default=1)")); + add_option(new SimpleOption("normalise_intersections", + "Normalize polygon intersections so that interpolation weights sum to 1.")); + add_option(new SimpleOption("validate", + "Enable extra validations at cost of performance. For debugging purpose.")); + add_option(new SimpleOption("matrix_free", "Do not store matrix for consecutive interpolations")); + + add_option( + new SimpleOption("statistics.intersection", "Enable extra statistics on polygon intersections")); + add_option(new SimpleOption("statistics.accuracy", + "Enable extra statistics, comparing result with initial condition")); + add_option( + new SimpleOption("statistics.conservation", "Enable extra statistics computing mass conservation")); + + add_option(new eckit::option::Separator("Output options")); + + add_option(new SimpleOption( + "output-gmsh", "Output gmsh files src_mesh.msh, tgt_mesh.msh, src_field.msh, tgt_field.msh")); + add_option(new SimpleOption("gmsh.coordinates", "Mesh coordinates [xy,lonlat,xyz]")); + add_option(new SimpleOption("gmsh.ghost", "output of ghost")); + + add_option(new SimpleOption("output-json", "Output json file with run information")); + add_option(new SimpleOption("json.file", "File path for json output")); + + add_option(new eckit::option::Separator("Initial condition options")); + + add_option(new SimpleOption( + "init", "Setup initial source field [ constant, spherical_harmonic, vortex_rollup (default) ]")); + add_option(new SimpleOption("vortex_rollup.t", "Value that controls vortex rollup (default = 0.5)")); + add_option(new SimpleOption("constant.value", "Value that is assigned in case init==constant)")); + add_option(new SimpleOption("spherical_harmonic.n", "total wave number 'n' of a spherical harmonic")); + add_option(new SimpleOption("spherical_harmonic.m", "zonal wave number 'm' of a spherical harmonic")); + } + + struct Timers { + using StopWatch = atlas::runtime::trace::StopWatch; + StopWatch target_setup; + StopWatch source_setup; + StopWatch initial_condition; + StopWatch interpolation_setup; + StopWatch interpolation_execute; + } timers; +}; + +std::function get_init(const AtlasTool::Args& args) { + std::string init; + args.get("init", init = "vortex_rollup"); + if (init == "vortex_rollup") { + double t; + args.get("vortex_rollup.t", t = 1.); + return [t](const PointLonLat& p) { return util::function::vortex_rollup(p.lon(), p.lat(), t); }; + } + else if (init == "spherical_harmonic") { + int n = 2; + int m = 2; + args.get("spherical_harmonic.n", n); + args.get("spherical_harmonic.m", m); + + bool caching = true; // true -> warning not thread-safe + util::function::SphericalHarmonic Y(n, m, caching); + return [Y](const PointLonLat& p) { return Y(p.lon(), p.lat()); }; + } + else if (init == "constant") { + double value; + args.get("constant.value", value = 1.); + return [value](const PointLonLat&) { return value; }; + } + else { + if (args.has("init")) { + Log::error() << "Bad value for \"init\": \"" << init << "\" not recognised." << std::endl; + ATLAS_NOTIMPLEMENTED; + } + } + ATLAS_THROW_EXCEPTION("Should not be here"); +} + +int AtlasParallelInterpolation::execute(const AtlasTool::Args& args) { + auto src_grid = Grid{args.getString("source.grid", "H16")}; + auto tgt_grid = Grid{args.getString("target.grid", "H32")}; + + auto create_functionspace = [&](Mesh& mesh, int halo, std::string type) -> FunctionSpace { + if (type.empty()) { + type = "NodeColumns"; + if (mesh.grid().type() == "healpix" || mesh.grid().type() == "cubedsphere") { + type = "CellColumns"; + } + } + if (type == "CellColumns") { + if (!mesh.cells().has_field("lonlat")) { + mesh::actions::Build2DCellCentres{"lonlat"}(mesh); + } + return functionspace::CellColumns(mesh, option::halo(halo)); + } + else if (type == "NodeColumns") { + return functionspace::NodeColumns(mesh, option::halo(halo)); + } + ATLAS_THROW_EXCEPTION("FunctionSpace " << type << " is not recognized."); + }; + + timers.target_setup.start(); + auto tgt_mesh = Mesh{tgt_grid}; + auto tgt_functionspace = + create_functionspace(tgt_mesh, args.getLong("target.halo", 0), args.getString("target.functionspace", "")); + auto tgt_field = tgt_functionspace.createField(); + timers.target_setup.stop(); + + timers.source_setup.start(); + auto src_meshgenerator = MeshGenerator{src_grid.meshgenerator() | option::halo(2)}; + auto src_partitioner = grid::MatchingPartitioner{tgt_mesh}; + auto src_mesh = src_meshgenerator.generate(src_grid, src_partitioner); + auto src_functionspace = + create_functionspace(src_mesh, args.getLong("source.halo", 2), args.getString("source.functionspace", "")); + auto src_field = src_functionspace.createField(); + timers.source_setup.stop(); + + { + ATLAS_TRACE("Initial condition"); + timers.initial_condition.start(); + const auto lonlat = array::make_view(src_functionspace.lonlat()); + auto src_view = array::make_view(src_field); + auto f = get_init(args); + for (idx_t n = 0; n < lonlat.shape(0); ++n) { + src_view(n) = f(PointLonLat{lonlat(n, LON), lonlat(n, LAT)}); + } + src_field.set_dirty(false); + timers.initial_condition.start(); + } + + + timers.interpolation_setup.start(); + auto interpolation = + Interpolation(option::type("conservative-spherical-polygon") | args, src_functionspace, tgt_functionspace); + timers.interpolation_setup.stop(); + + + timers.interpolation_execute.start(); + auto metadata = interpolation.execute(src_field, tgt_field); + timers.interpolation_execute.stop(); + + // API not yet acceptable + Field src_conservation_field; + { + using Statistics = interpolation::method::ConservativeSphericalPolygonInterpolation::Statistics; + Statistics stats(metadata); + if (args.getBool("statistics.accuracy", false)) { + stats.accuracy(interpolation, tgt_field, get_init(args)); + } + if (args.getBool("statistics.conservation", false)) { + // compute difference field + src_conservation_field = stats.diff(interpolation, src_field, tgt_field); + } + } + + + Log::info() << "interpolation metadata: \n"; + { + eckit::JSON json(Log::info(), eckit::JSON::Formatting::indent(2)); + json << metadata; + } + Log::info() << std::endl; + + if (args.getBool("output-gmsh", false)) { + if (args.getBool("gmsh.ghost", false)) { + ATLAS_TRACE("halo exchange target"); + tgt_field.haloExchange(); + } + util::Config config(args.getSubConfiguration("gmsh")); + output::Gmsh{"src_mesh.msh", config}.write(src_mesh); + output::Gmsh{"src_field.msh", config}.write(src_field); + output::Gmsh{"tgt_mesh.msh", config}.write(tgt_mesh); + output::Gmsh{"tgt_field.msh", config}.write(tgt_field); + if (src_conservation_field) { + output::Gmsh{"src_conservation_field.msh", config}.write(src_conservation_field); + } + } + + if (args.getBool("output-json", false)) { + util::Config output; + output.set("setup.source.grid", args.getString("source.grid")); + output.set("setup.target.grid", args.getString("target.grid")); + output.set("setup.source.functionspace", src_functionspace.type()); + output.set("setup.target.functionspace", tgt_functionspace.type()); + output.set("setup.source.halo", args.getLong("source.halo", 2)); + output.set("setup.target.halo", args.getLong("target.halo", 0)); + output.set("setup.interpolation.order", args.getInt("order", 1)); + output.set("setup.interpolation.normalise_intersections", args.getBool("normalise_intersections", false)); + output.set("setup.interpolation.validate", args.getBool("validate", false)); + output.set("setup.interpolation.matrix_free", args.getBool("matrix-free", false)); + output.set("setup.init", args.getString("init", "vortex_rollup")); + + output.set("runtime.mpi", mpi::size()); + output.set("runtime.omp", atlas_omp_get_max_threads()); + output.set("atlas.build_type", ATLAS_BUILD_TYPE); + + output.set("timings.target.setup", timers.target_setup.elapsed()); + output.set("timings.source.setup", timers.source_setup.elapsed()); + output.set("timings.initial_condition", timers.initial_condition.elapsed()); + output.set("timings.interpolation.setup", timers.interpolation_setup.elapsed()); + output.set("timings.interpolation.execute", timers.interpolation_execute.elapsed()); + + output.set("interpolation", metadata); + + eckit::PathName json_filepath(args.getString("json.file", "out.json")); + std::ostringstream ss; + eckit::JSON json(ss, eckit::JSON::Formatting::indent(4)); + json << output; + + eckit::FileStream file(json_filepath, "w"); + std::string str = ss.str(); + file.write(str.data(), str.size()); + file.close(); + } + + + return success(); +} + +} // namespace atlas + + +int main(int argc, char* argv[]) { + atlas::AtlasParallelInterpolation tool(argc, argv); + return tool.start(); +} diff --git a/src/tests/interpolation/CMakeLists.txt b/src/tests/interpolation/CMakeLists.txt index 11b9e85be..f2cbda9a2 100644 --- a/src/tests/interpolation/CMakeLists.txt +++ b/src/tests/interpolation/CMakeLists.txt @@ -13,6 +13,12 @@ ecbuild_add_test( TARGET atlas_test_Quad3D ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ) +ecbuild_add_test( TARGET atlas_test_interpolation_conservative + SOURCES test_interpolation_conservative.cc + LIBS atlas + ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} +) + ecbuild_add_test( TARGET atlas_test_Quad2D SOURCES test_Quad2D.cc LIBS atlas diff --git a/src/tests/interpolation/test_interpolation_conservative.cc b/src/tests/interpolation/test_interpolation_conservative.cc new file mode 100644 index 000000000..92b426ff0 --- /dev/null +++ b/src/tests/interpolation/test_interpolation_conservative.cc @@ -0,0 +1,205 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + + +#include + +#include "eckit/geometry/Sphere.h" +#include "eckit/types/FloatCompare.h" + +#include "atlas/array.h" +#include "atlas/array/MakeView.h" +#include "atlas/field.h" +#include "atlas/grid.h" +#include "atlas/interpolation.h" +#include "atlas/interpolation/method/unstructured/ConservativeSphericalPolygonInterpolation.h" +#include "atlas/mesh.h" +#include "atlas/mesh/Mesh.h" +#include "atlas/meshgenerator.h" +#include "atlas/option.h" +#include "atlas/util/Config.h" + +#include "tests/AtlasTestEnvironment.h" + + +namespace atlas { +namespace test { + +using ConservativeMethod = interpolation::method::ConservativeSphericalPolygonInterpolation; +using Statistics = ConservativeMethod::Statistics; + +void do_remapping_test(Grid src_grid, Grid tgt_grid, std::function func, + Statistics& remap_stat_1, Statistics& remap_stat_2) { + Log::info().indent(); + // setup conservative remap: compute weights, polygon intersection, etc + util::Config config("type", "conservative-spherical-polygon"); + config.set("order", 1); + config.set("validate", true); + config.set("statistics.intersection", true); + config.set("statistics.conservation", true); + + auto conservative_interpolation = Interpolation(config, src_grid, tgt_grid); + Log::info() << conservative_interpolation << std::endl; + + // create source field from analytic function "func" + const auto& src_fs = conservative_interpolation.source(); + const auto& tgt_fs = conservative_interpolation.target(); + auto src_field = src_fs.createField(); + auto tgt_field = tgt_fs.createField(); + auto src_vals = array::make_view(src_field); + auto tgt_vals = array::make_view(tgt_field); + + { + ATLAS_TRACE("initial condition"); + // A bit of a hack here... + ConservativeMethod& consMethod = dynamic_cast(*conservative_interpolation.get()); + for (idx_t spt = 0; spt < src_vals.size(); ++spt) { + auto p = consMethod.src_points(spt); + PointLonLat pll; + eckit::geometry::Sphere::convertCartesianToSpherical(1., p, pll); + src_vals(spt) = func(pll); + } + } + + // project source field to target mesh in 1st order + remap_stat_1 = conservative_interpolation.execute(src_field, tgt_field); + remap_stat_1.accuracy(conservative_interpolation, tgt_field, func); + + ATLAS_TRACE_SCOPE("test caching") { + // We can create the interpolation without polygon intersections + auto cache = interpolation::Cache(conservative_interpolation); + // cache = ConservativeMethod::Cache + MatrixCache (1st order) + util::Config cfg(option::type("conservative-spherical-polygon")); + config.set("validate", true); + { + ATLAS_TRACE("cached -> 1st order using cached matrix"); + cfg.set("matrix_free", false); + cfg.set("order", 1); + auto interpolation = Interpolation(cfg, src_grid, tgt_grid, cache); + Log::info() << interpolation << std::endl; + interpolation.execute(src_field, tgt_field); + } + { + ATLAS_TRACE("cached -> 1st order constructing new matrix"); + cfg.set("matrix_free", false); + cfg.set("order", 1); + auto cache_without_matrix = + ConservativeMethod::Cache(cache); // to mimick when cache was created with matrix_free option + auto interpolation = Interpolation(cfg, src_grid, tgt_grid, cache_without_matrix); + Log::info() << interpolation << std::endl; + interpolation.execute(src_field, tgt_field); + } + { + ATLAS_TRACE("cached -> 1st order matrix-free"); + cfg.set("matrix_free", true); + cfg.set("order", 1); + auto interpolation = Interpolation(cfg, src_grid, tgt_grid, cache); + Log::info() << interpolation << std::endl; + interpolation.execute(src_field, tgt_field); + } + auto cache_2 = interpolation::Cache{}; + { + ATLAS_TRACE("cached -> 2nd order constructing new matrix"); + cfg.set("matrix_free", false); + cfg.set("order", 2); + auto interpolation = Interpolation(cfg, src_grid, tgt_grid, cache); + Log::info() << interpolation << std::endl; + interpolation.execute(src_field, tgt_field); + cache_2 = interpolation.createCache(); + } + { + ATLAS_TRACE("cached -> 2nd order matrix-free"); + cfg.set("matrix_free", true); + cfg.set("order", 2); + auto interpolation = Interpolation(cfg, src_grid, tgt_grid, cache); + Log::info() << interpolation << std::endl; + interpolation.execute(src_field, tgt_field); + } + { + ATLAS_TRACE("cached -> 2nd order using cached matrix"); + cfg.set("matrix_free", false); + cfg.set("order", 2); + auto interpolation = Interpolation(cfg, src_grid, tgt_grid, cache_2); + Log::info() << interpolation << std::endl; + interpolation.execute(src_field, tgt_field); + } + } + + + { + // project source field to target mesh in 2nd order + config.set("order", 2); + conservative_interpolation = Interpolation(config, src_grid, tgt_grid); + Log::info() << conservative_interpolation << std::endl; + remap_stat_2 = conservative_interpolation.execute(src_field, tgt_field); + remap_stat_2.accuracy(conservative_interpolation, tgt_field, func); + } +} + +void check(const Statistics remap_stat_1, Statistics remap_stat_2, std::array tol) { + auto improvement = [](double& e, double& r) { return 100. * (r - e) / r; }; + double err; + // check polygon intersections + err = remap_stat_1.errors[Statistics::Errors::GEO_DIFF]; + Log::info() << "Polygon area computation improvement: " << improvement(err, tol[0]) << " %" << std::endl; + EXPECT(err < tol[0]); + err = remap_stat_1.errors[Statistics::Errors::GEO_L1]; + Log::info() << "Polygon intersection improvement : " << improvement(err, tol[1]) << " %" << std::endl; + EXPECT(err < tol[1]); + + // check remap accuracy + err = remap_stat_1.errors[Statistics::Errors::REMAP_L2]; + Log::info() << "1st order accuracy improvement : " << improvement(err, tol[2]) << " %" << std::endl; + EXPECT(err < tol[2]); + err = remap_stat_2.errors[Statistics::Errors::REMAP_L2]; + Log::info() << "2nd order accuracy improvement : " << improvement(err, tol[3]) << " %" << std::endl; + EXPECT(err < tol[3]); + + // check mass conservation + err = remap_stat_1.errors[Statistics::Errors::REMAP_CONS]; + Log::info() << "1st order conservation improvement : " << improvement(err, tol[4]) << " %" << std::endl; + EXPECT(err < tol[4]); + err = remap_stat_2.errors[Statistics::Errors::REMAP_CONS]; + Log::info() << "2nd order conservation improvement : " << improvement(err, tol[5]) << " %" << std::endl + << std::endl; + EXPECT(err < tol[5]); + Log::info().unindent(); +} + +CASE("test_interpolation_conservative") { +#if 1 + SECTION("analytic constfunc") { + auto func = [](const PointLonLat& p) { return 1.; }; + Statistics remap_stat_1; + Statistics remap_stat_2; + do_remapping_test(Grid("H47"), Grid("H48"), func, remap_stat_1, remap_stat_2); + check(remap_stat_1, remap_stat_2, {1.e-13, 5.e-8, 2.9e-6, 2.9e-6, 5.5e-5, 5.5e-5}); + } + + SECTION("analytic Y_2^2 as in Jones(1998)") { + auto func = [](const PointLonLat& p) { + double cos = std::cos(0.025 * p[0]); + return 2. + cos * cos * std::cos(2 * 0.025 * p[1]); + }; + Statistics remap_stat_1; + Statistics remap_stat_2; + do_remapping_test(Grid("H47"), Grid("H48"), func, remap_stat_1, remap_stat_2); + check(remap_stat_1, remap_stat_2, {1.e-13, 5.e-8, 4.8e-4, 1.1e-4, 8.9e-5, 1.1e-4}); + } +#endif +} + +} // namespace test +} // namespace atlas + + +int main(int argc, char** argv) { + return atlas::test::run(argc, argv); +} diff --git a/src/tests/mesh/CMakeLists.txt b/src/tests/mesh/CMakeLists.txt index 7dd3a7007..7a495d050 100644 --- a/src/tests/mesh/CMakeLists.txt +++ b/src/tests/mesh/CMakeLists.txt @@ -114,8 +114,6 @@ if( TEST atlas_test_cubedsphere_meshgen ) set_tests_properties( atlas_test_cubedsphere_meshgen PROPERTIES TIMEOUT 30 ) endif() - - ecbuild_add_executable( TARGET atlas_test_mesh_reorder SOURCES test_mesh_reorder.cc LIBS atlas diff --git a/src/tests/mesh/test_mesh_node2cell.cc b/src/tests/mesh/test_mesh_node2cell.cc index 5e01a7af1..0b5cf1e0b 100644 --- a/src/tests/mesh/test_mesh_node2cell.cc +++ b/src/tests/mesh/test_mesh_node2cell.cc @@ -337,6 +337,21 @@ CASE("test_node2cell_with_halo") { } } +CASE("test_node2cell_AtlasGrids") { + std::vector gridnames{"N16", "O2", "F2", "L2", "H2"}; + for (auto& gridname : gridnames) { + auto grid = Grid(gridname); + Mesh mesh = MeshGenerator(grid.meshgenerator()).generate(grid); + mesh::actions::build_parallel_fields(mesh); + mesh::actions::build_periodic_boundaries(mesh); + mesh::actions::build_halo(mesh, 1); + mesh::actions::build_node_to_cell_connectivity(mesh); + for (idx_t jnode = 0; jnode < mesh.nodes().size(); ++jnode) { + EXPECT(mesh.nodes().cell_connectivity().cols(jnode) > 0); + } + } +} + //----------------------------------------------------------------------------- } // namespace test From 0207eb3eb3ebafbcf9b19216d609622de5e831d6 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Fri, 20 May 2022 08:41:33 +0000 Subject: [PATCH 24/53] Fix compilation of static library and tests using eckit::testing --- src/atlas/projection/detail/VariableResolutionProjection.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/atlas/projection/detail/VariableResolutionProjection.cc b/src/atlas/projection/detail/VariableResolutionProjection.cc index 712b7f266..b8cf952e8 100644 --- a/src/atlas/projection/detail/VariableResolutionProjection.cc +++ b/src/atlas/projection/detail/VariableResolutionProjection.cc @@ -20,7 +20,6 @@ #include "atlas/runtime/Exception.h" #include "atlas/util/Config.h" #include "atlas/util/Constants.h" -#include "eckit/testing/Test.h" /** * Projection for LAM stretching @@ -224,7 +223,7 @@ void VariableResolutionProjectionT::checkvalue(const double& epsilon, if (value_check > epsilon || value_check < (-1. * epsilon)) { std::string err_message; std::string str = std::to_string(value_check); - throw eckit::BadValue("USER defined limits not in the middle of the area " + str, Here()); + throw_Exception("USER defined limits not in the middle of the area " + str, Here()); } } From 56830b12f8f9ca7221744a064543fae6f377fedc Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 6 Jun 2022 09:42:30 +0100 Subject: [PATCH 25/53] Prevent multiple atlas::Library::initialize() --- src/atlas/library/Library.cc | 5 +++++ src/atlas/library/Library.h | 1 + 2 files changed, 6 insertions(+) diff --git a/src/atlas/library/Library.cc b/src/atlas/library/Library.cc index 7e6f5f5fd..93b4a872a 100644 --- a/src/atlas/library/Library.cc +++ b/src/atlas/library/Library.cc @@ -238,6 +238,10 @@ void Library::initialise(int argc, char** argv) { void Library::initialise(const eckit::Parametrisation& config) { + if (initialized_) { + return; + } + initialized_ = true; if (config.has("log")) { config.get("log.info", info_); config.get("log.trace", trace_); @@ -336,6 +340,7 @@ void Library::finalise() { warning_ = false; warning_channel_.reset(new eckit::Channel(new eckit::PrefixTarget("ATLAS_WARNING"))); } + initialized_ = false; } eckit::Channel& Library::infoChannel() const { diff --git a/src/atlas/library/Library.h b/src/atlas/library/Library.h index f1f15cf85..4d3516099 100644 --- a/src/atlas/library/Library.h +++ b/src/atlas/library/Library.h @@ -89,6 +89,7 @@ class Library : public eckit::system::Library { protected: virtual const void* addr() const override; + bool initialized_{false}; bool debug_{false}; bool info_{true}; bool warning_{true}; From da6315e663c4b01c27992a54852557b9de1842d9 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 6 Jun 2022 09:43:19 +0100 Subject: [PATCH 26/53] Allow to pass CellColumns functionspace to Gmsh::write(field,functionspace) --- src/atlas/output/detail/GmshIO.cc | 10 ++++++++++ src/atlas/output/detail/GmshIO.h | 6 ++++++ 2 files changed, 16 insertions(+) diff --git a/src/atlas/output/detail/GmshIO.cc b/src/atlas/output/detail/GmshIO.cc index 4c4d84d65..223d895e4 100644 --- a/src/atlas/output/detail/GmshIO.cc +++ b/src/atlas/output/detail/GmshIO.cc @@ -1239,6 +1239,13 @@ void GmshIO::write_delegate(const Field& field, const functionspace::NoFunctionS // ---------------------------------------------------------------------------- +void GmshIO::write_delegate(const Field& field, const functionspace::CellColumns& functionspace, + const eckit::PathName& file_path, GmshIO::openmode mode) const { + FieldSet fieldset; + fieldset.add(field); + write_delegate(fieldset, functionspace, file_path, mode); +} + // ---------------------------------------------------------------------------- void GmshIO::write_delegate(const Field& field, const functionspace::StructuredColumns& functionspace, const PathName& file_path, openmode mode) const { @@ -1438,6 +1445,9 @@ void GmshIO::write(const Field& field, const FunctionSpace& funcspace, const eck else if (functionspace::StructuredColumns(funcspace)) { write_delegate(field, functionspace::StructuredColumns(funcspace), file_path, mode); } + else if (functionspace::CellColumns(funcspace)) { + write_delegate(field, functionspace::CellColumns(funcspace), file_path, mode); + } else { ATLAS_NOTIMPLEMENTED; } diff --git a/src/atlas/output/detail/GmshIO.h b/src/atlas/output/detail/GmshIO.h index 4a21c24e7..81a972729 100644 --- a/src/atlas/output/detail/GmshIO.h +++ b/src/atlas/output/detail/GmshIO.h @@ -128,6 +128,12 @@ class GmshIO { void write_delegate(const Field& field, const functionspace::NodeColumns&, const eckit::PathName& file_path, openmode mode = std::ios::out) const; + /// Write field to file using Cells functionspace + /// Depending on argument "mode", the fields will be appended, + /// or existing file will be overwritten + void write_delegate(const Field& field, const functionspace::CellColumns&, const eckit::PathName& file_path, + openmode mode = std::ios::out) const; + /// Write field to file using Nodes functionspace /// Depending on argument "mode", the fields will be appended, /// or existing file will be overwritten From b3ba40b1b674f4aaa0daa4c5eb51fcb53240ef7b Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 6 Jun 2022 11:43:56 +0100 Subject: [PATCH 27/53] Fortran API: atlas_MeshGenerator%generate(grid, partitioner) --- .../detail/MeshGeneratorInterface.cc | 18 ++++++++++++++++ .../detail/MeshGeneratorInterface.h | 13 ++++++++++++ .../mesh/atlas_MeshGenerator_module.F90 | 21 ++++++++++++++++++- 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/src/atlas/meshgenerator/detail/MeshGeneratorInterface.cc b/src/atlas/meshgenerator/detail/MeshGeneratorInterface.cc index 8f8ac7c2b..05fe092f1 100644 --- a/src/atlas/meshgenerator/detail/MeshGeneratorInterface.cc +++ b/src/atlas/meshgenerator/detail/MeshGeneratorInterface.cc @@ -11,6 +11,7 @@ #include "atlas/meshgenerator/detail/MeshGeneratorInterface.h" #include "atlas/grid/Distribution.h" #include "atlas/grid/Grid.h" +#include "atlas/grid/Partitioner.h" #include "atlas/mesh/Mesh.h" #include "atlas/meshgenerator.h" #include "atlas/meshgenerator/detail/MeshGeneratorImpl.h" @@ -85,6 +86,23 @@ Mesh::Implementation* atlas__MeshGenerator__generate__grid(const MeshGenerator:: m->detach(); return m; } + +Mesh::Implementation* atlas__MeshGenerator__generate__grid_partitioner( + const MeshGenerator::Implementation* This, const Grid::Implementation* grid, + const grid::Partitioner::Implementation* partitioner) { + ATLAS_ASSERT(This != nullptr, "Cannot access uninitialised atlas_MeshGenerator"); + ATLAS_ASSERT(grid != nullptr, "Cannot access uninitialised atlas_Grid"); + ATLAS_ASSERT(partitioner != nullptr, "Cannot access uninitialised atlas_Partitioner"); + + Mesh::Implementation* m; + { + Mesh mesh = This->generate(Grid(grid), grid::Partitioner(partitioner)); + mesh.get()->attach(); + m = mesh.get(); + } + m->detach(); + return m; +} } // extern "C" //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/atlas/meshgenerator/detail/MeshGeneratorInterface.h b/src/atlas/meshgenerator/detail/MeshGeneratorInterface.h index 40d85970a..e7a2a6dbf 100644 --- a/src/atlas/meshgenerator/detail/MeshGeneratorInterface.h +++ b/src/atlas/meshgenerator/detail/MeshGeneratorInterface.h @@ -37,6 +37,16 @@ class MeshImpl; } // namespace detail } // namespace mesh } // namespace atlas +namespace atlas { +namespace grid { +namespace detail { +namespace partitioner { +class Partitioner; +} // namespace partitioner +} // namespace detail +} // namespace grid +using PartitionerImpl = grid::detail::partitioner::Partitioner; +} // namespace atlas namespace atlas { @@ -54,6 +64,9 @@ mesh::detail::MeshImpl* atlas__MeshGenerator__generate__grid_griddist(const Mesh const GridImpl* grid, const grid::DistributionImpl* distribution); mesh::detail::MeshImpl* atlas__MeshGenerator__generate__grid(const MeshGeneratorImpl* This, const GridImpl* grid); +mesh::detail::MeshImpl* atlas__MeshGenerator__generate__grid_partitioner(const MeshGeneratorImpl* This, + const GridImpl* grid, + const PartitionerImpl* partitioner); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/atlas_f/mesh/atlas_MeshGenerator_module.F90 b/src/atlas_f/mesh/atlas_MeshGenerator_module.F90 index 54c8dae6f..454cec07e 100644 --- a/src/atlas_f/mesh/atlas_MeshGenerator_module.F90 +++ b/src/atlas_f/mesh/atlas_MeshGenerator_module.F90 @@ -28,7 +28,10 @@ module atlas_MeshGenerator_module !------------------------------------------------------------------------------ TYPE, extends(fckit_owned_object) :: atlas_MeshGenerator contains - procedure, public :: generate => atlas_MeshGenerator__generate + procedure, private :: atlas_MeshGenerator__generate + procedure, private :: atlas_MeshGenerator__generate_partitioner + generic :: generate => atlas_MeshGenerator__generate, atlas_MeshGenerator__generate_partitioner + #if FCKIT_FINAL_NOT_INHERITING final :: atlas_MeshGenerator__final_auto #endif @@ -105,6 +108,22 @@ function atlas_MeshGenerator__generate(this,grid,distribution) result(mesh) call mesh%return() end function +function atlas_MeshGenerator__generate_partitioner(this,grid,partitioner) result(mesh) + use atlas_MeshGenerator_c_binding + use atlas_Grid_module, only: atlas_Grid + use atlas_Partitioner_module, only: atlas_Partitioner + use atlas_Mesh_module, only: atlas_Mesh + type(atlas_Mesh) :: mesh + class(atlas_MeshGenerator), intent(in) :: this + class(atlas_Grid), intent(in) :: grid + class(atlas_Partitioner), intent(in) :: partitioner + call mesh%reset_c_ptr() ! Somehow needed with PGI/16.7 and build-type "bit" + mesh = atlas_Mesh( atlas__MeshGenerator__generate__grid_partitioner( & + this%CPTR_PGIBUG_A,grid%CPTR_PGIBUG_A,partitioner%CPTR_PGIBUG_A) ) + call mesh%return() +end function + + !------------------------------------------------------------------------------- #if FCKIT_FINAL_NOT_INHERITING From fdf34b50f05ec6c5ceb8b0d6db84feb4de9e6f3a Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 6 Jun 2022 10:17:47 +0100 Subject: [PATCH 28/53] Default StructuredMeshGenerator partitioner is equal_regions instead of trans/ectrans --- src/atlas/meshgenerator/detail/StructuredMeshGenerator.cc | 3 --- src/tests/trans/CMakeLists.txt | 4 ++-- src/tests/trans/fctest_trans.F90 | 5 ++++- src/tests/trans/test_trans.cc | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/atlas/meshgenerator/detail/StructuredMeshGenerator.cc b/src/atlas/meshgenerator/detail/StructuredMeshGenerator.cc index a32d43a9e..dee84557e 100644 --- a/src/atlas/meshgenerator/detail/StructuredMeshGenerator.cc +++ b/src/atlas/meshgenerator/detail/StructuredMeshGenerator.cc @@ -133,9 +133,6 @@ StructuredMeshGenerator::StructuredMeshGenerator(const eckit::Parametrisation& p partitioner = "equal_regions"; } } - else if (grid::Partitioner::exists("trans")) { - partitioner = "trans"; - } else { partitioner = "equal_regions"; } diff --git a/src/tests/trans/CMakeLists.txt b/src/tests/trans/CMakeLists.txt index eabe73a0a..7d0fd0d0f 100644 --- a/src/tests/trans/CMakeLists.txt +++ b/src/tests/trans/CMakeLists.txt @@ -14,7 +14,7 @@ if( HAVE_FCTEST ) LINKER_LANGUAGE Fortran SOURCES fctest_trans.F90 LIBS atlas_f - CONDITION eckit_HAVE_MPI AND transi_HAVE_MPI + CONDITION eckit_HAVE_MPI AND ( transi_HAVE_MPI OR ectrans_HAVE_MPI ) MPI 4 ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ) @@ -40,7 +40,7 @@ endif() ecbuild_add_test( TARGET atlas_test_trans MPI 4 SOURCES test_trans.cc - CONDITION atlas_HAVE_TRANS AND eckit_HAVE_MPI AND transi_HAVE_MPI + CONDITION atlas_HAVE_TRANS AND eckit_HAVE_MPI AND ( transi_HAVE_MPI OR ectrans_HAVE_MPI ) LIBS atlas transi ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} # There seems to be a issue with vd2uv raising FE_INVALID, only on specific arch (bamboo-leap42-gnu63) diff --git a/src/tests/trans/fctest_trans.F90 b/src/tests/trans/fctest_trans.F90 index bd6b488c5..c6fc56528 100644 --- a/src/tests/trans/fctest_trans.F90 +++ b/src/tests/trans/fctest_trans.F90 @@ -61,6 +61,7 @@ end module fctest_atlas_trans_fixture type(atlas_StructuredGrid) :: grid type(atlas_StructuredGrid) :: trans_grid type(atlas_MeshGenerator) :: meshgenerator + type(atlas_Partitioner) :: partitioner type(atlas_Mesh) :: mesh type(atlas_Trans) :: trans type(atlas_mesh_Nodes) :: nodes @@ -87,8 +88,10 @@ end module fctest_atlas_trans_fixture FCTEST_CHECK_EQUAL( grid%owners(), 1 ) + partitioner = atlas_Partitioner(type="trans") + meshgenerator = atlas_MeshGenerator() - mesh = meshgenerator%generate(grid) + mesh = meshgenerator%generate(grid,partitioner) call meshgenerator%final() diff --git a/src/tests/trans/test_trans.cc b/src/tests/trans/test_trans.cc index 961b2d13e..2271092e0 100644 --- a/src/tests/trans/test_trans.cc +++ b/src/tests/trans/test_trans.cc @@ -310,7 +310,7 @@ CASE("test_spectral_fields") { Grid g("O48"); StructuredMeshGenerator generate(atlas::util::Config("angle", 0)("triangulate", false)); - Mesh m = generate(g); + Mesh m = generate(g, grid::Partitioner("trans")); trans::Trans trans(g, 47); @@ -463,7 +463,7 @@ CASE("test_trans_using_grid") { CASE("test_trans_using_functionspace_NodeColumns") { Log::info() << "test_trans_using_functionspace_NodeColumns" << std::endl; - functionspace::NodeColumns gp(MeshGenerator("structured").generate(Grid("O48"))); + functionspace::NodeColumns gp(MeshGenerator("structured").generate(Grid("O48"), grid::Partitioner("trans"))); functionspace::Spectral sp(47); trans::Trans trans(gp, sp); From 52dfe0febeebe6bbc3d96b7a64ef215dc1d5999f Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 6 Jun 2022 09:48:22 +0100 Subject: [PATCH 29/53] Deprecate Trans naming of 'ifs' or 'trans' in favour of 'ectrans' --- src/atlas/functionspace/Spectral.cc | 2 +- .../detail/partitioner/TransPartitioner.cc | 2 ++ .../detail/partitioner/TransPartitioner.h | 2 +- .../detail/RegularMeshGenerator.cc | 4 ++-- .../detail/StructuredMeshGenerator.cc | 3 ++- src/atlas/trans/VorDivToUV.cc | 2 +- src/atlas/trans/detail/TransFactory.cc | 2 +- .../trans/ifs/LegendreCacheCreatorIFS.cc | 9 +++++---- src/atlas/trans/ifs/TransIFS.cc | 14 ++++++++----- src/atlas/trans/ifs/TransIFSNodeColumns.cc | 7 +++++-- .../trans/ifs/TransIFSStructuredColumns.cc | 8 ++++++-- src/atlas/trans/ifs/VorDivToUVIFS.cc | 5 +++-- src/atlas/trans/local/TransLocal.cc | 6 +++++- .../benchmark_trans/atlas-benchmark-trans.cc | 10 +++++----- src/tests/trans/fctest_trans.F90 | 12 +++++------ src/tests/trans/test_trans.cc | 20 +++++++++---------- src/tests/trans/test_transgeneral.cc | 10 +++++----- 17 files changed, 69 insertions(+), 49 deletions(-) diff --git a/src/atlas/functionspace/Spectral.cc b/src/atlas/functionspace/Spectral.cc index f517723d6..3564f542c 100644 --- a/src/atlas/functionspace/Spectral.cc +++ b/src/atlas/functionspace/Spectral.cc @@ -90,7 +90,7 @@ class Spectral::Parallelisation { return array::make_view(trans_->nasm0, array::make_shape(trans_->nsmax + 1)); } - std::string distribution() const { return "trans"; } + std::string distribution() const { return "ectrans"; } operator ::Trans_t*() const { return trans_.get(); } std::shared_ptr<::Trans_t> trans_; }; diff --git a/src/atlas/grid/detail/partitioner/TransPartitioner.cc b/src/atlas/grid/detail/partitioner/TransPartitioner.cc index f7e5639cb..1395e6f3a 100644 --- a/src/atlas/grid/detail/partitioner/TransPartitioner.cc +++ b/src/atlas/grid/detail/partitioner/TransPartitioner.cc @@ -117,6 +117,8 @@ int TransPartitioner::nb_regions(int b) const { } // namespace atlas namespace { +atlas::grid::detail::partitioner::PartitionerBuilder __ecTrans( + "ectrans"); atlas::grid::detail::partitioner::PartitionerBuilder __Trans( "trans"); atlas::grid::detail::partitioner::PartitionerBuilder __TransIFS( diff --git a/src/atlas/grid/detail/partitioner/TransPartitioner.h b/src/atlas/grid/detail/partitioner/TransPartitioner.h index fed759c4c..b15c75be1 100644 --- a/src/atlas/grid/detail/partitioner/TransPartitioner.h +++ b/src/atlas/grid/detail/partitioner/TransPartitioner.h @@ -44,7 +44,7 @@ class TransPartitioner : public Partitioner { int nb_regions(int b) const; - virtual std::string type() const { return "trans"; } + virtual std::string type() const { return "ectrans"; } private: size_t nbands_; diff --git a/src/atlas/meshgenerator/detail/RegularMeshGenerator.cc b/src/atlas/meshgenerator/detail/RegularMeshGenerator.cc index c3ae70d5b..07f36849b 100644 --- a/src/atlas/meshgenerator/detail/RegularMeshGenerator.cc +++ b/src/atlas/meshgenerator/detail/RegularMeshGenerator.cc @@ -95,8 +95,8 @@ void RegularMeshGenerator::configure_defaults() { // This options sets the default partitioner std::string partitioner; - if (grid::Partitioner::exists("trans") && mpi::size() > 1) { - partitioner = "trans"; + if (grid::Partitioner::exists("ectrans") && mpi::size() > 1) { + partitioner = "ectrans"; } else { partitioner = "checkerboard"; diff --git a/src/atlas/meshgenerator/detail/StructuredMeshGenerator.cc b/src/atlas/meshgenerator/detail/StructuredMeshGenerator.cc index dee84557e..41ba410ce 100644 --- a/src/atlas/meshgenerator/detail/StructuredMeshGenerator.cc +++ b/src/atlas/meshgenerator/detail/StructuredMeshGenerator.cc @@ -179,6 +179,7 @@ void StructuredMeshGenerator::configure_defaults() { } void StructuredMeshGenerator::generate(const Grid& grid, Mesh& mesh) const { + ATLAS_TRACE(); ATLAS_ASSERT(!mesh.generated()); const StructuredGrid rg = StructuredGrid(grid); @@ -191,7 +192,7 @@ void StructuredMeshGenerator::generate(const Grid& grid, Mesh& mesh) const { std::string partitioner_type = "equal_regions"; options.get("partitioner", partitioner_type); - if (partitioner_type == "trans") { + if (partitioner_type == "ectrans") { if (rg.ny() % 2 == 1) { partitioner_type = "equal_regions"; // Odd number of latitudes } diff --git a/src/atlas/trans/VorDivToUV.cc b/src/atlas/trans/VorDivToUV.cc index 56e54cf8f..7610aea29 100644 --- a/src/atlas/trans/VorDivToUV.cc +++ b/src/atlas/trans/VorDivToUV.cc @@ -24,7 +24,7 @@ // For factory registration only #if ATLAS_HAVE_TRANS #include "atlas/trans/ifs/VorDivToUVIFS.h" -#define TRANS_DEFAULT "ifs" +#define TRANS_DEFAULT "ectrans" #else #define TRANS_DEFAULT "local" #endif diff --git a/src/atlas/trans/detail/TransFactory.cc b/src/atlas/trans/detail/TransFactory.cc index 924f6d59b..1380b092e 100644 --- a/src/atlas/trans/detail/TransFactory.cc +++ b/src/atlas/trans/detail/TransFactory.cc @@ -54,7 +54,7 @@ void force_link() { namespace { struct default_backend { #if ATLAS_HAVE_TRANS - std::string value = "ifs"; + std::string value = "ectrans"; #else std::string value = "local"; #endif diff --git a/src/atlas/trans/ifs/LegendreCacheCreatorIFS.cc b/src/atlas/trans/ifs/LegendreCacheCreatorIFS.cc index 290d0ecb4..73622fe06 100644 --- a/src/atlas/trans/ifs/LegendreCacheCreatorIFS.cc +++ b/src/atlas/trans/ifs/LegendreCacheCreatorIFS.cc @@ -25,8 +25,9 @@ namespace atlas { namespace trans { namespace { -static LegendreCacheCreatorBuilder builder("ifs"); -} +static LegendreCacheCreatorBuilder builder_ifs("ifs"); // Deprecated +static LegendreCacheCreatorBuilder builder_ectrans("ectrans"); +} // namespace namespace { @@ -126,11 +127,11 @@ LegendreCacheCreatorIFS::LegendreCacheCreatorIFS(const Grid& grid, int truncatio grid_(grid), truncation_(truncation), config_(config) {} void LegendreCacheCreatorIFS::create(const std::string& path) const { - Trans(grid_, truncation_, config_ | option::type("ifs") | option::write_legendre(path)); + Trans(grid_, truncation_, config_ | option::type("ectrans") | option::write_legendre(path)); } Cache LegendreCacheCreatorIFS::create() const { - return TransCache(Trans(grid_, truncation_, config_ | option::type("ifs"))); + return TransCache(Trans(grid_, truncation_, config_ | option::type("ectrans"))); } size_t LegendreCacheCreatorIFS::estimate() const { diff --git a/src/atlas/trans/ifs/TransIFS.cc b/src/atlas/trans/ifs/TransIFS.cc index ba0b874bf..ec9b0f6de 100644 --- a/src/atlas/trans/ifs/TransIFS.cc +++ b/src/atlas/trans/ifs/TransIFS.cc @@ -47,8 +47,9 @@ namespace atlas { namespace trans { namespace { -static TransBuilderGrid builder("ifs", "ifs"); -} +static TransBuilderGrid builder_ifs("ifs", "ifs"); // deprecated +static TransBuilderGrid builder_ectrans("ectrans", "ectrans"); +} // namespace class TransParameters { public: @@ -156,6 +157,7 @@ void TransIFS::dirtrans(const Field& gpfield, Field& spfield, const eckit::Confi } void TransIFS::dirtrans(const FieldSet& gpfields, FieldSet& spfields, const eckit::Configuration& config) const { + ATLAS_TRACE(); assert_spectral_functionspace(spfields); std::string functionspace(fieldset_functionspace(gpfields)); @@ -173,6 +175,7 @@ void TransIFS::dirtrans(const FieldSet& gpfields, FieldSet& spfields, const ecki } void TransIFS::invtrans(const Field& spfield, Field& gpfield, const eckit::Configuration& config) const { + ATLAS_TRACE(); ATLAS_ASSERT(Spectral(spfield.functionspace())); if (StructuredColumns(gpfield.functionspace())) { __invtrans(Spectral(spfield.functionspace()), spfield, StructuredColumns(gpfield.functionspace()), gpfield, @@ -187,6 +190,7 @@ void TransIFS::invtrans(const Field& spfield, Field& gpfield, const eckit::Confi } void TransIFS::invtrans(const FieldSet& spfields, FieldSet& gpfields, const eckit::Configuration& config) const { + ATLAS_TRACE(); assert_spectral_functionspace(spfields); std::string functionspace(fieldset_functionspace(gpfields)); @@ -742,9 +746,9 @@ namespace trans { void TransIFS::assertCompatibleDistributions(const FunctionSpace& gp, const FunctionSpace& /*sp*/) const { std::string gp_dist = gp.distribution(); - if (gp_dist != "trans" && // distribution computed by TransPartitioner - gp_dist != "serial" && // serial distribution always works - gp_dist != "custom") { // trust user that he knows what he is doing + if (gp_dist != "ectrans" && // distribution computed by TransPartitioner + gp_dist != "serial" && // serial distribution always works + gp_dist != "custom") { // trust user that he knows what he is doing throw_Exception(gp.type() + " functionspace has unsupported distribution (" + gp_dist + ") " "to do spectral transforms. Please " diff --git a/src/atlas/trans/ifs/TransIFSNodeColumns.cc b/src/atlas/trans/ifs/TransIFSNodeColumns.cc index 07dc2d9ec..547795e8e 100644 --- a/src/atlas/trans/ifs/TransIFSNodeColumns.cc +++ b/src/atlas/trans/ifs/TransIFSNodeColumns.cc @@ -30,8 +30,11 @@ TransIFSNodeColumns::TransIFSNodeColumns(const Cache& cache, const functionspace TransIFSNodeColumns::~TransIFSNodeColumns() = default; namespace { -static TransBuilderFunctionSpace builder("ifs(NodeColumns,Spectral)", "ifs"); -} +static TransBuilderFunctionSpace builder_ifs("ifs(NodeColumns,Spectral)", "ifs"); +// Deprecated, use below + +static TransBuilderFunctionSpace builder_ectrans("ectrans(NodeColumns,Spectral)", "ectrans"); +} // namespace } // namespace trans } // namespace atlas diff --git a/src/atlas/trans/ifs/TransIFSStructuredColumns.cc b/src/atlas/trans/ifs/TransIFSStructuredColumns.cc index f49ea7932..c670a73d1 100644 --- a/src/atlas/trans/ifs/TransIFSStructuredColumns.cc +++ b/src/atlas/trans/ifs/TransIFSStructuredColumns.cc @@ -32,8 +32,12 @@ TransIFSStructuredColumns::TransIFSStructuredColumns(const Cache& cache, const f TransIFSStructuredColumns::~TransIFSStructuredColumns() = default; namespace { -static TransBuilderFunctionSpace builder("ifs(StructuredColumns,Spectral)", "ifs"); -} +static TransBuilderFunctionSpace builder_ifs("ifs(StructuredColumns,Spectral)", "ifs"); +// Deprecated, use below + +static TransBuilderFunctionSpace builder_ectrans("ectrans(StructuredColumns,Spectral)", + "ectrans"); +} // namespace } // namespace trans } // namespace atlas diff --git a/src/atlas/trans/ifs/VorDivToUVIFS.cc b/src/atlas/trans/ifs/VorDivToUVIFS.cc index 9efc0fbf9..b32e4fdf9 100644 --- a/src/atlas/trans/ifs/VorDivToUVIFS.cc +++ b/src/atlas/trans/ifs/VorDivToUVIFS.cc @@ -28,8 +28,9 @@ namespace atlas { namespace trans { namespace { -static VorDivToUVBuilder builder("ifs"); -} +static VorDivToUVBuilder builder_ifs("ifs"); // Deprecated +static VorDivToUVBuilder builder_ectrans("ectrans"); +} // namespace namespace { void trans_check(const int code, const char* msg, const eckit::CodeLocation& location) { diff --git a/src/atlas/trans/local/TransLocal.cc b/src/atlas/trans/local/TransLocal.cc index be932c04b..2ef699c1f 100644 --- a/src/atlas/trans/local/TransLocal.cc +++ b/src/atlas/trans/local/TransLocal.cc @@ -750,7 +750,9 @@ const functionspace::Spectral& TransLocal::spectral() const { void TransLocal::invtrans(const Field& spfield, Field& gpfield, const eckit::Configuration& config) const { // VERY PRELIMINARY IMPLEMENTATION WITHOUT ANY GUARANTEES - int nb_scalar_fields = 1; + int nb_scalar_fields = 1; + ATLAS_ASSERT(spfield.rank() == 1, "Only rank-1 fields supported at the moment"); + ATLAS_ASSERT(gpfield.rank() == 1, "Only rank-1 fields supported at the moment"); const auto scalar_spectra = array::make_view(spfield); auto gp_fields = array::make_view(gpfield); @@ -802,6 +804,8 @@ void gp_transpose(const int nb_size, const int nb_fields, const double gp_tmp[], void TransLocal::invtrans_vordiv2wind(const Field& spvor, const Field& spdiv, Field& gpwind, const eckit::Configuration& config) const { // VERY PRELIMINARY IMPLEMENTATION WITHOUT ANY GUARANTEES + ATLAS_ASSERT(spvor.rank() == 1, "Only rank-1 fields supported at the moment"); + ATLAS_ASSERT(spdiv.rank() == 1, "Only rank-1 fields supported at the moment"); int nb_vordiv_fields = 1; const auto vorticity_spectra = array::make_view(spvor); const auto divergence_spectra = array::make_view(spdiv); diff --git a/src/sandbox/benchmark_trans/atlas-benchmark-trans.cc b/src/sandbox/benchmark_trans/atlas-benchmark-trans.cc index f505eb75c..a258399da 100644 --- a/src/sandbox/benchmark_trans/atlas-benchmark-trans.cc +++ b/src/sandbox/benchmark_trans/atlas-benchmark-trans.cc @@ -128,15 +128,15 @@ int Tool::execute(const Args& args) { Log::error() << "Atlas was not compiled with support for '" << types[0] << "' backend." << std::endl; return failed(); } - if (not domain.global() && types[0] == "ifs") { - Log::error() << "The 'ifs' backend for Trans can only use global domains." << std::endl; + if (not domain.global() && types[0] == "ectrans") { + Log::error() << "The 'ectrans' backend for Trans can only use global domains." << std::endl; return failed(); } } else { types.emplace_back("local"); - if (trans::Trans::hasBackend("ifs") && domain.global()) { - types.emplace_back("ifs"); + if (trans::Trans::hasBackend("ectrans") && domain.global()) { + types.emplace_back("ectrans"); } } @@ -175,7 +175,7 @@ int Tool::execute(const Args& args) { } std::map> linalg_backends{ - {"ifs", {"lapack"}}, {"local", {"generic", "openmp", "lapack", "eigen"}}}; + {"ectrans", {"lapack"}}, {"local", {"generic", "openmp", "lapack", "eigen"}}}; if (args.has("matrix_multiply")) { std::string backend = args.getString("matrix_multiply"); linalg_backends["local"] = {backend}; diff --git a/src/tests/trans/fctest_trans.F90 b/src/tests/trans/fctest_trans.F90 index c6fc56528..739acf409 100644 --- a/src/tests/trans/fctest_trans.F90 +++ b/src/tests/trans/fctest_trans.F90 @@ -47,9 +47,9 @@ end module fctest_atlas_trans_fixture TEST( test_trans_backend ) type(atlas_Trans) :: trans FCTEST_CHECK( trans%has_backend("local") ) - FCTEST_CHECK( trans%has_backend("ifs") ) - if( trans%has_backend("ifs") ) then - FCTEST_CHECK_EQUAL( trans%backend(), "ifs" ) + FCTEST_CHECK( trans%has_backend("ectrans") ) + if( trans%has_backend("ectrans") ) then + FCTEST_CHECK_EQUAL( trans%backend(), "ectrans" ) else FCTEST_CHECK_EQUAL( trans%backend(), "local" ) endif @@ -88,7 +88,7 @@ end module fctest_atlas_trans_fixture FCTEST_CHECK_EQUAL( grid%owners(), 1 ) - partitioner = atlas_Partitioner(type="trans") + partitioner = atlas_Partitioner(type="ectrans") meshgenerator = atlas_MeshGenerator() mesh = meshgenerator%generate(grid,partitioner) @@ -264,7 +264,7 @@ end module fctest_atlas_trans_fixture grid = atlas_StructuredGrid("O24") trans = atlas_Trans(grid,truncation) - partitioner = atlas_Partitioner(type="ifs") + partitioner = atlas_Partitioner(type="ectrans") gridpoints_fs = atlas_functionspace_StructuredColumns(grid,partitioner) scalarfield1 = gridpoints_fs%create_field(name="scalar1",kind=atlas_real(c_double),levels=nlev) scalarfield2 = gridpoints_fs%create_field(name="scalar2",kind=atlas_real(c_double)) @@ -350,7 +350,7 @@ end module fctest_atlas_trans_fixture grid = atlas_StructuredGrid("O24") trans = atlas_Trans(grid,23) -partitioner = atlas_Partitioner("ifs") +partitioner = atlas_Partitioner("ectrans") gridpoints = atlas_functionspace_StructuredColumns(grid,partitioner) spectral = atlas_functionspace_Spectral(trans) diff --git a/src/tests/trans/test_trans.cc b/src/tests/trans/test_trans.cc index 2271092e0..2ae749319 100644 --- a/src/tests/trans/test_trans.cc +++ b/src/tests/trans/test_trans.cc @@ -110,7 +110,7 @@ void read_rspecg(Field spec) { //----------------------------------------------------------------------------- CASE("test_trans_distribution_matches_atlas") { - EXPECT(grid::Partitioner::exists("trans")); + EXPECT(grid::Partitioner::exists("ectrans")); // Create grid and trans object Grid g("N80"); @@ -310,7 +310,7 @@ CASE("test_spectral_fields") { Grid g("O48"); StructuredMeshGenerator generate(atlas::util::Config("angle", 0)("triangulate", false)); - Mesh m = generate(g, grid::Partitioner("trans")); + Mesh m = generate(g, grid::Partitioner("ectrans")); trans::Trans trans(g, 47); @@ -344,7 +344,7 @@ CASE("test_nomesh") { trans::Trans trans(g, 47); functionspace::Spectral spectral(trans); - functionspace::StructuredColumns gridpoints(g, grid::Partitioner("trans")); + functionspace::StructuredColumns gridpoints(g, grid::Partitioner("ectrans")); Field spfg = spectral.createField(option::name("spf") | option::global()); Field spf = spectral.createField(option::name("spf")); @@ -422,7 +422,7 @@ CASE("test_trans_factory") { trans::TransFactory::list(Log::info()); Log::info() << std::endl; - functionspace::StructuredColumns gp(Grid("O48"), grid::Partitioner("trans")); + functionspace::StructuredColumns gp(Grid("O48"), grid::Partitioner("ectrans")); functionspace::Spectral sp(47); trans::Trans trans1 = trans::Trans(gp, sp); @@ -437,7 +437,7 @@ CASE("test_trans_using_grid") { trans::Trans trans(Grid("O48"), 47); - functionspace::StructuredColumns gp(trans.grid(), grid::Partitioner("trans")); + functionspace::StructuredColumns gp(trans.grid(), grid::Partitioner("ectrans")); functionspace::Spectral sp(trans.truncation()); Field spf = sp.createField(option::name("spf")); @@ -463,7 +463,7 @@ CASE("test_trans_using_grid") { CASE("test_trans_using_functionspace_NodeColumns") { Log::info() << "test_trans_using_functionspace_NodeColumns" << std::endl; - functionspace::NodeColumns gp(MeshGenerator("structured").generate(Grid("O48"), grid::Partitioner("trans"))); + functionspace::NodeColumns gp(MeshGenerator("structured").generate(Grid("O48"), grid::Partitioner("ectrans"))); functionspace::Spectral sp(47); trans::Trans trans(gp, sp); @@ -491,7 +491,7 @@ CASE("test_trans_using_functionspace_NodeColumns") { CASE("test_trans_using_functionspace_StructuredColumns") { Log::info() << "test_trans_using_functionspace_StructuredColumns" << std::endl; - functionspace::StructuredColumns gp(Grid("O48"), grid::Partitioner("trans")); + functionspace::StructuredColumns gp(Grid("O48"), grid::Partitioner("ectrans")); functionspace::Spectral sp(47); trans::Trans trans(gp, sp); @@ -554,8 +554,8 @@ CASE("test_trans_VorDivToUV") { Log::info() << std::endl; // With IFS - if (trans::VorDivToUVFactory::has("ifs")) { - trans::VorDivToUV vordiv_to_UV(truncation, option::type("ifs")); + if (trans::VorDivToUVFactory::has("ectrans")) { + trans::VorDivToUV vordiv_to_UV(truncation, option::type("ectrans")); EXPECT(vordiv_to_UV.truncation() == truncation); std::vector field_U(nfld * nspec2); @@ -597,7 +597,7 @@ CASE("test_trans_VorDivToUV") { CASE("ATLAS-256: Legendre coefficient expected unique identifiers") { if (mpi::comm().size() == 1) { util::Config options; - options.set(option::type("ifs")); + options.set(option::type("ectrans")); options.set("flt", false); auto uids = { diff --git a/src/tests/trans/test_transgeneral.cc b/src/tests/trans/test_transgeneral.cc index 33072cffa..4213c52e8 100644 --- a/src/tests/trans/test_transgeneral.cc +++ b/src/tests/trans/test_transgeneral.cc @@ -505,7 +505,7 @@ CASE("test_trans_vordiv_with_translib") { //int trc = ndgl - 1; // linear int trc = ndgl / 2. - 1; // cubic #if ATLAS_HAVE_TRANS - trans::Trans transIFS(g, trc, util::Config("type", "ifs")); + trans::Trans transIFS(g, trc, util::Config("type", "ectrans")); double rav = 0.; // compute average rms error of trans library in rav #endif trans::Trans transLocal1(g, trc, util::Config("type", "local")); @@ -653,7 +653,7 @@ CASE("test_trans_hires") { double tolerance = 1.e-13; //#if ATLAS_HAVE_TRANS - // //std::string transTypes[4] = {"localopt", "localopt2", "Local", "ifs"}; + // //std::string transTypes[4] = {"localopt", "localopt2", "Local", "ectrans"}; // //std::string transTypes[2] = {"localopt2", "Local"}; // //std::string transTypes[3] = {"Local", "localopt2", "localopt"}; // //std::string transTypes[1] = {"Local"}; @@ -662,10 +662,10 @@ CASE("test_trans_hires") { //#endif std::vector trans_configs; std::vector transTypes = {"local"}; - if (trans::Trans::hasBackend("ifs")) { - transTypes.emplace_back("ifs"); + if (trans::Trans::hasBackend("ectrans")) { + transTypes.emplace_back("ectrans"); } - std::map> backends{{"ifs", {"lapack"}}, + std::map> backends{{"ectrans", {"lapack"}}, {"local", {"generic", "openmp", "eigen"}}}; //Domain testdomain =F ZonalBandDomain( {-90., 90.} ); From adf3d6ca59bcedab6e24bc8bdbb7142d2a5a286a Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 7 Jun 2022 15:33:11 +0100 Subject: [PATCH 30/53] TransIFS: allow gp_field with halo --- src/atlas/trans/ifs/TransIFS.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/atlas/trans/ifs/TransIFS.cc b/src/atlas/trans/ifs/TransIFS.cc index ec9b0f6de..6fc9db81d 100644 --- a/src/atlas/trans/ifs/TransIFS.cc +++ b/src/atlas/trans/ifs/TransIFS.cc @@ -1162,8 +1162,8 @@ void TransIFS::__dirtrans(const StructuredColumns& gp, const Field& gpfield, con if (compute_nfld(gpfield) != compute_nfld(spfield)) { throw_Exception("dirtrans: different number of gridpoint fields than spectral fields", Here()); } - if ((int)gpfield.shape(0) != ngptot()) { - throw_Exception("dirtrans: slowest moving index must be ngptot", Here()); + if ((int)gpfield.shape(0) < ngptot()) { + throw_Exception("dirtrans: slowest moving index must be >= ngptot", Here()); } const int nfld = compute_nfld(gpfield); @@ -1536,10 +1536,10 @@ void TransIFS::__invtrans(const functionspace::Spectral& sp, const Field& spfiel ATLAS_ASSERT(gpfield.functionspace() == 0 || functionspace::StructuredColumns(gpfield.functionspace())); ATLAS_ASSERT(spfield.functionspace() == 0 || functionspace::Spectral(spfield.functionspace())); if (compute_nfld(gpfield) != compute_nfld(spfield)) { - throw_Exception("dirtrans: different number of gridpoint fields than spectral fields", Here()); + throw_Exception("invtrans: different number of gridpoint fields than spectral fields", Here()); } - if ((int)gpfield.shape(0) != ngptot()) { - throw_Exception("dirtrans: slowest moving index must be ngptot", Here()); + if ((int)gpfield.shape(0) < ngptot()) { + throw_Exception("invtrans: slowest moving index must be >= ngptot", Here()); } const int nfld = compute_nfld(gpfield); @@ -1587,8 +1587,8 @@ void TransIFS::__invtrans_adj(const functionspace::Spectral& sp, Field& spfield, if (compute_nfld(gpfield) != compute_nfld(spfield)) { throw_Exception("dirtrans: different number of gridpoint fields than spectral fields", Here()); } - if ((int)gpfield.shape(0) != ngptot()) { - throw_Exception("dirtrans: slowest moving index must be ngptot", Here()); + if ((int)gpfield.shape(0) < ngptot()) { + throw_Exception("dirtrans: slowest moving index must be >= ngptot", Here()); } const int nfld = compute_nfld(gpfield); From de9efc559cd0f3ba468fa4b45266d3a264435b80 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 7 Jun 2022 15:33:43 +0100 Subject: [PATCH 31/53] Output: allow chaining of multiple write() --- src/atlas/output/Output.cc | 15 ++++++++++----- src/atlas/output/Output.h | 10 +++++----- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/atlas/output/Output.cc b/src/atlas/output/Output.cc index de012d25b..4a02e6338 100644 --- a/src/atlas/output/Output.cc +++ b/src/atlas/output/Output.cc @@ -55,28 +55,33 @@ Output::Output(const std::string& key, std::ostream& stream, const eckit::Parame Handle(detail::OutputFactory::build(key, stream, params)) {} /// Write mesh file -void Output::write(const Mesh& m, const eckit::Parametrisation& c) const { +const Output& Output::write(const Mesh& m, const eckit::Parametrisation& c) const { get()->write(m, c); + return *this; } /// Write field to file -void Output::write(const Field& f, const eckit::Parametrisation& c) const { +const Output& Output::write(const Field& f, const eckit::Parametrisation& c) const { get()->write(f, c); + return *this; } /// Write fieldset to file using FunctionSpace -void Output::write(const FieldSet& f, const eckit::Parametrisation& c) const { +const Output& Output::write(const FieldSet& f, const eckit::Parametrisation& c) const { get()->write(f, c); + return *this; } /// Write field to file using Functionspace -void Output::write(const Field& f, const FunctionSpace& fs, const eckit::Parametrisation& c) const { +const Output& Output::write(const Field& f, const FunctionSpace& fs, const eckit::Parametrisation& c) const { get()->write(f, fs, c); + return *this; } /// Write fieldset to file using FunctionSpace -void Output::write(const FieldSet& f, const FunctionSpace& fs, const eckit::Parametrisation& c) const { +const Output& Output::write(const FieldSet& f, const FunctionSpace& fs, const eckit::Parametrisation& c) const { get()->write(f, fs, c); + return *this; } namespace detail { diff --git a/src/atlas/output/Output.h b/src/atlas/output/Output.h index 5223b96ff..6e7e7e6b8 100644 --- a/src/atlas/output/Output.h +++ b/src/atlas/output/Output.h @@ -94,19 +94,19 @@ class Output : DOXYGEN_HIDE(public util::ObjectHandle) { Output(const std::string&, std::ostream&, const eckit::Parametrisation& = util::NoConfig()); /// Write mesh file - void write(const Mesh&, const eckit::Parametrisation& = util::NoConfig()) const; + const Output& write(const Mesh&, const eckit::Parametrisation& = util::NoConfig()) const; /// Write field to file - void write(const Field&, const eckit::Parametrisation& = util::NoConfig()) const; + const Output& write(const Field&, const eckit::Parametrisation& = util::NoConfig()) const; /// Write fieldset to file using FunctionSpace - void write(const FieldSet&, const eckit::Parametrisation& = util::NoConfig()) const; + const Output& write(const FieldSet&, const eckit::Parametrisation& = util::NoConfig()) const; /// Write field to file using Functionspace - void write(const Field&, const FunctionSpace&, const eckit::Parametrisation& = util::NoConfig()) const; + const Output& write(const Field&, const FunctionSpace&, const eckit::Parametrisation& = util::NoConfig()) const; /// Write fieldset to file using FunctionSpace - void write(const FieldSet&, const FunctionSpace&, const eckit::Parametrisation& = util::NoConfig()) const; + const Output& write(const FieldSet&, const FunctionSpace&, const eckit::Parametrisation& = util::NoConfig()) const; }; namespace detail { From a8931a791fcb25c7d7e2631d3c3e1329daf6edc2 Mon Sep 17 00:00:00 2001 From: Marek Wlasak Date: Thu, 16 Jun 2022 11:15:01 +0100 Subject: [PATCH 32/53] extending the interface for structured columns interface --- src/atlas/trans/ifs/TransIFS.cc | 1249 +++++++++++++++++++++---------- src/atlas/trans/ifs/TransIFS.h | 51 +- 2 files changed, 890 insertions(+), 410 deletions(-) diff --git a/src/atlas/trans/ifs/TransIFS.cc b/src/atlas/trans/ifs/TransIFS.cc index 6fc9db81d..21b3fe97d 100644 --- a/src/atlas/trans/ifs/TransIFS.cc +++ b/src/atlas/trans/ifs/TransIFS.cc @@ -245,15 +245,28 @@ void TransIFS::invtrans_adj(const FieldSet& gpfields, FieldSet& spfields, const void TransIFS::invtrans_grad(const Field& spfield, Field& gradfield, const eckit::Configuration& config) const { ATLAS_ASSERT(Spectral(spfield.functionspace())); ATLAS_ASSERT(NodeColumns(gradfield.functionspace())); - __invtrans_grad(Spectral(spfield.functionspace()), spfield, NodeColumns(gradfield.functionspace()), gradfield, - config); + if (StructuredColumns(gradfield.functionspace())) { + __invtrans_grad(Spectral(spfield.functionspace()), spfield, StructuredColumns(gradfield.functionspace()), gradfield, + config); + } + else if (NodeColumns(gradfield.functionspace())) { + __invtrans_grad(Spectral(spfield.functionspace()), spfield, NodeColumns(gradfield.functionspace()), gradfield, + config); + } + else { + ATLAS_NOTIMPLEMENTED; + } } void TransIFS::invtrans_grad(const FieldSet& spfields, FieldSet& gradfields, const eckit::Configuration& config) const { assert_spectral_functionspace(spfields); std::string functionspace(fieldset_functionspace(gradfields)); - if (functionspace == NodeColumns::type()) { + if (functionspace == StructuredColumns::type()) { + __invtrans_grad(Spectral(spfields[0].functionspace()), spfields, StructuredColumns(gradfields[0].functionspace()), + gradfields, config); + } + else if (functionspace == NodeColumns::type()) { __invtrans_grad(Spectral(spfields[0].functionspace()), spfields, NodeColumns(gradfields[0].functionspace()), gradfields, config); } @@ -267,8 +280,17 @@ void TransIFS::invtrans_grad(const FieldSet& spfields, FieldSet& gradfields, con void TransIFS::invtrans_grad_adj(const Field& gradfield, Field& spfield, const eckit::Configuration& config) const { ATLAS_ASSERT(Spectral(spfield.functionspace())); ATLAS_ASSERT(NodeColumns(gradfield.functionspace())); - __invtrans_grad_adj(Spectral(spfield.functionspace()), spfield, NodeColumns(gradfield.functionspace()), gradfield, - config); + if (StructuredColumns(gradfield.functionspace())) { + __invtrans_grad_adj(Spectral(spfield.functionspace()), spfield, StructuredColumns(gradfield.functionspace()), + gradfield, config); + } + else if (NodeColumns(gradfield.functionspace())) { + __invtrans_grad_adj(Spectral(spfield.functionspace()), spfield, NodeColumns(gradfield.functionspace()), + gradfield, config); + } + else { + ATLAS_NOTIMPLEMENTED; + } } void TransIFS::invtrans_grad_adj(const FieldSet& gradfields, FieldSet& spfields, @@ -276,7 +298,11 @@ void TransIFS::invtrans_grad_adj(const FieldSet& gradfields, FieldSet& spfields, assert_spectral_functionspace(spfields); std::string functionspace(fieldset_functionspace(gradfields)); - if (functionspace == NodeColumns::type()) { + if (functionspace == StructuredColumns::type()) { + __invtrans_grad_adj(Spectral(spfields[0].functionspace()), spfields, StructuredColumns(gradfields[0].functionspace()), + gradfields, config); + } + else if (functionspace == NodeColumns::type()) { __invtrans_grad_adj(Spectral(spfields[0].functionspace()), spfields, NodeColumns(gradfields[0].functionspace()), gradfields, config); } @@ -291,18 +317,35 @@ void TransIFS::dirtrans_wind2vordiv(const Field& gpwind, Field& spvor, Field& sp const eckit::Configuration& config) const { ATLAS_ASSERT(Spectral(spvor.functionspace())); ATLAS_ASSERT(Spectral(spdiv.functionspace())); - ATLAS_ASSERT(NodeColumns(gpwind.functionspace())); - __dirtrans_wind2vordiv(NodeColumns(gpwind.functionspace()), gpwind, Spectral(spvor.functionspace()), spvor, spdiv, + if (StructuredColumns(gpwind.functionspace())) { + __dirtrans_wind2vordiv(StructuredColumns(gpwind.functionspace()), gpwind, Spectral(spvor.functionspace()), spvor, spdiv, + config); + } + else if (NodeColumns(gpwind.functionspace())) { + __dirtrans_wind2vordiv(NodeColumns(gpwind.functionspace()), gpwind, Spectral(spvor.functionspace()), spvor, spdiv, config); + } + else { + ATLAS_NOTIMPLEMENTED; + } } + void TransIFS::invtrans_vordiv2wind(const Field& spvor, const Field& spdiv, Field& gpwind, const eckit::Configuration& config) const { ATLAS_ASSERT(Spectral(spvor.functionspace())); ATLAS_ASSERT(Spectral(spdiv.functionspace())); - ATLAS_ASSERT(NodeColumns(gpwind.functionspace())); - __invtrans_vordiv2wind(Spectral(spvor.functionspace()), spvor, spdiv, NodeColumns(gpwind.functionspace()), gpwind, - config); + if (StructuredColumns(gpwind.functionspace())) { + __invtrans_vordiv2wind(Spectral(spvor.functionspace()), spvor, spdiv, StructuredColumns(gpwind.functionspace()), + gpwind, config); + } + else if (NodeColumns(gpwind.functionspace())) { + __invtrans_vordiv2wind(Spectral(spvor.functionspace()), spvor, spdiv, NodeColumns(gpwind.functionspace()), + gpwind, config); + } + else { + ATLAS_NOTIMPLEMENTED; + } } @@ -311,8 +354,17 @@ void TransIFS::invtrans_vordiv2wind_adj(const Field& gpwind, Field& spvor, Field ATLAS_ASSERT(Spectral(spvor.functionspace())); ATLAS_ASSERT(Spectral(spdiv.functionspace())); ATLAS_ASSERT(NodeColumns(gpwind.functionspace())); - __invtrans_vordiv2wind_adj(Spectral(spvor.functionspace()), spvor, spdiv, NodeColumns(gpwind.functionspace()), - gpwind, config); + if (StructuredColumns(gpwind.functionspace())) { + __invtrans_vordiv2wind_adj(Spectral(spvor.functionspace()), spvor, spdiv, StructuredColumns(gpwind.functionspace()), + gpwind, config); + } + else if (NodeColumns(gpwind.functionspace())) { + __invtrans_vordiv2wind_adj(Spectral(spvor.functionspace()), spvor, spdiv, NodeColumns(gpwind.functionspace()), + gpwind, config); + } + else { + ATLAS_NOTIMPLEMENTED; + } } // -------------------------------------------------------------------------------------------- @@ -508,13 +560,16 @@ struct PackStructuredColumns { PackStructuredColumns(LocalView& rgpview): rgpview_(rgpview), f(0) {} - void operator()(const StructuredColumns& sc, const Field& field) { + void operator()(const StructuredColumns& sc, const Field& field, idx_t components = 0) { switch (field.rank()) { case 1: - pack_1(sc, field); + pack_1(sc, field, components); break; case 2: - pack_2(sc, field); + pack_2(sc, field, components); + break; + case 3: + pack_3(sc, field, components); break; default: ATLAS_DEBUG_VAR(field.rank()); @@ -523,7 +578,7 @@ struct PackStructuredColumns { } } - void pack_1(const StructuredColumns& sc, const Field& field) { + void pack_1(const StructuredColumns& sc, const Field& field, idx_t) { auto gpfield = make_view(field); for (idx_t jnode = 0; jnode < sc.sizeOwned(); ++jnode) { @@ -531,7 +586,7 @@ struct PackStructuredColumns { } ++f; } - void pack_2(const StructuredColumns& sc, const Field& field) { + void pack_2(const StructuredColumns& sc, const Field& field, idx_t) { auto gpfield = make_view(field); const idx_t nvars = gpfield.shape(1); for (idx_t jvar = 0; jvar < nvars; ++jvar) { @@ -541,6 +596,21 @@ struct PackStructuredColumns { ++f; } } + void pack_3(const StructuredColumns& sc, const Field& field, idx_t components) { + auto gpfield = make_view(field); + if (not components) { + components = gpfield.shape(2); + } + for (idx_t jcomp = 0; jcomp < components; ++jcomp) { + const idx_t nvars = gpfield.shape(1); + for (idx_t jvar = 0; jvar < nvars; ++jvar) { + for (idx_t jnode = 0; jnode < sc.sizeOwned(); ++jnode) { + rgpview_(f, jnode) = gpfield(jnode, jvar, jcomp); + } + ++f; + } + } + } }; struct PackSpectral { @@ -662,13 +732,16 @@ struct UnpackStructuredColumns { UnpackStructuredColumns(const LocalView& rgpview): rgpview_(rgpview), f(0) {} - void operator()(const StructuredColumns& sc, Field& field) { + void operator()(const StructuredColumns& sc, Field& field, int components = 0) { switch (field.rank()) { case 1: - unpack_1(sc, field); + unpack_1(sc, field, components); break; case 2: - unpack_2(sc, field); + unpack_2(sc, field, components); + break; + case 3: + unpack_3(sc, field, components); break; default: ATLAS_DEBUG_VAR(field.rank()); @@ -677,14 +750,14 @@ struct UnpackStructuredColumns { } } - void unpack_1(const StructuredColumns& sc, Field& field) { + void unpack_1(const StructuredColumns& sc, Field& field, idx_t) { auto gpfield = make_view(field); for (idx_t jnode = 0; jnode < sc.sizeOwned(); ++jnode) { gpfield(jnode) = rgpview_(f, jnode); } ++f; } - void unpack_2(const StructuredColumns& sc, Field& field) { + void unpack_2(const StructuredColumns& sc, Field& field, idx_t) { auto gpfield = make_view(field); const idx_t nvars = gpfield.shape(1); for (idx_t jvar = 0; jvar < nvars; ++jvar) { @@ -694,6 +767,20 @@ struct UnpackStructuredColumns { ++f; } } + void unpack_3(const StructuredColumns& sc, Field& field, idx_t components) { + auto gpfield = make_view(field); + if (not components) { + components = gpfield.shape(2); + } + for (idx_t jcomp = 0; jcomp < components; ++jcomp) { + for (idx_t jlev = 0; jlev < gpfield.shape(1); ++jlev) { + for (idx_t jnode = 0; jnode < sc.sizeOwned(); ++jnode) { + gpfield(jnode, jlev, jcomp) = rgpview_(f, jnode); + } + ++f; + } + } + } }; struct UnpackSpectral { @@ -1093,62 +1180,6 @@ void TransIFS::ctor_lonlat(const long nlon, const long nlat, long truncation, co TRANS_CHECK(::trans_setup(trans_.get())); } -// -------------------------------------------------------------------------------------------- - -void TransIFS::__dirtrans(const functionspace::NodeColumns& gp, const Field& gpfield, const Spectral& sp, - Field& spfield, const eckit::Configuration& config) const { - FieldSet gpfields; - gpfields.add(gpfield); - FieldSet spfields; - spfields.add(spfield); - __dirtrans(gp, gpfields, sp, spfields, config); -} - -// -------------------------------------------------------------------------------------------- - -void TransIFS::__dirtrans(const functionspace::NodeColumns& gp, const FieldSet& gpfields, const Spectral& sp, - FieldSet& spfields, const eckit::Configuration&) const { - assertCompatibleDistributions(gp, sp); - - // Count total number of fields and do sanity checks - const int nfld = compute_nfld(gpfields); - const int trans_sp_nfld = compute_nfld(spfields); - - if (nfld != trans_sp_nfld) { - throw_Exception("dirtrans: different number of gridpoint fields than spectral fields", Here()); - } - - // Arrays Trans expects - std::vector rgp(nfld * ngptot()); - std::vector rsp(nspec2() * nfld); - auto rgpview = LocalView(rgp.data(), make_shape(nfld, ngptot())); - auto rspview = LocalView(rsp.data(), make_shape(nspec2(), nfld)); - - // Pack gridpoints - { - PackNodeColumns pack(rgpview, gp); - for (idx_t jfld = 0; jfld < gpfields.size(); ++jfld) { - pack(gpfields[jfld]); - } - } - - // Do transform - { - struct ::DirTrans_t transform = ::new_dirtrans(trans_.get()); - transform.nscalar = nfld; - transform.rgp = rgp.data(); - transform.rspscalar = rsp.data(); - TRANS_CHECK(::trans_dirtrans(&transform)); - } - - // Unpack the spectral fields - { - UnpackSpectral unpack(rspview); - for (idx_t jfld = 0; jfld < spfields.size(); ++jfld) { - unpack(spfields[jfld]); - } - } -} // -------------------------------------------------------------------------------------------- @@ -1195,6 +1226,19 @@ void TransIFS::__dirtrans(const StructuredColumns& gp, const Field& gpfield, con } } +// -------------------------------------------------------------------------------------------- + +void TransIFS::__dirtrans(const functionspace::NodeColumns& gp, const Field& gpfield, const Spectral& sp, + Field& spfield, const eckit::Configuration& config) const { + FieldSet gpfields; + gpfields.add(gpfield); + FieldSet spfields; + spfields.add(spfield); + __dirtrans(gp, gpfields, sp, spfields, config); +} + +// -------------------------------------------------------------------------------------------- + void TransIFS::__dirtrans(const StructuredColumns& gp, const FieldSet& gpfields, const Spectral& sp, FieldSet& spfields, const eckit::Configuration&) const { assertCompatibleDistributions(gp, sp); @@ -1246,196 +1290,207 @@ void TransIFS::__dirtrans(const StructuredColumns& gp, const FieldSet& gpfields, // -------------------------------------------------------------------------------------------- -void TransIFS::__invtrans_grad(const Spectral& sp, const Field& spfield, const functionspace::NodeColumns& gp, - Field& gradfield, const eckit::Configuration& config) const { - FieldSet spfields; - spfields.add(spfield); - FieldSet gradfields; - gradfields.add(gradfield); - __invtrans_grad(sp, spfields, gp, gradfields, config); -} - -void TransIFS::__invtrans_grad(const Spectral& sp, const FieldSet& spfields, const functionspace::NodeColumns& gp, - FieldSet& gradfields, const eckit::Configuration& config) const { +void TransIFS::__dirtrans(const functionspace::NodeColumns& gp, const FieldSet& gpfields, const Spectral& sp, + FieldSet& spfields, const eckit::Configuration&) const { assertCompatibleDistributions(gp, sp); // Count total number of fields and do sanity checks - const int nb_gridpoint_field = compute_nfld(gradfields); - const int nfld = compute_nfld(spfields); + const int nfld = compute_nfld(gpfields); + const int trans_sp_nfld = compute_nfld(spfields); - if (nb_gridpoint_field != 2 * nfld) { // factor 2 because N-S and E-W derivatives - throw_Exception( - "invtrans_grad: different number of gridpoint " - "fields than spectral fields", - Here()); + if (nfld != trans_sp_nfld) { + throw_Exception("dirtrans: different number of gridpoint fields than spectral fields", Here()); } // Arrays Trans expects - // Allocate space for - std::vector rgp(3 * nfld * ngptot()); // (scalars) + (NS ders) + (EW ders) + std::vector rgp(nfld * ngptot()); std::vector rsp(nspec2() * nfld); - auto rgpview = LocalView(rgp.data(), make_shape(3 * nfld, ngptot())); + auto rgpview = LocalView(rgp.data(), make_shape(nfld, ngptot())); auto rspview = LocalView(rsp.data(), make_shape(nspec2(), nfld)); - - // Pack spectral fields + // Pack gridpoints { - PackSpectral pack(rspview); - for (idx_t jfld = 0; jfld < spfields.size(); ++jfld) { - pack(spfields[jfld]); + PackNodeColumns pack(rgpview, gp); + for (idx_t jfld = 0; jfld < gpfields.size(); ++jfld) { + pack(gpfields[jfld]); } } // Do transform { - struct ::InvTrans_t transform = ::new_invtrans(trans_.get()); + struct ::DirTrans_t transform = ::new_dirtrans(trans_.get()); transform.nscalar = nfld; transform.rgp = rgp.data(); transform.rspscalar = rsp.data(); - transform.lscalarders = true; - - TRANS_CHECK(::trans_invtrans(&transform)); + TRANS_CHECK(::trans_dirtrans(&transform)); } - // Unpack the gridpoint fields + // Unpack the spectral fields { - mesh::IsGhostNode is_ghost(gp.nodes()); - int f = nfld; // skip to where derivatives start - for (idx_t dim = 0; dim < 2; ++dim) { - for (idx_t jfld = 0; jfld < gradfields.size(); ++jfld) { - const idx_t nb_nodes = gradfields[jfld].shape(0); - const idx_t nlev = gradfields[jfld].levels(); - if (nlev) { - auto field = make_view(gradfields[jfld]); - for (idx_t jlev = 0; jlev < nlev; ++jlev) { - int n = 0; - for (idx_t jnode = 0; jnode < nb_nodes; ++jnode) { - if (!is_ghost(jnode)) { - field(jnode, jlev, 1 - dim) = rgpview(f, n); - ++n; - } - } - ATLAS_ASSERT(n == ngptot()); - } - } - else { - auto field = make_view(gradfields[jfld]); - int n = 0; - for (idx_t jnode = 0; jnode < nb_nodes; ++jnode) { - if (!is_ghost(jnode)) { - field(jnode, 1 - dim) = rgpview(f, n); - ++n; - } - } - ATLAS_ASSERT(n == ngptot()); - } - ++f; - } + UnpackSpectral unpack(rspview); + for (idx_t jfld = 0; jfld < spfields.size(); ++jfld) { + unpack(spfields[jfld]); } } } -//--------------------------------------------------------------------------------------------- - -void TransIFS::__invtrans_grad_adj(const Spectral& sp, Field& spfield, const functionspace::NodeColumns& gp, - const Field& gradfield, const eckit::Configuration& config) const { - FieldSet spfields; - spfields.add(spfield); - FieldSet gradfields; - gradfields.add(gradfield); - __invtrans_grad_adj(sp, spfields, gp, gradfields, config); -} +// -------------------------------------------------------------------------------------------- -void TransIFS::__invtrans_grad_adj(const Spectral& sp, FieldSet& spfields, const functionspace::NodeColumns& gp, - const FieldSet& gradfields, const eckit::Configuration& config) const { -#if ATLAS_HAVE_ECTRANS || defined(TRANS_HAVE_INVTRANS_ADJ) +void TransIFS::__dirtrans_wind2vordiv(const functionspace::StructuredColumns& gp, + const Field& gpwind, const Spectral& sp, + Field& spvor, Field& spdiv, + const eckit::Configuration&) const { assertCompatibleDistributions(gp, sp); // Count total number of fields and do sanity checks - const idx_t nfld = compute_nfld(gradfields); - for (idx_t jfld = 0; jfld < gradfields.size(); ++jfld) { - const Field& f = gradfields[jfld]; - ATLAS_ASSERT(f.functionspace() == 0 || functionspace::StructuredColumns(f.functionspace())); + const size_t nfld = compute_nfld(spvor); + if (spdiv.shape(0) != spvor.shape(0)) { + throw_Exception("dirtrans:vorticity not compatible with divergence for 1st dimension", Here()); + } + if (spdiv.shape(1) != spvor.shape(1)) { + throw_Exception("dirtrans:vorticity not compatible with divergence for 2st dimension", Here()); + } + const size_t nwindfld = compute_nfld(gpwind); + if (nwindfld != 2 * nfld && nwindfld != 3 * nfld) { + throw_Exception("dirtrans:wind field is not compatible with vorticity, divergence.", Here()); } - const int trans_sp_nfld = compute_nfld(spfields); + if (spdiv.shape(0) != nspec2()) { + std::stringstream msg; + msg << "dirtrans: Spectral vorticity and divergence have wrong dimension: " + "nspec2 " + << spdiv.shape(0) << " should be " << nspec2(); + throw_Exception(msg.str(), Here()); + } - if (nfld != trans_sp_nfld) { - throw_Exception("dirtrans: different number of gridpoint fields than spectral fields", Here()); + if (spvor.size() == 0) { + throw_Exception("dirtrans: spectral vorticity field is empty."); + } + if (spdiv.size() == 0) { + throw_Exception("dirtrans: spectral divergence field is empty."); } + // Arrays Trans expects - std::vector rgp(nfld * ngptot()); - std::vector rsp(nspec2() * nfld); - auto rgpview = LocalView(rgp.data(), make_shape(nfld, ngptot())); - auto rspview = LocalView(rsp.data(), make_shape(nspec2(), nfld)); + std::vector rgp(2 * nfld * ngptot()); + std::vector rspvor(nspec2() * nfld); + std::vector rspdiv(nspec2() * nfld); + auto rgpview = LocalView(rgp.data(), make_shape(2 * nfld, ngptot())); + auto rspvorview = LocalView(rspvor.data(), make_shape(nspec2(), nfld)); + auto rspdivview = LocalView(rspdiv.data(), make_shape(nspec2(), nfld)); // Pack gridpoints { - PackNodeColumns pack(rgpview, gp); - for (idx_t jfld = 0; jfld < gradfields.size(); ++jfld) { - pack(gradfields[jfld]); - } + PackStructuredColumns pack(rgpview); + int wind_components = 2; + pack(gpwind.functionspace(), gpwind, wind_components); } // Do transform { - struct ::InvTransAdj_t transform = ::new_invtrans_adj(trans_.get()); - transform.nscalar = nfld; - transform.rgp = rgp.data(); - transform.rspscalar = rsp.data(); - transform.lscalarders = true; + struct ::DirTrans_t transform = ::new_dirtrans(trans_.get()); + transform.nvordiv = int(nfld); + transform.rgp = rgp.data(); + transform.rspvor = rspvor.data(); + transform.rspdiv = rspdiv.data(); - TRANS_CHECK(::trans_invtrans_adj(&transform)); + ATLAS_ASSERT(transform.rspvor); + ATLAS_ASSERT(transform.rspdiv); + TRANS_CHECK(::trans_dirtrans(&transform)); } - // Unpack the spectral fields - { - UnpackSpectral unpack(rspview); - for (idx_t jfld = 0; jfld < spfields.size(); ++jfld) { - unpack(spfields[jfld]); - } - } -#else - ATLAS_NOTIMPLEMENTED; -#endif + // Pack spectral fields + UnpackSpectral unpack_vor(rspvorview); + unpack_vor(spvor); + UnpackSpectral unpack_div(rspdivview); + unpack_div(spdiv); } -// -------------------------------------------------------------------------------------------- +// ----------------------------------------------------------------------------------------------- -void TransIFS::__invtrans(const Spectral& sp, const Field& spfield, const functionspace::NodeColumns& gp, - Field& gpfield, const eckit::Configuration& config) const { - FieldSet spfields; - spfields.add(spfield); - FieldSet gpfields; - gpfields.add(gpfield); - __invtrans(sp, spfields, gp, gpfields, config); -} +void TransIFS::__dirtrans_wind2vordiv(const functionspace::NodeColumns& gp, const Field& gpwind, + const Spectral& sp, + Field& spvor, Field& spdiv, const eckit::Configuration&) const { + assertCompatibleDistributions(gp, sp); -// -------------------------------------------------------------------------------------------- + // Count total number of fields and do sanity checks + const size_t nfld = compute_nfld(spvor); + if (spdiv.shape(0) != spvor.shape(0)) { + throw_Exception("dirtrans: vorticity not compatible with divergence for 1st dimension.", Here()); + } + if (spdiv.shape(1) != spvor.shape(1)) { + throw_Exception("dirtrans: vorticity not compatible with divergence for 2nd dimension.", Here()); + } + const size_t nwindfld = compute_nfld(gpwind); + if (nwindfld != 2 * nfld && nwindfld != 3 * nfld) { + throw_Exception("dirtrans: wind field is not compatible with vorticity, divergence.", Here()); + } -void TransIFS::__invtrans_adj(const Spectral& sp, Field& spfield, const functionspace::NodeColumns& gp, - const Field& gpfield, const eckit::Configuration& config) const { - FieldSet spfields; - spfields.add(spfield); - FieldSet gpfields; - gpfields.add(gpfield); - __invtrans_adj(sp, spfields, gp, gpfields, config); + if (spdiv.shape(0) != nspec2()) { + std::stringstream msg; + msg << "dirtrans: Spectral vorticity and divergence have wrong dimension: " + "nspec2 " + << spdiv.shape(0) << " should be " << nspec2(); + throw_Exception(msg.str(), Here()); + } + + if (spvor.size() == 0) { + throw_Exception("dirtrans: spectral vorticity field is empty."); + } + if (spdiv.size() == 0) { + throw_Exception("dirtrans: spectral divergence field is empty."); + } + + // Arrays Trans expects + std::vector rgp(2 * nfld * ngptot()); + std::vector rspvor(nspec2() * nfld); + std::vector rspdiv(nspec2() * nfld); + auto rgpview = LocalView(rgp.data(), make_shape(2 * nfld, ngptot())); + auto rspvorview = LocalView(rspvor.data(), make_shape(nspec2(), nfld)); + auto rspdivview = LocalView(rspdiv.data(), make_shape(nspec2(), nfld)); + + // Pack gridpoints + { + PackNodeColumns pack(rgpview, gp); + int wind_components = 2; + pack(gpwind, wind_components); + } + + // Do transform + { + struct ::DirTrans_t transform = ::new_dirtrans(trans_.get()); + transform.nvordiv = int(nfld); + transform.rgp = rgp.data(); + transform.rspvor = rspvor.data(); + transform.rspdiv = rspdiv.data(); + + ATLAS_ASSERT(transform.rspvor); + ATLAS_ASSERT(transform.rspdiv); + TRANS_CHECK(::trans_dirtrans(&transform)); + } + + // Pack spectral fields + UnpackSpectral unpack_vor(rspvorview); + unpack_vor(spvor); + UnpackSpectral unpack_div(rspdivview); + unpack_div(spdiv); } // -------------------------------------------------------------------------------------------- -void TransIFS::__invtrans(const Spectral& sp, const FieldSet& spfields, const functionspace::NodeColumns& gp, - FieldSet& gpfields, const eckit::Configuration& config) const { +void TransIFS::__invtrans(const functionspace::Spectral& sp, const Field& spfield, + const functionspace::StructuredColumns& gp, Field& gpfield, + const eckit::Configuration& config) const { assertCompatibleDistributions(gp, sp); - - // Count total number of fields and do sanity checks - const int nfld = compute_nfld(gpfields); - const int nb_spectral_fields = compute_nfld(spfields); - - if (nfld != nb_spectral_fields) { + ATLAS_ASSERT(gpfield.functionspace() == 0 || functionspace::StructuredColumns(gpfield.functionspace())); + ATLAS_ASSERT(spfield.functionspace() == 0 || functionspace::Spectral(spfield.functionspace())); + if (compute_nfld(gpfield) != compute_nfld(spfield)) { throw_Exception("invtrans: different number of gridpoint fields than spectral fields", Here()); } + if ((int)gpfield.shape(0) < ngptot()) { + throw_Exception("invtrans: slowest moving index must be >= ngptot", Here()); + } + const int nfld = compute_nfld(gpfield); // Arrays Trans expects std::vector rgp(nfld * ngptot()); @@ -1446,9 +1501,7 @@ void TransIFS::__invtrans(const Spectral& sp, const FieldSet& spfields, const fu // Pack spectral fields { PackSpectral pack(rspview); - for (idx_t jfld = 0; jfld < spfields.size(); ++jfld) { - pack(spfields[jfld]); - } + pack(spfield); } // Do transform @@ -1460,88 +1513,94 @@ void TransIFS::__invtrans(const Spectral& sp, const FieldSet& spfields, const fu TRANS_CHECK(::trans_invtrans(&transform)); } - // Unpack the gridpoint fields + // Unpack gridpoint fields { - UnpackNodeColumns unpack(rgpview, gp); - for (idx_t jfld = 0; jfld < gpfields.size(); ++jfld) { - unpack(gpfields[jfld]); - } + UnpackStructuredColumns unpack(rgpview); + unpack(gp, gpfield); } } // -------------------------------------------------------------------------------------------- -void TransIFS::__invtrans_adj(const Spectral& sp, FieldSet& spfields, const functionspace::NodeColumns& gp, - const FieldSet& gpfields, const eckit::Configuration& config) const { -#if ATLAS_HAVE_ECTRANS || defined(TRANS_HAVE_INVTRANS_ADJ) +void TransIFS::__invtrans(const Spectral& sp, const Field& spfield, const functionspace::NodeColumns& gp, + Field& gpfield, const eckit::Configuration& config) const { + FieldSet spfields; + spfields.add(spfield); + FieldSet gpfields; + gpfields.add(gpfield); + __invtrans(sp, spfields, gp, gpfields, config); +} + +// -------------------------------------------------------------------------------------------- +void TransIFS::__invtrans(const functionspace::Spectral& sp, const FieldSet& spfields, + const functionspace::StructuredColumns& gp, FieldSet& gpfields, + const eckit::Configuration& config) const { assertCompatibleDistributions(gp, sp); // Count total number of fields and do sanity checks - const idx_t nfld = compute_nfld(gpfields); + const int nfld = compute_nfld(gpfields); for (idx_t jfld = 0; jfld < gpfields.size(); ++jfld) { const Field& f = gpfields[jfld]; ATLAS_ASSERT(f.functionspace() == 0 || functionspace::StructuredColumns(f.functionspace())); } - const int trans_sp_nfld = compute_nfld(spfields); + const int nb_spectral_fields = compute_nfld(spfields); - if (nfld != trans_sp_nfld) { - throw_Exception("invtrans_adj: different number of gridpoint fields than spectral fields", Here()); + if (nfld != nb_spectral_fields) { + std::stringstream msg; + msg << "invtrans: different number of gridpoint fields than spectral fields" + << "[ " << nfld << " != " << nb_spectral_fields << " ]"; + throw_Exception(msg.str(), Here()); } + // Arrays Trans expects std::vector rgp(nfld * ngptot()); std::vector rsp(nspec2() * nfld); auto rgpview = LocalView(rgp.data(), make_shape(nfld, ngptot())); auto rspview = LocalView(rsp.data(), make_shape(nspec2(), nfld)); - // Pack gridpoints + // Pack spectral fields { - PackNodeColumns pack(rgpview, gp); - for (idx_t jfld = 0; jfld < gpfields.size(); ++jfld) { - pack(gpfields[jfld]); + PackSpectral pack(rspview); + for (idx_t jfld = 0; jfld < spfields.size(); ++jfld) { + pack(spfields[jfld]); } } // Do transform { - struct ::InvTransAdj_t transform = ::new_invtrans_adj(trans_.get()); - transform.nscalar = int(nfld); - transform.rgp = rgp.data(); - transform.rspscalar = rsp.data(); + struct ::InvTrans_t transform = ::new_invtrans(trans_.get()); + transform.nscalar = nfld; + transform.rgp = rgp.data(); + transform.rspscalar = rsp.data(); - TRANS_CHECK(::trans_invtrans_adj(&transform)); + TRANS_CHECK(::trans_invtrans(&transform)); } - // Unpack the spectral fields + // Unpack the gridpoint fields { - UnpackSpectral unpack(rspview); - for (idx_t jfld = 0; jfld < spfields.size(); ++jfld) { - unpack(spfields[jfld]); + UnpackStructuredColumns unpack(rgpview); + for (idx_t jfld = 0; jfld < gpfields.size(); ++jfld) { + unpack(gp, gpfields[jfld]); } } -#else - ATLAS_NOTIMPLEMENTED; -#endif } - // -------------------------------------------------------------------------------------------- -void TransIFS::__invtrans(const functionspace::Spectral& sp, const Field& spfield, - const functionspace::StructuredColumns& gp, Field& gpfield, - const eckit::Configuration& config) const { +void TransIFS::__invtrans(const Spectral& sp, const FieldSet& spfields, const functionspace::NodeColumns& gp, + FieldSet& gpfields, const eckit::Configuration& config) const { assertCompatibleDistributions(gp, sp); - ATLAS_ASSERT(gpfield.functionspace() == 0 || functionspace::StructuredColumns(gpfield.functionspace())); - ATLAS_ASSERT(spfield.functionspace() == 0 || functionspace::Spectral(spfield.functionspace())); - if (compute_nfld(gpfield) != compute_nfld(spfield)) { + + // Count total number of fields and do sanity checks + const int nfld = compute_nfld(gpfields); + const int nb_spectral_fields = compute_nfld(spfields); + + if (nfld != nb_spectral_fields) { throw_Exception("invtrans: different number of gridpoint fields than spectral fields", Here()); } - if ((int)gpfield.shape(0) < ngptot()) { - throw_Exception("invtrans: slowest moving index must be >= ngptot", Here()); - } - const int nfld = compute_nfld(gpfield); // Arrays Trans expects std::vector rgp(nfld * ngptot()); @@ -1552,7 +1611,9 @@ void TransIFS::__invtrans(const functionspace::Spectral& sp, const Field& spfiel // Pack spectral fields { PackSpectral pack(rspview); - pack(spfield); + for (idx_t jfld = 0; jfld < spfields.size(); ++jfld) { + pack(spfields[jfld]); + } } // Do transform @@ -1564,33 +1625,420 @@ void TransIFS::__invtrans(const functionspace::Spectral& sp, const Field& spfiel TRANS_CHECK(::trans_invtrans(&transform)); } - // Unpack gridpoint fields + // Unpack the gridpoint fields { - UnpackStructuredColumns unpack(rgpview); - unpack(gp, gpfield); + UnpackNodeColumns unpack(rgpview, gp); + for (idx_t jfld = 0; jfld < gpfields.size(); ++jfld) { + unpack(gpfields[jfld]); + } } } - // -------------------------------------------------------------------------------------------- -void TransIFS::__invtrans_adj(const functionspace::Spectral& sp, Field& spfield, - const functionspace::StructuredColumns& gp, const Field& gpfield, - const eckit::Configuration& config) const { -#if ATLAS_HAVE_ECTRANS || defined(TRANS_HAVE_INVTRANS_ADJ) +void TransIFS::__invtrans_grad(const Spectral& sp, const Field& spfield, const functionspace::StructuredColumns& gp, + Field& gradfield, const eckit::Configuration& config) const { + FieldSet spfields; + spfields.add(spfield); + FieldSet gradfields; + gradfields.add(gradfield); + __invtrans_grad(sp, spfields, gp, gradfields, config); +} - ATLAS_ASSERT(gpfield.functionspace() == 0 || functionspace::StructuredColumns(gpfield.functionspace())); - ATLAS_ASSERT(spfield.functionspace() == 0 || functionspace::Spectral(spfield.functionspace())); +// -------------------------------------------------------------------------------------------- + +void TransIFS::__invtrans_grad(const Spectral& sp, const Field& spfield, const functionspace::NodeColumns& gp, + Field& gradfield, const eckit::Configuration& config) const { + FieldSet spfields; + spfields.add(spfield); + FieldSet gradfields; + gradfields.add(gradfield); + __invtrans_grad(sp, spfields, gp, gradfields, config); +} + +// -------------------------------------------------------------------------------------------- + +void TransIFS::__invtrans_grad(const Spectral& sp, const FieldSet& spfields, + const functionspace::StructuredColumns& gp, + FieldSet& gradfields, + const eckit::Configuration& config) const { + assertCompatibleDistributions(gp, sp); + + // Count total number of fields and do sanity checks + const int nb_gridpoint_field = compute_nfld(gradfields); + const int nfld = compute_nfld(spfields); + + if (nb_gridpoint_field != 2 * nfld) { // factor 2 because N-S and E-W derivatives + throw_Exception( + "invtrans_grad: different number of gridpoint " + "fields than spectral fields", + Here()); + } + + // Arrays Trans expects + // Allocate space for + std::vector rgp(3 * nfld * ngptot()); // (scalars) + (NS ders) + (EW ders) + std::vector rsp(nspec2() * nfld); + auto rgpview = LocalView(rgp.data(), make_shape(3 * nfld, ngptot())); + auto rspview = LocalView(rsp.data(), make_shape(nspec2(), nfld)); + + + // Pack spectral fields + { + PackSpectral pack(rspview); + for (idx_t jfld = 0; jfld < spfields.size(); ++jfld) { + pack(spfields[jfld]); + } + } + + // Do transform + { + struct ::InvTrans_t transform = ::new_invtrans(trans_.get()); + transform.nscalar = nfld; + transform.rgp = rgp.data(); + transform.rspscalar = rsp.data(); + transform.lscalarders = true; + + TRANS_CHECK(::trans_invtrans(&transform)); + } + + // Unpack the gridpoint fields + { + int f = nfld; // skip to where derivatives start + for (idx_t dim = 0; dim < 2; ++dim) { + for (idx_t jfld = 0; jfld < gradfields.size(); ++jfld) { + const idx_t nb_nodes = StructuredColumns(gradfields[jfld].functionspace()).sizeOwned(); + const idx_t nlev = gradfields[jfld].levels(); + if (nlev) { + auto field = make_view(gradfields[jfld]); + for (idx_t jlev = 0; jlev < nlev; ++jlev) { + for (idx_t jnode = 0; jnode < nb_nodes; ++jnode) { + field(jnode, jlev, 1 - dim) = rgpview(f, jnode); + } + } + } + else { + auto field = make_view(gradfields[jfld]); + for (idx_t jnode = 0; jnode < nb_nodes; ++jnode) { + field(jnode, 1 - dim) = rgpview(f, jnode); + } + } + ++f; + } + } + } +} + +// -------------------------------------------------------------------------------------------- + +void TransIFS::__invtrans_grad(const Spectral& sp, const FieldSet& spfields, + const functionspace::NodeColumns& gp, + FieldSet& gradfields, const eckit::Configuration& config) const { + assertCompatibleDistributions(gp, sp); + + // Count total number of fields and do sanity checks + const int nb_gridpoint_field = compute_nfld(gradfields); + const int nfld = compute_nfld(spfields); + + if (nb_gridpoint_field != 2 * nfld) { // factor 2 because N-S and E-W derivatives + throw_Exception( + "invtrans_grad: different number of gridpoint " + "fields than spectral fields", + Here()); + } + + // Arrays Trans expects + // Allocate space for + std::vector rgp(3 * nfld * ngptot()); // (scalars) + (NS ders) + (EW ders) + std::vector rsp(nspec2() * nfld); + auto rgpview = LocalView(rgp.data(), make_shape(3 * nfld, ngptot())); + auto rspview = LocalView(rsp.data(), make_shape(nspec2(), nfld)); + + + // Pack spectral fields + { + PackSpectral pack(rspview); + for (idx_t jfld = 0; jfld < spfields.size(); ++jfld) { + pack(spfields[jfld]); + } + } + + // Do transform + { + struct ::InvTrans_t transform = ::new_invtrans(trans_.get()); + transform.nscalar = nfld; + transform.rgp = rgp.data(); + transform.rspscalar = rsp.data(); + transform.lscalarders = true; + + TRANS_CHECK(::trans_invtrans(&transform)); + } + + // Unpack the gridpoint fields + { + mesh::IsGhostNode is_ghost(gp.nodes()); + int f = nfld; // skip to where derivatives start + for (idx_t dim = 0; dim < 2; ++dim) { + for (idx_t jfld = 0; jfld < gradfields.size(); ++jfld) { + const idx_t nb_nodes = gradfields[jfld].shape(0); + const idx_t nlev = gradfields[jfld].levels(); + if (nlev) { + auto field = make_view(gradfields[jfld]); + for (idx_t jlev = 0; jlev < nlev; ++jlev) { + int n = 0; + for (idx_t jnode = 0; jnode < nb_nodes; ++jnode) { + if (!is_ghost(jnode)) { + field(jnode, jlev, 1 - dim) = rgpview(f, n); + ++n; + } + } + ATLAS_ASSERT(n == ngptot()); + } + } + else { + auto field = make_view(gradfields[jfld]); + int n = 0; + for (idx_t jnode = 0; jnode < nb_nodes; ++jnode) { + if (!is_ghost(jnode)) { + field(jnode, 1 - dim) = rgpview(f, n); + ++n; + } + } + ATLAS_ASSERT(n == ngptot()); + } + ++f; + } + } + } +} + +// -------------------------------------------------------------------------------------------- +void TransIFS::__invtrans_vordiv2wind(const Spectral& sp, const Field& spvor, const Field& spdiv, + const functionspace::StructuredColumns& gp, Field& gpwind, + const eckit::Configuration&) const { + assertCompatibleDistributions(gp, sp); + + // Count total number of fields and do sanity checks + const int nfld = compute_nfld(spvor); + if (spdiv.shape(0) != spvor.shape(0)) { + throw_Exception("invtrans: vorticity not compatible with divergence.", Here()); + } + if (spdiv.shape(1) != spvor.shape(1)) { + throw_Exception("invtrans: vorticity not compatible with divergence.", Here()); + } + const int nwindfld = compute_nfld(gpwind); + if (nwindfld != 2 * nfld && nwindfld != 3 * nfld) { + throw_Exception("invtrans: wind field is not compatible with vorticity, divergence.", Here()); + } + + if (spdiv.shape(0) != nspec2()) { + std::stringstream msg; + msg << "invtrans: Spectral vorticity and divergence have wrong dimension: " + "nspec2 " + << spdiv.shape(0) << " should be " << nspec2(); + throw_Exception(msg.str(), Here()); + } + + ATLAS_ASSERT(spvor.rank() == 2); + ATLAS_ASSERT(spdiv.rank() == 2); + if (spvor.size() == 0) { + throw_Exception("invtrans: spectral vorticity field is empty."); + } + if (spdiv.size() == 0) { + throw_Exception("invtrans: spectral divergence field is empty."); + } + + // Arrays Trans expects + std::vector rgp(2 * nfld * ngptot()); + std::vector rspvor(nspec2() * nfld); + std::vector rspdiv(nspec2() * nfld); + auto rgpview = LocalView(rgp.data(), make_shape(2 * nfld, ngptot())); + auto rspvorview = LocalView(rspvor.data(), make_shape(nspec2(), nfld)); + auto rspdivview = LocalView(rspdiv.data(), make_shape(nspec2(), nfld)); + + // Pack spectral fields + PackSpectral pack_vor(rspvorview); + pack_vor(spvor); + PackSpectral pack_div(rspdivview); + pack_div(spdiv); + + // Do transform + { + struct ::InvTrans_t transform = ::new_invtrans(trans_.get()); + transform.nvordiv = nfld; + transform.rgp = rgp.data(); + transform.rspvor = rspvor.data(); + transform.rspdiv = rspdiv.data(); + + ATLAS_ASSERT(transform.rspvor); + ATLAS_ASSERT(transform.rspdiv); + TRANS_CHECK(::trans_invtrans(&transform)); + } + + // Unpack the gridpoint fields + { + UnpackStructuredColumns unpack(rgpview); + int wind_components = 2; + unpack(gp, gpwind, wind_components); + } +} + +//--------------------------------------------------------------------------------------------- + +void TransIFS::__invtrans_vordiv2wind(const Spectral& sp, const Field& spvor, const Field& spdiv, + const functionspace::NodeColumns& gp, Field& gpwind, + const eckit::Configuration&) const { + assertCompatibleDistributions(gp, sp); + + // Count total number of fields and do sanity checks + const int nfld = compute_nfld(spvor); + if (spdiv.shape(0) != spvor.shape(0)) { + throw_Exception("invtrans: vorticity not compatible with divergence.", Here()); + } + if (spdiv.shape(1) != spvor.shape(1)) { + throw_Exception("invtrans: vorticity not compatible with divergence.", Here()); + } + const int nwindfld = compute_nfld(gpwind); + if (nwindfld != 2 * nfld && nwindfld != 3 * nfld) { + throw_Exception("invtrans: wind field is not compatible with vorticity, divergence.", Here()); + } + + if (spdiv.shape(0) != nspec2()) { + std::stringstream msg; + msg << "invtrans: Spectral vorticity and divergence have wrong dimension: " + "nspec2 " + << spdiv.shape(0) << " should be " << nspec2(); + throw_Exception(msg.str(), Here()); + } + + ATLAS_ASSERT(spvor.rank() == 2); + ATLAS_ASSERT(spdiv.rank() == 2); + if (spvor.size() == 0) { + throw_Exception("invtrans: spectral vorticity field is empty."); + } + if (spdiv.size() == 0) { + throw_Exception("invtrans: spectral divergence field is empty."); + } + + // Arrays Trans expects + std::vector rgp(2 * nfld * ngptot()); + std::vector rspvor(nspec2() * nfld); + std::vector rspdiv(nspec2() * nfld); + auto rgpview = LocalView(rgp.data(), make_shape(2 * nfld, ngptot())); + auto rspvorview = LocalView(rspvor.data(), make_shape(nspec2(), nfld)); + auto rspdivview = LocalView(rspdiv.data(), make_shape(nspec2(), nfld)); + + // Pack spectral fields + PackSpectral pack_vor(rspvorview); + pack_vor(spvor); + PackSpectral pack_div(rspdivview); + pack_div(spdiv); + + // Do transform + { + struct ::InvTrans_t transform = ::new_invtrans(trans_.get()); + transform.nvordiv = nfld; + transform.rgp = rgp.data(); + transform.rspvor = rspvor.data(); + transform.rspdiv = rspdiv.data(); + + ATLAS_ASSERT(transform.rspvor); + ATLAS_ASSERT(transform.rspdiv); + TRANS_CHECK(::trans_invtrans(&transform)); + } + + // Unpack the gridpoint fields + { + UnpackNodeColumns unpack(rgpview, gp); + int wind_components = 2; + unpack(gpwind, wind_components); + } +} + +// -------------------------------------------------------------------------------------------- + +void TransIFS::__invtrans_adj(const functionspace::Spectral& sp, Field& spfield, + const functionspace::StructuredColumns& gp, const Field& gpfield, + const eckit::Configuration& config) const { +#if ATLAS_HAVE_ECTRANS || defined(TRANS_HAVE_INVTRANS_ADJ) + + ATLAS_ASSERT(gpfield.functionspace() == 0 || functionspace::StructuredColumns(gpfield.functionspace())); + ATLAS_ASSERT(spfield.functionspace() == 0 || functionspace::Spectral(spfield.functionspace())); + + assertCompatibleDistributions(gp, sp); + + if (compute_nfld(gpfield) != compute_nfld(spfield)) { + throw_Exception("dirtrans: different number of gridpoint fields than spectral fields", Here()); + } + if ((int)gpfield.shape(0) < ngptot()) { + throw_Exception("dirtrans: slowest moving index must be >= ngptot", Here()); + } + const int nfld = compute_nfld(gpfield); + + // Arrays Trans expects + std::vector rgp(nfld * ngptot()); + std::vector rsp(nspec2() * nfld); + auto rgpview = LocalView(rgp.data(), make_shape(nfld, ngptot())); + auto rspview = LocalView(rsp.data(), make_shape(nspec2(), nfld)); + + // Pack gridpoints + { + PackStructuredColumns pack(rgpview); + pack(gp, gpfield); + } + + // Do transform + { + struct ::InvTransAdj_t transform = ::new_invtrans_adj(trans_.get()); + transform.nscalar = nfld; + transform.rgp = rgp.data(); + transform.rspscalar = rsp.data(); + TRANS_CHECK(::trans_invtrans_adj(&transform)); + } + + // Unpack spectral + { + UnpackSpectral unpack(rspview); + unpack(spfield); + } + +#else + ATLAS_NOTIMPLEMENTED; +#endif +} + +// -------------------------------------------------------------------------------------------- + +void TransIFS::__invtrans_adj(const Spectral& sp, Field& spfield, const functionspace::NodeColumns& gp, + const Field& gpfield, const eckit::Configuration& config) const { + FieldSet spfields; + spfields.add(spfield); + FieldSet gpfields; + gpfields.add(gpfield); + __invtrans_adj(sp, spfields, gp, gpfields, config); +} + +// -------------------------------------------------------------------------------------------- + +void TransIFS::__invtrans_adj(const functionspace::Spectral& sp, FieldSet& spfields, + const functionspace::StructuredColumns& gp, const FieldSet& gpfields, + const eckit::Configuration& config) const { +#if ATLAS_HAVE_ECTRANS || defined(TRANS_HAVE_INVTRANS_ADJ) + + assertCompatibleDistributions(gp, sp); + + // Count total number of fields and do sanity checks + const idx_t nfld = compute_nfld(gpfields); + for (idx_t jfld = 0; jfld < gpfields.size(); ++jfld) { + const Field& f = gpfields[jfld]; + ATLAS_ASSERT(f.functionspace() == 0 || functionspace::StructuredColumns(f.functionspace())); + } - assertCompatibleDistributions(gp, sp); + const int trans_sp_nfld = compute_nfld(spfields); - if (compute_nfld(gpfield) != compute_nfld(spfield)) { - throw_Exception("dirtrans: different number of gridpoint fields than spectral fields", Here()); - } - if ((int)gpfield.shape(0) < ngptot()) { - throw_Exception("dirtrans: slowest moving index must be >= ngptot", Here()); + if (nfld != trans_sp_nfld) { + throw_Exception("invtrans_adj: different number of gridpoint fields than spectral fields", Here()); } - const int nfld = compute_nfld(gpfield); // Arrays Trans expects std::vector rgp(nfld * ngptot()); @@ -1601,22 +2049,27 @@ void TransIFS::__invtrans_adj(const functionspace::Spectral& sp, Field& spfield, // Pack gridpoints { PackStructuredColumns pack(rgpview); - pack(gp, gpfield); + for (idx_t jfld = 0; jfld < gpfields.size(); ++jfld) { + pack(gp, gpfields[jfld]); + } } // Do transform { struct ::InvTransAdj_t transform = ::new_invtrans_adj(trans_.get()); - transform.nscalar = nfld; + transform.nscalar = int(nfld); transform.rgp = rgp.data(); transform.rspscalar = rsp.data(); + TRANS_CHECK(::trans_invtrans_adj(&transform)); } - // Unpack spectral + // Unpack the spectral fields { UnpackSpectral unpack(rspview); - unpack(spfield); + for (idx_t jfld = 0; jfld < spfields.size(); ++jfld) { + unpack(spfields[jfld]); + } } #else @@ -1626,81 +2079,104 @@ void TransIFS::__invtrans_adj(const functionspace::Spectral& sp, Field& spfield, // -------------------------------------------------------------------------------------------- -void TransIFS::__invtrans(const functionspace::Spectral& sp, const FieldSet& spfields, - const functionspace::StructuredColumns& gp, FieldSet& gpfields, - const eckit::Configuration& config) const { +void TransIFS::__invtrans_adj(const Spectral& sp, FieldSet& spfields, const functionspace::NodeColumns& gp, + const FieldSet& gpfields, const eckit::Configuration& config) const { +#if ATLAS_HAVE_ECTRANS || defined(TRANS_HAVE_INVTRANS_ADJ) + assertCompatibleDistributions(gp, sp); // Count total number of fields and do sanity checks - const int nfld = compute_nfld(gpfields); + const idx_t nfld = compute_nfld(gpfields); for (idx_t jfld = 0; jfld < gpfields.size(); ++jfld) { const Field& f = gpfields[jfld]; ATLAS_ASSERT(f.functionspace() == 0 || functionspace::StructuredColumns(f.functionspace())); } - const int nb_spectral_fields = compute_nfld(spfields); + const int trans_sp_nfld = compute_nfld(spfields); - if (nfld != nb_spectral_fields) { - std::stringstream msg; - msg << "invtrans: different number of gridpoint fields than spectral fields" - << "[ " << nfld << " != " << nb_spectral_fields << " ]"; - throw_Exception(msg.str(), Here()); + if (nfld != trans_sp_nfld) { + throw_Exception("invtrans_adj: different number of gridpoint fields than spectral fields", Here()); } - // Arrays Trans expects std::vector rgp(nfld * ngptot()); std::vector rsp(nspec2() * nfld); auto rgpview = LocalView(rgp.data(), make_shape(nfld, ngptot())); auto rspview = LocalView(rsp.data(), make_shape(nspec2(), nfld)); - // Pack spectral fields + // Pack gridpoints { - PackSpectral pack(rspview); - for (idx_t jfld = 0; jfld < spfields.size(); ++jfld) { - pack(spfields[jfld]); + PackNodeColumns pack(rgpview, gp); + for (idx_t jfld = 0; jfld < gpfields.size(); ++jfld) { + pack(gpfields[jfld]); } } // Do transform { - struct ::InvTrans_t transform = ::new_invtrans(trans_.get()); - transform.nscalar = nfld; - transform.rgp = rgp.data(); - transform.rspscalar = rsp.data(); + struct ::InvTransAdj_t transform = ::new_invtrans_adj(trans_.get()); + transform.nscalar = int(nfld); + transform.rgp = rgp.data(); + transform.rspscalar = rsp.data(); - TRANS_CHECK(::trans_invtrans(&transform)); + TRANS_CHECK(::trans_invtrans_adj(&transform)); } - // Unpack the gridpoint fields + // Unpack the spectral fields { - UnpackStructuredColumns unpack(rgpview); - for (idx_t jfld = 0; jfld < gpfields.size(); ++jfld) { - unpack(gp, gpfields[jfld]); + UnpackSpectral unpack(rspview); + for (idx_t jfld = 0; jfld < spfields.size(); ++jfld) { + unpack(spfields[jfld]); } } +#else + ATLAS_NOTIMPLEMENTED; +#endif } +//--------------------------------------------------------------------------------------------- -void TransIFS::__invtrans_adj(const functionspace::Spectral& sp, FieldSet& spfields, - const functionspace::StructuredColumns& gp, const FieldSet& gpfields, - const eckit::Configuration& config) const { -#if ATLAS_HAVE_ECTRANS || defined(TRANS_HAVE_INVTRANS_ADJ) +void TransIFS::__invtrans_grad_adj(const Spectral& sp, Field& spfield, + const functionspace::StructuredColumns& gp, + const Field& gradfield, const eckit::Configuration& config) const { + FieldSet spfields; + spfields.add(spfield); + FieldSet gradfields; + gradfields.add(gradfield); + __invtrans_grad_adj(sp, spfields, gp, gradfields, config); +} + +//--------------------------------------------------------------------------------------------- + +void TransIFS::__invtrans_grad_adj(const Spectral& sp, Field& spfield, + const functionspace::NodeColumns& gp, + const Field& gradfield, const eckit::Configuration& config) const { + FieldSet spfields; + spfields.add(spfield); + FieldSet gradfields; + gradfields.add(gradfield); + __invtrans_grad_adj(sp, spfields, gp, gradfields, config); +} + +//--------------------------------------------------------------------------------------------- +void TransIFS::__invtrans_grad_adj(const Spectral& sp, FieldSet& spfields, + const functionspace::StructuredColumns& gp, + const FieldSet& gradfields, const eckit::Configuration& config) const { +#if ATLAS_HAVE_ECTRANS || defined(TRANS_HAVE_INVTRANS_ADJ) assertCompatibleDistributions(gp, sp); // Count total number of fields and do sanity checks - const idx_t nfld = compute_nfld(gpfields); - for (idx_t jfld = 0; jfld < gpfields.size(); ++jfld) { - const Field& f = gpfields[jfld]; + const idx_t nfld = compute_nfld(gradfields); + for (idx_t jfld = 0; jfld < gradfields.size(); ++jfld) { + const Field& f = gradfields[jfld]; ATLAS_ASSERT(f.functionspace() == 0 || functionspace::StructuredColumns(f.functionspace())); } const int trans_sp_nfld = compute_nfld(spfields); if (nfld != trans_sp_nfld) { - throw_Exception("invtrans_adj: different number of gridpoint fields than spectral fields", Here()); + throw_Exception("__invtrans_grad_adj: different number of gridpoint fields than spectral fields", Here()); } - // Arrays Trans expects std::vector rgp(nfld * ngptot()); std::vector rsp(nspec2() * nfld); @@ -1710,17 +2186,18 @@ void TransIFS::__invtrans_adj(const functionspace::Spectral& sp, FieldSet& spfie // Pack gridpoints { PackStructuredColumns pack(rgpview); - for (idx_t jfld = 0; jfld < gpfields.size(); ++jfld) { - pack(gp, gpfields[jfld]); + for (idx_t jfld = 0; jfld < gradfields.size(); ++jfld) { + pack(gp, gradfields[jfld]); } } // Do transform { struct ::InvTransAdj_t transform = ::new_invtrans_adj(trans_.get()); - transform.nscalar = int(nfld); + transform.nscalar = nfld; transform.rgp = rgp.data(); transform.rspscalar = rsp.data(); + transform.lscalarders = true; TRANS_CHECK(::trans_invtrans_adj(&transform)); } @@ -1732,115 +2209,103 @@ void TransIFS::__invtrans_adj(const functionspace::Spectral& sp, FieldSet& spfie unpack(spfields[jfld]); } } - #else ATLAS_NOTIMPLEMENTED; #endif } +//--------------------------------------------------------------------------------------------- -// ----------------------------------------------------------------------------------------------- - -void TransIFS::__dirtrans_wind2vordiv(const functionspace::NodeColumns& gp, const Field& gpwind, const Spectral& sp, - Field& spvor, Field& spdiv, const eckit::Configuration&) const { +void TransIFS::__invtrans_grad_adj(const Spectral& sp, FieldSet& spfields, const functionspace::NodeColumns& gp, + const FieldSet& gradfields, const eckit::Configuration& config) const { +#if ATLAS_HAVE_ECTRANS || defined(TRANS_HAVE_INVTRANS_ADJ) assertCompatibleDistributions(gp, sp); // Count total number of fields and do sanity checks - const size_t nfld = compute_nfld(spvor); - if (spdiv.shape(0) != spvor.shape(0)) { - throw_Exception("invtrans: vorticity not compatible with divergence.", Here()); - } - if (spdiv.shape(1) != spvor.shape(1)) { - throw_Exception("invtrans: vorticity not compatible with divergence.", Here()); - } - const size_t nwindfld = compute_nfld(gpwind); - if (nwindfld != 2 * nfld && nwindfld != 3 * nfld) { - throw_Exception("dirtrans: wind field is not compatible with vorticity, divergence.", Here()); + const idx_t nfld = compute_nfld(gradfields); + for (idx_t jfld = 0; jfld < gradfields.size(); ++jfld) { + const Field& f = gradfields[jfld]; + ATLAS_ASSERT(f.functionspace() == 0 || functionspace::StructuredColumns(f.functionspace())); } - if (spdiv.shape(0) != nspec2()) { - std::stringstream msg; - msg << "dirtrans: Spectral vorticity and divergence have wrong dimension: " - "nspec2 " - << spdiv.shape(0) << " should be " << nspec2(); - throw_Exception(msg.str(), Here()); - } + const int trans_sp_nfld = compute_nfld(spfields); - if (spvor.size() == 0) { - throw_Exception("dirtrans: spectral vorticity field is empty."); - } - if (spdiv.size() == 0) { - throw_Exception("dirtrans: spectral divergence field is empty."); + if (nfld != trans_sp_nfld) { + throw_Exception("dirtrans: different number of gridpoint fields than spectral fields", Here()); } - // Arrays Trans expects - std::vector rgp(2 * nfld * ngptot()); - std::vector rspvor(nspec2() * nfld); - std::vector rspdiv(nspec2() * nfld); - auto rgpview = LocalView(rgp.data(), make_shape(2 * nfld, ngptot())); - auto rspvorview = LocalView(rspvor.data(), make_shape(nspec2(), nfld)); - auto rspdivview = LocalView(rspdiv.data(), make_shape(nspec2(), nfld)); + std::vector rgp(nfld * ngptot()); + std::vector rsp(nspec2() * nfld); + auto rgpview = LocalView(rgp.data(), make_shape(nfld, ngptot())); + auto rspview = LocalView(rsp.data(), make_shape(nspec2(), nfld)); // Pack gridpoints { PackNodeColumns pack(rgpview, gp); - int wind_components = 2; - pack(gpwind, wind_components); + for (idx_t jfld = 0; jfld < gradfields.size(); ++jfld) { + pack(gradfields[jfld]); + } } // Do transform { - struct ::DirTrans_t transform = ::new_dirtrans(trans_.get()); - transform.nvordiv = int(nfld); - transform.rgp = rgp.data(); - transform.rspvor = rspvor.data(); - transform.rspdiv = rspdiv.data(); + struct ::InvTransAdj_t transform = ::new_invtrans_adj(trans_.get()); + transform.nscalar = nfld; + transform.rgp = rgp.data(); + transform.rspscalar = rsp.data(); + transform.lscalarders = true; - ATLAS_ASSERT(transform.rspvor); - ATLAS_ASSERT(transform.rspdiv); - TRANS_CHECK(::trans_dirtrans(&transform)); + TRANS_CHECK(::trans_invtrans_adj(&transform)); } - // Pack spectral fields - UnpackSpectral unpack_vor(rspvorview); - unpack_vor(spvor); - UnpackSpectral unpack_div(rspdivview); - unpack_div(spdiv); + // Unpack the spectral fields + { + UnpackSpectral unpack(rspview); + for (idx_t jfld = 0; jfld < spfields.size(); ++jfld) { + unpack(spfields[jfld]); + } + } +#else + ATLAS_NOTIMPLEMENTED; +#endif } -void TransIFS::__invtrans_vordiv2wind(const Spectral& sp, const Field& spvor, const Field& spdiv, - const functionspace::NodeColumns& gp, Field& gpwind, - const eckit::Configuration&) const { +// -------------------------------------------------------------------------------------------- + +void TransIFS::__invtrans_vordiv2wind_adj(const Spectral& sp, Field& spvor, Field& spdiv, + const functionspace::StructuredColumns& gp, + const Field& gpwind, + const eckit::Configuration&) const { +#if ATLAS_HAVE_ECTRANS || defined(TRANS_HAVE_INVTRANS_ADJ) + assertCompatibleDistributions(gp, sp); // Count total number of fields and do sanity checks - const int nfld = compute_nfld(spvor); + const size_t nfld = compute_nfld(spvor); if (spdiv.shape(0) != spvor.shape(0)) { - throw_Exception("invtrans: vorticity not compatible with divergence.", Here()); + throw_Exception("invtrans_vordiv2wind_adj: vorticity not compatible with divergence.", Here()); } if (spdiv.shape(1) != spvor.shape(1)) { - throw_Exception("invtrans: vorticity not compatible with divergence.", Here()); + throw_Exception("invtrans_vordiv2wind_adj: vorticity not compatible with divergence.", Here()); } - const int nwindfld = compute_nfld(gpwind); + const size_t nwindfld = compute_nfld(gpwind); if (nwindfld != 2 * nfld && nwindfld != 3 * nfld) { - throw_Exception("invtrans: wind field is not compatible with vorticity, divergence.", Here()); + throw_Exception("dirtrans: wind field is not compatible with vorticity, divergence.", Here()); } if (spdiv.shape(0) != nspec2()) { std::stringstream msg; - msg << "invtrans: Spectral vorticity and divergence have wrong dimension: " + msg << "invtrans_vordiv2wind_adj: Spectral vorticity and divergence have wrong dimension: " "nspec2 " << spdiv.shape(0) << " should be " << nspec2(); throw_Exception(msg.str(), Here()); } - ATLAS_ASSERT(spvor.rank() == 2); - ATLAS_ASSERT(spdiv.rank() == 2); if (spvor.size() == 0) { - throw_Exception("invtrans: spectral vorticity field is empty."); + throw_Exception("invtrans_vordiv2wind_adj: spectral vorticity field is empty."); } if (spdiv.size() == 0) { - throw_Exception("invtrans: spectral divergence field is empty."); + throw_Exception("invtrans_vordiv2wind_adj: spectral divergence field is empty."); } // Arrays Trans expects @@ -1851,31 +2316,35 @@ void TransIFS::__invtrans_vordiv2wind(const Spectral& sp, const Field& spvor, co auto rspvorview = LocalView(rspvor.data(), make_shape(nspec2(), nfld)); auto rspdivview = LocalView(rspdiv.data(), make_shape(nspec2(), nfld)); - // Pack spectral fields - PackSpectral pack_vor(rspvorview); - pack_vor(spvor); - PackSpectral pack_div(rspdivview); - pack_div(spdiv); + // Pack gridpoints + { + PackStructuredColumns pack(rgpview); + int wind_components = 2; + pack(gp, gpwind, wind_components); + } // Do transform { - struct ::InvTrans_t transform = ::new_invtrans(trans_.get()); - transform.nvordiv = nfld; - transform.rgp = rgp.data(); - transform.rspvor = rspvor.data(); - transform.rspdiv = rspdiv.data(); + struct ::InvTransAdj_t transform = ::new_invtrans_adj(trans_.get()); + transform.nvordiv = int(nfld); + transform.rgp = rgp.data(); + transform.rspvor = rspvor.data(); + transform.rspdiv = rspdiv.data(); ATLAS_ASSERT(transform.rspvor); ATLAS_ASSERT(transform.rspdiv); - TRANS_CHECK(::trans_invtrans(&transform)); + TRANS_CHECK(::trans_invtrans_adj(&transform)); } - // Unpack the gridpoint fields - { - UnpackNodeColumns unpack(rgpview, gp); - int wind_components = 2; - unpack(gpwind, wind_components); - } + // Pack spectral fields + UnpackSpectral unpack_vor(rspvorview); + unpack_vor(spvor); + UnpackSpectral unpack_div(rspdivview); + unpack_div(spdiv); + +#else + ATLAS_NOTIMPLEMENTED; +#endif } void TransIFS::__invtrans_vordiv2wind_adj(const Spectral& sp, Field& spvor, Field& spdiv, diff --git a/src/atlas/trans/ifs/TransIFS.h b/src/atlas/trans/ifs/TransIFS.h index 377aa1738..44a447e1a 100644 --- a/src/atlas/trans/ifs/TransIFS.h +++ b/src/atlas/trans/ifs/TransIFS.h @@ -235,65 +235,76 @@ class TransIFS : public trans::TransImpl { public: void __dirtrans(const functionspace::StructuredColumns&, const Field& gpfield, const functionspace::Spectral&, Field& spfield, const eckit::Configuration& = util::NoConfig()) const; - void __dirtrans(const functionspace::StructuredColumns&, const FieldSet& gpfields, const functionspace::Spectral&, - FieldSet& spfields, const eckit::Configuration& = util::NoConfig()) const; - void __dirtrans(const functionspace::NodeColumns&, const Field& gpfield, const functionspace::Spectral&, Field& spfield, const eckit::Configuration& = util::NoConfig()) const; + + void __dirtrans(const functionspace::StructuredColumns&, const FieldSet& gpfields, const functionspace::Spectral&, + FieldSet& spfields, const eckit::Configuration& = util::NoConfig()) const; void __dirtrans(const functionspace::NodeColumns&, const FieldSet& gpfields, const functionspace::Spectral&, FieldSet& spfields, const eckit::Configuration& = util::NoConfig()) const; + void __dirtrans_wind2vordiv(const functionspace::StructuredColumns&, const Field& gpwind, const functionspace::Spectral&, + Field& spvor, Field& spdiv, const eckit::Configuration& = util::NoConfig()) const; void __dirtrans_wind2vordiv(const functionspace::NodeColumns&, const Field& gpwind, const functionspace::Spectral&, Field& spvor, Field& spdiv, const eckit::Configuration& = util::NoConfig()) const; void __invtrans(const functionspace::Spectral&, const Field& spfield, const functionspace::StructuredColumns&, Field& gpfield, const eckit::Configuration& = util::NoConfig()) const; - - void __invtrans(const functionspace::Spectral&, const FieldSet& spfields, const functionspace::StructuredColumns&, - FieldSet& gpfields, const eckit::Configuration& = util::NoConfig()) const; - void __invtrans(const functionspace::Spectral&, const Field& spfield, const functionspace::NodeColumns&, Field& gpfield, const eckit::Configuration& = util::NoConfig()) const; + void __invtrans(const functionspace::Spectral&, const FieldSet& spfields, const functionspace::StructuredColumns&, + FieldSet& gpfields, const eckit::Configuration& = util::NoConfig()) const; void __invtrans(const functionspace::Spectral&, const FieldSet& spfields, const functionspace::NodeColumns&, FieldSet& gpfields, const eckit::Configuration& = util::NoConfig()) const; - void __invtrans_vordiv2wind(const functionspace::Spectral&, const Field& spvor, const Field& spdiv, - const functionspace::NodeColumns&, Field& gpwind, - const eckit::Configuration& = util::NoConfig()) const; - + void __invtrans_grad(const functionspace::Spectral& sp, const Field& spfield, const functionspace::StructuredColumns& gp, + Field& gradfield, const eckit::Configuration& = util::NoConfig()) const; void __invtrans_grad(const functionspace::Spectral& sp, const Field& spfield, const functionspace::NodeColumns& gp, Field& gradfield, const eckit::Configuration& = util::NoConfig()) const; + void __invtrans_grad(const functionspace::Spectral& sp, const FieldSet& spfields, + const functionspace::StructuredColumns& gp, FieldSet& gradfields, + const eckit::Configuration& = util::NoConfig()) const; void __invtrans_grad(const functionspace::Spectral& sp, const FieldSet& spfields, const functionspace::NodeColumns& gp, FieldSet& gradfields, const eckit::Configuration& = util::NoConfig()) const; + void __invtrans_vordiv2wind(const functionspace::Spectral&, const Field& spvor, const Field& spdiv, + const functionspace::StructuredColumns&, Field& gpwind, + const eckit::Configuration& = util::NoConfig()) const; + void __invtrans_vordiv2wind(const functionspace::Spectral&, const Field& spvor, const Field& spdiv, + const functionspace::NodeColumns&, Field& gpwind, + const eckit::Configuration& = util::NoConfig()) const; void __invtrans_adj(const functionspace::Spectral&, Field& spfield, const functionspace::StructuredColumns&, const Field& gpfield, const eckit::Configuration& = util::NoConfig()) const; - - void __invtrans_adj(const functionspace::Spectral&, FieldSet& spfields, const functionspace::StructuredColumns&, - const FieldSet& gpfields, const eckit::Configuration& = util::NoConfig()) const; - void __invtrans_adj(const functionspace::Spectral&, Field& spfield, const functionspace::NodeColumns&, const Field& gpfield, const eckit::Configuration& = util::NoConfig()) const; + void __invtrans_adj(const functionspace::Spectral&, FieldSet& spfields, const functionspace::StructuredColumns&, + const FieldSet& gpfields, const eckit::Configuration& = util::NoConfig()) const; void __invtrans_adj(const functionspace::Spectral&, FieldSet& spfields, const functionspace::NodeColumns&, const FieldSet& gpfields, const eckit::Configuration& = util::NoConfig()) const; - void __invtrans_vordiv2wind_adj(const functionspace::Spectral&, Field& spvor, Field& spdiv, - const functionspace::NodeColumns&, const Field& gpwind, - const eckit::Configuration& = util::NoConfig()) const; - + void __invtrans_grad_adj(const functionspace::Spectral& sp, Field& spfield, const functionspace::StructuredColumns& gp, + const Field& gradfield, const eckit::Configuration& = util::NoConfig()) const; void __invtrans_grad_adj(const functionspace::Spectral& sp, Field& spfield, const functionspace::NodeColumns& gp, const Field& gradfield, const eckit::Configuration& = util::NoConfig()) const; + void __invtrans_grad_adj(const functionspace::Spectral& sp, FieldSet& spfields, + const functionspace::StructuredColumns& gp, const FieldSet& gradfields, + const eckit::Configuration& = util::NoConfig()) const; void __invtrans_grad_adj(const functionspace::Spectral& sp, FieldSet& spfields, const functionspace::NodeColumns& gp, const FieldSet& gradfields, const eckit::Configuration& = util::NoConfig()) const; - + void __invtrans_vordiv2wind_adj(const functionspace::Spectral&, Field& spvor, Field& spdiv, + const functionspace::StructuredColumns&, const Field& gpwind, + const eckit::Configuration& = util::NoConfig()) const; + void __invtrans_vordiv2wind_adj(const functionspace::Spectral&, Field& spvor, Field& spdiv, + const functionspace::NodeColumns&, const Field& gpwind, + const eckit::Configuration& = util::NoConfig()) const; public: void specnorm(const int nb_fields, const double spectra[], double norms[], int rank = 0) const; From 20b0e0de240407aa29629fd9f3a9de47c12317b3 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Thu, 7 Jul 2022 18:45:30 +0100 Subject: [PATCH 33/53] Apply clang-format --- src/atlas/trans/ifs/TransIFS.cc | 61 ++++++++++++++------------------- src/atlas/trans/ifs/TransIFS.h | 16 +++++---- 2 files changed, 36 insertions(+), 41 deletions(-) diff --git a/src/atlas/trans/ifs/TransIFS.cc b/src/atlas/trans/ifs/TransIFS.cc index 21b3fe97d..3f15c2efe 100644 --- a/src/atlas/trans/ifs/TransIFS.cc +++ b/src/atlas/trans/ifs/TransIFS.cc @@ -246,12 +246,12 @@ void TransIFS::invtrans_grad(const Field& spfield, Field& gradfield, const eckit ATLAS_ASSERT(Spectral(spfield.functionspace())); ATLAS_ASSERT(NodeColumns(gradfield.functionspace())); if (StructuredColumns(gradfield.functionspace())) { - __invtrans_grad(Spectral(spfield.functionspace()), spfield, StructuredColumns(gradfield.functionspace()), gradfield, - config); + __invtrans_grad(Spectral(spfield.functionspace()), spfield, StructuredColumns(gradfield.functionspace()), + gradfield, config); } else if (NodeColumns(gradfield.functionspace())) { - __invtrans_grad(Spectral(spfield.functionspace()), spfield, NodeColumns(gradfield.functionspace()), gradfield, - config); + __invtrans_grad(Spectral(spfield.functionspace()), spfield, NodeColumns(gradfield.functionspace()), gradfield, + config); } else { ATLAS_NOTIMPLEMENTED; @@ -263,8 +263,8 @@ void TransIFS::invtrans_grad(const FieldSet& spfields, FieldSet& gradfields, con std::string functionspace(fieldset_functionspace(gradfields)); if (functionspace == StructuredColumns::type()) { - __invtrans_grad(Spectral(spfields[0].functionspace()), spfields, StructuredColumns(gradfields[0].functionspace()), - gradfields, config); + __invtrans_grad(Spectral(spfields[0].functionspace()), spfields, + StructuredColumns(gradfields[0].functionspace()), gradfields, config); } else if (functionspace == NodeColumns::type()) { __invtrans_grad(Spectral(spfields[0].functionspace()), spfields, NodeColumns(gradfields[0].functionspace()), @@ -299,8 +299,8 @@ void TransIFS::invtrans_grad_adj(const FieldSet& gradfields, FieldSet& spfields, std::string functionspace(fieldset_functionspace(gradfields)); if (functionspace == StructuredColumns::type()) { - __invtrans_grad_adj(Spectral(spfields[0].functionspace()), spfields, StructuredColumns(gradfields[0].functionspace()), - gradfields, config); + __invtrans_grad_adj(Spectral(spfields[0].functionspace()), spfields, + StructuredColumns(gradfields[0].functionspace()), gradfields, config); } else if (functionspace == NodeColumns::type()) { __invtrans_grad_adj(Spectral(spfields[0].functionspace()), spfields, NodeColumns(gradfields[0].functionspace()), @@ -318,12 +318,12 @@ void TransIFS::dirtrans_wind2vordiv(const Field& gpwind, Field& spvor, Field& sp ATLAS_ASSERT(Spectral(spvor.functionspace())); ATLAS_ASSERT(Spectral(spdiv.functionspace())); if (StructuredColumns(gpwind.functionspace())) { - __dirtrans_wind2vordiv(StructuredColumns(gpwind.functionspace()), gpwind, Spectral(spvor.functionspace()), spvor, spdiv, - config); + __dirtrans_wind2vordiv(StructuredColumns(gpwind.functionspace()), gpwind, Spectral(spvor.functionspace()), + spvor, spdiv, config); } else if (NodeColumns(gpwind.functionspace())) { - __dirtrans_wind2vordiv(NodeColumns(gpwind.functionspace()), gpwind, Spectral(spvor.functionspace()), spvor, spdiv, - config); + __dirtrans_wind2vordiv(NodeColumns(gpwind.functionspace()), gpwind, Spectral(spvor.functionspace()), spvor, + spdiv, config); } else { ATLAS_NOTIMPLEMENTED; @@ -355,8 +355,8 @@ void TransIFS::invtrans_vordiv2wind_adj(const Field& gpwind, Field& spvor, Field ATLAS_ASSERT(Spectral(spdiv.functionspace())); ATLAS_ASSERT(NodeColumns(gpwind.functionspace())); if (StructuredColumns(gpwind.functionspace())) { - __invtrans_vordiv2wind_adj(Spectral(spvor.functionspace()), spvor, spdiv, StructuredColumns(gpwind.functionspace()), - gpwind, config); + __invtrans_vordiv2wind_adj(Spectral(spvor.functionspace()), spvor, spdiv, + StructuredColumns(gpwind.functionspace()), gpwind, config); } else if (NodeColumns(gpwind.functionspace())) { __invtrans_vordiv2wind_adj(Spectral(spvor.functionspace()), spvor, spdiv, NodeColumns(gpwind.functionspace()), @@ -1336,9 +1336,8 @@ void TransIFS::__dirtrans(const functionspace::NodeColumns& gp, const FieldSet& // -------------------------------------------------------------------------------------------- -void TransIFS::__dirtrans_wind2vordiv(const functionspace::StructuredColumns& gp, - const Field& gpwind, const Spectral& sp, - Field& spvor, Field& spdiv, +void TransIFS::__dirtrans_wind2vordiv(const functionspace::StructuredColumns& gp, const Field& gpwind, + const Spectral& sp, Field& spvor, Field& spdiv, const eckit::Configuration&) const { assertCompatibleDistributions(gp, sp); @@ -1407,8 +1406,7 @@ void TransIFS::__dirtrans_wind2vordiv(const functionspace::StructuredColumns& gp // ----------------------------------------------------------------------------------------------- -void TransIFS::__dirtrans_wind2vordiv(const functionspace::NodeColumns& gp, const Field& gpwind, - const Spectral& sp, +void TransIFS::__dirtrans_wind2vordiv(const functionspace::NodeColumns& gp, const Field& gpwind, const Spectral& sp, Field& spvor, Field& spdiv, const eckit::Configuration&) const { assertCompatibleDistributions(gp, sp); @@ -1658,10 +1656,8 @@ void TransIFS::__invtrans_grad(const Spectral& sp, const Field& spfield, const f // -------------------------------------------------------------------------------------------- -void TransIFS::__invtrans_grad(const Spectral& sp, const FieldSet& spfields, - const functionspace::StructuredColumns& gp, - FieldSet& gradfields, - const eckit::Configuration& config) const { +void TransIFS::__invtrans_grad(const Spectral& sp, const FieldSet& spfields, const functionspace::StructuredColumns& gp, + FieldSet& gradfields, const eckit::Configuration& config) const { assertCompatibleDistributions(gp, sp); // Count total number of fields and do sanity checks @@ -1713,7 +1709,7 @@ void TransIFS::__invtrans_grad(const Spectral& sp, const FieldSet& spfields, auto field = make_view(gradfields[jfld]); for (idx_t jlev = 0; jlev < nlev; ++jlev) { for (idx_t jnode = 0; jnode < nb_nodes; ++jnode) { - field(jnode, jlev, 1 - dim) = rgpview(f, jnode); + field(jnode, jlev, 1 - dim) = rgpview(f, jnode); } } } @@ -1731,8 +1727,7 @@ void TransIFS::__invtrans_grad(const Spectral& sp, const FieldSet& spfields, // -------------------------------------------------------------------------------------------- -void TransIFS::__invtrans_grad(const Spectral& sp, const FieldSet& spfields, - const functionspace::NodeColumns& gp, +void TransIFS::__invtrans_grad(const Spectral& sp, const FieldSet& spfields, const functionspace::NodeColumns& gp, FieldSet& gradfields, const eckit::Configuration& config) const { assertCompatibleDistributions(gp, sp); @@ -2135,8 +2130,7 @@ void TransIFS::__invtrans_adj(const Spectral& sp, FieldSet& spfields, const func //--------------------------------------------------------------------------------------------- -void TransIFS::__invtrans_grad_adj(const Spectral& sp, Field& spfield, - const functionspace::StructuredColumns& gp, +void TransIFS::__invtrans_grad_adj(const Spectral& sp, Field& spfield, const functionspace::StructuredColumns& gp, const Field& gradfield, const eckit::Configuration& config) const { FieldSet spfields; spfields.add(spfield); @@ -2147,8 +2141,7 @@ void TransIFS::__invtrans_grad_adj(const Spectral& sp, Field& spfield, //--------------------------------------------------------------------------------------------- -void TransIFS::__invtrans_grad_adj(const Spectral& sp, Field& spfield, - const functionspace::NodeColumns& gp, +void TransIFS::__invtrans_grad_adj(const Spectral& sp, Field& spfield, const functionspace::NodeColumns& gp, const Field& gradfield, const eckit::Configuration& config) const { FieldSet spfields; spfields.add(spfield); @@ -2159,8 +2152,7 @@ void TransIFS::__invtrans_grad_adj(const Spectral& sp, Field& spfield, //--------------------------------------------------------------------------------------------- -void TransIFS::__invtrans_grad_adj(const Spectral& sp, FieldSet& spfields, - const functionspace::StructuredColumns& gp, +void TransIFS::__invtrans_grad_adj(const Spectral& sp, FieldSet& spfields, const functionspace::StructuredColumns& gp, const FieldSet& gradfields, const eckit::Configuration& config) const { #if ATLAS_HAVE_ECTRANS || defined(TRANS_HAVE_INVTRANS_ADJ) assertCompatibleDistributions(gp, sp); @@ -2273,9 +2265,8 @@ void TransIFS::__invtrans_grad_adj(const Spectral& sp, FieldSet& spfields, const // -------------------------------------------------------------------------------------------- void TransIFS::__invtrans_vordiv2wind_adj(const Spectral& sp, Field& spvor, Field& spdiv, - const functionspace::StructuredColumns& gp, - const Field& gpwind, - const eckit::Configuration&) const { + const functionspace::StructuredColumns& gp, const Field& gpwind, + const eckit::Configuration&) const { #if ATLAS_HAVE_ECTRANS || defined(TRANS_HAVE_INVTRANS_ADJ) assertCompatibleDistributions(gp, sp); diff --git a/src/atlas/trans/ifs/TransIFS.h b/src/atlas/trans/ifs/TransIFS.h index 44a447e1a..7fff8eed7 100644 --- a/src/atlas/trans/ifs/TransIFS.h +++ b/src/atlas/trans/ifs/TransIFS.h @@ -243,8 +243,9 @@ class TransIFS : public trans::TransImpl { void __dirtrans(const functionspace::NodeColumns&, const FieldSet& gpfields, const functionspace::Spectral&, FieldSet& spfields, const eckit::Configuration& = util::NoConfig()) const; - void __dirtrans_wind2vordiv(const functionspace::StructuredColumns&, const Field& gpwind, const functionspace::Spectral&, - Field& spvor, Field& spdiv, const eckit::Configuration& = util::NoConfig()) const; + void __dirtrans_wind2vordiv(const functionspace::StructuredColumns&, const Field& gpwind, + const functionspace::Spectral&, Field& spvor, Field& spdiv, + const eckit::Configuration& = util::NoConfig()) const; void __dirtrans_wind2vordiv(const functionspace::NodeColumns&, const Field& gpwind, const functionspace::Spectral&, Field& spvor, Field& spdiv, const eckit::Configuration& = util::NoConfig()) const; @@ -258,8 +259,9 @@ class TransIFS : public trans::TransImpl { void __invtrans(const functionspace::Spectral&, const FieldSet& spfields, const functionspace::NodeColumns&, FieldSet& gpfields, const eckit::Configuration& = util::NoConfig()) const; - void __invtrans_grad(const functionspace::Spectral& sp, const Field& spfield, const functionspace::StructuredColumns& gp, - Field& gradfield, const eckit::Configuration& = util::NoConfig()) const; + void __invtrans_grad(const functionspace::Spectral& sp, const Field& spfield, + const functionspace::StructuredColumns& gp, Field& gradfield, + const eckit::Configuration& = util::NoConfig()) const; void __invtrans_grad(const functionspace::Spectral& sp, const Field& spfield, const functionspace::NodeColumns& gp, Field& gradfield, const eckit::Configuration& = util::NoConfig()) const; @@ -287,8 +289,9 @@ class TransIFS : public trans::TransImpl { void __invtrans_adj(const functionspace::Spectral&, FieldSet& spfields, const functionspace::NodeColumns&, const FieldSet& gpfields, const eckit::Configuration& = util::NoConfig()) const; - void __invtrans_grad_adj(const functionspace::Spectral& sp, Field& spfield, const functionspace::StructuredColumns& gp, - const Field& gradfield, const eckit::Configuration& = util::NoConfig()) const; + void __invtrans_grad_adj(const functionspace::Spectral& sp, Field& spfield, + const functionspace::StructuredColumns& gp, const Field& gradfield, + const eckit::Configuration& = util::NoConfig()) const; void __invtrans_grad_adj(const functionspace::Spectral& sp, Field& spfield, const functionspace::NodeColumns& gp, const Field& gradfield, const eckit::Configuration& = util::NoConfig()) const; @@ -305,6 +308,7 @@ class TransIFS : public trans::TransImpl { void __invtrans_vordiv2wind_adj(const functionspace::Spectral&, Field& spvor, Field& spdiv, const functionspace::NodeColumns&, const Field& gpwind, const eckit::Configuration& = util::NoConfig()) const; + public: void specnorm(const int nb_fields, const double spectra[], double norms[], int rank = 0) const; From 9eb3d7e5a7c91d5abf3f8add23000d08afca829d Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Fri, 8 Jul 2022 12:03:04 +0100 Subject: [PATCH 34/53] Add Fortran API to PointCloud constructor, accepting ghost field --- .../functionspace/detail/PointCloudInterface.cc | 6 ++++++ src/atlas/functionspace/detail/PointCloudInterface.h | 3 +++ .../atlas_functionspace_PointCloud_module.F90 | 12 ++++++++++++ 3 files changed, 21 insertions(+) diff --git a/src/atlas/functionspace/detail/PointCloudInterface.cc b/src/atlas/functionspace/detail/PointCloudInterface.cc index 840f8a54a..fe7d53e69 100644 --- a/src/atlas/functionspace/detail/PointCloudInterface.cc +++ b/src/atlas/functionspace/detail/PointCloudInterface.cc @@ -30,6 +30,12 @@ const detail::PointCloud* atlas__functionspace__PointCloud__new__lonlat(const Fi return new detail::PointCloud(Field(lonlat)); } +const detail::PointCloud* atlas__functionspace__PointCloud__new__lonlat_ghost(const field::FieldImpl* lonlat, + const field::FieldImpl* ghost) { + return new detail::PointCloud(Field(lonlat), Field(ghost)); +} + + const detail::PointCloud* atlas__functionspace__PointCloud__new__grid(const Grid::Implementation* grid) { return new detail::PointCloud(Grid(grid)); } diff --git a/src/atlas/functionspace/detail/PointCloudInterface.h b/src/atlas/functionspace/detail/PointCloudInterface.h index 3123dd74d..34f0cacf1 100644 --- a/src/atlas/functionspace/detail/PointCloudInterface.h +++ b/src/atlas/functionspace/detail/PointCloudInterface.h @@ -38,6 +38,9 @@ extern "C" { const detail::PointCloud* atlas__functionspace__PointCloud__new__lonlat(const field::FieldImpl* lonlat); +const detail::PointCloud* atlas__functionspace__PointCloud__new__lonlat_ghost(const field::FieldImpl* lonlat, + const field::FieldImpl* ghost); + const detail::PointCloud* atlas__functionspace__PointCloud__new__grid(const GridImpl* grid); void atlas__functionspace__PointCloud__delete(detail::PointCloud* This); diff --git a/src/atlas_f/functionspace/atlas_functionspace_PointCloud_module.F90 b/src/atlas_f/functionspace/atlas_functionspace_PointCloud_module.F90 index 59f46b089..f30896ac1 100644 --- a/src/atlas_f/functionspace/atlas_functionspace_PointCloud_module.F90 +++ b/src/atlas_f/functionspace/atlas_functionspace_PointCloud_module.F90 @@ -53,6 +53,7 @@ module atlas_functionspace_PointCloud_module interface atlas_functionspace_PointCloud module procedure ctor_cptr module procedure ctor_lonlat + module procedure ctor_lonlat_ghost module procedure ctor_grid end interface @@ -84,6 +85,17 @@ function ctor_lonlat(lonlat) result(this) !------------------------------------------------------------------------------ +function ctor_lonlat_ghost(lonlat,ghost) result(this) + use atlas_functionspace_PointCloud_c_binding + type(atlas_functionspace_PointCloud) :: this + class(atlas_Field), intent(in) :: lonlat + class(atlas_Field), intent(in) :: ghost + call this%reset_c_ptr( atlas__functionspace__PointCloud__new__lonlat_ghost( lonlat%CPTR_PGIBUG_A, ghost%CPTR_PGIBUG_A ) ) + call this%return() +end function + +!------------------------------------------------------------------------------ + function ctor_grid(grid) result(this) use atlas_functionspace_PointCloud_c_binding type(atlas_functionspace_PointCloud) :: this From 4c142d7cc9cd8ab18af76e85812191e6dd4a7031 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 19 Jul 2022 11:10:01 +0000 Subject: [PATCH 35/53] Tool to generate sandbox project --- tools/generate-sandbox-project.sh | 151 ++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100755 tools/generate-sandbox-project.sh diff --git a/tools/generate-sandbox-project.sh b/tools/generate-sandbox-project.sh new file mode 100755 index 000000000..ea80b0cf6 --- /dev/null +++ b/tools/generate-sandbox-project.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash + +# (C) Copyright 2022 ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. + + +set -e + +function help { +echo "$(basename ${BASH_SOURCE[0]}) --project PROJECT_DIR [--cxx] [--fortran] [--build]" +echo "" +echo "DECRIPTION" +echo " Tool that generates a simple CMake project (not with ecbuild)" +echo " with a \"main\" program that is linked to atlas." +echo " Note: It is intended for quickly prototyping or testing atlas features." +echo " For more complicated projects, please see" +echo " https://sites.ecmwf.int/docs/atlas/getting_started/linking" +echo "" +echo "OPTIONS" +echo " --help Print this help" +echo " --project Path of project directory to be generated" +echo " --c++ , --cxx Create C++ project (default if not specified)" +echo " --fortran Create Fortran project instead of C++" +echo " --build Attempt to build template project in 'build'" +echo " subdirectory of generated project" +} + +LANGUAGES=C +WITH_CXX=false +WITH_Fortran=false +WITH_build=false + +if [ $# -eq 0 ]; then + help + exit 1 +fi + +while test $# -gt 0; do + + case "$1" in + --project) + PROJECT_DIR=$2 + PROJECT_NAME=$(basename $PROJECT_DIR) + shift # past argument + shift # past value + ;; + --c++|--cxx) + WITH_CXX=true + LANGUAGES="${LANGUAGES} CXX" + shift + ;; + --fortran) + WITH_Fortran=true + LANGUAGES="${LANGUAGES} Fortran" + shift + ;; + --build) + WITH_build=true + shift + ;; + --help|-h) + help + shift + ;; + *) + # unknown option + help + break + ;; + esac +done + +if ( $WITH_Fortran && $WITH_CXX ) ; then + echo "ERROR: Both --cxx and --fortran were supplied. Choose one only." + exit 1 +fi + +if ( ! $WITH_Fortran && ! $WITH_CXX ) ; then + echo "No language specified. Defaulting to use C++." + WITH_CXX=true +fi + +mkdir -p $PROJECT_DIR +cd $PROJECT_DIR +cat << EOF > CMakeLists.txt +cmake_minimum_required( VERSION 3.18 ) + +project( ${PROJECT_NAME} VERSION 0.0.0 LANGUAGES ${LANGUAGES} ) + +find_package( atlas REQUIRED ) + +EOF +if ( $WITH_CXX ) ; then + cat <<- EOF >> CMakeLists.txt + add_executable( main ) + target_sources( main PUBLIC main.cc ) + target_link_libraries( main PUBLIC atlas ) + EOF + cat <<- EOF > main.cc + #include "atlas/library.h" + #include "atlas/runtime/Log.h" + int main(int argc, char* argv[]) { + atlas::initialize(argc,argv); + atlas::Log::info() << "Hello from C++" << std::endl; + atlas::finalize(); + return 0; + } + EOF +elif ( $WITH_Fortran ) ; then + cat <<- EOF >> CMakeLists.txt + add_executable( main ) + target_sources( main PUBLIC main.F90 ) + target_link_libraries( main PUBLIC atlas_f ) + EOF + cat <<- EOF > main.F90 + program main + use atlas_module + implicit none + call atlas_initialize() + call atlas_log%info("Hello from Fortran") + call atlas_finalize() + end program + EOF +fi + +echo "Template project ${PROJECT_NAME} has been created in ${PROJECT_DIR}" + +if ( $WITH_build ) ; then + mkdir -p build + cd build + + echo "" + echo "Project ${PROJECT_NAME} is being configured in $(pwd)" + + cmake .. + + echo "" + echo "Project ${PROJECT_NAME} is being built in $(pwd)" + + make VERBOSE=1 + + echo "" + echo "Running program main in $(pwd)" + echo "" + ./main +fi From a6bc31ca8e7f0fd4b354d98c3f5fb7f34efc66d2 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 19 Jul 2022 13:12:56 +0100 Subject: [PATCH 36/53] Fixup previous commit --- tools/generate-sandbox-project.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/generate-sandbox-project.sh b/tools/generate-sandbox-project.sh index ea80b0cf6..76961ac40 100755 --- a/tools/generate-sandbox-project.sh +++ b/tools/generate-sandbox-project.sh @@ -51,12 +51,12 @@ while test $# -gt 0; do ;; --c++|--cxx) WITH_CXX=true - LANGUAGES="${LANGUAGES} CXX" + LANGUAGES="${LANGUAGES} CXX" shift ;; --fortran) WITH_Fortran=true - LANGUAGES="${LANGUAGES} Fortran" + LANGUAGES="${LANGUAGES} Fortran" shift ;; --build) @@ -69,7 +69,7 @@ while test $# -gt 0; do ;; *) # unknown option - help + help break ;; esac @@ -83,6 +83,7 @@ fi if ( ! $WITH_Fortran && ! $WITH_CXX ) ; then echo "No language specified. Defaulting to use C++." WITH_CXX=true + LANGUAGES="${LANGUAGES} CXX" fi mkdir -p $PROJECT_DIR @@ -95,6 +96,7 @@ project( ${PROJECT_NAME} VERSION 0.0.0 LANGUAGES ${LANGUAGES} ) find_package( atlas REQUIRED ) EOF + if ( $WITH_CXX ) ; then cat <<- EOF >> CMakeLists.txt add_executable( main ) From d0af5085b47bf31d2d2c5c3d8a9f7e60dece7ec9 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Thu, 21 Jul 2022 17:00:28 +0100 Subject: [PATCH 37/53] ATLAS-365 Fix use of ATLAS_LINALG_DENSE_BACKEND --- src/atlas/linalg/dense/Backend.cc | 38 ++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/src/atlas/linalg/dense/Backend.cc b/src/atlas/linalg/dense/Backend.cc index f72a43611..0c58a7a36 100644 --- a/src/atlas/linalg/dense/Backend.cc +++ b/src/atlas/linalg/dense/Backend.cc @@ -32,6 +32,24 @@ namespace linalg { namespace dense { namespace { + +util::Config to_config(const std::string& type) { + util::Config b; + std::vector tokens; + eckit::Tokenizer{'.'}(type, tokens); + ATLAS_ASSERT(tokens.size() <= 2); + ATLAS_ASSERT(tokens.size() > 0); + if (tokens.size() == 1) { + b.set("type", type); + } + else { + util::Config b; + b.set("type", tokens[0]); + b.set("backend", tokens[1]); + } + return b; +} + struct backends { std::map map_; std::string current_backend_; @@ -44,19 +62,9 @@ struct backends { void set(const std::string& current_backend) { current_backend_ = current_backend; } dense::Backend& get(const std::string& type) { + ATLAS_ASSERT(!type.empty()); if (map_.find(type) == map_.end()) { - std::vector tokens; - eckit::Tokenizer{'.'}(type, tokens); - ATLAS_ASSERT(tokens.size() <= 2); - if (tokens.size() == 1) { - map_.emplace(type, util::Config("type", type)); - } - else { - util::Config b; - b.set("type", tokens[0]); - b.set("backend", tokens[1]); - map_.emplace(type, b); - } + map_.emplace(type, to_config(type)); } return map_[type]; } @@ -82,8 +90,12 @@ struct backends { else { current_backend_ = backend::eckit_linalg::type(); } + map_.emplace("default", util::Config("type", current_backend_)); + } + else { + current_backend_ = configured; + map_.emplace("default", to_config(configured)); } - map_.emplace("default", util::Config("type", current_backend_)); } }; } // namespace From aacf0f4d4d638284a68df065be4694e4062e4382 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Thu, 21 Jul 2022 17:02:51 +0100 Subject: [PATCH 38/53] ATLAS-366 Add debug output to TransLocal --- src/atlas/trans/local/TransLocal.cc | 81 +++++++++++++++++++++++++---- 1 file changed, 71 insertions(+), 10 deletions(-) diff --git a/src/atlas/trans/local/TransLocal.cc b/src/atlas/trans/local/TransLocal.cc index 2ef699c1f..4eb5c790b 100644 --- a/src/atlas/trans/local/TransLocal.cc +++ b/src/atlas/trans/local/TransLocal.cc @@ -18,6 +18,7 @@ #include "eckit/config/YAMLConfiguration.h" #include "eckit/eckit.h" #include "eckit/io/DataHandle.h" +#include "eckit/linalg/LinearAlgebra.h" #include "eckit/log/Bytes.h" #include "eckit/log/JSON.h" #include "eckit/types/FloatCompare.h" @@ -88,11 +89,10 @@ class TransParameters { static const std::map string_to_FFT = {{"OFF", static_cast(option::FFT::OFF)}, {"FFTW", static_cast(option::FFT::FFTW)}}; #ifdef ATLAS_HAVE_FFTW - std::string fft_default = "FFTW"; + return string_to_FFT.at(config_.getString("fft", "FFTW")); #else - std::string fft_default = "OFF"; + return string_to_FFT.at("OFF"); #endif - return string_to_FFT.at(config_.getString("fft", fft_default)); } std::string matrix_multiply() const { return config_.getString("matrix_multiply", ""); } @@ -208,7 +208,7 @@ void alloc_aligned(double*& ptr, size_t n, const char* msg) { void free_aligned(double*& ptr, const char* msg) { ATLAS_ASSERT(msg); - Log::debug() << "TransLocal: dellocating '" << msg << "'" << std::endl; + Log::debug() << "TransLocal: deallocating '" << msg << "'" << std::endl; free_aligned(ptr); } @@ -216,6 +216,36 @@ size_t add_padding(size_t n) { return size_t(std::ceil(n / 8.)) * 8; } +std::string detect_linalg_backend(const std::string& linalg_backend_) { + linalg::dense::Backend linalg_backend = linalg::dense::Backend{linalg_backend_}; + if (linalg_backend.type() == linalg::dense::backend::eckit_linalg::type()) { + std::string backend; + linalg_backend.get("backend", backend); + if (backend.empty() || backend == "default") { +#if ATLAS_ECKIT_HAVE_ECKIT_585 + return eckit::linalg::LinearAlgebraDense::backend().name(); +#else + return eckit::linalg::LinearAlgebra::backend().name(); +#endif + } + return backend; + } + return linalg_backend.type(); +}; + +bool using_eckit_default_backend(const std::string& linalg_backend_) { + linalg::dense::Backend linalg_backend = linalg::dense::Backend{linalg_backend_}; + if (linalg_backend.type() == linalg::dense::backend::eckit_linalg::type()) { + std::string backend; + linalg_backend.get("backend", backend); + if (backend.empty() || backend == "default") { + return true; + } + } + return false; +}; + + } // namespace int fourier_truncation(const int truncation, // truncation @@ -283,6 +313,11 @@ TransLocal::TransLocal(const Cache& cache, const Grid& grid, const Domain& domai linalg_backend_(TransParameters{config}.matrix_multiply()), warning_(TransParameters{config}.warning()) { ATLAS_TRACE("TransLocal constructor"); + + if (mpi::size() > 1) { + ATLAS_THROW_EXCEPTION("TransLocal is not implemented for more than 1 MPI task."); + } + double fft_threshold = 0.0; // fraction of latitudes of the full grid down to which FFT is used. // This threshold needs to be adjusted depending on the dgemm and FFT performance of the machine // on which this code is running! @@ -495,6 +530,30 @@ TransLocal::TransLocal(const Cache& cache, const Grid& grid, const Domain& domai } Log::info() << std::endl;*/ + + Log::debug() << "TransLocal set up with:\n" + << " - grid: " << grid.name() << '\n' + << " - truncation: " << truncation << '\n'; + if (not domain.global()) { + Log::debug() << " - domain: " << domain << '\n'; + } + if (GlobalDomain(domain)) { + if (GlobalDomain(domain).west() != 0.) { + Log::debug() << " - global domain with modified west: " << GlobalDomain(domain).west() << '\n'; + } + } + Log::debug() << " - fft: " << std::boolalpha << useFFT_ << '\n'; + Log::debug() << " - linalg_backend: "; + if (using_eckit_default_backend(linalg_backend_)) { + Log::debug() << "eckit_linalg default (currently \"" << detect_linalg_backend(linalg_backend_) + << "\" but could be changed after setup, check invtrans debug output)" << '\n'; + } + else { + Log::debug() << detect_linalg_backend(linalg_backend_) << '\n'; + } + Log::debug() << " - legendre_cache: " << std::boolalpha << bool(legendre_cache_) << std::endl; + + // precomputations for Legendre polynomials: { const auto nlatsLeg = size_t(nlatsLeg_); @@ -533,8 +592,8 @@ TransLocal::TransLocal(const Cache& cache, const Grid& grid, const Domain& domai legendre_asym_ = legendre.read(size_asym); } else { - alloc_aligned(legendre_sym_, size_sym, "symmetric"); - alloc_aligned(legendre_asym_, size_asym, "asymmetric"); + alloc_aligned(legendre_sym_, size_sym, "Legendre coeffs symmetric"); + alloc_aligned(legendre_asym_, size_asym, "Legendre coeffs asymmetric"); } ATLAS_TRACE_SCOPE("Legendre precomputations (structured)") { @@ -629,7 +688,7 @@ TransLocal::TransLocal(const Cache& cache, const Grid& grid, const Domain& domai << "WARNING: Spectral transform results may contain aliasing errors. This will be addressed soon." << std::endl; - alloc_aligned(fourier_, 2 * (truncation_ + 1) * nlonsMax, "Fourier coeffs."); + alloc_aligned(fourier_, 2 * (truncation_ + 1) * nlonsMax, "Fourier coeffs"); #if !TRANSLOCAL_DGEMM2 { ATLAS_TRACE("Fourier precomputations (NoFFT)"); @@ -870,9 +929,10 @@ void TransLocal::invtrans_legendre(const int truncation, const int nlats, const const eckit::Configuration&) const { // Legendre transform: { + Log::debug() << "TransLocal::invtrans_legendre: Legendre GEMM with \"" << detect_linalg_backend(linalg_backend_) + << "\" using " << nlatsLegReduced_ - nlat0_[0] << " latitudes out of " << nlatsGlobal_ / 2 + << std::endl; linalg::dense::Backend linalg_backend{linalg_backend_}; - Log::debug() << "Legendre dgemm: using " << nlatsLegReduced_ - nlat0_[0] << " latitudes out of " - << nlatsGlobal_ / 2 << std::endl; ATLAS_TRACE("Inverse Legendre Transform (GEMM)"); for (int jm = 0; jm <= truncation_; jm++) { size_t size_sym = num_n(truncation_ + 1, jm, true); @@ -1071,7 +1131,8 @@ void TransLocal::invtrans_fourier_regular(const int nlats, const int nlons, cons #if !TRANSLOCAL_DGEMM2 // dgemm-method 1 { - ATLAS_TRACE("Inverse Fourier Transform (NoFFT,matrix_multiply=" + std::string(linalg_backend) + ")"); + ATLAS_TRACE("Inverse Fourier Transform (NoFFT,matrix_multiply=" + detect_linalg_backend(linalg_backend_) + + ")"); linalg::Matrix A(fourier_, nlons, (truncation_ + 1) * 2); linalg::Matrix B(scl_fourier, (truncation_ + 1) * 2, nb_fields * nlats); linalg::Matrix C(gp_fields, nlons, nb_fields * nlats); From 526cc1d537049332c53e6495d39cb5b6263c7e19 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 27 Jul 2022 10:57:09 +0100 Subject: [PATCH 39/53] ATLAS-367 Avoid eckit preprocessor definitions --- src/CMakeLists.txt | 13 +++++++++++++ src/atlas/io/Exceptions.cc | 4 ++-- src/atlas/io/detail/Encoder.h | 4 ++-- src/atlas/io/detail/Endian.h | 8 ++++---- src/atlas/io/detail/sfinae.h | 1 - src/atlas/library/Library.cc | 26 ++++++++++++++++++-------- src/atlas/library/defines.h.in | 5 +++++ 7 files changed, 44 insertions(+), 17 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 74c80f31f..88dc8c11e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -69,6 +69,19 @@ if( CMAKE_BUILD_TYPE MATCHES "Debug" ) set( atlas_BUILD_TYPE_DEBUG 1 ) endif() +check_cxx_source_compiles( "#include \n int main() { char * type; int status; char * r = abi::__cxa_demangle(type, 0, 0, &status); }" + ATLAS_HAVE_CXXABI_H ) + +test_big_endian( _BIG_ENDIAN ) + +if( _BIG_ENDIAN ) + set( ATLAS_BIG_ENDIAN 1 ) + set( ATLAS_LITTLE_ENDIAN 0 ) +else() + set( ATLAS_BIG_ENDIAN 0 ) + set( ATLAS_LITTLE_ENDIAN 1 ) +endif() + ecbuild_parse_version( ${eckit_VERSION} PREFIX ATLAS_ECKIT ) math( EXPR ATLAS_ECKIT_VERSION_INT "( 10000 * ${ATLAS_ECKIT_VERSION_MAJOR} ) + ( 100 * ${ATLAS_ECKIT_VERSION_MINOR} ) + ${ATLAS_ECKIT_VERSION_PATCH}" ) diff --git a/src/atlas/io/Exceptions.cc b/src/atlas/io/Exceptions.cc index e9c9e92a7..2464a7996 100644 --- a/src/atlas/io/Exceptions.cc +++ b/src/atlas/io/Exceptions.cc @@ -11,7 +11,7 @@ #include "Exceptions.h" #include "eckit/eckit_config.h" -#ifdef eckit_HAVE_CXXABI_H +#if ATLAS_HAVE_CXXABI_H #include #endif @@ -24,7 +24,7 @@ namespace io { //--------------------------------------------------------------------------------------------------------------------- std::string demangle(const char* name) { -#ifdef eckit_HAVE_CXXABI_H +#if ATLAS_HAVE_CXXABI_H int status = -4; std::unique_ptr res{abi::__cxa_demangle(name, nullptr, nullptr, &status), std::free}; diff --git a/src/atlas/io/detail/Encoder.h b/src/atlas/io/detail/Encoder.h index 6439be133..e9c8dba04 100644 --- a/src/atlas/io/detail/Encoder.h +++ b/src/atlas/io/detail/Encoder.h @@ -95,12 +95,12 @@ class Encoder { }; inline size_t encode_metadata(const Encoder& encoder, atlas::io::Metadata& metadata) { - ATLAS_ASSERT(encoder); + ASSERT(encoder); return encoder.self_->encode_metadata_(metadata); } inline void encode_data(const Encoder& encoder, atlas::io::Data& out) { - ATLAS_ASSERT(encoder); + ASSERT(encoder); ATLAS_TRACE(); encoder.self_->encode_data_(out); } diff --git a/src/atlas/io/detail/Endian.h b/src/atlas/io/detail/Endian.h index b84d965d2..fd3a0e9cc 100644 --- a/src/atlas/io/detail/Endian.h +++ b/src/atlas/io/detail/Endian.h @@ -10,7 +10,7 @@ #pragma once -#include "eckit/eckit.h" +#include "atlas/library/defines.h" namespace atlas { namespace io { @@ -19,14 +19,14 @@ enum class Endian { little = 0, big = 1, -#if ECKIT_BIG_ENDIAN +#if ATLAS_BIG_ENDIAN native = big, swapped = little -#elif ECKIT_LITTLE_ENDIAN +#elif ATLAS_LITTLE_ENDIAN native = little, swapped = big #else -#error Neither ECKIT_BIG_ENDIAN nor ECKIT_LITTLE_ENDIAN equals true +#error Neither ATLAS_BIG_ENDIAN nor ATLAS_LITTLE_ENDIAN equals true #endif }; diff --git a/src/atlas/io/detail/sfinae.h b/src/atlas/io/detail/sfinae.h index da324f945..63e7269d4 100644 --- a/src/atlas/io/detail/sfinae.h +++ b/src/atlas/io/detail/sfinae.h @@ -11,7 +11,6 @@ #pragma once #include "atlas/io/detail/TypeTraits.h" -#include "atlas/runtime/Exception.h" namespace atlas { namespace io { diff --git a/src/atlas/library/Library.cc b/src/atlas/library/Library.cc index 93b4a872a..1b58a2b1e 100644 --- a/src/atlas/library/Library.cc +++ b/src/atlas/library/Library.cc @@ -13,8 +13,6 @@ #include #include -#include // for dynamic loading (should be delegated to eckit) - #include "eckit/config/LibEcKit.h" #include "eckit/config/Resource.h" #include "eckit/eckit.h" @@ -29,6 +27,22 @@ #include "eckit/types/Types.h" #include "eckit/utils/Translator.h" +#if ATLAS_ECKIT_HAVE_ECKIT_585 +#include "eckit/linalg/LinearAlgebraDense.h" +namespace { +static bool feature_MKL() { + return eckit::linalg::LinearAlgebraDense::hasBackend("mkl"); +} +} // namespace +#else +#include "eckit/linalg/LinearAlgebra.h" +namespace { +static bool feature_MKL() { + return eckit::linalg::LinearAlgebra::hasBackend("mkl"); +} +} // namespace +#endif + #include "atlas/library/FloatingPointExceptions.h" #include "atlas/library/Plugin.h" #include "atlas/library/config.h" @@ -426,12 +440,8 @@ void Library::Information::print(std::ostream& out) const { bool feature_BoundsChecking(ATLAS_ARRAYVIEW_BOUNDS_CHECKING); bool feature_Init_sNaN(ATLAS_INIT_SNAN); bool feature_MPI(false); -#if ECKIT_HAVE_MPI +#if ATLAS_HAVE_MPI feature_MPI = true; -#endif - bool feature_MKL(false); -#if ECKIT_HAVE_MKL - feature_MKL = true; #endif std::string array_data_store = "Native"; #if ATLAS_HAVE_GRIDTOOLS_STORAGE @@ -449,7 +459,7 @@ void Library::Information::print(std::ostream& out) const { << " Trans : " << str(feature_Trans) << '\n' << " FFTW : " << str(feature_FFTW) << '\n' << " Eigen : " << str(feature_Eigen) << '\n' - << " MKL : " << str(feature_MKL) << '\n' + << " MKL : " << str(feature_MKL()) << '\n' << " Tesselation : " << str(feature_Tesselation) << '\n' << " ArrayDataStore : " << array_data_store << '\n' << " idx_t : " << ATLAS_BITS_LOCAL << " bit integer" << '\n' diff --git a/src/atlas/library/defines.h.in b/src/atlas/library/defines.h.in index 3316a349e..926ad64f0 100644 --- a/src/atlas/library/defines.h.in +++ b/src/atlas/library/defines.h.in @@ -22,6 +22,7 @@ #define ATLAS_HAVE_FORTRAN @atlas_HAVE_FORTRAN@ #define ATLAS_HAVE_EIGEN @atlas_HAVE_EIGEN@ #define ATLAS_HAVE_FFTW @atlas_HAVE_FFTW@ +#define ATLAS_HAVE_MPI @atlas_HAVE_MPI@ #define ATLAS_BITS_GLOBAL @ATLAS_BITS_GLOBAL@ #define ATLAS_ARRAYVIEW_BOUNDS_CHECKING @atlas_HAVE_BOUNDSCHECKING@ #define ATLAS_INDEXVIEW_BOUNDS_CHECKING @atlas_HAVE_BOUNDSCHECKING@ @@ -38,6 +39,10 @@ #define ATLAS_BUILD_TYPE_RELEASE @atlas_BUILD_TYPE_RELEASE@ #define ATLAS_ECKIT_VERSION_INT @ATLAS_ECKIT_VERSION_INT@ +#cmakedefine01 ATLAS_HAVE_CXXABI_H +#cmakedefine01 ATLAS_LITTLE_ENDIAN +#cmakedefine01 ATLAS_BIG_ENDIAN + #ifdef __CUDACC__ #define ATLAS_HOST_DEVICE __host__ __device__ #define ATLAS_DEVICE __device__ From 63068476f4bd8bc52f0c8b09d849223d02cd19d7 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 27 Jul 2022 11:23:16 +0100 Subject: [PATCH 40/53] Gmsh info shows remote_index --- src/atlas/output/detail/GmshIO.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/atlas/output/detail/GmshIO.cc b/src/atlas/output/detail/GmshIO.cc index 223d895e4..74a893e8b 100644 --- a/src/atlas/output/detail/GmshIO.cc +++ b/src/atlas/output/detail/GmshIO.cc @@ -1154,8 +1154,8 @@ void GmshIO::write(const Mesh& mesh, const PathName& file_path) const { lat(n) = lonlat(n, 1); } write(fieldset, function_space, mesh_info, std::ios_base::out); - std::vector extra_fields = {"partition", "water", "dual_volumes", - "dual_delta_sph", "ghost", "halo"}; + std::vector extra_fields = {"partition", "water", "dual_volumes", "dual_delta_sph", + "ghost", "halo", "remote_index"}; for (auto& f : extra_fields) { if (nodes.has_field(f)) { write(nodes.field(f), function_space, mesh_info, std::ios_base::app); From d40658b28d0eed62b67aea8bc49b4bfff9171ec4 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 27 Jul 2022 11:23:59 +0100 Subject: [PATCH 41/53] Add convenience constructor Mesh::Mesh(Grid,Partitioner) --- src/atlas/mesh/Mesh.cc | 11 +++++++++++ src/atlas/mesh/Mesh.h | 8 ++++++++ 2 files changed, 19 insertions(+) diff --git a/src/atlas/mesh/Mesh.cc b/src/atlas/mesh/Mesh.cc index 7c58124e1..0e02781de 100644 --- a/src/atlas/mesh/Mesh.cc +++ b/src/atlas/mesh/Mesh.cc @@ -29,6 +29,17 @@ Mesh::Mesh(const Grid& grid): get()->detach(); } +Mesh::Mesh(const Grid& grid, const grid::Partitioner& partitioner): + Handle([&]() { + auto meshgenerator = MeshGenerator{grid.meshgenerator()}; + auto mesh = meshgenerator.generate(grid, partitioner); + mesh.get()->attach(); + return mesh.get(); + }()) { + get()->detach(); +} + + Mesh::Mesh(eckit::Stream& stream): Handle(new Implementation(stream)) {} //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/atlas/mesh/Mesh.h b/src/atlas/mesh/Mesh.h index 6051cd58f..27ffc59af 100644 --- a/src/atlas/mesh/Mesh.h +++ b/src/atlas/mesh/Mesh.h @@ -24,6 +24,12 @@ class Projection; class Grid; } // namespace atlas +namespace atlas { +namespace grid { +class Partitioner; +} +} // namespace atlas + namespace atlas { namespace util { class Metadata; @@ -69,6 +75,8 @@ class Mesh : DOXYGEN_HIDE(public util::ObjectHandle) { /// @brief Generate a mesh from a Grid with recommended mesh generator and partitioner strategy Mesh(const Grid&); + Mesh(const Grid&, const grid::Partitioner&); + /// @brief Construct a mesh from a Stream (serialization) explicit Mesh(eckit::Stream&); From 4f98a26d13585a7d8cd2756a446ea35bedab450a Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 27 Jul 2022 11:24:41 +0100 Subject: [PATCH 42/53] CellColumns to build on-demand lonlat cell centres if not existing --- src/atlas/functionspace/CellColumns.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/atlas/functionspace/CellColumns.cc b/src/atlas/functionspace/CellColumns.cc index 760bc04b5..7da6a2080 100644 --- a/src/atlas/functionspace/CellColumns.cc +++ b/src/atlas/functionspace/CellColumns.cc @@ -20,6 +20,7 @@ #include "atlas/mesh/HybridElements.h" #include "atlas/mesh/IsGhostNode.h" #include "atlas/mesh/Mesh.h" +#include "atlas/mesh/actions/Build2DCellCentres.h" #include "atlas/mesh/actions/BuildHalo.h" #include "atlas/mesh/actions/BuildParallelFields.h" #include "atlas/mesh/actions/BuildPeriodicBoundaries.h" @@ -564,6 +565,9 @@ const parallel::Checksum& CellColumns::checksum() const { } Field CellColumns::lonlat() const { + if (!mesh_.cells().has_field("lonlat")) { + mesh::actions::Build2DCellCentres("lonlat")(const_cast(mesh_)); + } return mesh_.cells().field("lonlat"); } From cd233c59adaff4328c03b8a6b4f6f6b517658daf Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 27 Jul 2022 11:26:02 +0100 Subject: [PATCH 43/53] Enable GridBoxMethod via the StructuredColumns function spaces --- src/atlas/interpolation/method/knn/GridBoxMethod.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/atlas/interpolation/method/knn/GridBoxMethod.cc b/src/atlas/interpolation/method/knn/GridBoxMethod.cc index 5ac390f47..24357f60b 100644 --- a/src/atlas/interpolation/method/knn/GridBoxMethod.cc +++ b/src/atlas/interpolation/method/knn/GridBoxMethod.cc @@ -18,6 +18,7 @@ #include "eckit/log/ProgressTimer.h" #include "eckit/types/FloatCompare.h" +#include "atlas/functionspace.h" #include "atlas/grid.h" #include "atlas/parallel/mpi/mpi.h" #include "atlas/runtime/Exception.h" @@ -45,7 +46,12 @@ void GridBoxMethod::print(std::ostream& out) const { } -void GridBoxMethod::do_setup(const FunctionSpace& /*source*/, const FunctionSpace& /*target*/) { +void GridBoxMethod::do_setup(const FunctionSpace& source, const FunctionSpace& target) { + if (functionspace::StructuredColumns(source) && functionspace::StructuredColumns(target)) { + do_setup(functionspace::StructuredColumns(source).grid(), functionspace::StructuredColumns(target).grid(), + Cache()); + return; + } ATLAS_NOTIMPLEMENTED; } @@ -93,7 +99,7 @@ void GridBoxMethod::do_setup(const Grid& source, const Grid& target, const Cache ATLAS_TRACE("GridBoxMethod::setup()"); if (mpi::size() > 1) { - ATLAS_NOTIMPLEMENTED; + ATLAS_THROW_EXCEPTION("Not implemented for MPI-parallel runs"); } ATLAS_ASSERT(source); From 88fe282479367748f80494049f82d728e1a0fca2 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 27 Jul 2022 18:31:03 +0100 Subject: [PATCH 44/53] Cleanup use of ecbuild_add_library --- cmake/features/FFTW.cmake | 5 +++ cmake/features/PROJ.cmake | 5 +++ cmake/features/TESSELATION.cmake | 7 ++- src/atlas/CMakeLists.txt | 76 +++++++++++--------------------- src/atlas_f/CMakeLists.txt | 46 ++++++++----------- 5 files changed, 60 insertions(+), 79 deletions(-) diff --git a/cmake/features/FFTW.cmake b/cmake/features/FFTW.cmake index 5fb7e9d97..a913d38c0 100644 --- a/cmake/features/FFTW.cmake +++ b/cmake/features/FFTW.cmake @@ -3,3 +3,8 @@ ecbuild_add_option( FEATURE FFTW DESCRIPTION "Support for fftw" REQUIRED_PACKAGES "FFTW COMPONENTS double QUIET" ) + +if( NOT HAVE_FFTW ) + unset( FFTW_LIBRARIES ) + unset( FFTW_INCLUDES ) +endif() diff --git a/cmake/features/PROJ.cmake b/cmake/features/PROJ.cmake index bb4de078b..ff94d127b 100644 --- a/cmake/features/PROJ.cmake +++ b/cmake/features/PROJ.cmake @@ -49,3 +49,8 @@ ecbuild_add_option( FEATURE PROJ DESCRIPTION "PROJ-based projections" DEFAULT OFF CONDITION PROJ_FOUND ) + +if( NOT HAVE_PROJ ) + unset( PROJ_LIBRARIES ) + unset( PROJ_INCLUDE_DIRS ) +endif() diff --git a/cmake/features/TESSELATION.cmake b/cmake/features/TESSELATION.cmake index e9f424493..12b4ebe1c 100644 --- a/cmake/features/TESSELATION.cmake +++ b/cmake/features/TESSELATION.cmake @@ -8,7 +8,7 @@ ecbuild_add_option( FEATURE TESSELATION "CGAL QUIET" "Boost VERSION 1.45.0 QUIET" ) -if( atlas_HAVE_TESSELATION ) +if( HAVE_TESSELATION ) list( APPEND CGAL_INCLUDE_DIRS ${Boost_INCLUDE_DIRS} ) if ( TARGET CGAL::CGAL ) list( APPEND CGAL_LIBRARIES CGAL::CGAL ${CGAL_3RD_PARTY_LIBRARIES} ${GMP_LIBRARIES} ${MPFR_LIBRARIES} ${Boost_THREAD_LIBRARY} ${Boost_SYSTEM_LIBRARY} ) @@ -24,3 +24,8 @@ if( atlas_HAVE_TESSELATION ) list( APPEND CGAL_LIBRARIES ${CGAL_LIBRARY} ${CGAL_3RD_PARTY_LIBRARIES} ${GMP_LIBRARIES} ${MPFR_LIBRARIES} ${Boost_THREAD_LIBRARY} ${Boost_SYSTEM_LIBRARY} ) endif() endif() + +if( NOT HAVE_TESSELATION ) + unset( CGAL_LIBRARIES ) + unset( CGAL_INCLUDE_DIRS ) +endif() diff --git a/src/atlas/CMakeLists.txt b/src/atlas/CMakeLists.txt index 9d2201fa0..14850fd4e 100644 --- a/src/atlas/CMakeLists.txt +++ b/src/atlas/CMakeLists.txt @@ -916,60 +916,36 @@ ecbuild_add_library( TARGET atlas SOURCES ${source_list} -) - -if( atlas_HAVE_FORTRAN ) - target_link_libraries( atlas PRIVATE fckit ) -endif() -if( atlas_HAVE_TRANS ) - target_link_libraries( atlas PRIVATE transi ) -endif() - -if( atlas_HAVE_PROJ ) - target_link_libraries( atlas PRIVATE ${PROJ_LIBRARIES} ) - target_include_directories( atlas PRIVATE ${PROJ_INCLUDE_DIRS} ) -endif() - -target_link_libraries( atlas PUBLIC - eckit - eckit_geometry - eckit_linalg - eckit_maths - eckit_mpi - eckit_option -) - -target_include_directories( atlas PUBLIC + PRIVATE_LIBS + $<${atlas_HAVE_FORTRAN}:fckit> + $<${atlas_HAVE_TRANS}:transi> + $<${atlas_HAVE_ACC}:atlas_acc_support> + ${CGAL_LIBRARIES} + ${FFTW_LIBRARIES} + ${PROJ_LIBRARIES} + + PUBLIC_LIBS + eckit + eckit_geometry + eckit_linalg + eckit_maths + eckit_mpi + eckit_option + $<${atlas_HAVE_EIGEN}:Eigen3::Eigen> + $<${atlas_HAVE_OMP_CXX}:OpenMP::OpenMP_CXX> + $<${atlas_HAVE_GRIDTOOLS_STORAGE}:GridTools::gridtools> + + PRIVATE_INCLUDES + ${CGAL_INCLUDE_DIRS} + ${FFTW_INCLUDES} + ${PROJ_INCLUDE_DIRS} + + PUBLIC_INCLUDES $ $ $ -) -if( atlas_HAVE_TESSELATION ) - target_link_libraries( atlas PRIVATE ${CGAL_LIBRARIES} ) - target_include_directories( atlas PRIVATE ${CGAL_INCLUDE_DIRS} ) -endif() - -if( atlas_HAVE_FFTW ) - target_link_libraries( atlas PRIVATE ${FFTW_LIBRARIES} ) - target_include_directories( atlas PRIVATE ${FFTW_INCLUDES} ) -endif() - -if( atlas_HAVE_EIGEN ) - target_link_libraries( atlas PUBLIC Eigen3::Eigen ) -endif() - -if( atlas_HAVE_OMP_CXX ) - target_link_libraries( atlas PUBLIC OpenMP::OpenMP_CXX ) -endif() - -if( atlas_HAVE_GRIDTOOLS_STORAGE ) - target_link_libraries( atlas PUBLIC GridTools::gridtools ) -endif() - -if( atlas_HAVE_ACC ) - target_link_libraries( atlas PRIVATE atlas_acc_support ) -endif() +) target_compile_features( atlas PUBLIC cxx_std_11 ) diff --git a/src/atlas_f/CMakeLists.txt b/src/atlas_f/CMakeLists.txt index 40f68f2e1..7833e5d4c 100644 --- a/src/atlas_f/CMakeLists.txt +++ b/src/atlas_f/CMakeLists.txt @@ -204,36 +204,26 @@ ecbuild_add_library( TARGET atlas_f runtime/atlas_trace.cc runtime/atlas_Trace_module.F90 - PRIVATE_INCLUDES - ${FCKIT_INCLUDE_DIRS} + PUBLIC_LIBS + $ + fckit + + PRIVATE_LIBS + $<${atlas_HAVE_OMP_Fortran}:OpenMP::OpenMP_Fortran> + + PUBLIC_INCLUDES + $ + $ + $ + $ + $ + $ + + PRIVATE_INCLUDES + $ + $ ) -target_link_libraries( atlas_f PUBLIC - $ ) - -target_link_libraries( atlas_f PUBLIC - fckit ) - -if( atlas_HAVE_OMP_Fortran ) - target_link_libraries( atlas_f PRIVATE OpenMP::OpenMP_Fortran ) -endif() - -target_include_directories( atlas_f PUBLIC - $ - $ - $ - $ - $ - $ -) -target_include_directories( atlas_f PRIVATE - $ - $ -) -if( ECKIT_INCLUDE_DIRS ) - target_include_directories( atlas_f PRIVATE ${ECKIT_INCLUDE_DIRS} ) -endif() - fckit_target_preprocess_fypp( atlas_f DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/atlas_f.h.in From 84387b45e3816625512cb9a40fa37e500b774fca Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Thu, 28 Jul 2022 13:46:48 +0100 Subject: [PATCH 45/53] Protect definition of byte --- src/atlas/array/DataType.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/atlas/array/DataType.h b/src/atlas/array/DataType.h index a8485ef01..016adcb59 100644 --- a/src/atlas/array/DataType.h +++ b/src/atlas/array/DataType.h @@ -19,6 +19,8 @@ #if __cplusplus >= 201703L #include #else +#ifndef STD_BYTE_DEFINED +#define STD_BYTE_DEFINED namespace std { #ifdef _CRAYC struct byte { @@ -31,6 +33,7 @@ enum class byte : unsigned char #endif } // namespace std #endif +#endif //------------------------------------------------------------------------------------------------------ From 5f731463c3cfee3de9a02cdc2bbba6b8b8169e89 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Thu, 28 Jul 2022 13:50:26 +0100 Subject: [PATCH 46/53] ATLAS-368 atlas-io embedded as standalone project, only depending on eckit --- CMakeLists.txt | 4 + atlas-io/AUTHORS | 8 + atlas-io/CHANGELOG.md | 0 atlas-io/CMakeLists.txt | 71 ++ atlas-io/LICENSE | 190 +++++ atlas-io/README.md | 7 + atlas-io/cmake/atlas-io-import.cmake.in | 5 + atlas-io/src/CMakeLists.txt | 2 + atlas-io/src/atlas_io/CMakeLists.txt | 91 +++ .../io => atlas-io/src/atlas_io}/Data.cc | 16 +- .../atlas/io => atlas-io/src/atlas_io}/Data.h | 0 .../src/atlas_io}/Exceptions.cc | 2 +- .../io => atlas-io/src/atlas_io}/Exceptions.h | 0 .../src/atlas_io}/FileStream.cc | 19 +- .../io => atlas-io/src/atlas_io}/FileStream.h | 2 +- .../io => atlas-io/src/atlas_io}/Metadata.cc | 10 +- atlas-io/src/atlas_io/Metadata.h | 97 +++ .../src/atlas_io}/ReadRequest.cc | 25 +- .../src/atlas_io}/ReadRequest.h | 4 +- .../io => atlas-io/src/atlas_io}/Record.cc | 22 +- .../io => atlas-io/src/atlas_io}/Record.h | 6 +- .../src/atlas_io}/RecordItem.cc | 11 +- .../io => atlas-io/src/atlas_io}/RecordItem.h | 6 +- .../src/atlas_io}/RecordItemReader.cc | 30 +- .../src/atlas_io}/RecordItemReader.h | 6 +- .../src/atlas_io}/RecordPrinter.cc | 32 +- .../src/atlas_io}/RecordPrinter.h | 13 +- .../src/atlas_io}/RecordReader.cc | 8 +- .../src/atlas_io}/RecordReader.h | 12 +- .../src/atlas_io}/RecordWriter.cc | 26 +- .../src/atlas_io}/RecordWriter.h | 30 +- .../io => atlas-io/src/atlas_io}/Session.cc | 10 +- .../io => atlas-io/src/atlas_io}/Session.h | 4 +- .../io => atlas-io/src/atlas_io}/Stream.cc | 15 +- .../io => atlas-io/src/atlas_io}/Stream.h | 0 atlas-io/src/atlas_io/Trace.cc | 37 + atlas-io/src/atlas_io/Trace.h | 61 ++ atlas-io/src/atlas_io/atlas-io.h | 127 ++++ atlas-io/src/atlas_io/atlas_compat.h | 1 + atlas-io/src/atlas_io/detail/Assert.h | 15 + .../src/atlas_io}/detail/Base64.cc | 0 .../src/atlas_io}/detail/Base64.h | 0 atlas-io/src/atlas_io/detail/BlackMagic.h | 101 +++ .../src/atlas_io}/detail/Checksum.cc | 7 +- .../src/atlas_io}/detail/Checksum.h | 0 .../src/atlas_io}/detail/DataInfo.h | 4 +- atlas-io/src/atlas_io/detail/DataType.cc | 38 + atlas-io/src/atlas_io/detail/DataType.h | 414 +++++++++++ .../src/atlas_io}/detail/Decoder.cc | 6 +- .../src/atlas_io}/detail/Decoder.h | 7 +- .../src/atlas_io}/detail/Defaults.h | 0 atlas-io/src/atlas_io/detail/Encoder.cc | 31 + .../src/atlas_io}/detail/Encoder.h | 26 +- .../src/atlas_io}/detail/Endian.h | 16 +- .../src/atlas_io}/detail/Link.cc | 2 +- .../src/atlas_io}/detail/Link.h | 0 atlas-io/src/atlas_io/detail/NoConfig.h | 29 + .../src/atlas_io}/detail/ParsedRecord.h | 4 +- .../src/atlas_io}/detail/RecordInfo.h | 4 +- .../src/atlas_io}/detail/RecordSections.h | 6 +- .../src/atlas_io}/detail/Reference.h | 8 +- .../src/atlas_io}/detail/StaticAssert.h | 2 +- .../src/atlas_io}/detail/Time.cc | 2 - .../src/atlas_io}/detail/Time.h | 0 .../src/atlas_io}/detail/Type.h | 0 .../src/atlas_io}/detail/TypeTraits.h | 0 .../src/atlas_io}/detail/Version.h | 0 atlas-io/src/atlas_io/detail/defines.h.in | 21 + .../src/atlas_io}/detail/sfinae.h | 2 +- .../io => atlas-io/src/atlas_io}/detail/tag.h | 0 .../src/atlas_io}/print/Bytes.cc | 0 .../src/atlas_io}/print/Bytes.h | 0 .../src/atlas_io}/print/JSONFormat.cc | 8 +- .../src/atlas_io}/print/JSONFormat.h | 9 +- .../src/atlas_io}/print/TableFormat.cc | 50 +- .../src/atlas_io}/print/TableFormat.h | 10 +- .../src/atlas_io}/types/array.h | 8 +- .../atlas_io}/types/array/ArrayMetadata.cc | 18 +- .../src/atlas_io}/types/array/ArrayMetadata.h | 28 +- .../atlas_io}/types/array/ArrayReference.cc | 5 +- .../atlas_io}/types/array/ArrayReference.h | 6 +- .../types/array/adaptors/StdArrayAdaptor.h | 11 +- .../types/array/adaptors/StdVectorAdaptor.h | 11 +- .../src/atlas_io}/types/scalar.cc | 26 +- .../src/atlas_io}/types/scalar.h | 4 +- .../src/atlas_io}/types/string.h | 9 +- atlas-io/src/tools/CMakeLists.txt | 12 + atlas-io/src/tools/atlas-io-list.cc | 215 ++++++ atlas-io/tests/CMakeLists.txt | 37 + atlas-io/tests/TestEnvironment.h | 309 +++++++++ atlas-io/tests/test_io_encoding.cc | 652 ++++++++++++++++++ atlas-io/tests/test_io_record.cc | 596 ++++++++++++++++ atlas-io/tests/test_io_stream.cc | 211 ++++++ cmake/features/CXX17.cmake | 5 + src/CMakeLists.txt | 13 - src/apps/CMakeLists.txt | 10 - src/apps/atlas-io-list.cc | 108 --- src/atlas/CMakeLists.txt | 31 +- .../array/adaptors => }/ArrayAdaptor.cc | 17 +- .../{types/array/adaptors => }/ArrayAdaptor.h | 0 src/atlas/io/Metadata.h | 61 -- .../array/adaptors => }/VectorAdaptor.h | 6 +- src/atlas/io/atlas-io.h | 116 +--- src/atlas/library/Library.cc | 11 + src/atlas/library/defines.h.in | 4 - src/atlas/runtime/trace/Timings.cc | 21 +- src/tests/AtlasTestEnvironment.h | 3 +- src/tests/io/test_io_encoding.cc | 10 +- src/tests/io/test_io_record.cc | 4 + src/tests/io/test_io_stream.cc | 5 +- tools/apply-clang-format.sh | 10 +- 111 files changed, 3791 insertions(+), 634 deletions(-) create mode 100644 atlas-io/AUTHORS create mode 100644 atlas-io/CHANGELOG.md create mode 100644 atlas-io/CMakeLists.txt create mode 100644 atlas-io/LICENSE create mode 100644 atlas-io/README.md create mode 100644 atlas-io/cmake/atlas-io-import.cmake.in create mode 100644 atlas-io/src/CMakeLists.txt create mode 100644 atlas-io/src/atlas_io/CMakeLists.txt rename {src/atlas/io => atlas-io/src/atlas_io}/Data.cc (90%) rename {src/atlas/io => atlas-io/src/atlas_io}/Data.h (100%) rename {src/atlas/io => atlas-io/src/atlas_io}/Exceptions.cc (98%) rename {src/atlas/io => atlas-io/src/atlas_io}/Exceptions.h (100%) rename {src/atlas/io => atlas-io/src/atlas_io}/FileStream.cc (90%) rename {src/atlas/io => atlas-io/src/atlas_io}/FileStream.h (98%) rename {src/atlas/io => atlas-io/src/atlas_io}/Metadata.cc (92%) create mode 100644 atlas-io/src/atlas_io/Metadata.h rename {src/atlas/io => atlas-io/src/atlas_io}/ReadRequest.cc (89%) rename {src/atlas/io => atlas-io/src/atlas_io}/ReadRequest.h (96%) rename {src/atlas/io => atlas-io/src/atlas_io}/Record.cc (95%) rename {src/atlas/io => atlas-io/src/atlas_io}/Record.h (94%) rename {src/atlas/io => atlas-io/src/atlas_io}/RecordItem.cc (94%) rename {src/atlas/io => atlas-io/src/atlas_io}/RecordItem.h (96%) rename {src/atlas/io => atlas-io/src/atlas_io}/RecordItemReader.cc (90%) rename {src/atlas/io => atlas-io/src/atlas_io}/RecordItemReader.h (93%) rename {src/atlas/io => atlas-io/src/atlas_io}/RecordPrinter.cc (75%) rename {src/atlas/io => atlas-io/src/atlas_io}/RecordPrinter.h (80%) rename {src/atlas/io => atlas-io/src/atlas_io}/RecordReader.cc (90%) rename {src/atlas/io => atlas-io/src/atlas_io}/RecordReader.h (86%) rename {src/atlas/io => atlas-io/src/atlas_io}/RecordWriter.cc (94%) rename {src/atlas/io => atlas-io/src/atlas_io}/RecordWriter.h (79%) rename {src/atlas/io => atlas-io/src/atlas_io}/Session.cc (94%) rename {src/atlas/io => atlas-io/src/atlas_io}/Session.h (94%) rename {src/atlas/io => atlas-io/src/atlas_io}/Stream.cc (84%) rename {src/atlas/io => atlas-io/src/atlas_io}/Stream.h (100%) create mode 100644 atlas-io/src/atlas_io/Trace.cc create mode 100644 atlas-io/src/atlas_io/Trace.h create mode 100644 atlas-io/src/atlas_io/atlas-io.h create mode 100644 atlas-io/src/atlas_io/atlas_compat.h create mode 100644 atlas-io/src/atlas_io/detail/Assert.h rename {src/atlas/io => atlas-io/src/atlas_io}/detail/Base64.cc (100%) rename {src/atlas/io => atlas-io/src/atlas_io}/detail/Base64.h (100%) create mode 100644 atlas-io/src/atlas_io/detail/BlackMagic.h rename {src/atlas/io => atlas-io/src/atlas_io}/detail/Checksum.cc (92%) rename {src/atlas/io => atlas-io/src/atlas_io}/detail/Checksum.h (100%) rename {src/atlas/io => atlas-io/src/atlas_io}/detail/DataInfo.h (95%) create mode 100644 atlas-io/src/atlas_io/detail/DataType.cc create mode 100644 atlas-io/src/atlas_io/detail/DataType.h rename {src/atlas/io => atlas-io/src/atlas_io}/detail/Decoder.cc (89%) rename {src/atlas/io => atlas-io/src/atlas_io}/detail/Decoder.h (92%) rename {src/atlas/io => atlas-io/src/atlas_io}/detail/Defaults.h (100%) create mode 100644 atlas-io/src/atlas_io/detail/Encoder.cc rename {src/atlas/io => atlas-io/src/atlas_io}/detail/Encoder.h (82%) rename {src/atlas/io => atlas-io/src/atlas_io}/detail/Endian.h (66%) rename {src/atlas/io => atlas-io/src/atlas_io}/detail/Link.cc (97%) rename {src/atlas/io => atlas-io/src/atlas_io}/detail/Link.h (100%) create mode 100644 atlas-io/src/atlas_io/detail/NoConfig.h rename {src/atlas/io => atlas-io/src/atlas_io}/detail/ParsedRecord.h (94%) rename {src/atlas/io => atlas-io/src/atlas_io}/detail/RecordInfo.h (90%) rename {src/atlas/io => atlas-io/src/atlas_io}/detail/RecordSections.h (98%) rename {src/atlas/io => atlas-io/src/atlas_io}/detail/Reference.h (89%) rename {src/atlas/io => atlas-io/src/atlas_io}/detail/StaticAssert.h (98%) rename {src/atlas/io => atlas-io/src/atlas_io}/detail/Time.cc (99%) rename {src/atlas/io => atlas-io/src/atlas_io}/detail/Time.h (100%) rename {src/atlas/io => atlas-io/src/atlas_io}/detail/Type.h (100%) rename {src/atlas/io => atlas-io/src/atlas_io}/detail/TypeTraits.h (100%) rename {src/atlas/io => atlas-io/src/atlas_io}/detail/Version.h (100%) create mode 100644 atlas-io/src/atlas_io/detail/defines.h.in rename {src/atlas/io => atlas-io/src/atlas_io}/detail/sfinae.h (98%) rename {src/atlas/io => atlas-io/src/atlas_io}/detail/tag.h (100%) rename {src/atlas/io => atlas-io/src/atlas_io}/print/Bytes.cc (100%) rename {src/atlas/io => atlas-io/src/atlas_io}/print/Bytes.h (100%) rename {src/atlas/io => atlas-io/src/atlas_io}/print/JSONFormat.cc (91%) rename {src/atlas/io => atlas-io/src/atlas_io}/print/JSONFormat.h (80%) rename {src/atlas/io => atlas-io/src/atlas_io}/print/TableFormat.cc (87%) rename {src/atlas/io => atlas-io/src/atlas_io}/print/TableFormat.h (85%) rename {src/atlas/io => atlas-io/src/atlas_io}/types/array.h (59%) rename {src/atlas/io => atlas-io/src/atlas_io}/types/array/ArrayMetadata.cc (79%) rename {src/atlas/io => atlas-io/src/atlas_io}/types/array/ArrayMetadata.h (65%) rename {src/atlas/io => atlas-io/src/atlas_io}/types/array/ArrayReference.cc (96%) rename {src/atlas/io => atlas-io/src/atlas_io}/types/array/ArrayReference.h (93%) rename {src/atlas/io => atlas-io/src/atlas_io}/types/array/adaptors/StdArrayAdaptor.h (89%) rename {src/atlas/io => atlas-io/src/atlas_io}/types/array/adaptors/StdVectorAdaptor.h (87%) rename {src/atlas/io => atlas-io/src/atlas_io}/types/scalar.cc (88%) rename {src/atlas/io => atlas-io/src/atlas_io}/types/scalar.h (97%) rename {src/atlas/io => atlas-io/src/atlas_io}/types/string.h (85%) create mode 100644 atlas-io/src/tools/CMakeLists.txt create mode 100644 atlas-io/src/tools/atlas-io-list.cc create mode 100644 atlas-io/tests/CMakeLists.txt create mode 100644 atlas-io/tests/TestEnvironment.h create mode 100644 atlas-io/tests/test_io_encoding.cc create mode 100644 atlas-io/tests/test_io_record.cc create mode 100644 atlas-io/tests/test_io_stream.cc create mode 100644 cmake/features/CXX17.cmake delete mode 100644 src/apps/atlas-io-list.cc rename src/atlas/io/{types/array/adaptors => }/ArrayAdaptor.cc (81%) rename src/atlas/io/{types/array/adaptors => }/ArrayAdaptor.h (100%) delete mode 100644 src/atlas/io/Metadata.h rename src/atlas/io/{types/array/adaptors => }/VectorAdaptor.h (93%) diff --git a/CMakeLists.txt b/CMakeLists.txt index d1ecc198f..86a8ffc4d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,6 +32,9 @@ endif() ecbuild_debug( " eckit_FEATURES : [${eckit_FEATURES}]" ) +add_subdirectory(atlas-io) +find_package(atlas-io) + ################################################################################ # Features that can be enabled / disabled with -DENABLE_ @@ -52,6 +55,7 @@ include( features/INCLUDE_WHAT_YOU_USE ) include( features/INIT_SNAN ) include( features/DOCS ) include( features/ATLAS_RUN ) +include( features/CXX17 ) ################################################################################ # sources diff --git a/atlas-io/AUTHORS b/atlas-io/AUTHORS new file mode 100644 index 000000000..e4c77d93c --- /dev/null +++ b/atlas-io/AUTHORS @@ -0,0 +1,8 @@ +Authors +======= + +- Willem Deconinck + +Thanks for contributions from +============================= + diff --git a/atlas-io/CHANGELOG.md b/atlas-io/CHANGELOG.md new file mode 100644 index 000000000..e69de29bb diff --git a/atlas-io/CMakeLists.txt b/atlas-io/CMakeLists.txt new file mode 100644 index 000000000..b01ca4a0a --- /dev/null +++ b/atlas-io/CMakeLists.txt @@ -0,0 +1,71 @@ +# (C) Copyright 2021 ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. + +############################################################################################ +# Atlas-IO + +cmake_minimum_required( VERSION 3.12 FATAL_ERROR ) + +find_package( ecbuild 3.4 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild ) + +################################################################################ +# Initialise project Atlas + +project( atlas-io VERSION ${atlas_VERSION} LANGUAGES CXX ) + +################################################################################ +# Required packages + +ecbuild_find_package( NAME eckit REQUIRED ) + +ecbuild_debug( " eckit_FEATURES : [${eckit_FEATURES}]" ) + +################################################################################ +# Features that can be enabled / disabled with -DENABLE_ + +ecbuild_add_option( FEATURE CXX17 + DESCRIPTION "Use C++17 standard" + DEFAULT OFF ) + +################################################################################ +# sources + +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +check_cxx_source_compiles( "#include \n int main() { char * type; int status; char * r = abi::__cxa_demangle(type, 0, 0, &status); }" + ATLAS_IO_HAVE_CXXABI_H ) + +test_big_endian( _BIG_ENDIAN ) + +if( _BIG_ENDIAN ) + set( ATLAS_IO_BIG_ENDIAN 1 ) + set( ATLAS_IO_LITTLE_ENDIAN 0 ) +else() + set( ATLAS_IO_BIG_ENDIAN 0 ) + set( ATLAS_IO_LITTLE_ENDIAN 1 ) +endif() + + +add_subdirectory( src ) +add_subdirectory( tests ) + +################################################################################ +# Export and summarize + +ecbuild_add_resources( + TARGET atlas-io-others + SOURCES_PACK + README.md + CHANGELOG.md + LICENSE +) + +ecbuild_install_project( NAME Atlas-IO ) +ecbuild_print_summary() + diff --git a/atlas-io/LICENSE b/atlas-io/LICENSE new file mode 100644 index 000000000..af6ca0287 --- /dev/null +++ b/atlas-io/LICENSE @@ -0,0 +1,190 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 1996-2018 ECMWF + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/atlas-io/README.md b/atlas-io/README.md new file mode 100644 index 000000000..fe0780d70 --- /dev/null +++ b/atlas-io/README.md @@ -0,0 +1,7 @@ +atlas-io +======== + +An IO API for serializing arbitrary data, supporting compression. +The API contains a message format (like GRIB). +Multiple messages can be written to file or sent over network. + diff --git a/atlas-io/cmake/atlas-io-import.cmake.in b/atlas-io/cmake/atlas-io-import.cmake.in new file mode 100644 index 000000000..43cf6e5d7 --- /dev/null +++ b/atlas-io/cmake/atlas-io-import.cmake.in @@ -0,0 +1,5 @@ + +include( CMakeFindDependencyMacro ) + +## eckit +find_dependency( eckit HINTS ${CMAKE_CURRENT_LIST_DIR}/../eckit @eckit_DIR@ @eckit_BINARY_DIR@ ) diff --git a/atlas-io/src/CMakeLists.txt b/atlas-io/src/CMakeLists.txt new file mode 100644 index 000000000..b13eb90bd --- /dev/null +++ b/atlas-io/src/CMakeLists.txt @@ -0,0 +1,2 @@ +add_subdirectory(atlas_io) +add_subdirectory(tools) diff --git a/atlas-io/src/atlas_io/CMakeLists.txt b/atlas-io/src/atlas_io/CMakeLists.txt new file mode 100644 index 000000000..623ecb177 --- /dev/null +++ b/atlas-io/src/atlas_io/CMakeLists.txt @@ -0,0 +1,91 @@ +configure_file( detail/defines.h.in detail/defines.h ) + +ecbuild_add_library( TARGET atlas_io + + INSTALL_HEADERS ALL + HEADER_DESTINATION include/atlas_io + PUBLIC_LIBS eckit + PUBLIC_INCLUDES + $ + $ + + SOURCES + atlas-io.h + Data.cc + Data.h + detail/Assert.h + detail/Base64.cc + detail/Base64.h + detail/Checksum.h + detail/Checksum.cc + detail/DataInfo.h + detail/DataType.cc + detail/DataType.h + detail/Decoder.cc + detail/Decoder.h + detail/Defaults.h + detail/Encoder.cc + detail/Encoder.h + detail/Endian.h + detail/Link.cc + detail/Link.h + detail/ParsedRecord.h + detail/RecordInfo.h + detail/RecordSections.h + detail/Reference.h + detail/sfinae.h + detail/StaticAssert.h + detail/tag.h + detail/Time.cc + detail/Time.h + detail/Type.h + detail/TypeTraits.h + detail/Version.h + Exceptions.cc + Exceptions.h + FileStream.cc + FileStream.h + Metadata.cc + Metadata.h + print/TableFormat.cc + print/TableFormat.h + print/JSONFormat.cc + print/JSONFormat.h + print/Bytes.cc + print/Bytes.h + ReadRequest.cc + ReadRequest.h + Record.cc + Record.h + RecordItem.cc + RecordItem.h + RecordItemReader.cc + RecordItemReader.h + RecordPrinter.cc + RecordPrinter.h + RecordReader.cc + RecordReader.h + RecordWriter.cc + RecordWriter.h + Session.cc + Session.h + Stream.cc + Stream.h + Trace.cc + Trace.h + types/array.h + types/array/ArrayMetadata.cc + types/array/ArrayMetadata.h + types/array/ArrayReference.cc + types/array/ArrayReference.h + types/array/adaptors/StdArrayAdaptor.h + types/array/adaptors/StdVectorAdaptor.h + types/string.h + types/scalar.h + types/scalar.cc + ${CMAKE_CURRENT_BINARY_DIR}/detail/defines.h +) + +target_compile_features( atlas_io PUBLIC + cxx_std_11 + $<${HAVE_CXX17}:cxx_std_17> ) diff --git a/src/atlas/io/Data.cc b/atlas-io/src/atlas_io/Data.cc similarity index 90% rename from src/atlas/io/Data.cc rename to atlas-io/src/atlas_io/Data.cc index 0ed4fa498..2a5c4f9b8 100644 --- a/src/atlas/io/Data.cc +++ b/atlas-io/src/atlas_io/Data.cc @@ -14,10 +14,10 @@ #include "eckit/utils/Compressor.h" -#include "atlas/io/Stream.h" -#include "atlas/io/detail/Checksum.h" -#include "atlas/runtime/Exception.h" -#include "atlas/runtime/Trace.h" +#include "atlas_io/Stream.h" +#include "atlas_io/Trace.h" +#include "atlas_io/detail/Assert.h" +#include "atlas_io/detail/Checksum.h" namespace atlas { namespace io { @@ -27,9 +27,9 @@ namespace io { Data::Data(void* p, size_t size): buffer_(p, size), size_(size) {} std::uint64_t Data::write(Stream& out) const { - ATLAS_TRACE(); + ATLAS_IO_TRACE(); if (size()) { - ATLAS_ASSERT(buffer_.size() >= size()); + ATLAS_IO_ASSERT(buffer_.size() >= size()); return out.write(buffer_.data(), size()); } return 0; @@ -45,7 +45,7 @@ std::uint64_t Data::read(Stream& in, size_t size) { void Data::compress(const std::string& compression) { - ATLAS_TRACE("compress(" + compression + ")"); + ATLAS_IO_TRACE("compress(" + compression + ")"); if (size_) { auto compressor = std::unique_ptr(eckit::CompressorFactory::instance().build(compression)); if (dynamic_cast(compressor.get())) { @@ -58,7 +58,7 @@ void Data::compress(const std::string& compression) { } void Data::decompress(const std::string& compression, size_t uncompressed_size) { - ATLAS_TRACE("decompress(" + compression + ")"); + ATLAS_IO_TRACE("decompress(" + compression + ")"); auto compressor = std::unique_ptr(eckit::CompressorFactory::instance().build(compression)); if (dynamic_cast(compressor.get())) { diff --git a/src/atlas/io/Data.h b/atlas-io/src/atlas_io/Data.h similarity index 100% rename from src/atlas/io/Data.h rename to atlas-io/src/atlas_io/Data.h diff --git a/src/atlas/io/Exceptions.cc b/atlas-io/src/atlas_io/Exceptions.cc similarity index 98% rename from src/atlas/io/Exceptions.cc rename to atlas-io/src/atlas_io/Exceptions.cc index 2464a7996..43886633d 100644 --- a/src/atlas/io/Exceptions.cc +++ b/atlas-io/src/atlas_io/Exceptions.cc @@ -10,7 +10,7 @@ #include "Exceptions.h" -#include "eckit/eckit_config.h" +#include "atlas_io/detail/defines.h" #if ATLAS_HAVE_CXXABI_H #include #endif diff --git a/src/atlas/io/Exceptions.h b/atlas-io/src/atlas_io/Exceptions.h similarity index 100% rename from src/atlas/io/Exceptions.h rename to atlas-io/src/atlas_io/Exceptions.h diff --git a/src/atlas/io/FileStream.cc b/atlas-io/src/atlas_io/FileStream.cc similarity index 90% rename from src/atlas/io/FileStream.cc rename to atlas-io/src/atlas_io/FileStream.cc index 6ac548916..889b31749 100644 --- a/src/atlas/io/FileStream.cc +++ b/atlas-io/src/atlas_io/FileStream.cc @@ -8,14 +8,13 @@ * nor does it submit to any jurisdiction. */ -#include "atlas/io/FileStream.h" +#include "atlas_io/FileStream.h" #include "eckit/io/FileHandle.h" #include "eckit/io/PooledHandle.h" -#include "atlas/io/Session.h" -#include "atlas/runtime/Exception.h" -#include "atlas/runtime/Trace.h" +#include "atlas_io/Session.h" +#include "atlas_io/Trace.h" namespace atlas { namespace io { @@ -31,11 +30,11 @@ namespace { /// * read: for reading /// * write: for writing, will overwrite existing file /// * append: for appending implemented via write and seek to eof. -/// - ATLAS_TRACE recording +/// - ATLAS_IO_TRACE recording class FileHandle : public eckit::FileHandle { public: FileHandle(const eckit::PathName& path, char openmode): eckit::FileHandle(path, openmode == 'a' /*overwrite*/) { - ATLAS_TRACE("FileHandle::open(" + eckit::FileHandle::path() + "," + openmode + ")"); + ATLAS_IO_TRACE("FileHandle::open(" + eckit::FileHandle::path() + "," + openmode + ")"); if (openmode == 'r') { openForRead(); } @@ -50,7 +49,7 @@ class FileHandle : public eckit::FileHandle { void close() override { if (not closed_) { - ATLAS_TRACE("FileHandle::close(" + path() + ")"); + ATLAS_IO_TRACE("FileHandle::close(" + path() + ")"); eckit::FileHandle::close(); closed_ = true; } @@ -79,15 +78,15 @@ class FileHandle : public eckit::FileHandle { /// /// Main difference with eckit::PooledHandle /// - Automatic opening and closing of file -/// - ATLAS_TRACE recording +/// - ATLAS_IO_TRACE recording class PooledHandle : public eckit::PooledHandle { public: PooledHandle(const eckit::PathName& path): eckit::PooledHandle(path), path_(path) { - ATLAS_TRACE("PooledHandle::open(" + path_.baseName() + ")"); + ATLAS_IO_TRACE("PooledHandle::open(" + path_.baseName() + ")"); openForRead(); } ~PooledHandle() override { - ATLAS_TRACE("PooledHandle::close(" + path_.baseName() + ")"); + ATLAS_IO_TRACE("PooledHandle::close(" + path_.baseName() + ")"); close(); } eckit::PathName path_; diff --git a/src/atlas/io/FileStream.h b/atlas-io/src/atlas_io/FileStream.h similarity index 98% rename from src/atlas/io/FileStream.h rename to atlas-io/src/atlas_io/FileStream.h index e0b937beb..6a0c563a8 100644 --- a/src/atlas/io/FileStream.h +++ b/atlas-io/src/atlas_io/FileStream.h @@ -14,7 +14,7 @@ #include "eckit/filesystem/PathName.h" -#include "atlas/io/Stream.h" +#include "atlas_io/Stream.h" namespace eckit { class DataHandle; diff --git a/src/atlas/io/Metadata.cc b/atlas-io/src/atlas_io/Metadata.cc similarity index 92% rename from src/atlas/io/Metadata.cc rename to atlas-io/src/atlas_io/Metadata.cc index 2e0418131..44f0a6400 100644 --- a/src/atlas/io/Metadata.cc +++ b/atlas-io/src/atlas_io/Metadata.cc @@ -13,12 +13,14 @@ #include #include #include + #include "eckit/log/JSON.h" -#include "atlas/runtime/Exception.h" +#include "atlas_io/atlas_compat.h" +#include "atlas_io/detail/Assert.h" -#include "atlas/io/Exceptions.h" -#include "atlas/io/types/array/ArrayReference.h" +#include "atlas_io/Exceptions.h" +#include "atlas_io/types/array/ArrayReference.h" namespace atlas { namespace io { @@ -59,7 +61,7 @@ void write(const atlas::io::Metadata& metadata, atlas::io::Stream& out) { void Metadata::link(Metadata&& linked) { std::string initial_link = link(); - ATLAS_ASSERT(initial_link.size()); + ATLAS_IO_ASSERT(initial_link.size()); data = std::move(linked.data); record = std::move(linked.record); diff --git a/atlas-io/src/atlas_io/Metadata.h b/atlas-io/src/atlas_io/Metadata.h new file mode 100644 index 000000000..dce380105 --- /dev/null +++ b/atlas-io/src/atlas_io/Metadata.h @@ -0,0 +1,97 @@ +/* + * (C) Copyright 2020 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#pragma once + +#include +#include + +#include "atlas_io/Stream.h" +#include "atlas_io/detail/Checksum.h" +#include "atlas_io/detail/DataInfo.h" +#include "atlas_io/detail/Endian.h" +#include "atlas_io/detail/Link.h" +#include "atlas_io/detail/RecordInfo.h" +#include "atlas_io/detail/Type.h" + +#include "eckit/config/LocalConfiguration.h" +#include "eckit/value/Value.h" + + +namespace atlas { +namespace io { + +class Metadata; +class Stream; + +//--------------------------------------------------------------------------------------------------------------------- + +size_t uncompressed_size(const atlas::io::Metadata& m); + +//--------------------------------------------------------------------------------------------------------------------- + +class Metadata : public eckit::LocalConfiguration { +public: + using eckit::LocalConfiguration::LocalConfiguration; + + Metadata(): eckit::LocalConfiguration() {} + + Link link() const { return Link{getString("link", "")}; } + + Type type() const { return Type{getString("type", "")}; } + + void link(atlas::io::Metadata&&); + + std::string json() const; + + DataInfo data; + RecordInfo record; + + + // extended LocalConfiguration: + using eckit::LocalConfiguration::set; + Metadata& set(const eckit::LocalConfiguration& other) { + eckit::Value& root = const_cast(get()); + auto& other_root = other.get(); + std::vector other_keys; + eckit::fromValue(other_keys, other_root.keys()); + for (auto& key : other_keys) { + root[key] = other_root[key]; + } + return *this; + } + + /// @brief Constructor immediately setting a value. + template + Metadata(const std::string& name, const ValueT& value) { + set(name, value); + } + + /// @brief Constructor starting from a Configuration + Metadata(const eckit::Configuration& other): eckit::LocalConfiguration(other) {} + + + Metadata& remove(const std::string& name) { + eckit::Value& root = const_cast(get()); + root.remove(name); + return *this; + } +}; + +//--------------------------------------------------------------------------------------------------------------------- + +void write(const atlas::io::Metadata&, std::ostream& out); + +void write(const atlas::io::Metadata&, atlas::io::Stream& out); + +//--------------------------------------------------------------------------------------------------------------------- + +} // namespace io +} // namespace atlas diff --git a/src/atlas/io/ReadRequest.cc b/atlas-io/src/atlas_io/ReadRequest.cc similarity index 89% rename from src/atlas/io/ReadRequest.cc rename to atlas-io/src/atlas_io/ReadRequest.cc index bc4112433..030b2e5cd 100644 --- a/src/atlas/io/ReadRequest.cc +++ b/atlas-io/src/atlas_io/ReadRequest.cc @@ -10,13 +10,14 @@ #include "ReadRequest.h" -#include "atlas/io/Exceptions.h" -#include "atlas/io/RecordItemReader.h" -#include "atlas/io/detail/Checksum.h" -#include "atlas/io/detail/Defaults.h" -#include "atlas/runtime/Exception.h" -#include "atlas/runtime/Log.h" -#include "atlas/runtime/Trace.h" +#include "eckit/log/Log.h" + +#include "atlas_io/Exceptions.h" +#include "atlas_io/RecordItemReader.h" +#include "atlas_io/Trace.h" +#include "atlas_io/detail/Assert.h" +#include "atlas_io/detail/Checksum.h" +#include "atlas_io/detail/Defaults.h" namespace atlas { namespace io { @@ -32,7 +33,7 @@ static std::string stream_path(Stream stream) { ReadRequest::ReadRequest(const std::string& URI, atlas::io::Decoder* decoder): uri_(URI), decoder_(decoder), item_(new RecordItem()) { do_checksum_ = defaults::checksum_read(); - ATLAS_ASSERT(uri_.size()); + ATLAS_IO_ASSERT(uri_.size()); } ReadRequest::ReadRequest(Stream stream, size_t offset, const std::string& key, Decoder* decoder): @@ -43,7 +44,7 @@ ReadRequest::ReadRequest(Stream stream, size_t offset, const std::string& key, D decoder_(decoder), item_(new RecordItem()) { do_checksum_ = defaults::checksum_read(); - ATLAS_ASSERT(stream_); + ATLAS_IO_ASSERT(stream_); } //--------------------------------------------------------------------------------------------------------------------- @@ -66,7 +67,7 @@ ReadRequest::ReadRequest(ReadRequest&& other): ReadRequest::~ReadRequest() { if (item_) { if (not finished_) { - Log::error() << "Request for " << uri_ << " was not completed." << std::endl; + eckit::Log::error() << "Request for " << uri_ << " was not completed." << std::endl; } } } @@ -126,7 +127,7 @@ void ReadRequest::decode() { decompress(); io::decode(item_->metadata(), item_->data(), *decoder_); if (item_->data().size()) { - ATLAS_TRACE_SCOPE("deallocate"); + ATLAS_IO_TRACE_SCOPE("deallocate"); item_->clear(); } else { @@ -137,7 +138,7 @@ void ReadRequest::decode() { //--------------------------------------------------------------------------------------------------------------------- void ReadRequest::wait() { - ATLAS_TRACE("ReadRequest::wait(" + uri_ + ")"); + ATLAS_IO_TRACE("ReadRequest::wait(" + uri_ + ")"); if (item_) { if (not finished_) { read(); diff --git a/src/atlas/io/ReadRequest.h b/atlas-io/src/atlas_io/ReadRequest.h similarity index 96% rename from src/atlas/io/ReadRequest.h rename to atlas-io/src/atlas_io/ReadRequest.h index c655c4436..b510147fa 100644 --- a/src/atlas/io/ReadRequest.h +++ b/atlas-io/src/atlas_io/ReadRequest.h @@ -14,8 +14,8 @@ #include #include -#include "atlas/io/RecordItem.h" -#include "atlas/io/detail/Decoder.h" +#include "atlas_io/RecordItem.h" +#include "atlas_io/detail/Decoder.h" namespace atlas { namespace io { diff --git a/src/atlas/io/Record.cc b/atlas-io/src/atlas_io/Record.cc similarity index 95% rename from src/atlas/io/Record.cc rename to atlas-io/src/atlas_io/Record.cc index 4a3297fb9..9eff80b01 100644 --- a/src/atlas/io/Record.cc +++ b/atlas-io/src/atlas_io/Record.cc @@ -8,18 +8,16 @@ * nor does it submit to any jurisdiction. */ -#include "atlas/io/Record.h" +#include "atlas_io/Record.h" #include "eckit/config/YAMLConfiguration.h" #include "eckit/filesystem/URI.h" -#include "atlas/runtime/Exception.h" -#include "atlas/runtime/Log.h" -#include "atlas/runtime/Trace.h" - -#include "atlas/io/Exceptions.h" -#include "atlas/io/detail/ParsedRecord.h" -#include "atlas/io/detail/Version.h" +#include "atlas_io/Exceptions.h" +#include "atlas_io/Trace.h" +#include "atlas_io/detail/Assert.h" +#include "atlas_io/detail/ParsedRecord.h" +#include "atlas_io/detail/Version.h" namespace atlas { namespace io { @@ -58,7 +56,7 @@ Endian RecordHead::endian() const { else if (magic_number == 3523477504) { return Endian::swapped; } - throw_Exception("Mixed endianness is not supported", Here()); + throw Exception("Mixed endianness is not supported", Here()); } //--------------------------------------------------------------------------------------------------------------------- @@ -85,7 +83,7 @@ bool Record::empty() const { const Metadata& Record::metadata(const std::string& key) const { if (record_->items.find(key) == record_->items.end()) { - ATLAS_THROW_EXCEPTION("Record does not contain key \"" << key << "\""); + throw Exception("Record does not contain key \"" + key + "\"", Here()); } return record_->items.at(key); } @@ -139,7 +137,7 @@ Record& Record::read(Stream& in, bool read_to_end) { return *this; } - ATLAS_TRACE("read_metadata"); + ATLAS_IO_TRACE("read_metadata"); auto& r = record_->head; auto rbegin = in.position(); @@ -206,7 +204,7 @@ Record& Record::read(Stream& in, bool read_to_end) { throw DataCorruption(err.str()); } - ATLAS_ASSERT(r.metadata_format == "yaml"); + ATLAS_IO_ASSERT(r.metadata_format == "yaml"); Metadata metadata = eckit::YAMLConfiguration(metadata_str); record_->keys = metadata.keys(); for (const auto& key : record_->keys) { diff --git a/src/atlas/io/Record.h b/atlas-io/src/atlas_io/Record.h similarity index 94% rename from src/atlas/io/Record.h rename to atlas-io/src/atlas_io/Record.h index d14d5ea70..b543c5827 100644 --- a/src/atlas/io/Record.h +++ b/atlas-io/src/atlas_io/Record.h @@ -15,9 +15,9 @@ #include #include -#include "atlas/io/detail/Endian.h" -#include "atlas/io/detail/Time.h" -#include "atlas/io/detail/Version.h" +#include "atlas_io/detail/Endian.h" +#include "atlas_io/detail/Time.h" +#include "atlas_io/detail/Version.h" namespace atlas { namespace io { diff --git a/src/atlas/io/RecordItem.cc b/atlas-io/src/atlas_io/RecordItem.cc similarity index 94% rename from src/atlas/io/RecordItem.cc rename to atlas-io/src/atlas_io/RecordItem.cc index de695bcdc..c163d5ce4 100644 --- a/src/atlas/io/RecordItem.cc +++ b/atlas-io/src/atlas_io/RecordItem.cc @@ -12,7 +12,8 @@ #include "eckit/filesystem/URI.h" -#include "atlas/runtime/Exception.h" +#include "atlas_io/atlas_compat.h" +#include "atlas_io/detail/Assert.h" namespace atlas { namespace io { @@ -22,8 +23,8 @@ namespace io { RecordItem::URI::URI(const std::string& _uri) { eckit::URI uri{_uri}; - ATLAS_ASSERT(uri.scheme() == "file"); - ATLAS_ASSERT(not uri.query("key").empty()); + ATLAS_IO_ASSERT(uri.scheme() == "file"); + ATLAS_IO_ASSERT(not uri.query("key").empty()); path = uri.path(); offset = 0; @@ -95,7 +96,7 @@ void RecordItem::clear() { //--------------------------------------------------------------------------------------------------------------------- void RecordItem::decompress() { - ATLAS_ASSERT(not empty()); + ATLAS_IO_ASSERT(not empty()); if (metadata().data.compressed()) { data_.decompress(metadata().data.compression(), metadata().data.size()); } @@ -105,7 +106,7 @@ void RecordItem::decompress() { //--------------------------------------------------------------------------------------------------------------------- void RecordItem::compress() { - ATLAS_ASSERT(not empty()); + ATLAS_IO_ASSERT(not empty()); if (not metadata().data.compressed() && metadata().data.compression() != "none") { data_.compress(metadata().data.compression()); metadata_->data.compressed(true); diff --git a/src/atlas/io/RecordItem.h b/atlas-io/src/atlas_io/RecordItem.h similarity index 96% rename from src/atlas/io/RecordItem.h rename to atlas-io/src/atlas_io/RecordItem.h index a8d8eb33f..994cbe9b3 100644 --- a/src/atlas/io/RecordItem.h +++ b/atlas-io/src/atlas_io/RecordItem.h @@ -14,9 +14,9 @@ #include #include -#include "atlas/io/Data.h" -#include "atlas/io/Metadata.h" -#include "atlas/io/detail/tag.h" +#include "atlas_io/Data.h" +#include "atlas_io/Metadata.h" +#include "atlas_io/detail/tag.h" namespace atlas { namespace io { diff --git a/src/atlas/io/RecordItemReader.cc b/atlas-io/src/atlas_io/RecordItemReader.cc similarity index 90% rename from src/atlas/io/RecordItemReader.cc rename to atlas-io/src/atlas_io/RecordItemReader.cc index 79888dd4a..9ef960d62 100644 --- a/src/atlas/io/RecordItemReader.cc +++ b/atlas-io/src/atlas_io/RecordItemReader.cc @@ -10,16 +10,14 @@ #include "RecordItemReader.h" -#include "atlas/io/Exceptions.h" -#include "atlas/io/FileStream.h" -#include "atlas/io/Record.h" -#include "atlas/io/Session.h" -#include "atlas/io/detail/ParsedRecord.h" -#include "atlas/io/detail/RecordSections.h" - -#include "atlas/runtime/Exception.h" -#include "atlas/runtime/Log.h" -#include "atlas/runtime/Trace.h" +#include "atlas_io/Exceptions.h" +#include "atlas_io/FileStream.h" +#include "atlas_io/Record.h" +#include "atlas_io/Session.h" +#include "atlas_io/Trace.h" +#include "atlas_io/detail/Assert.h" +#include "atlas_io/detail/ParsedRecord.h" +#include "atlas_io/detail/RecordSections.h" namespace atlas { namespace io { @@ -49,7 +47,7 @@ inline Struct read_struct(IStream& in) { //--------------------------------------------------------------------------------------------------------------------- static Data read_data(const Record& record, int data_section_index, Stream in) { - ATLAS_TRACE("read_data(data_section=" + std::to_string(data_section_index) + ")"); + ATLAS_IO_TRACE("read_data(data_section=" + std::to_string(data_section_index) + ")"); if (data_section_index == 0) { return atlas::io::Data(); } @@ -70,7 +68,7 @@ static Data read_data(const Record& record, int data_section_index, Stream in) { if (data.read(in, data_size) != data_size) { throw InvalidRecord("Data section is not valid"); } - ATLAS_ASSERT(data.size() == data_size); + ATLAS_IO_ASSERT(data.size() == data_size); } auto data_end = atlas::io::read_struct(in); if (not data_end.valid()) { @@ -119,7 +117,7 @@ static Record read_record(Stream in, size_t offset) { //--------------------------------------------------------------------------------------------------------------------- RecordItemReader::RecordItemReader(Stream in, size_t offset, const std::string& key): in_(in), uri_{"", offset, key} { - ATLAS_TRACE("RecordItemReader(Stream,offset,key"); + ATLAS_IO_TRACE("RecordItemReader(Stream,offset,key"); record_ = read_record(in, uri_.offset); if (not record_.has(uri_.key)) { @@ -170,7 +168,7 @@ void RecordItemReader::read(RecordItem& item) { //--------------------------------------------------------------------------------------------------------------------- void RecordItemReader::read(Metadata& metadata, bool follow_links) { - ATLAS_TRACE("RecordItemReader::read_metadata(" + uri_.path + ":" + uri_.key + ")"); + ATLAS_IO_TRACE("RecordItemReader::read_metadata(" + uri_.path + ":" + uri_.key + ")"); metadata = record_.metadata(uri_.key); @@ -186,7 +184,7 @@ void RecordItemReader::read(Metadata& metadata, bool follow_links) { //--------------------------------------------------------------------------------------------------------------------- static void read_from_stream(Record record, Stream in, const std::string& key, io::Metadata& metadata, io::Data& data) { - ATLAS_TRACE("RecordItemReader::read( Stream, " + key + ")"); + ATLAS_IO_TRACE("RecordItemReader::read( Stream, " + key + ")"); metadata = record.metadata(key); @@ -207,7 +205,7 @@ void RecordItemReader::read(io::Metadata& metadata, io::Data& data) { return; } - ATLAS_TRACE("RecordItemReader::read(" + uri_.path + ":" + uri_.key + ")"); + ATLAS_IO_TRACE("RecordItemReader::read(" + uri_.path + ":" + uri_.key + ")"); metadata = record_.metadata(uri_.key); diff --git a/src/atlas/io/RecordItemReader.h b/atlas-io/src/atlas_io/RecordItemReader.h similarity index 93% rename from src/atlas/io/RecordItemReader.h rename to atlas-io/src/atlas_io/RecordItemReader.h index 07f019b66..55cc4cdfa 100644 --- a/src/atlas/io/RecordItemReader.h +++ b/atlas-io/src/atlas_io/RecordItemReader.h @@ -12,9 +12,9 @@ #include -#include "atlas/io/Record.h" -#include "atlas/io/RecordItem.h" -#include "atlas/io/Stream.h" +#include "atlas_io/Record.h" +#include "atlas_io/RecordItem.h" +#include "atlas_io/Stream.h" namespace atlas { namespace io { diff --git a/src/atlas/io/RecordPrinter.cc b/atlas-io/src/atlas_io/RecordPrinter.cc similarity index 75% rename from src/atlas/io/RecordPrinter.cc rename to atlas-io/src/atlas_io/RecordPrinter.cc index 225628df1..e12ef5abd 100644 --- a/src/atlas/io/RecordPrinter.cc +++ b/atlas-io/src/atlas_io/RecordPrinter.cc @@ -12,33 +12,37 @@ #include -#include "atlas/io/FileStream.h" -#include "atlas/io/print/JSONFormat.h" -#include "atlas/io/print/TableFormat.h" -#include "atlas/runtime/Exception.h" +#include "atlas_io/Exceptions.h" +#include "atlas_io/FileStream.h" +#include "atlas_io/detail/Assert.h" +#include "atlas_io/print/JSONFormat.h" +#include "atlas_io/print/TableFormat.h" +#include "atlas_io/atlas_compat.h" namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- -RecordPrinter::RecordPrinter(const eckit::PathName& path, const util::Config& config): RecordPrinter(path, 0, config) {} +RecordPrinter::RecordPrinter(const eckit::PathName& path, const eckit::Configuration& config): + RecordPrinter(path, 0, config) {} //--------------------------------------------------------------------------------------------------------------------- -RecordPrinter::RecordPrinter(const eckit::PathName& path, const std::uint64_t offset, const util::Config& config): +RecordPrinter::RecordPrinter(const eckit::PathName& path, const std::uint64_t offset, + const eckit::Configuration& config): RecordPrinter(Record::URI{path, offset}, config) {} //--------------------------------------------------------------------------------------------------------------------- -RecordPrinter::RecordPrinter(const Record::URI& ref, const util::Config& config): +RecordPrinter::RecordPrinter(const Record::URI& ref, const eckit::Configuration& config): uri_(ref), record_(Session::record(ref.path, ref.offset)) { if (record_.empty()) { auto in = InputFileStream(uri_.path); in.seek(uri_.offset); record_.read(in, true); - ATLAS_ASSERT(not record_.empty()); + ATLAS_IO_ASSERT(not record_.empty()); } config.get("format", options_.format); @@ -60,7 +64,7 @@ RecordPrinter::RecordPrinter(const Record::URI& ref, const util::Config& config) for (auto& supported_format : supported_formats) { s << "\n - " << supported_format; } - throw_Exception(s.str(), Here()); + throw Exception(s.str(), Here()); } } } @@ -68,17 +72,19 @@ RecordPrinter::RecordPrinter(const Record::URI& ref, const util::Config& config) //--------------------------------------------------------------------------------------------------------------------- void RecordPrinter::print(std::ostream& out) const { + eckit::LocalConfiguration config; + config.set("details", options_.details); if (options_.format == "json") { - JSONFormat{uri_, util::Config("details", options_.details)}.print(out); + JSONFormat{uri_, config}.print(out); } else if (options_.format == "yaml") { - JSONFormat{uri_, util::Config("details", options_.details)}.print(out); + JSONFormat{uri_, config}.print(out); } else if (options_.format == "table") { - TableFormat{uri_, util::Config("details", options_.details)}.print(out); + TableFormat{uri_, config}.print(out); } else { - ATLAS_THROW_EXCEPTION("Cannot print record: Unrecognized format " << options_.format << "."); + throw Exception("Cannot print record: Unrecognized format " + options_.format + ".", Here()); } } diff --git a/src/atlas/io/RecordPrinter.h b/atlas-io/src/atlas_io/RecordPrinter.h similarity index 80% rename from src/atlas/io/RecordPrinter.h rename to atlas-io/src/atlas_io/RecordPrinter.h index 75302ef16..a05d6919c 100644 --- a/src/atlas/io/RecordPrinter.h +++ b/atlas-io/src/atlas_io/RecordPrinter.h @@ -14,11 +14,12 @@ #include #include +#include "eckit/config/Configuration.h" #include "eckit/filesystem/PathName.h" -#include "atlas/io/Record.h" -#include "atlas/io/Session.h" -#include "atlas/util/Config.h" +#include "atlas_io/Record.h" +#include "atlas_io/Session.h" +#include "atlas_io/detail/NoConfig.h" namespace atlas { namespace io { @@ -27,11 +28,11 @@ namespace io { class RecordPrinter { public: - RecordPrinter(const Record::URI&, const util::Config& = util::NoConfig()); + RecordPrinter(const Record::URI&, const eckit::Configuration& = NoConfig()); - RecordPrinter(const eckit::PathName&, const util::Config& = util::NoConfig()); + RecordPrinter(const eckit::PathName&, const eckit::Configuration& = NoConfig()); - RecordPrinter(const eckit::PathName&, std::uint64_t offset, const util::Config& = util::NoConfig()); + RecordPrinter(const eckit::PathName&, std::uint64_t offset, const eckit::Configuration& = NoConfig()); Record record() const { return record_; } diff --git a/src/atlas/io/RecordReader.cc b/atlas-io/src/atlas_io/RecordReader.cc similarity index 90% rename from src/atlas/io/RecordReader.cc rename to atlas-io/src/atlas_io/RecordReader.cc index 80d2edfe7..d64171da4 100644 --- a/src/atlas/io/RecordReader.cc +++ b/atlas-io/src/atlas_io/RecordReader.cc @@ -10,8 +10,8 @@ #include "RecordReader.h" -#include "atlas/io/Metadata.h" -#include "atlas/io/RecordItemReader.h" +#include "atlas_io/Metadata.h" +#include "atlas_io/RecordItemReader.h" namespace atlas { namespace io { @@ -43,6 +43,10 @@ RecordItem::URI RecordReader::uri(const std::string& key) const { //--------------------------------------------------------------------------------------------------------------------- +void RecordReader::trace(const std::string&, const char* file, int line, const char* func) {} + +//--------------------------------------------------------------------------------------------------------------------- + void RecordReader::wait(const std::string& key) { request(key).wait(); } diff --git a/src/atlas/io/RecordReader.h b/atlas-io/src/atlas_io/RecordReader.h similarity index 86% rename from src/atlas/io/RecordReader.h rename to atlas-io/src/atlas_io/RecordReader.h index 157882d0c..6082ddc67 100644 --- a/src/atlas/io/RecordReader.h +++ b/atlas-io/src/atlas_io/RecordReader.h @@ -14,9 +14,9 @@ #include #include -#include "atlas/io/Metadata.h" -#include "atlas/io/ReadRequest.h" -#include "atlas/io/Session.h" +#include "atlas_io/Metadata.h" +#include "atlas_io/ReadRequest.h" +#include "atlas_io/Session.h" namespace atlas { namespace io { @@ -33,10 +33,10 @@ class RecordReader { template ReadRequest& read(const std::string& key, T& value) { - ATLAS_TRACE("read(" + key + ")"); + trace("read(" + key + ")", __FILE__, __LINE__, __func__); if (stream_) { - ATLAS_TRACE("stream"); + trace("stream", __FILE__, __LINE__, __func__); requests_.emplace(key, ReadRequest{stream_, offset_, key, value}); } else { @@ -63,6 +63,8 @@ class RecordReader { RecordItem::URI uri(const std::string& key) const; + void trace(const std::string&, const char* file, int line, const char* func); + private: Session session_; diff --git a/src/atlas/io/RecordWriter.cc b/atlas-io/src/atlas_io/RecordWriter.cc similarity index 94% rename from src/atlas/io/RecordWriter.cc rename to atlas-io/src/atlas_io/RecordWriter.cc index ff5394b5f..b8299ef18 100644 --- a/src/atlas/io/RecordWriter.cc +++ b/atlas-io/src/atlas_io/RecordWriter.cc @@ -8,16 +8,16 @@ * nor does it submit to any jurisdiction. */ -#include "atlas/io/RecordWriter.h" +#include "atlas_io/RecordWriter.h" -#include "atlas/io/Exceptions.h" -#include "atlas/io/RecordWriter.h" -#include "atlas/io/detail/Checksum.h" -#include "atlas/io/detail/Defaults.h" -#include "atlas/io/detail/Encoder.h" -#include "atlas/io/detail/RecordSections.h" -#include "atlas/runtime/Exception.h" +#include "atlas_io/Exceptions.h" +#include "atlas_io/RecordWriter.h" +#include "atlas_io/Trace.h" +#include "atlas_io/detail/Checksum.h" +#include "atlas_io/detail/Defaults.h" +#include "atlas_io/detail/Encoder.h" +#include "atlas_io/detail/RecordSections.h" namespace atlas { namespace io { @@ -44,7 +44,7 @@ inline void write_string(OStream& out, const std::string& s) { //--------------------------------------------------------------------------------------------------------------------- size_t RecordWriter::write(Stream out) const { - ATLAS_TRACE("RecordWriter::write"); + ATLAS_IO_TRACE("RecordWriter::write"); RecordHead r; auto begin_of_record = out.position(); @@ -59,7 +59,7 @@ size_t RecordWriter::write(Stream out) const { // Metadata section // ---------------- - ATLAS_TRACE_SCOPE("metadata section") { + ATLAS_IO_TRACE_SCOPE("metadata section") { r.metadata_offset = position(); atlas::io::write_struct(out, RecordMetadataSection::Begin()); auto metadata_str = metadata(); @@ -84,7 +84,7 @@ size_t RecordWriter::write(Stream out) const { // Data sections // ------------- - ATLAS_TRACE_SCOPE("data sections") { + ATLAS_IO_TRACE_SCOPE("data sections") { size_t i{0}; for (auto& key : keys_) { auto& encoder = encoders_.at(key); @@ -157,7 +157,7 @@ void RecordWriter::checksum(bool on) { //--------------------------------------------------------------------------------------------------------------------- -void RecordWriter::set(const RecordWriter::Key& key, Link&& link, const util::Config&) { +void RecordWriter::set(const RecordWriter::Key& key, Link&& link, const eckit::Configuration&) { keys_.emplace_back(key); encoders_[key] = std::move(Encoder{link}); info_.emplace(key, DataInfo{}); @@ -165,7 +165,7 @@ void RecordWriter::set(const RecordWriter::Key& key, Link&& link, const util::Co //--------------------------------------------------------------------------------------------------------------------- -void RecordWriter::set(const RecordWriter::Key& key, Encoder&& encoder, const util::Config& config) { +void RecordWriter::set(const RecordWriter::Key& key, Encoder&& encoder, const eckit::Configuration& config) { DataInfo info; if (encoder.encodes_data()) { ++nb_data_sections_; diff --git a/src/atlas/io/RecordWriter.h b/atlas-io/src/atlas_io/RecordWriter.h similarity index 79% rename from src/atlas/io/RecordWriter.h rename to atlas-io/src/atlas_io/RecordWriter.h index ca321c5b2..ef30883bd 100644 --- a/src/atlas/io/RecordWriter.h +++ b/atlas-io/src/atlas_io/RecordWriter.h @@ -15,15 +15,17 @@ #include -#include "atlas/io/FileStream.h" -#include "atlas/io/RecordItem.h" -#include "atlas/io/Stream.h" -#include "atlas/io/detail/Encoder.h" -#include "atlas/io/detail/Reference.h" -#include "atlas/io/detail/TypeTraits.h" -#include "atlas/io/types/array/ArrayReference.h" +#include "atlas_io/FileStream.h" +#include "atlas_io/RecordItem.h" +#include "atlas_io/Stream.h" +#include "atlas_io/detail/Encoder.h" +#include "atlas_io/detail/NoConfig.h" +#include "atlas_io/detail/Reference.h" +#include "atlas_io/detail/TypeTraits.h" -#include "atlas/io/detail/Defaults.h" +#include "atlas_io/types/array/ArrayReference.h" + +#include "atlas_io/detail/Defaults.h" namespace atlas { @@ -57,32 +59,32 @@ class RecordWriter { // -- set( Key, Value ) where Value can be a variety of things /// @brief Add link to other record item (RecordItem::URI) - void set(const Key&, Link&&, const util::Config& = util::NoConfig()); + void set(const Key&, Link&&, const eckit::Configuration& = NoConfig()); /// @brief Add item to record - void set(const Key&, Encoder&&, const util::Config& = util::NoConfig()); + void set(const Key&, Encoder&&, const eckit::Configuration& = NoConfig()); /// @brief Add item to record template = 0> - void set(const Key& key, Value&& value, const util::Config& config = util::NoConfig()) { + void set(const Key& key, Value&& value, const eckit::Configuration& config = NoConfig()) { set(key, Encoder{std::move(value)}, config); } /// @brief Add item to record template - void set(const Key& key, const Reference& value, const util::Config& config = util::NoConfig()) { + void set(const Key& key, const Reference& value, const eckit::Configuration& config = NoConfig()) { set(key, std::move(value), config); } /// @brief Add item to record template = 0> - void set(const Key& key, const Value& value, const util::Config& config = util::NoConfig()) { + void set(const Key& key, const Value& value, const eckit::Configuration& config = NoConfig()) { set(key, RecordItem(interprete(value)), config); } /// @brief Add item to record template = 0> - void set(const Key& key, const Value& value, const util::Config& config = util::NoConfig()) { + void set(const Key& key, const Value& value, const eckit::Configuration& config = NoConfig()) { set(key, Encoder{value}, config); } diff --git a/src/atlas/io/Session.cc b/atlas-io/src/atlas_io/Session.cc similarity index 94% rename from src/atlas/io/Session.cc rename to atlas-io/src/atlas_io/Session.cc index c3e69a64b..6ce9756ba 100644 --- a/src/atlas/io/Session.cc +++ b/atlas-io/src/atlas_io/Session.cc @@ -19,7 +19,9 @@ #include "eckit/filesystem/PathName.h" -#include "atlas/runtime/Exception.h" +#include "atlas_io/Exceptions.h" +#include "atlas_io/atlas_compat.h" +#include "atlas_io/detail/Assert.h" namespace atlas { namespace io { @@ -91,7 +93,7 @@ void ActiveSession::store(Stream stream) { SessionImpl& ActiveSession::current() { lock_guard lock(mutex_); if (count_ == 0) { - ATLAS_THROW_EXCEPTION("No atlas::io session is currently active"); + throw Exception("No atlas::io session is currently active", Here()); } return *session_; } @@ -101,7 +103,7 @@ SessionImpl& ActiveSession::current() { void ActiveSession::push() { lock_guard lock(mutex_); if (count_ == 0) { - ATLAS_ASSERT(session_ == nullptr); + ATLAS_IO_ASSERT(session_ == nullptr); session_.reset(new SessionImpl()); } ++count_; @@ -112,7 +114,7 @@ void ActiveSession::push() { void ActiveSession::pop() { lock_guard lock(mutex_); if (count_ == 0) { - ATLAS_THROW_EXCEPTION("No atlas::io session is currently active"); + throw Exception("No atlas::io session is currently active", Here()); } --count_; if (count_ == 0) { diff --git a/src/atlas/io/Session.h b/atlas-io/src/atlas_io/Session.h similarity index 94% rename from src/atlas/io/Session.h rename to atlas-io/src/atlas_io/Session.h index 6827bf2ff..4ab4f146b 100644 --- a/src/atlas/io/Session.h +++ b/atlas-io/src/atlas_io/Session.h @@ -13,8 +13,8 @@ #include #include -#include "atlas/io/Record.h" -#include "atlas/io/Stream.h" +#include "atlas_io/Record.h" +#include "atlas_io/Stream.h" namespace atlas { namespace io { diff --git a/src/atlas/io/Stream.cc b/atlas-io/src/atlas_io/Stream.cc similarity index 84% rename from src/atlas/io/Stream.cc rename to atlas-io/src/atlas_io/Stream.cc index 1f5559fd8..a69cdd3b7 100644 --- a/src/atlas/io/Stream.cc +++ b/atlas-io/src/atlas_io/Stream.cc @@ -8,11 +8,12 @@ * nor does it submit to any jurisdiction. */ -#include "atlas/io/Stream.h" +#include "atlas_io/Stream.h" #include "eckit/io/DataHandle.h" -#include "atlas/runtime/Exception.h" +#include "atlas_io/atlas_compat.h" +#include "atlas_io/detail/Assert.h" namespace atlas { namespace io { @@ -28,27 +29,27 @@ Stream::Stream(std::shared_ptr datahandle): shared_(datahandl Stream::Stream(const Stream& other) = default; eckit::DataHandle& Stream::datahandle() { - ATLAS_ASSERT(ptr_ != nullptr); + ATLAS_IO_ASSERT(ptr_ != nullptr); return *ptr_; } uint64_t Stream::seek(uint64_t offset) { - ATLAS_ASSERT(ptr_ != nullptr); + ATLAS_IO_ASSERT(ptr_ != nullptr); return std::uint64_t(ptr_->seek(static_cast(offset))); } uint64_t Stream::position() { - ATLAS_ASSERT(ptr_ != nullptr); + ATLAS_IO_ASSERT(ptr_ != nullptr); return std::uint64_t(ptr_->position()); } uint64_t Stream::write(const void* data, size_t length) { - ATLAS_ASSERT(ptr_ != nullptr); + ATLAS_IO_ASSERT(ptr_ != nullptr); return std::uint64_t(ptr_->write(data, static_cast(length))); } uint64_t Stream::read(void* data, size_t length) { - ATLAS_ASSERT(ptr_ != nullptr); + ATLAS_IO_ASSERT(ptr_ != nullptr); return std::uint64_t(ptr_->read(data, static_cast(length))); } diff --git a/src/atlas/io/Stream.h b/atlas-io/src/atlas_io/Stream.h similarity index 100% rename from src/atlas/io/Stream.h rename to atlas-io/src/atlas_io/Stream.h diff --git a/atlas-io/src/atlas_io/Trace.cc b/atlas-io/src/atlas_io/Trace.cc new file mode 100644 index 000000000..468626cae --- /dev/null +++ b/atlas-io/src/atlas_io/Trace.cc @@ -0,0 +1,37 @@ +/* + * (C) Copyright 2020 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#include "Trace.h" + +#include "eckit/log/CodeLocation.h" + +namespace atlas { +namespace io { + +atlas::io::Trace::Trace(const eckit::CodeLocation& loc) { + for (auto& hook : TraceHookRegistry::instance().hooks) { + hooks_.emplace_back(hook(loc, loc.func())); + } +} + +Trace::Trace(const eckit::CodeLocation& loc, const std::string& title) { + for (auto& hook : TraceHookRegistry::instance().hooks) { + hooks_.emplace_back(hook(loc, title)); + } +} + +Trace::Trace(const eckit::CodeLocation& loc, const std::string& title, const Labels& labels) { + for (auto& hook : TraceHookRegistry::instance().hooks) { + hooks_.emplace_back(hook(loc, title)); + } +} + +} // namespace io +} // namespace atlas diff --git a/atlas-io/src/atlas_io/Trace.h b/atlas-io/src/atlas_io/Trace.h new file mode 100644 index 000000000..bf622edc0 --- /dev/null +++ b/atlas-io/src/atlas_io/Trace.h @@ -0,0 +1,61 @@ +/* + * (C) Copyright 2020 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#pragma once + + +#include +#include +#include +#include + +namespace eckit { +class CodeLocation; +} + +namespace atlas { +namespace io { + +struct TraceHook { + TraceHook() = default; + virtual ~TraceHook() = default; +}; + +struct TraceHookRegistry { + using TraceHookBuilder = std::function(const eckit::CodeLocation&, const std::string&)>; + std::vector hooks; + static TraceHookRegistry& instance() { + static TraceHookRegistry instance; + return instance; + } + static void add(TraceHookBuilder&& hook) { instance().hooks.emplace_back(hook); } + static void add(const TraceHookBuilder& hook) { instance().hooks.emplace_back(hook); } + +private: + TraceHookRegistry() = default; +}; + +struct Trace { + using Labels = std::vector; + Trace(const eckit::CodeLocation& loc); + Trace(const eckit::CodeLocation& loc, const std::string& title); + Trace(const eckit::CodeLocation& loc, const std::string& title, const Labels& labels); + +private: + std::vector> hooks_; +}; + +} // namespace io +} // namespace atlas + +#include "atlas_io/detail/BlackMagic.h" + +#define ATLAS_IO_TRACE(...) __ATLAS_IO_TYPE(::atlas::io::Trace, Here() __ATLAS_IO_COMMA_ARGS(__VA_ARGS__)) +#define ATLAS_IO_TRACE_SCOPE(...) __ATLAS_IO_TYPE_SCOPE(::atlas::io::Trace, Here() __ATLAS_IO_COMMA_ARGS(__VA_ARGS__)) diff --git a/atlas-io/src/atlas_io/atlas-io.h b/atlas-io/src/atlas_io/atlas-io.h new file mode 100644 index 000000000..c56adccd1 --- /dev/null +++ b/atlas-io/src/atlas_io/atlas-io.h @@ -0,0 +1,127 @@ +/* + * (C) Copyright 2020 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#pragma once + +#include +#include +#include + +#include "atlas_io/detail/Link.h" +#include "atlas_io/detail/Reference.h" +#include "atlas_io/detail/StaticAssert.h" +#include "atlas_io/detail/sfinae.h" + +#include "atlas_io/Exceptions.h" +#include "atlas_io/FileStream.h" +#include "atlas_io/Record.h" +#include "atlas_io/RecordItemReader.h" +#include "atlas_io/RecordPrinter.h" +#include "atlas_io/RecordReader.h" +#include "atlas_io/RecordWriter.h" +#include "atlas_io/Session.h" +#include "atlas_io/Stream.h" +#include "atlas_io/Trace.h" + +#include "atlas_io/types/array.h" +#include "atlas_io/types/scalar.h" +#include "atlas_io/types/string.h" + + +namespace atlas { +namespace io { + +//--------------------------------------------------------------------------------------------------------------------- + +inline Link link(const std::string& uri) { + return Link{uri}; +} + +//--------------------------------------------------------------------------------------------------------------------- + +template = 0> +Reference ref(const T& x, tag::enable_static_assert = tag::enable_static_assert()) { + static_assert(is_encodable(), + "in atlas::io::ref(const Value&)" + "\n" + "\n Static assertion failed" + "\n -----------------------" + "\n" + "\n Cannot encode values of referenced type." + "\n" + "\n Implement the functions" + "\n" + "\n void encode_data(const Value& in, atlas::io::Data& out);" + "\n size_t encode_metadata(const Value& value, atlas::io::Metadata& metadata);" + "\n" + "\n or alternatively a conversion function to atlas::io::types::ArrayView" + "\n" + "\n void interprete(const Value& in, atlas::io::types::ArrayView& out)" + "\n" + "\n Rules of argument-dependent-lookup apply." + "\n --> Functions need to be declared in namespace of any of the arguments." + "\n" + "\n Note, turn this into a runtime exception by calling this function instead:" + "\n" + "\n atlas::io::ref(const T&, atlas::io::no_static_assert() )" + "\n"); + return Reference(x); +} + + +template = 0> +Reference ref(const T& x, tag::disable_static_assert) { + if (not is_encodable()) { + throw NotEncodable(x); + } + return Reference(x); +} + + +template = 0> +ArrayReference ref(const T& x, tag::enable_static_assert = tag::enable_static_assert()) { + ArrayReference w; + interprete(x, w); + return w; +} + +//--------------------------------------------------------------------------------------------------------------------- + +template +RecordItem copy(T&& value, tag::disable_static_assert) { + return RecordItem(std::forward(value), tag::disable_static_assert()); +} + +template +RecordItem copy(T&& value) { + return RecordItem(std::forward(value)); +} + +//--------------------------------------------------------------------------------------------------------------------- + +template +void encode(const T& in, atlas::io::Metadata& metadata, atlas::io::Data& data, + tag::enable_static_assert = tag::enable_static_assert()) { + auto referenced = ref(in, tag::enable_static_assert()); + sfinae::encode_metadata(referenced, metadata); + sfinae::encode_data(referenced, data); +} + +template +void encode(const T& in, atlas::io::Metadata& metadata, atlas::io::Data& data, tag::disable_static_assert) { + auto referenced = ref(in, tag::disable_static_assert()); + sfinae::encode_metadata(referenced, metadata); + sfinae::encode_data(referenced, data); +} + +//--------------------------------------------------------------------------------------------------------------------- + +} // namespace io +} // namespace atlas diff --git a/atlas-io/src/atlas_io/atlas_compat.h b/atlas-io/src/atlas_io/atlas_compat.h new file mode 100644 index 000000000..6f70f09be --- /dev/null +++ b/atlas-io/src/atlas_io/atlas_compat.h @@ -0,0 +1 @@ +#pragma once diff --git a/atlas-io/src/atlas_io/detail/Assert.h b/atlas-io/src/atlas_io/detail/Assert.h new file mode 100644 index 000000000..153ce60f0 --- /dev/null +++ b/atlas-io/src/atlas_io/detail/Assert.h @@ -0,0 +1,15 @@ +/* + * (C) Copyright 2020 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#pragma once + +#include "eckit/exception/Exceptions.h" +#define ATLAS_IO_ASSERT(X) ASSERT(X) +#define ATLAS_IO_ASSERT_MSG(X, M) ASSERT_MSG(X, M) diff --git a/src/atlas/io/detail/Base64.cc b/atlas-io/src/atlas_io/detail/Base64.cc similarity index 100% rename from src/atlas/io/detail/Base64.cc rename to atlas-io/src/atlas_io/detail/Base64.cc diff --git a/src/atlas/io/detail/Base64.h b/atlas-io/src/atlas_io/detail/Base64.h similarity index 100% rename from src/atlas/io/detail/Base64.h rename to atlas-io/src/atlas_io/detail/Base64.h diff --git a/atlas-io/src/atlas_io/detail/BlackMagic.h b/atlas-io/src/atlas_io/detail/BlackMagic.h new file mode 100644 index 000000000..a889c5e19 --- /dev/null +++ b/atlas-io/src/atlas_io/detail/BlackMagic.h @@ -0,0 +1,101 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#pragma once + +// This file contains preprocessor black magic. It contains macros that +// can return the number of arguments passed + +//----------------------------------------------------------------------------------------------------------- +// Public + +/// Returns the number of passed arguments +#define __ATLAS_IO_NARG(...) + +/// Splice a and b together +#define __ATLAS_IO_SPLICE(a, b) + +#define __ATLAS_IO_STRINGIFY(a) a + +#define __ATLAS_IO_TYPE(Type, ...) +#define __ATLAS_IO_TYPE_SCOPE(Type, ...) + +//----------------------------------------------------------------------------------------------------------- +// Details + +// Undefine these, to be redefined further down. +#undef __ATLAS_IO_NARG +#undef __ATLAS_IO_SPLICE +#undef __ATLAS_IO_TYPE +#undef __ATLAS_IO_TYPE_SCOPE + +#define __ATLAS_IO_REVERSE 5, 4, 3, 2, 1, 0 +#define __ATLAS_IO_ARGN(_1, _2, _3, _4, _5, N, ...) N +#define __ATLAS_IO_NARG__(dummy, ...) __ATLAS_IO_ARGN(__VA_ARGS__) +#define __ATLAS_IO_NARG_(...) __ATLAS_IO_NARG__(dummy, ##__VA_ARGS__, __ATLAS_IO_REVERSE) +#define __ATLAS_IO_SPLICE(a, b) __ATLAS_IO_SPLICE_1(a, b) +#define __ATLAS_IO_SPLICE_1(a, b) __ATLAS_IO_SPLICE_2(a, b) +#define __ATLAS_IO_SPLICE_2(a, b) a##b + +#define __ATLAS_IO_ARG16(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, ...) _15 +#define __ATLAS_IO_HAS_COMMA(...) __ATLAS_IO_ARG16(__VA_ARGS__, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0) +#define __ATLAS_IO_TRIGGER_PARENTHESIS(...) , +#define __ATLAS_IO_ISEMPTY(...) \ + __ATLAS_IO_ISEMPTY_(/* test if there is just one argument, eventually an empty \ + one */ \ + __ATLAS_IO_HAS_COMMA(__VA_ARGS__), /* test if \ + _TRIGGER_PARENTHESIS_ \ + together with the \ + argument adds a comma */ \ + __ATLAS_IO_HAS_COMMA( \ + __ATLAS_IO_TRIGGER_PARENTHESIS __VA_ARGS__), /* test if the argument together with \ + a parenthesis adds a comma */ \ + __ATLAS_IO_HAS_COMMA(__VA_ARGS__(/*empty*/)), /* test if placing it between \ + __ATLAS_IO_TRIGGER_PARENTHESIS and the \ + parenthesis adds a comma */ \ + __ATLAS_IO_HAS_COMMA(__ATLAS_IO_TRIGGER_PARENTHESIS __VA_ARGS__(/*empty*/))) + +#define __ATLAS_IO_PASTE5(_0, _1, _2, _3, _4) _0##_1##_2##_3##_4 +#define __ATLAS_IO_ISEMPTY_(_0, _1, _2, _3) \ + __ATLAS_IO_HAS_COMMA(__ATLAS_IO_PASTE5(__ATLAS_IO_IS_EMPTY_CASE_, _0, _1, _2, _3)) +#define __ATLAS_IO_IS_EMPTY_CASE_0001 , + +#define __ATLAS_IO_NARG(...) __ATLAS_IO_SPLICE(__ATLAS_IO_CALL_NARG_, __ATLAS_IO_ISEMPTY(__VA_ARGS__))(__VA_ARGS__) +#define __ATLAS_IO_CALL_NARG_1(...) 0 +#define __ATLAS_IO_CALL_NARG_0 __ATLAS_IO_NARG_ + +#define __ATLAS_IO_COMMA_ARGS(...) \ + __ATLAS_IO_SPLICE(__ATLAS_IO_COMMA_ARGS_, __ATLAS_IO_ISEMPTY(__VA_ARGS__))(__VA_ARGS__) +#define __ATLAS_IO_COMMA_ARGS_1(...) +#define __ATLAS_IO_COMMA_ARGS_0(...) , __VA_ARGS__ + +#define __ATLAS_IO_ARGS_OR_DUMMY(...) \ + __ATLAS_IO_SPLICE(__ATLAS_IO_ARGS_OR_DUMMY_, __ATLAS_IO_ISEMPTY(__VA_ARGS__)) \ + (__VA_ARGS__) +#define __ATLAS_IO_ARGS_OR_DUMMY_0(...) __VA_ARGS__ +#define __ATLAS_IO_ARGS_OR_DUMMY_1(...) 0 + +#define __ATLAS_IO_TYPE(Type, ...) \ + __ATLAS_IO_SPLICE(__ATLAS_IO_TYPE_, __ATLAS_IO_ISEMPTY(__VA_ARGS__)) \ + (Type, __ATLAS_IO_ARGS_OR_DUMMY(__VA_ARGS__)) +#define __ATLAS_IO_TYPE_1(Type, dummy) Type __ATLAS_IO_SPLICE(__variable_, __LINE__) +#define __ATLAS_IO_TYPE_0(Type, ...) Type __ATLAS_IO_SPLICE(__variable_, __LINE__)(__VA_ARGS__) + +#define __ATLAS_IO_TYPE_SCOPE(Type, ...) \ + __ATLAS_IO_SPLICE(__ATLAS_IO_TYPE_SCOPE_, __ATLAS_IO_ISEMPTY(__VA_ARGS__)) \ + (Type, __ATLAS_IO_ARGS_OR_DUMMY(__VA_ARGS__)) +#define __ATLAS_IO_TYPE_SCOPE_1(Type, ...) \ + for (bool __ATLAS_IO_SPLICE(__done_, __LINE__) = false; __ATLAS_IO_SPLICE(__done_, __LINE__) != true;) \ + for (Type __ATLAS_IO_SPLICE(__variable_, __LINE__); __ATLAS_IO_SPLICE(__done_, __LINE__) != true; \ + __ATLAS_IO_SPLICE(__done_, __LINE__) = true) +#define __ATLAS_IO_TYPE_SCOPE_0(Type, ...) \ + for (bool __ATLAS_IO_SPLICE(__done_, __LINE__) = false; __ATLAS_IO_SPLICE(__done_, __LINE__) != true;) \ + for (Type __ATLAS_IO_SPLICE(__variable_, __LINE__)(__VA_ARGS__); __ATLAS_IO_SPLICE(__done_, __LINE__) != true; \ + __ATLAS_IO_SPLICE(__done_, __LINE__) = true) diff --git a/src/atlas/io/detail/Checksum.cc b/atlas-io/src/atlas_io/detail/Checksum.cc similarity index 92% rename from src/atlas/io/detail/Checksum.cc rename to atlas-io/src/atlas_io/detail/Checksum.cc index 168ecb8ab..ced4c37d2 100644 --- a/src/atlas/io/detail/Checksum.cc +++ b/atlas-io/src/atlas_io/detail/Checksum.cc @@ -15,9 +15,8 @@ #include "eckit/utils/Hash.h" #include "eckit/utils/Tokenizer.h" -#include "atlas/io/detail/Defaults.h" -#include "atlas/runtime/Exception.h" -#include "atlas/runtime/Trace.h" +#include "atlas_io/Trace.h" +#include "atlas_io/detail/Defaults.h" namespace atlas { namespace io { @@ -59,7 +58,7 @@ std::string checksum(const void* buffer, size_t size, const std::string& algorit auto hash = [&](const std::string& alg) -> std::string { std::unique_ptr hasher(eckit::HashFactory::instance().build(alg)); - ATLAS_TRACE("checksum(" + alg + ")"); + ATLAS_IO_TRACE("checksum(" + alg + ")"); return std::string(alg) + ":" + hasher->compute(buffer, long(size)); }; diff --git a/src/atlas/io/detail/Checksum.h b/atlas-io/src/atlas_io/detail/Checksum.h similarity index 100% rename from src/atlas/io/detail/Checksum.h rename to atlas-io/src/atlas_io/detail/Checksum.h diff --git a/src/atlas/io/detail/DataInfo.h b/atlas-io/src/atlas_io/detail/DataInfo.h similarity index 95% rename from src/atlas/io/detail/DataInfo.h rename to atlas-io/src/atlas_io/detail/DataInfo.h index 3f06017c5..75f77cc7a 100644 --- a/src/atlas/io/detail/DataInfo.h +++ b/atlas-io/src/atlas_io/detail/DataInfo.h @@ -14,8 +14,8 @@ #include -#include "atlas/io/detail/Checksum.h" -#include "atlas/io/detail/Endian.h" +#include "atlas_io/detail/Checksum.h" +#include "atlas_io/detail/Endian.h" namespace atlas { namespace io { diff --git a/atlas-io/src/atlas_io/detail/DataType.cc b/atlas-io/src/atlas_io/detail/DataType.cc new file mode 100644 index 000000000..d3ff3553b --- /dev/null +++ b/atlas-io/src/atlas_io/detail/DataType.cc @@ -0,0 +1,38 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#include "DataType.h" + +#include + +#include "atlas_io/Exceptions.h" + +//------------------------------------------------------------------------------------------------------ + +namespace atlas { +namespace io { + +void DataType::throw_not_recognised(kind_t kind) { + std::stringstream msg; + msg << "kind [" << kind << "] not recognised."; + throw Exception(msg.str(), Here()); +} + +void DataType::throw_not_recognised(std::string datatype) { + std::stringstream msg; + msg << "datatype [" << datatype << "] not recognised."; + throw Exception(msg.str(), Here()); +} + + +//------------------------------------------------------------------------------------------------------ + +} // namespace io +} // namespace atlas diff --git a/atlas-io/src/atlas_io/detail/DataType.h b/atlas-io/src/atlas_io/detail/DataType.h new file mode 100644 index 000000000..40600cdb4 --- /dev/null +++ b/atlas-io/src/atlas_io/detail/DataType.h @@ -0,0 +1,414 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#pragma once + +#include + +//------------------------------------------------------------------------------------------------------ + +// For type safety we want to use std::byte for the DataType "BYTE", but it is a C++17 feature. +// Backport std::byte here without any operations +#if __cplusplus >= 201703L +#include +#else +#ifndef STD_BYTE_DEFINED +#define STD_BYTE_DEFINED +namespace std { +#ifdef _CRAYC +struct byte { + unsigned char byte_; +}; +#else +enum class byte : unsigned char +{ +}; +#endif +} // namespace std +#endif +#endif + +//------------------------------------------------------------------------------------------------------ + +namespace atlas { +namespace io { + +class DataType { +public: + typedef long kind_t; + static const kind_t KIND_BYTE = 1; + static const kind_t KIND_INT32 = -4; + static const kind_t KIND_INT64 = -8; + static const kind_t KIND_REAL32 = 4; + static const kind_t KIND_REAL64 = 8; + static const kind_t KIND_UINT64 = -16; + + template + static DataType create(); + + static DataType byte() { return DataType(KIND_BYTE); } + static DataType int32() { return DataType(KIND_INT32); } + static DataType int64() { return DataType(KIND_INT64); } + static DataType real32() { return DataType(KIND_REAL32); } + static DataType real64() { return DataType(KIND_REAL64); } + static DataType uint64() { return DataType(KIND_UINT64); } + + template + static kind_t kind(); + template + static kind_t kind(const DATATYPE&); + + template + static std::string str(); + template + static std::string str(const DATATYPE); + + static kind_t str_to_kind(const std::string&); + static std::string kind_to_str(kind_t); + static bool kind_valid(kind_t); + +private: + static std::string byte_str() { return "byte"; } + static std::string int32_str() { return "int32"; } + static std::string int64_str() { return "int64"; } + static std::string real32_str() { return "real32"; } + static std::string real64_str() { return "real64"; } + static std::string uint64_str() { return "uint64"; } + + [[noreturn]] static void throw_not_recognised(kind_t); + [[noreturn]] static void throw_not_recognised(std::string datatype); + +public: + DataType(const std::string&); + DataType(long); + DataType(const DataType&); + DataType& operator=(const DataType&); + std::string str() const { return kind_to_str(kind_); } + kind_t kind() const { return kind_; } + size_t size() const { return (kind_ == KIND_UINT64) ? 8 : std::abs(kind_); } + + friend bool operator==(DataType dt1, DataType dt2); + friend bool operator!=(DataType dt1, DataType dt2); + friend bool operator==(DataType dt, kind_t kind); + friend bool operator!=(DataType dt, kind_t kind); + friend bool operator==(kind_t kind, DataType dt); + friend bool operator!=(kind_t kind, DataType dt2); + +private: + kind_t kind_; +}; + +template <> +inline std::string DataType::str() { + return byte_str(); +} +template <> +inline std::string DataType::str() { + return byte_str(); +} +template <> +inline std::string DataType::str() { + static_assert(sizeof(int) == 4, ""); + return int32_str(); +} +template <> +inline std::string DataType::str() { + static_assert(sizeof(int) == 4, ""); + return int32_str(); +} +template <> +inline std::string DataType::str() { + static_assert(sizeof(long) == 8, ""); + return int64_str(); +} +template <> +inline std::string DataType::str() { + static_assert(sizeof(long) == 8, ""); + return int64_str(); +} +template <> +inline std::string DataType::str() { + static_assert(sizeof(long long) == 8, ""); + return int64_str(); +} +template <> +inline std::string DataType::str() { + static_assert(sizeof(long long) == 8, ""); + return int64_str(); +} +template <> +inline std::string DataType::str() { + static_assert(sizeof(float) == 4, ""); + return real32_str(); +} +template <> +inline std::string DataType::str() { + static_assert(sizeof(float) == 4, ""); + return real32_str(); +} +template <> +inline std::string DataType::str() { + static_assert(sizeof(double) == 8, ""); + return real64_str(); +} +template <> +inline std::string DataType::str() { + static_assert(sizeof(double) == 8, ""); + return real64_str(); +} +template <> +inline std::string DataType::str() { + static_assert(sizeof(unsigned long) == 8, ""); + return uint64_str(); +} +template <> +inline std::string DataType::str() { + static_assert(sizeof(unsigned long) == 8, ""); + return uint64_str(); +} + +template <> +inline std::string DataType::str() { + static_assert(sizeof(unsigned long long) == 8, ""); + return uint64_str(); +} +template <> +inline std::string DataType::str() { + static_assert(sizeof(unsigned long long) == 8, ""); + return uint64_str(); +} +template <> +inline std::string DataType::str(const int&) { + return str(); +} +template <> +inline std::string DataType::str(const long&) { + return str(); +} +template <> +inline std::string DataType::str(const long long&) { + return str(); +} +template <> +inline std::string DataType::str(const unsigned long&) { + return str(); +} +template <> +inline std::string DataType::str(const unsigned long long&) { + return str(); +} +template <> +inline std::string DataType::str(const float&) { + return str(); +} +template <> +inline std::string DataType::str(const double&) { + return str(); +} +template <> +inline DataType::kind_t DataType::kind() { + static_assert(sizeof(std::byte) == 1, ""); + return KIND_BYTE; +} +template <> +inline DataType::kind_t DataType::kind() { + static_assert(sizeof(std::byte) == 1, ""); + return KIND_BYTE; +} +template <> +inline DataType::kind_t DataType::kind() { + static_assert(sizeof(int) == 4, ""); + return KIND_INT32; +} +template <> +inline DataType::kind_t DataType::kind() { + static_assert(sizeof(int) == 4, ""); + return KIND_INT32; +} +template <> +inline DataType::kind_t DataType::kind() { + static_assert(sizeof(long) == 8, ""); + return KIND_INT64; +} +template <> +inline DataType::kind_t DataType::kind() { + static_assert(sizeof(long) == 8, ""); + return KIND_INT64; +} +template <> +inline DataType::kind_t DataType::kind() { + static_assert(sizeof(long long) == 8, ""); + return KIND_INT64; +} +template <> +inline DataType::kind_t DataType::kind() { + static_assert(sizeof(long long) == 8, ""); + return KIND_INT64; +} +template <> +inline DataType::kind_t DataType::kind() { + static_assert(sizeof(unsigned long) == 8, ""); + return KIND_UINT64; +} +template <> +inline DataType::kind_t DataType::kind() { + static_assert(sizeof(unsigned long) == 8, ""); + return KIND_UINT64; +} +template <> +inline DataType::kind_t DataType::kind() { + static_assert(sizeof(unsigned long long) == 8, ""); + return KIND_UINT64; +} +template <> +inline DataType::kind_t DataType::kind() { + static_assert(sizeof(unsigned long long) == 8, ""); + return KIND_UINT64; +} +template <> +inline DataType::kind_t DataType::kind() { + static_assert(sizeof(float) == 4, ""); + return KIND_REAL32; +} +template <> +inline DataType::kind_t DataType::kind() { + static_assert(sizeof(float) == 4, ""); + return KIND_REAL32; +} +template <> +inline DataType::kind_t DataType::kind() { + static_assert(sizeof(double) == 8, ""); + return KIND_REAL64; +} +template <> +inline DataType::kind_t DataType::kind() { + static_assert(sizeof(double) == 8, ""); + return KIND_REAL64; +} +template <> +inline DataType::kind_t DataType::kind(const int&) { + return kind(); +} +template <> +inline DataType::kind_t DataType::kind(const long&) { + return kind(); +} +template <> +inline DataType::kind_t DataType::kind(const unsigned long&) { + return kind(); +} +template <> +inline DataType::kind_t DataType::kind(const float&) { + return kind(); +} +template <> +inline DataType::kind_t DataType::kind(const double&) { + return kind(); +} + +inline DataType::kind_t DataType::str_to_kind(const std::string& datatype) { + if (datatype == "int32") + return KIND_INT32; + else if (datatype == "int64") + return KIND_INT64; + else if (datatype == "uint64") + return KIND_UINT64; + else if (datatype == "real32") + return KIND_REAL32; + else if (datatype == "real64") + return KIND_REAL64; + else if (datatype == "byte") { + return KIND_BYTE; + } + else { + throw_not_recognised(datatype); + } +} +inline std::string DataType::kind_to_str(kind_t kind) { + switch (kind) { + case KIND_INT32: + return int32_str(); + case KIND_INT64: + return int64_str(); + case KIND_UINT64: + return uint64_str(); + case KIND_REAL32: + return real32_str(); + case KIND_REAL64: + return real64_str(); + case KIND_BYTE: + return byte_str(); + default: + throw_not_recognised(kind); + } +} +inline bool DataType::kind_valid(kind_t kind) { + switch (kind) { + case KIND_BYTE: + case KIND_INT32: + case KIND_INT64: + case KIND_UINT64: + case KIND_REAL32: + case KIND_REAL64: + return true; + default: + return false; + } +} + +inline DataType::DataType(const DataType& other): kind_(other.kind_) {} + +inline DataType& DataType::operator=(const DataType& other) { + kind_ = other.kind_; + return *this; +} + +inline DataType::DataType(const std::string& datatype): kind_(str_to_kind(datatype)) {} + +inline DataType::DataType(long kind): kind_(kind) {} + +inline bool operator==(DataType dt1, DataType dt2) { + return dt1.kind_ == dt2.kind_; +} + +inline bool operator!=(DataType dt1, DataType dt2) { + return dt1.kind_ != dt2.kind_; +} + +inline bool operator==(DataType dt, DataType::kind_t kind) { + return dt.kind_ == kind; +} + +inline bool operator!=(DataType dt, DataType::kind_t kind) { + return dt.kind_ != kind; +} + +inline bool operator==(DataType::kind_t kind, DataType dt) { + return dt.kind_ == kind; +} + +inline bool operator!=(DataType::kind_t kind, DataType dt) { + return dt.kind_ != kind; +} + +template +inline DataType DataType::create() { + return DataType(DataType::kind()); +} + +template +inline DataType make_datatype() { + return DataType(DataType::kind()); +} + +//------------------------------------------------------------------------------------------------------ + +} // namespace io +} // namespace atlas diff --git a/src/atlas/io/detail/Decoder.cc b/atlas-io/src/atlas_io/detail/Decoder.cc similarity index 89% rename from src/atlas/io/detail/Decoder.cc rename to atlas-io/src/atlas_io/detail/Decoder.cc index 32df4fc44..309e0e6d4 100644 --- a/src/atlas/io/detail/Decoder.cc +++ b/atlas-io/src/atlas_io/detail/Decoder.cc @@ -10,18 +10,18 @@ #include "Decoder.h" -#include "atlas/runtime/Trace.h" +#include "atlas_io/Trace.h" namespace atlas { namespace io { void decode(const atlas::io::Metadata& metadata, const atlas::io::Data& data, Decoder& decoder) { - ATLAS_TRACE("decode"); + ATLAS_IO_TRACE("decode"); decoder.self_->decode_(metadata, data); } void decode(const atlas::io::Metadata& metadata, const atlas::io::Data& data, Decoder&& decoder) { - ATLAS_TRACE_SCOPE("decode"); + ATLAS_IO_TRACE_SCOPE("decode"); decoder.self_->decode_(metadata, data); } diff --git a/src/atlas/io/detail/Decoder.h b/atlas-io/src/atlas_io/detail/Decoder.h similarity index 92% rename from src/atlas/io/detail/Decoder.h rename to atlas-io/src/atlas_io/detail/Decoder.h index f86e99610..c1f754481 100644 --- a/src/atlas/io/detail/Decoder.h +++ b/atlas-io/src/atlas_io/detail/Decoder.h @@ -12,10 +12,9 @@ #include -#include "atlas/io/Data.h" -#include "atlas/io/Metadata.h" -#include "atlas/io/detail/TypeTraits.h" -#include "atlas/runtime/Trace.h" +#include "atlas_io/Data.h" +#include "atlas_io/Metadata.h" +#include "atlas_io/detail/TypeTraits.h" namespace atlas { namespace io { diff --git a/src/atlas/io/detail/Defaults.h b/atlas-io/src/atlas_io/detail/Defaults.h similarity index 100% rename from src/atlas/io/detail/Defaults.h rename to atlas-io/src/atlas_io/detail/Defaults.h diff --git a/atlas-io/src/atlas_io/detail/Encoder.cc b/atlas-io/src/atlas_io/detail/Encoder.cc new file mode 100644 index 000000000..9b8a1c8f2 --- /dev/null +++ b/atlas-io/src/atlas_io/detail/Encoder.cc @@ -0,0 +1,31 @@ +/* + * (C) Copyright 2020 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#include "Encoder.h" + +#include "atlas_io/Trace.h" + +namespace atlas { +namespace io { + +size_t encode_metadata(const Encoder& encoder, atlas::io::Metadata& metadata) { + ATLAS_IO_TRACE(); + ASSERT(encoder); + return encoder.self_->encode_metadata_(metadata); +} + +void encode_data(const Encoder& encoder, atlas::io::Data& out) { + ATLAS_IO_TRACE(); + ASSERT(encoder); + encoder.self_->encode_data_(out); +} + +} // namespace io +} // namespace atlas diff --git a/src/atlas/io/detail/Encoder.h b/atlas-io/src/atlas_io/detail/Encoder.h similarity index 82% rename from src/atlas/io/detail/Encoder.h rename to atlas-io/src/atlas_io/detail/Encoder.h index e9c8dba04..d6a7ae05f 100644 --- a/src/atlas/io/detail/Encoder.h +++ b/atlas-io/src/atlas_io/detail/Encoder.h @@ -15,14 +15,12 @@ #include #include -#include "atlas/io/Data.h" -#include "atlas/io/RecordItem.h" -#include "atlas/io/detail/DataInfo.h" -#include "atlas/io/detail/Link.h" -#include "atlas/io/detail/Reference.h" -#include "atlas/io/detail/TypeTraits.h" - -#include "atlas/runtime/Trace.h" +#include "atlas_io/Data.h" +#include "atlas_io/RecordItem.h" +#include "atlas_io/detail/DataInfo.h" +#include "atlas_io/detail/Link.h" +#include "atlas_io/detail/Reference.h" +#include "atlas_io/detail/TypeTraits.h" namespace atlas { namespace io { @@ -94,16 +92,8 @@ class Encoder { std::unique_ptr self_; }; -inline size_t encode_metadata(const Encoder& encoder, atlas::io::Metadata& metadata) { - ASSERT(encoder); - return encoder.self_->encode_metadata_(metadata); -} - -inline void encode_data(const Encoder& encoder, atlas::io::Data& out) { - ASSERT(encoder); - ATLAS_TRACE(); - encoder.self_->encode_data_(out); -} +size_t encode_metadata(const Encoder& encoder, atlas::io::Metadata& metadata); +void encode_data(const Encoder& encoder, atlas::io::Data& out); } // namespace io diff --git a/src/atlas/io/detail/Endian.h b/atlas-io/src/atlas_io/detail/Endian.h similarity index 66% rename from src/atlas/io/detail/Endian.h rename to atlas-io/src/atlas_io/detail/Endian.h index fd3a0e9cc..b886c0859 100644 --- a/src/atlas/io/detail/Endian.h +++ b/atlas-io/src/atlas_io/detail/Endian.h @@ -10,7 +10,15 @@ #pragma once -#include "atlas/library/defines.h" +#include "atlas_io/detail/defines.h" + +#ifndef ATLAS_IO_BIG_ENDIAN +#error ATLAS_IO_BIG_ENDIAN not defined +#endif + +#ifndef ATLAS_IO_LITTLE_ENDIAN +#error ATLAS_IO_LITTLE_ENDIAN not defined +#endif namespace atlas { namespace io { @@ -19,14 +27,14 @@ enum class Endian { little = 0, big = 1, -#if ATLAS_BIG_ENDIAN +#if ATLAS_IO_BIG_ENDIAN native = big, swapped = little -#elif ATLAS_LITTLE_ENDIAN +#elif ATLAS_IO_LITTLE_ENDIAN native = little, swapped = big #else -#error Neither ATLAS_BIG_ENDIAN nor ATLAS_LITTLE_ENDIAN equals true +#error Neither ATLAS_IO_BIG_ENDIAN nor ATLAS_IO_LITTLE_ENDIAN equals true #endif }; diff --git a/src/atlas/io/detail/Link.cc b/atlas-io/src/atlas_io/detail/Link.cc similarity index 97% rename from src/atlas/io/detail/Link.cc rename to atlas-io/src/atlas_io/detail/Link.cc index a6e9db109..0708cf7d8 100644 --- a/src/atlas/io/detail/Link.cc +++ b/atlas-io/src/atlas_io/detail/Link.cc @@ -12,7 +12,7 @@ #include "eckit/filesystem/PathName.h" -#include "atlas/io/RecordItem.h" +#include "atlas_io/RecordItem.h" namespace atlas { namespace io { diff --git a/src/atlas/io/detail/Link.h b/atlas-io/src/atlas_io/detail/Link.h similarity index 100% rename from src/atlas/io/detail/Link.h rename to atlas-io/src/atlas_io/detail/Link.h diff --git a/atlas-io/src/atlas_io/detail/NoConfig.h b/atlas-io/src/atlas_io/detail/NoConfig.h new file mode 100644 index 000000000..f87fb11b0 --- /dev/null +++ b/atlas-io/src/atlas_io/detail/NoConfig.h @@ -0,0 +1,29 @@ +/* + * (C) Copyright 2020 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#pragma once + +#include "eckit/config/LocalConfiguration.h" + +namespace atlas { +namespace io { + +//--------------------------------------------------------------------------------------------------------------------- + +class NoConfig : public eckit::LocalConfiguration { +public: + NoConfig() = default; + virtual ~NoConfig() = default; +}; + +//--------------------------------------------------------------------------------------------------------------------- + +} // namespace io +} // namespace atlas diff --git a/src/atlas/io/detail/ParsedRecord.h b/atlas-io/src/atlas_io/detail/ParsedRecord.h similarity index 94% rename from src/atlas/io/detail/ParsedRecord.h rename to atlas-io/src/atlas_io/detail/ParsedRecord.h index 25f5d102d..f55520de8 100644 --- a/src/atlas/io/detail/ParsedRecord.h +++ b/atlas-io/src/atlas_io/detail/ParsedRecord.h @@ -14,8 +14,8 @@ #include #include -#include "atlas/io/Metadata.h" -#include "atlas/io/detail/RecordSections.h" +#include "atlas_io/Metadata.h" +#include "atlas_io/detail/RecordSections.h" namespace atlas { namespace io { diff --git a/src/atlas/io/detail/RecordInfo.h b/atlas-io/src/atlas_io/detail/RecordInfo.h similarity index 90% rename from src/atlas/io/detail/RecordInfo.h rename to atlas-io/src/atlas_io/detail/RecordInfo.h index 06e48ea2a..3d29e3abf 100644 --- a/src/atlas/io/detail/RecordInfo.h +++ b/atlas-io/src/atlas_io/detail/RecordInfo.h @@ -10,8 +10,8 @@ #pragma once -#include "atlas/io/detail/Time.h" -#include "atlas/io/detail/Version.h" +#include "atlas_io/detail/Time.h" +#include "atlas_io/detail/Version.h" namespace atlas { namespace io { diff --git a/src/atlas/io/detail/RecordSections.h b/atlas-io/src/atlas_io/detail/RecordSections.h similarity index 98% rename from src/atlas/io/detail/RecordSections.h rename to atlas-io/src/atlas_io/detail/RecordSections.h index 3a612c827..9718a69d9 100644 --- a/src/atlas/io/detail/RecordSections.h +++ b/atlas-io/src/atlas_io/detail/RecordSections.h @@ -15,9 +15,9 @@ #include "eckit/types/FixedString.h" -#include "atlas/io/detail/Endian.h" -#include "atlas/io/detail/Time.h" -#include "atlas/io/detail/Version.h" +#include "atlas_io/detail/Endian.h" +#include "atlas_io/detail/Time.h" +#include "atlas_io/detail/Version.h" namespace atlas { namespace io { diff --git a/src/atlas/io/detail/Reference.h b/atlas-io/src/atlas_io/detail/Reference.h similarity index 89% rename from src/atlas/io/detail/Reference.h rename to atlas-io/src/atlas_io/detail/Reference.h index d7e01ccf0..8cd1040e6 100644 --- a/src/atlas/io/detail/Reference.h +++ b/atlas-io/src/atlas_io/detail/Reference.h @@ -10,11 +10,11 @@ #pragma once -#include "atlas/io/Data.h" -#include "atlas/io/Metadata.h" -#include "atlas/io/detail/sfinae.h" +#include "atlas_io/Data.h" +#include "atlas_io/Metadata.h" +#include "atlas_io/detail/sfinae.h" -#include "atlas/io/Exceptions.h" +#include "atlas_io/Exceptions.h" namespace atlas { namespace io { diff --git a/src/atlas/io/detail/StaticAssert.h b/atlas-io/src/atlas_io/detail/StaticAssert.h similarity index 98% rename from src/atlas/io/detail/StaticAssert.h rename to atlas-io/src/atlas_io/detail/StaticAssert.h index f8abf421a..5c0fca799 100644 --- a/src/atlas/io/detail/StaticAssert.h +++ b/atlas-io/src/atlas_io/detail/StaticAssert.h @@ -23,7 +23,7 @@ #if ATLAS_IO_STATIC_ASSERT #include -#include "atlas/io/detail/TypeTraits.h" +#include "atlas_io/detail/TypeTraits.h" namespace atlas { namespace io { diff --git a/src/atlas/io/detail/Time.cc b/atlas-io/src/atlas_io/detail/Time.cc similarity index 99% rename from src/atlas/io/detail/Time.cc rename to atlas-io/src/atlas_io/detail/Time.cc index 20b6e45a0..834ae7732 100644 --- a/src/atlas/io/detail/Time.cc +++ b/atlas-io/src/atlas_io/detail/Time.cc @@ -71,8 +71,6 @@ std::basic_ostream& operator<<(std::basic_ostream& #include "eckit/log/JSON.h" -#include "atlas/runtime/Log.h" - namespace atlas { namespace io { diff --git a/src/atlas/io/detail/Time.h b/atlas-io/src/atlas_io/detail/Time.h similarity index 100% rename from src/atlas/io/detail/Time.h rename to atlas-io/src/atlas_io/detail/Time.h diff --git a/src/atlas/io/detail/Type.h b/atlas-io/src/atlas_io/detail/Type.h similarity index 100% rename from src/atlas/io/detail/Type.h rename to atlas-io/src/atlas_io/detail/Type.h diff --git a/src/atlas/io/detail/TypeTraits.h b/atlas-io/src/atlas_io/detail/TypeTraits.h similarity index 100% rename from src/atlas/io/detail/TypeTraits.h rename to atlas-io/src/atlas_io/detail/TypeTraits.h diff --git a/src/atlas/io/detail/Version.h b/atlas-io/src/atlas_io/detail/Version.h similarity index 100% rename from src/atlas/io/detail/Version.h rename to atlas-io/src/atlas_io/detail/Version.h diff --git a/atlas-io/src/atlas_io/detail/defines.h.in b/atlas-io/src/atlas_io/detail/defines.h.in new file mode 100644 index 000000000..40d8132c8 --- /dev/null +++ b/atlas-io/src/atlas_io/detail/defines.h.in @@ -0,0 +1,21 @@ +#if 0 +/* + * (C) Copyright 2022 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ +// clang-format off +#endif + +#ifndef atlas_io_defines_h +#define atlas_io_defines_h + +#cmakedefine01 ATLAS_IO_HAVE_CXXABI_H +#cmakedefine01 ATLAS_IO_LITTLE_ENDIAN +#cmakedefine01 ATLAS_IO_BIG_ENDIAN + +#endif diff --git a/src/atlas/io/detail/sfinae.h b/atlas-io/src/atlas_io/detail/sfinae.h similarity index 98% rename from src/atlas/io/detail/sfinae.h rename to atlas-io/src/atlas_io/detail/sfinae.h index 63e7269d4..1510b3172 100644 --- a/src/atlas/io/detail/sfinae.h +++ b/atlas-io/src/atlas_io/detail/sfinae.h @@ -10,7 +10,7 @@ #pragma once -#include "atlas/io/detail/TypeTraits.h" +#include "atlas_io/detail/TypeTraits.h" namespace atlas { namespace io { diff --git a/src/atlas/io/detail/tag.h b/atlas-io/src/atlas_io/detail/tag.h similarity index 100% rename from src/atlas/io/detail/tag.h rename to atlas-io/src/atlas_io/detail/tag.h diff --git a/src/atlas/io/print/Bytes.cc b/atlas-io/src/atlas_io/print/Bytes.cc similarity index 100% rename from src/atlas/io/print/Bytes.cc rename to atlas-io/src/atlas_io/print/Bytes.cc diff --git a/src/atlas/io/print/Bytes.h b/atlas-io/src/atlas_io/print/Bytes.h similarity index 100% rename from src/atlas/io/print/Bytes.h rename to atlas-io/src/atlas_io/print/Bytes.h diff --git a/src/atlas/io/print/JSONFormat.cc b/atlas-io/src/atlas_io/print/JSONFormat.cc similarity index 91% rename from src/atlas/io/print/JSONFormat.cc rename to atlas-io/src/atlas_io/print/JSONFormat.cc index dd0659526..51446a160 100644 --- a/src/atlas/io/print/JSONFormat.cc +++ b/atlas-io/src/atlas_io/print/JSONFormat.cc @@ -12,14 +12,14 @@ #include "eckit/log/JSON.h" -#include "atlas/io/Record.h" -#include "atlas/io/RecordItemReader.h" -#include "atlas/io/Session.h" +#include "atlas_io/Record.h" +#include "atlas_io/RecordItemReader.h" +#include "atlas_io/Session.h" namespace atlas { namespace io { -JSONFormat::JSONFormat(const Record::URI& record, const util::Config& config): +JSONFormat::JSONFormat(const Record::URI& record, const eckit::Configuration& config): record_(Session::record(record.path, record.offset)) { for (const auto& key : record_.keys()) { items_.emplace(key, Metadata()); diff --git a/src/atlas/io/print/JSONFormat.h b/atlas-io/src/atlas_io/print/JSONFormat.h similarity index 80% rename from src/atlas/io/print/JSONFormat.h rename to atlas-io/src/atlas_io/print/JSONFormat.h index 6b7167929..bb93e22eb 100644 --- a/src/atlas/io/print/JSONFormat.h +++ b/atlas-io/src/atlas_io/print/JSONFormat.h @@ -14,9 +14,10 @@ #include #include -#include "atlas/io/Metadata.h" -#include "atlas/io/Record.h" -#include "atlas/util/Config.h" +#include "eckit/config/Configuration.h" + +#include "atlas_io/Metadata.h" +#include "atlas_io/Record.h" namespace atlas { namespace io { @@ -24,7 +25,7 @@ namespace io { class JSONFormat { public: - JSONFormat(const Record::URI& record, const util::Config& config); + JSONFormat(const Record::URI& record, const eckit::Configuration&); void print(std::ostream&) const; diff --git a/src/atlas/io/print/TableFormat.cc b/atlas-io/src/atlas_io/print/TableFormat.cc similarity index 87% rename from src/atlas/io/print/TableFormat.cc rename to atlas-io/src/atlas_io/print/TableFormat.cc index d463d9b2c..968257341 100644 --- a/src/atlas/io/print/TableFormat.cc +++ b/atlas-io/src/atlas_io/print/TableFormat.cc @@ -12,15 +12,17 @@ #include -#include "atlas/io/Exceptions.h" -#include "atlas/io/Metadata.h" -#include "atlas/io/Record.h" -#include "atlas/io/RecordItemReader.h" -#include "atlas/io/Session.h" -#include "atlas/io/print/Bytes.h" -#include "atlas/io/types/array/ArrayReference.h" -#include "atlas/io/types/scalar.h" -#include "atlas/runtime/Exception.h" +#include "atlas_io/Exceptions.h" +#include "atlas_io/Metadata.h" +#include "atlas_io/Record.h" +#include "atlas_io/RecordItemReader.h" +#include "atlas_io/Session.h" +#include "atlas_io/detail/Assert.h" +#include "atlas_io/print/Bytes.h" +#include "atlas_io/types/array/ArrayReference.h" +#include "atlas_io/types/scalar.h" + +#include "atlas_io/atlas_compat.h" namespace atlas { namespace io { @@ -62,25 +64,25 @@ class ArrayMetadataPrettyPrint : public MetadataPrettyPrintBase { ArrayMetadataPrettyPrint(const Metadata& m): metadata_(m) {} void print(std::ostream& out) const override { std::string type = metadata_.getString("type"); - ATLAS_ASSERT(type == "array"); + ATLAS_IO_ASSERT(type == "array"); ArrayMetadata array(metadata_); out << std::setw(7) << std::left << array.datatype().str(); if (metadata_.has("value")) { out << ": "; std::string datatype = metadata_.getString("datatype"); - if (datatype == array::DataType::str()) { + if (datatype == DataType::str()) { print_value(out); } - else if (datatype == array::DataType::str()) { + else if (datatype == DataType::str()) { print_value(out); } - else if (datatype == array::DataType::str()) { + else if (datatype == DataType::str()) { print_value(out); } - else if (datatype == array::DataType::str()) { + else if (datatype == DataType::str()) { print_value(out); } - else if (datatype == array::DataType::str()) { + else if (datatype == DataType::str()) { print_value(out); } } @@ -105,7 +107,7 @@ class StringMetadataPrettyPrint : public MetadataPrettyPrintBase { StringMetadataPrettyPrint(const Metadata& m): metadata_(m) {} void print(std::ostream& out) const override { std::string type = metadata_.getString("type"); - ATLAS_ASSERT(type == "string"); + ATLAS_IO_ASSERT(type == "string"); std::string value = metadata_.getString("value"); if (value.size() <= 32) { out << value; @@ -131,23 +133,23 @@ class ScalarMetadataPrettyPrint : public MetadataPrettyPrintBase { } void print(std::ostream& out) const override { std::string type = metadata_.getString("type"); - ATLAS_ASSERT(type == "scalar"); + ATLAS_IO_ASSERT(type == "scalar"); std::string datatype = metadata_.getString("datatype"); std::string base64 = metadata_.getString("base64"); out << std::setw(7) << std::left << datatype << ": "; - if (datatype == array::DataType::str()) { + if (datatype == DataType::str()) { out << decode(); } - else if (datatype == array::DataType::str()) { + else if (datatype == DataType::str()) { out << decode(); } - else if (datatype == array::DataType::str()) { + else if (datatype == DataType::str()) { out << decode(); } - else if (datatype == array::DataType::str()) { + else if (datatype == DataType::str()) { out << decode(); } - else if (datatype == array::DataType::str()) { + else if (datatype == DataType::str()) { out << decode(); } } @@ -257,7 +259,7 @@ struct TablePrinter { }; -TableFormat::TableFormat(const Record::URI& record, const util::Config& config): +TableFormat::TableFormat(const Record::URI& record, const eckit::Parametrisation& config): record_(Session::record(record.path, record.offset)) { for (const auto& key : record_.keys()) { items_.emplace(key, Metadata()); @@ -268,7 +270,7 @@ TableFormat::TableFormat(const Record::URI& record, const util::Config& config): } void TableFormat::print(std::ostream& out) const { - ATLAS_ASSERT(not record_.empty()); + ATLAS_IO_ASSERT(not record_.empty()); TablePrinter table; table.column("name"); diff --git a/src/atlas/io/print/TableFormat.h b/atlas-io/src/atlas_io/print/TableFormat.h similarity index 85% rename from src/atlas/io/print/TableFormat.h rename to atlas-io/src/atlas_io/print/TableFormat.h index 07cde45fb..304daba72 100644 --- a/src/atlas/io/print/TableFormat.h +++ b/atlas-io/src/atlas_io/print/TableFormat.h @@ -16,10 +16,10 @@ #include #include -#include "atlas/io/Metadata.h" -#include "atlas/io/Record.h" -#include "atlas/io/RecordItemReader.h" -#include "atlas/io/Session.h" +#include "atlas_io/Metadata.h" +#include "atlas_io/Record.h" +#include "atlas_io/RecordItemReader.h" +#include "atlas_io/Session.h" namespace atlas { namespace io { @@ -44,7 +44,7 @@ class MetadataPrettyPrint { class TableFormat { public: - TableFormat(const Record::URI& record, const util::Config& config); + TableFormat(const Record::URI& record, const eckit::Parametrisation& config); void print(std::ostream&) const; diff --git a/src/atlas/io/types/array.h b/atlas-io/src/atlas_io/types/array.h similarity index 59% rename from src/atlas/io/types/array.h rename to atlas-io/src/atlas_io/types/array.h index e04996fd5..d2129e674 100644 --- a/src/atlas/io/types/array.h +++ b/atlas-io/src/atlas_io/types/array.h @@ -10,8 +10,6 @@ #pragma once -#include "atlas/io/types/array/ArrayReference.h" -#include "atlas/io/types/array/adaptors/ArrayAdaptor.h" -#include "atlas/io/types/array/adaptors/StdArrayAdaptor.h" -#include "atlas/io/types/array/adaptors/StdVectorAdaptor.h" -#include "atlas/io/types/array/adaptors/VectorAdaptor.h" +#include "atlas_io/types/array/ArrayReference.h" +#include "atlas_io/types/array/adaptors/StdArrayAdaptor.h" +#include "atlas_io/types/array/adaptors/StdVectorAdaptor.h" diff --git a/src/atlas/io/types/array/ArrayMetadata.cc b/atlas-io/src/atlas_io/types/array/ArrayMetadata.cc similarity index 79% rename from src/atlas/io/types/array/ArrayMetadata.cc rename to atlas-io/src/atlas_io/types/array/ArrayMetadata.cc index 971d95040..57ddef403 100644 --- a/src/atlas/io/types/array/ArrayMetadata.cc +++ b/atlas-io/src/atlas_io/types/array/ArrayMetadata.cc @@ -14,7 +14,9 @@ #include #include -#include "atlas/runtime/Exception.h" +#include "atlas_io/Exceptions.h" +#include "atlas_io/atlas_compat.h" +#include "atlas_io/detail/Assert.h" namespace atlas { namespace io { @@ -32,7 +34,9 @@ size_t encode_metadata(const ArrayMetadata& value, atlas::io::Metadata& out) { int ArrayMetadata::shape(int i) const { if (i >= rank()) { - throw_OutOfRange("shape", i, rank()); + throw Exception( + "ArrayMetadata::shape(i=" + std::to_string(i) + ") goes out of bounds. rank=" + std::to_string(rank()), + Here()); } return shape_[size_t(i)]; } @@ -42,11 +46,11 @@ int ArrayMetadata::shape(int i) const { ArrayMetadata::ArrayMetadata(const Metadata& metadata): datatype_(DataType::KIND_REAL64) /* circumvent absense of default constructor */ { std::string encoded_type; - ATLAS_ASSERT(metadata.get("type", encoded_type), "metadata is missing 'type'"); - ATLAS_ASSERT(encoded_type == type(), "metadata has unexpected type '" + encoded_type + "'"); - ATLAS_ASSERT(metadata.get("shape", shape_), "metadata is missing 'shape'"); + ATLAS_IO_ASSERT_MSG(metadata.get("type", encoded_type), "metadata is missing 'type'"); + ATLAS_IO_ASSERT_MSG(encoded_type == type(), "metadata has unexpected type '" + encoded_type + "'"); + ATLAS_IO_ASSERT_MSG(metadata.get("shape", shape_), "metadata is missing 'shape'"); std::string datatype_str; - ATLAS_ASSERT(metadata.get("datatype", datatype_str), "metadata is missing 'datatype'"); + ATLAS_IO_ASSERT_MSG(metadata.get("datatype", datatype_str), "metadata is missing 'datatype'"); datatype_ = DataType(datatype_str); } @@ -71,7 +75,7 @@ ArrayMetadata::ArrayMetadata(const ArrayMetadata& other): ArrayMetadata{other.da //--------------------------------------------------------------------------------------------------------------------- -ArrayMetadata::ArrayMetadata(ArrayMetadata&& other): shape_{std::move(other.shape_)}, datatype_{other.datatype_} {} +ArrayMetadata::ArrayMetadata(ArrayMetadata&& other): shape_(std::move(other.shape_)), datatype_{other.datatype_} {} //--------------------------------------------------------------------------------------------------------------------- diff --git a/src/atlas/io/types/array/ArrayMetadata.h b/atlas-io/src/atlas_io/types/array/ArrayMetadata.h similarity index 65% rename from src/atlas/io/types/array/ArrayMetadata.h rename to atlas-io/src/atlas_io/types/array/ArrayMetadata.h index eba34da33..4711afb44 100644 --- a/src/atlas/io/types/array/ArrayMetadata.h +++ b/atlas-io/src/atlas_io/types/array/ArrayMetadata.h @@ -13,19 +13,37 @@ #include #include -#include "atlas/array/ArrayShape.h" -#include "atlas/array/DataType.h" -#include "atlas/io/Metadata.h" +#include "atlas_io/Metadata.h" +#include "atlas_io/detail/DataType.h" namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- +class ArrayShape : public std::vector { +private: + using Base = std::vector; + +public: + ArrayShape() {} + ArrayShape(Base&& base): Base(std::forward(base)) {} + template + ArrayShape(std::initializer_list list): Base(list.begin(), list.end()) {} + template + ArrayShape(idx_t data[], size_t size): Base(data, data + size) {} + template + ArrayShape(const std::array& list): Base(list.begin(), list.end()) {} + template + ArrayShape(const std::vector& list): Base(list.begin(), list.end()) {} +}; + +//--------------------------------------------------------------------------------------------------------------------- + class ArrayMetadata { public: - using ArrayShape = array::ArrayShape; - using DataType = array::DataType; + using ArrayShape = io::ArrayShape; + using DataType = io::DataType; static std::string type() { return "array"; } diff --git a/src/atlas/io/types/array/ArrayReference.cc b/atlas-io/src/atlas_io/types/array/ArrayReference.cc similarity index 96% rename from src/atlas/io/types/array/ArrayReference.cc rename to atlas-io/src/atlas_io/types/array/ArrayReference.cc index 7f25b8784..f880a7f83 100644 --- a/src/atlas/io/types/array/ArrayReference.cc +++ b/atlas-io/src/atlas_io/types/array/ArrayReference.cc @@ -10,7 +10,8 @@ #include "ArrayReference.h" -#include "atlas/runtime/Exception.h" +#include "atlas_io/atlas_compat.h" +#include "atlas_io/detail/Assert.h" namespace atlas { namespace io { @@ -26,7 +27,7 @@ void encode_data(const ArrayReference& value, atlas::io::Data& out) { namespace { template void encode_metadata_value(const ArrayReference& value, atlas::io::Metadata& out) { - ATLAS_ASSERT(value.datatype() == array::make_datatype()); + ATLAS_IO_ASSERT(value.datatype() == make_datatype()); const T* array = reinterpret_cast(value.data()); std::vector vector(value.size()); std::copy(array, array + vector.size(), vector.begin()); diff --git a/src/atlas/io/types/array/ArrayReference.h b/atlas-io/src/atlas_io/types/array/ArrayReference.h similarity index 93% rename from src/atlas/io/types/array/ArrayReference.h rename to atlas-io/src/atlas_io/types/array/ArrayReference.h index d2ff9bb32..2f3cc4960 100644 --- a/src/atlas/io/types/array/ArrayReference.h +++ b/atlas-io/src/atlas_io/types/array/ArrayReference.h @@ -10,9 +10,9 @@ #pragma once -#include "atlas/io/Data.h" -#include "atlas/io/Metadata.h" -#include "atlas/io/types/array/ArrayMetadata.h" +#include "atlas_io/Data.h" +#include "atlas_io/Metadata.h" +#include "atlas_io/types/array/ArrayMetadata.h" namespace atlas { namespace io { diff --git a/src/atlas/io/types/array/adaptors/StdArrayAdaptor.h b/atlas-io/src/atlas_io/types/array/adaptors/StdArrayAdaptor.h similarity index 89% rename from src/atlas/io/types/array/adaptors/StdArrayAdaptor.h rename to atlas-io/src/atlas_io/types/array/adaptors/StdArrayAdaptor.h index 4809f116a..34a991c19 100644 --- a/src/atlas/io/types/array/adaptors/StdArrayAdaptor.h +++ b/atlas-io/src/atlas_io/types/array/adaptors/StdArrayAdaptor.h @@ -12,12 +12,11 @@ #include -#include "atlas/io/Data.h" -#include "atlas/io/Exceptions.h" -#include "atlas/io/Metadata.h" -#include "atlas/io/types/array/ArrayMetadata.h" -#include "atlas/io/types/array/ArrayReference.h" -#include "atlas/runtime/Exception.h" +#include "atlas_io/Data.h" +#include "atlas_io/Exceptions.h" +#include "atlas_io/Metadata.h" +#include "atlas_io/types/array/ArrayMetadata.h" +#include "atlas_io/types/array/ArrayReference.h" namespace std { diff --git a/src/atlas/io/types/array/adaptors/StdVectorAdaptor.h b/atlas-io/src/atlas_io/types/array/adaptors/StdVectorAdaptor.h similarity index 87% rename from src/atlas/io/types/array/adaptors/StdVectorAdaptor.h rename to atlas-io/src/atlas_io/types/array/adaptors/StdVectorAdaptor.h index 60642140d..18aaeb47f 100644 --- a/src/atlas/io/types/array/adaptors/StdVectorAdaptor.h +++ b/atlas-io/src/atlas_io/types/array/adaptors/StdVectorAdaptor.h @@ -12,12 +12,11 @@ #include -#include "atlas/io/Data.h" -#include "atlas/io/Exceptions.h" -#include "atlas/io/Metadata.h" -#include "atlas/io/types/array/ArrayMetadata.h" -#include "atlas/io/types/array/ArrayReference.h" -#include "atlas/runtime/Exception.h" +#include "atlas_io/Data.h" +#include "atlas_io/Exceptions.h" +#include "atlas_io/Metadata.h" +#include "atlas_io/types/array/ArrayMetadata.h" +#include "atlas_io/types/array/ArrayReference.h" namespace std { diff --git a/src/atlas/io/types/scalar.cc b/atlas-io/src/atlas_io/types/scalar.cc similarity index 88% rename from src/atlas/io/types/scalar.cc rename to atlas-io/src/atlas_io/types/scalar.cc index e8657c186..0e4ea1ea4 100644 --- a/src/atlas/io/types/scalar.cc +++ b/atlas-io/src/atlas_io/types/scalar.cc @@ -25,13 +25,13 @@ #include "eckit/utils/ByteSwap.h" -#include "atlas/array/DataType.h" -#include "atlas/runtime/Exception.h" -#include "atlas/runtime/Log.h" +#include "atlas_io/atlas_compat.h" +#include "atlas_io/detail/Assert.h" +#include "atlas_io/detail/DataType.h" -#include "atlas/io/Data.h" -#include "atlas/io/Metadata.h" -#include "atlas/io/detail/Base64.h" +#include "atlas_io/Data.h" +#include "atlas_io/Metadata.h" +#include "atlas_io/detail/Base64.h" namespace atlas { namespace io { @@ -41,15 +41,15 @@ namespace io { template void decode_scalar(const atlas::io::Metadata& metadata, T& value) { - ATLAS_ASSERT(metadata.getString("type") == "scalar"); - ATLAS_ASSERT(metadata.getString("datatype") == array::DataType::str()); + ATLAS_IO_ASSERT(metadata.getString("type") == "scalar"); + ATLAS_IO_ASSERT(metadata.getString("datatype") == DataType::str()); metadata.get("value", value); } template void decode_scalar_b64(const atlas::io::Metadata& metadata, T& value) { - ATLAS_ASSERT(metadata.getString("type") == "scalar"); - ATLAS_ASSERT(metadata.getString("datatype") == array::DataType::str()); + ATLAS_IO_ASSERT(metadata.getString("type") == "scalar"); + ATLAS_IO_ASSERT(metadata.getString("datatype") == DataType::str()); std::string base64 = metadata.getString("base64"); T value_ns = Base64::decode(base64); if (Endian::native == Endian::little) { @@ -65,19 +65,19 @@ void decode_scalar_b64(const atlas::io::Metadata& metadata, T& value) { template void encode_scalar_metadata(const T& value, atlas::io::Metadata& out) { out.set("type", "scalar"); - out.set("datatype", array::DataType::str()); + out.set("datatype", DataType::str()); out.set("value", value); } inline void encode_scalar_metadata(const unsigned long& value, atlas::io::Metadata& out) { out.set("type", "scalar"); - out.set("datatype", array::DataType::str()); + out.set("datatype", DataType::str()); out.set("value", size_t(value)); } inline void encode_scalar_metadata(const unsigned long long& value, atlas::io::Metadata& out) { out.set("type", "scalar"); - out.set("datatype", array::DataType::str()); + out.set("datatype", DataType::str()); out.set("value", size_t(value)); } diff --git a/src/atlas/io/types/scalar.h b/atlas-io/src/atlas_io/types/scalar.h similarity index 97% rename from src/atlas/io/types/scalar.h rename to atlas-io/src/atlas_io/types/scalar.h index 94593c6c5..c70a3a0ef 100644 --- a/src/atlas/io/types/scalar.h +++ b/atlas-io/src/atlas_io/types/scalar.h @@ -10,8 +10,8 @@ #pragma once -#include "atlas/io/Data.h" -#include "atlas/io/Metadata.h" +#include "atlas_io/Data.h" +#include "atlas_io/Metadata.h" namespace atlas { namespace io { diff --git a/src/atlas/io/types/string.h b/atlas-io/src/atlas_io/types/string.h similarity index 85% rename from src/atlas/io/types/string.h rename to atlas-io/src/atlas_io/types/string.h index e4a8e6d81..7afd755cc 100644 --- a/src/atlas/io/types/string.h +++ b/atlas-io/src/atlas_io/types/string.h @@ -12,9 +12,10 @@ #include -#include "atlas/io/Data.h" -#include "atlas/io/Metadata.h" -#include "atlas/runtime/Exception.h" +#include "atlas_io/Data.h" +#include "atlas_io/Metadata.h" +#include "atlas_io/atlas_compat.h" +#include "atlas_io/detail/Assert.h" namespace atlas { namespace io { @@ -30,7 +31,7 @@ inline size_t encode_metadata(const std::string& value, atlas::io::Metadata& out inline void encode_data(const std::string&, atlas::io::Data&) {} inline void decode(const atlas::io::Metadata& metadata, const atlas::io::Data&, std::string& value) { - ATLAS_ASSERT(metadata.getString("type") == "string"); + ATLAS_IO_ASSERT(metadata.getString("type") == "string"); metadata.get("value", value); } diff --git a/atlas-io/src/tools/CMakeLists.txt b/atlas-io/src/tools/CMakeLists.txt new file mode 100644 index 000000000..5c51d8d3b --- /dev/null +++ b/atlas-io/src/tools/CMakeLists.txt @@ -0,0 +1,12 @@ +# (C) Copyright 2013 ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. + +ecbuild_add_executable( + TARGET atlas-io-list + SOURCES atlas-io-list.cc + LIBS atlas_io eckit_option ) diff --git a/atlas-io/src/tools/atlas-io-list.cc b/atlas-io/src/tools/atlas-io-list.cc new file mode 100644 index 000000000..4550e131e --- /dev/null +++ b/atlas-io/src/tools/atlas-io-list.cc @@ -0,0 +1,215 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#include +#include +#include + +#include "eckit/option/CmdArgs.h" +#include "eckit/option/Separator.h" +#include "eckit/option/SimpleOption.h" +#include "eckit/option/VectorOption.h" +#include "eckit/runtime/Tool.h" + +#include "eckit/filesystem/PathName.h" + +#include "atlas_io/Exceptions.h" +#include "atlas_io/RecordPrinter.h" +#include "atlas_io/print/Bytes.h" + +//-------------------------------------------------------------------------------- + +using eckit::Log; + +class AtlasIOTool : public eckit::Tool { +public: + using Options = std::vector; + using Args = eckit::option::CmdArgs; + +protected: + virtual std::string indent() { return " "; } + virtual std::string briefDescription() { return ""; } + virtual std::string longDescription() { return ""; } + virtual std::string usage() { return name() + " [OPTION]... [--help,-h]"; } + + void add_option(eckit::option::Option* option) { options_.push_back(option); } + + virtual void help(std::ostream& out = Log::info()) { + auto indented = [&](const std::string& s) -> std::string { + std::string str = indent() + s; + size_t pos = 0; + while ((pos = str.find('\n', pos)) != std::string::npos) { + str.replace(pos, 1, '\n' + indent()); + ++pos; + } + return str; + }; + + out << "NAME\n" << indented(name()); + std::string brief = briefDescription(); + if (brief.size()) { + out << " - " << brief << '\n'; + } + + std::string usg = usage(); + if (usg.size()) { + out << '\n'; + out << "SYNOPSIS\n" << indented(usg) << '\n'; + } + std::string desc = longDescription(); + if (desc.size()) { + out << '\n'; + out << "DESCRIPTION\n" << indented(desc) << '\n'; + } + out << '\n'; + out << "OPTIONS\n"; + for (Options::const_iterator it = options_.begin(); it != options_.end(); ++it) { + std::stringstream s; + s << **it; + out << indented(s.str()) << "\n\n"; + } + out << std::flush; + } + + virtual int numberOfPositionalArguments() { return -1; } + virtual int minimumPositionalArguments() { return 0; } + + bool handle_help() { + for (int i = 1; i < argc(); ++i) { + if (argv(i) == "--help" || argv(i) == "-h") { + help(std::cout); + return true; + } + } + return false; + } + +public: + AtlasIOTool(int argc, char** argv): eckit::Tool(argc, argv) { + add_option(new eckit::option::SimpleOption("help", "Print this help")); + } + + int start() { + try { + if (handle_help()) { + return success(); + } + + if (argc() - 1 < minimumPositionalArguments()) { + Log::error() << "Usage: " << usage() << std::endl; + return failed(); + } + + Options opts = options_; + std::function dummy = [](const std::string&) {}; + Args args(dummy, opts, numberOfPositionalArguments(), minimumPositionalArguments() > 0); + + int err_code = execute(args); + return err_code; + } + catch (eckit::Exception& e) { + Log::error() << "** " << e.what() << " Caught in " << Here() << std::endl; + Log::error() << "** Exception terminates " << name() << std::endl; + } + catch (std::exception& e) { + Log::error() << "** " << e.what() << " Caught in " << Here() << std::endl; + Log::error() << "** Exception terminates " << name() << std::endl; + } + return failed(); + } + + void run() final {} // unused + + virtual int execute(const Args&) = 0; + + static constexpr int success() { return 0; } + static constexpr int failed() { return 1; } + +private: + Options options_; +}; + + +//---------------------------------------------------------------------------------------------------------------------- + +struct AtlasIOList : public AtlasIOTool { + std::string briefDescription() override { return "Inspection of atlas-io files"; } + std::string usage() override { return name() + " [OPTION]... [--help,-h]"; } + std::string longDescription() override { + return "Inspection of atlas-io files\n" + "\n" + " : path to atlas-io file"; + } + + AtlasIOList(int argc, char** argv): AtlasIOTool(argc, argv) { + add_option(new eckit::option::SimpleOption("format", "Output format")); + add_option(new eckit::option::SimpleOption("version", "Print version of records")); + add_option(new eckit::option::SimpleOption("details", "Print detailed information")); + } + int execute(const Args& args) override { + auto return_code = success(); + + using namespace atlas; + + // User sanity checks + if (args.count() == 0) { + Log::error() << "No file specified." << std::endl; + help(Log::error()); + return failed(); + } + + // Configuration + eckit::LocalConfiguration config; + config.set("format", args.getString("format", "table")); + config.set("details", args.getBool("details", false)); + + // Loop over files + for (size_t f = 0; f < args.count(); ++f) { + eckit::PathName file(args(f)); + if (!file.exists()) { + Log::error() << "File does not exist: " << file << std::endl; + return failed(); + } + auto filesize = size_t(file.size()); + + io::Session session; + + std::uint64_t pos = 0; + try { + while (pos < filesize) { + auto uri = io::Record::URI{file, pos}; + auto record = io::RecordPrinter{uri, config}; + + std::stringstream out; + out << "\n# " << uri.path << " [" << uri.offset << "] " + << "{ size: " << atlas::io::Bytes{record.size()}.str(0) << ", version: " << record.version() + << ", created: " << record.time() << " }"; + out << '\n' << (config.getString("format") == "table" ? "" : "---") << '\n'; + out << record << std::endl; + + std::cout << out.str(); + + pos += record.size(); + } + } + catch (const io::Exception& e) { + Log::error() << " ATLAS-IO-ERROR: " << e.what() << std::endl; + return_code = failed(); + } + } + return return_code; + } +}; + +//------------------------------------------------------------------------------------------------------ + +int main(int argc, char** argv) { + return AtlasIOList{argc, argv}.start(); +} diff --git a/atlas-io/tests/CMakeLists.txt b/atlas-io/tests/CMakeLists.txt new file mode 100644 index 000000000..128f046b9 --- /dev/null +++ b/atlas-io/tests/CMakeLists.txt @@ -0,0 +1,37 @@ +# (C) Copyright 2022 ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. + +ecbuild_add_test( TARGET atlas_io_test_encoding + SOURCES test_io_encoding.cc + LIBS atlas_io + ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} +) + +ecbuild_add_test( TARGET atlas_io_test_stream + SOURCES test_io_stream.cc + LIBS atlas_io + ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} +) + +ecbuild_add_executable( TARGET atlas_io_test_record + SOURCES test_io_record.cc + LIBS atlas_io + NOINSTALL +) + +foreach( algorithm none bzip2 aec lz4 snappy ) + string( TOUPPER ${algorithm} feature ) + if( eckit_HAVE_${feature} OR algorithm MATCHES "none" ) + ecbuild_add_test( TARGET atlas_io_test_record_COMPRESSION_${algorithm} + COMMAND atlas_test_io_record + ARGS --suffix ".${algorithm}" + ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ATLAS_IO_COMPRESSION=${algorithm} + ) + endif() +endforeach() + diff --git a/atlas-io/tests/TestEnvironment.h b/atlas-io/tests/TestEnvironment.h new file mode 100644 index 000000000..aa2b99e90 --- /dev/null +++ b/atlas-io/tests/TestEnvironment.h @@ -0,0 +1,309 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "eckit/config/LibEcKit.h" +#include "eckit/config/Resource.h" +#include "eckit/eckit.h" +#include "eckit/log/PrefixTarget.h" +#include "eckit/runtime/Main.h" +#include "eckit/testing/Test.h" +#include "eckit/types/Types.h" + +#include "atlas_io/atlas-io.h" +#include "atlas_io/detail/BlackMagic.h" + +namespace atlas { +namespace test { + +using eckit::types::is_approximately_equal; + +class Test; +static Test* current_test_{nullptr}; + +static size_t ATLAS_MAX_FAILED_EXPECTS() { + static size_t v = size_t(eckit::Resource("$ATLAS_MAX_FAILED_EXPECTS", 100)); + return v; +} + +class Test { + struct Failure { + std::string message; + eckit::CodeLocation location; + }; + +public: + Test(const std::string& description, const eckit::CodeLocation& location): + description_(description), location_(location) { + current_test_ = this; + } + ~Test() { current_test_ = nullptr; } + void expect_failed(const std::string& message, const eckit::CodeLocation& location) { + failures_.emplace_back(Failure{message, location}); + eckit::Log::error() << message << std::endl; + if (failures_.size() == ATLAS_MAX_FAILED_EXPECTS()) { + std::stringstream msg; + msg << "Maximum number of allowed EXPECTS have failed (${ATLAS_MAX_FAILED_EXPECTS}=" + << ATLAS_MAX_FAILED_EXPECTS() << ")."; + throw eckit::testing::TestException(msg.str(), location_); + } + } + bool failed() const { return failures_.size() > 0; } + void throw_on_failed_expects() { + if (failed()) { + std::stringstream msg; + msg << failures_.size() << " EXPECTS have failed"; + throw eckit::testing::TestException(msg.str(), location_); + } + } + const std::string& description() const { return description_; } + +private: + std::vector failures_; + std::string description_; + eckit::CodeLocation location_; +}; + +Test& current_test() { + ATLAS_IO_ASSERT(current_test_); + return *current_test_; +} + +//---------------------------------------------------------------------------------------------------------------------- + +#ifdef MAYBE_UNUSED +#elif defined(__GNUC__) +#define MAYBE_UNUSED __attribute__((unused)) +#else +#define MAYBE_UNUSED +#endif + +#ifdef EXPECT_EQ +#undef EXPECT_EQ +#endif +#ifdef EXPECT_APPROX_EQ +#undef EXPECT_APPROX_EQ +#endif + +#ifdef EXPECT +#undef EXPECT +#endif + +#define REQUIRE(expr) \ + do { \ + if (!(expr)) { \ + throw eckit::testing::TestException("EXPECT condition failed: " #expr, Here()); \ + } \ + } while (false) + +#define EXPECT(expr) \ + do { \ + if (!(expr)) { \ + current_test().expect_failed("EXPECT condition failed: " #expr, Here()); \ + } \ + } while (false) + +template +struct Printer { + static void print(std::ostream& out, const Value& v) { out << v; } +}; + +template <> +struct Printer { + static void print(std::ostream& out, const double& v) { out << std::fixed << std::setprecision(12) << v; } +}; + +template <> +struct Printer { + static void print(std::ostream& out, const eckit::CodeLocation& location) { + out << eckit::PathName{location.file()}.baseName() << " +" << location.line(); + } +}; + +template +struct PrintValue { + const Value& value; + PrintValue(const Value& v): value(v) {} + void print(std::ostream& out) const { Printer::print(out, value); } + friend std::ostream& operator<<(std::ostream& out, const PrintValue& v) { + v.print(out); + return out; + } +}; + +template +PrintValue print(const Value& v) { + return PrintValue(v); +} + +bool approx_eq(const float& v1, const float& v2) { + return is_approximately_equal(v1, v2); +} +bool approx_eq(const float& v1, const float& v2, const float& t) { + return is_approximately_equal(v1, v2, t); +} +bool approx_eq(const double& v1, const double& v2) { + return is_approximately_equal(v1, v2); +} +bool approx_eq(const double& v1, const double& v2, const double& t) { + return is_approximately_equal(v1, v2, t); +} +//bool approx_eq(const Point2& v1, const Point2& v2) { +// return approx_eq(v1[0], v2[0]) && approx_eq(v1[1], v2[1]); +//} +//bool approx_eq(const Point2& v1, const Point2& v2, const double& t) { +// return approx_eq(v1[0], v2[0], t) && approx_eq(v1[1], v2[1], t); +//} + +template +std::string expect_message(const std::string& condition, const T1& lhs, const T2& rhs, const eckit::CodeLocation& loc) { + std::stringstream msg; + msg << eckit::Colour::red << condition << " FAILED @ " << print(loc) << eckit::Colour::reset << "\n" + << eckit::Colour::red << " --> lhs = " << print(lhs) << eckit::Colour::reset << "\n" + << eckit::Colour::red << " --> rhs = " << print(rhs) << eckit::Colour::reset; + return msg.str(); +} + +#define EXPECT_EQ(lhs, rhs) \ + do { \ + if (!(lhs == rhs)) { \ + current_test().expect_failed(expect_message("EXPECT_EQ( " #lhs ", " #rhs " )", lhs, rhs, Here()), Here()); \ + } \ + } while (false) + +#define __EXPECT_APPROX_EQ(lhs, rhs) \ + do { \ + if (!(approx_eq(lhs, rhs))) { \ + current_test().expect_failed(expect_message("EXPECT_APPROX_EQ( " #lhs ", " #rhs " )", lhs, rhs, Here()), \ + Here()); \ + } \ + } while (false) + +#define __EXPECT_APPROX_EQ_TOL(lhs, rhs, tol) \ + do { \ + if (!(approx_eq(lhs, rhs, tol))) { \ + current_test().expect_failed( \ + expect_message("EXPECT_APPROX_EQ( " #lhs ", " #rhs ", " #tol " )", lhs, rhs, Here()), Here()); \ + } \ + } while (false) + +#define EXPECT_APPROX_EQ(...) __ATLAS_IO_SPLICE(__EXPECT_APPROX_EQ__, __ATLAS_IO_NARG(__VA_ARGS__))(__VA_ARGS__) +#define __EXPECT_APPROX_EQ__2 __EXPECT_APPROX_EQ +#define __EXPECT_APPROX_EQ__3 __EXPECT_APPROX_EQ_TOL + + +//---------------------------------------------------------------------------------------------------------------------- + + +namespace { +int digits(int number) { + int d = 0; + while (number) { + number /= 10; + d++; + } + return d; +} + +static std::string debug_prefix(const std::string& libname) { + std::string s = libname; + std::transform(s.begin(), s.end(), s.begin(), ::toupper); + s += "_DEBUG"; + return s; +} + +void debug_addTarget(eckit::LogTarget* target) { + for (std::string libname : eckit::system::Library::list()) { + const eckit::system::Library& lib = eckit::system::Library::lookup(libname); + if (lib.debug()) { + lib.debugChannel().addTarget(new eckit::PrefixTarget(debug_prefix(libname), target)); + } + } + if (eckit::Log::debug()) + eckit::Log::debug().addTarget(target); +} + +void debug_setTarget(eckit::LogTarget* target) { + for (std::string libname : eckit::system::Library::list()) { + const eckit::system::Library& lib = eckit::system::Library::lookup(libname); + if (lib.debug()) { + lib.debugChannel().setTarget(new eckit::PrefixTarget(debug_prefix(libname), target)); + } + } + if (eckit::Log::debug()) + eckit::Log::debug().setTarget(target); +} + +void debug_reset() { + for (std::string libname : eckit::system::Library::list()) { + const eckit::system::Library& lib = eckit::system::Library::lookup(libname); + if (lib.debug()) { + lib.debugChannel().reset(); + } + } + if (eckit::Log::debug()) + eckit::Log::debug().reset(); +} + +bool getEnv(const std::string& env, bool default_value) { + if (::getenv(env.c_str())) { + return eckit::Translator()(::getenv(env.c_str())); + } + return default_value; +} + +int getEnv(const std::string& env, int default_value) { + if (::getenv(env.c_str())) { + return eckit::Translator()(::getenv(env.c_str())); + } + return default_value; +} + +void setEnv(const std::string& env, bool value) { + constexpr int DO_NOT_REPLACE_IF_EXISTS = 0; + ::setenv(env.c_str(), eckit::Translator()(value).c_str(), DO_NOT_REPLACE_IF_EXISTS); +} + +} // namespace + +struct TestEnvironment { + TestEnvironment(int argc, char* argv[]) { eckit::Main::initialise(argc, argv); } + + ~TestEnvironment() {} +}; + + +//---------------------------------------------------------------------------------------------------------------------- + + +template +int run(int argc, char* argv[]) { + Environment env(argc, argv); + int errors = eckit::testing::run_tests(argc, argv, false); + return errors; +} + +int run(int argc, char* argv[]) { + return run(argc, argv); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace test +} // namespace atlas diff --git a/atlas-io/tests/test_io_encoding.cc b/atlas-io/tests/test_io_encoding.cc new file mode 100644 index 000000000..9a248d129 --- /dev/null +++ b/atlas-io/tests/test_io_encoding.cc @@ -0,0 +1,652 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#include +#include + +#include "atlas_io/atlas-io.h" + +#include "TestEnvironment.h" + +namespace atlas { +namespace test { + +using io::ArrayReference; + +// ------------------------------------------------------------------------------------------------------- + +struct UnencodableType { + std::string _; +}; + +// ------------------------------------------------------------------------------------------------------- + +// Example type that can be encoded / decoded with atlas::io +// The "operations" performed on this type are stored for unit-test purposes +class EncodableType { +public: + using Operations = std::vector; + + EncodableType(std::string s, std::shared_ptr operations = std::make_shared()): + str(s), ops(operations) { + ATLAS_IO_TRACE("EncodableType[" + str + "] construct"); + ops->push_back("constructor"); + } + + EncodableType(): ops(std::make_shared()) {} + + EncodableType(const EncodableType& other) { + // This constructor should not be called. + str = other.str; + ATLAS_IO_TRACE("EncodableType[" + str + "] copy constructor"); + ops = other.ops; + ops->push_back("copy constructor"); + } + + EncodableType(EncodableType&& other) { + str = std::move(other.str); + ATLAS_IO_TRACE("EncodableType[" + str + "] move constructor"); + ops = other.ops; + ops->push_back("move constructor"); + } + + + EncodableType& operator=(const EncodableType& rhs) { + // This assignment should not be called. + str = rhs.str; + ATLAS_IO_TRACE("EncodableType[" + str + "] assignment"); + ops = rhs.ops; + ops->push_back("assignment"); + return *this; + } + + EncodableType& operator=(const EncodableType&& rhs) { + // This assignment should not be called. + str = std::move(rhs.str); + ATLAS_IO_TRACE("EncodableType[" + str + "] move"); + ops = rhs.ops; + ops->push_back("move"); + return *this; + } + + + friend void encode_data(const EncodableType& in, atlas::io::Data& out) { + in.ops->push_back("encode_data"); + out.assign(in.str.data(), in.str.size()); + } + + friend size_t encode_metadata(const EncodableType& in, atlas::io::Metadata& metadata) { + in.ops->push_back("encode_metadata"); + metadata.set("type", "EncodableType"); + metadata.set("bytes", in.str.size()); + return in.str.size(); + } + + friend void decode(const atlas::io::Metadata&, const atlas::io::Data& b, EncodableType& out) { + out.ops->push_back("decode"); + const char* data = static_cast(b.data()); + out.str = std::string(data, data + b.size()); + } + + const std::vector& operations() const { return *ops; } + +private: + std::string str; + mutable std::shared_ptr> ops; +}; + +// ------------------------------------------------------------------------------------------------------- + +CASE("test exceptions") { + EXPECT(not(io::is_interpretable())); + EXPECT(not io::is_encodable()); + EXPECT(not io::can_encode_metadata()); + EXPECT(not io::can_encode_data()); + + UnencodableType in; + atlas::io::Data data; + atlas::io::Metadata metadata; + + EXPECT_THROWS_AS(io::ref(in, io::tag::disable_static_assert()), io::NotEncodable); + EXPECT_THROWS_AS(io::copy(in, io::tag::disable_static_assert()), io::NotEncodable); + EXPECT_THROWS_AS(io::encode(in, metadata, data, io::tag::disable_static_assert()), io::NotEncodable); +} + +// ------------------------------------------------------------------------------------------------------- + +CASE("encoding test::EncodableType") { + static_assert(not io::is_interpretable(), ""); + static_assert(io::is_encodable(), ""); + static_assert(io::is_decodable(), ""); + + const std::string encoded_string{"encoded string"}; + EncodableType in(encoded_string); + atlas::io::Data data; + atlas::io::Metadata metadata; + EXPECT_NO_THROW(encode(in, metadata, data)); + + EXPECT(metadata.type() == "EncodableType"); + EXPECT(data.size() == encoded_string.size()); + EXPECT(::memcmp(data, encoded_string.data(), encoded_string.size()) == 0); +} + + +// ------------------------------------------------------------------------------------------------------- + +CASE("encoding atlas::io::types::ArrayView") { + static_assert(not io::is_interpretable(), ""); + static_assert(io::can_encode_data(), ""); + static_assert(io::can_encode_metadata(), ""); + static_assert(io::is_encodable(), ""); +} + +// ------------------------------------------------------------------------------------------------------- + +template +void assert_StdVector() { + static_assert(io::is_interpretable, ArrayReference>(), ""); + static_assert(not io::can_encode_data>(), ""); + static_assert(not io::can_encode_metadata>(), ""); + static_assert(not io::is_encodable>(), ""); +} + +template +void encode_StdVector() { + std::vector in{1, 2, 3, 4, 5}; + + ArrayReference interpreted; + interprete(in, interpreted); + + atlas::io::Data data; + atlas::io::Metadata metadata; + + encode(interpreted, metadata, data); + + EXPECT(data.size() == in.size() * sizeof(T)); + EXPECT(::memcmp(in.data(), data.data(), data.size()) == 0); + EXPECT(metadata.type() == "array"); + EXPECT(metadata.getString("datatype") == atlas::io::DataType::str()); +} + +CASE("encoding std::vector") { + assert_StdVector(); + assert_StdVector(); + assert_StdVector(); + assert_StdVector(); + assert_StdVector(); + + encode_StdVector(); + encode_StdVector(); + encode_StdVector(); + encode_StdVector(); + + { + using T = std::byte; + std::bitset<8> bits; + std::vector in; + in.resize(5); + size_t n{0}; + for (auto& byte : in) { + bits.set(n++, true); + byte = *reinterpret_cast(&bits); + } + ArrayReference interpreted; + interprete(in, interpreted); + + atlas::io::Data data; + atlas::io::Metadata metadata; + + encode(interpreted, metadata, data); + + EXPECT(data.size() == in.size() * sizeof(T)); + EXPECT(::memcmp(in.data(), data.data(), data.size()) == 0); + EXPECT(metadata.type() == "array"); + EXPECT(metadata.getString("datatype") == atlas::io::DataType::str()); + } +} + +// ------------------------------------------------------------------------------------------------------- + + +template +void assert_StdArray() { + static_assert(io::is_interpretable, ArrayReference>(), ""); + static_assert(not io::can_encode_data>(), ""); + static_assert(not io::can_encode_metadata>(), ""); + static_assert(not io::is_encodable>(), ""); +} + +template +void encode_StdArray() { + std::array in{1, 2, 3, 4, 5}; + + ArrayReference interpreted; + interprete(in, interpreted); + + atlas::io::Data data; + atlas::io::Metadata metadata; + + encode(interpreted, metadata, data); + + EXPECT(data.size() == in.size() * sizeof(T)); + EXPECT(::memcmp(in.data(), data.data(), data.size()) == 0); + EXPECT(metadata.type() == "array"); + EXPECT(metadata.getString("datatype") == atlas::io::DataType::str()); +} + +CASE("encoding std::array") { + assert_StdArray(); + assert_StdArray(); + assert_StdArray(); + assert_StdArray(); + assert_StdArray(); + + encode_StdVector(); + encode_StdVector(); + encode_StdVector(); + encode_StdVector(); + + { + using T = std::byte; + std::bitset<8> bits; + std::vector in; + in.resize(5); + size_t n{0}; + for (auto& byte : in) { + bits.set(n++, true); + byte = *reinterpret_cast(&bits); + } + ArrayReference interpreted; + interprete(in, interpreted); + + atlas::io::Data data; + atlas::io::Metadata metadata; + + encode(interpreted, metadata, data); + + EXPECT(data.size() == in.size() * sizeof(T)); + EXPECT(::memcmp(in.data(), data.data(), data.size()) == 0); + EXPECT(metadata.type() == "array"); + EXPECT(metadata.getString("datatype") == atlas::io::DataType::str()); + } +} + +// ------------------------------------------------------------------------------------------------------- + +CASE("test Encoder") { + SECTION("default constructor") { + io::Encoder encoder; + EXPECT(encoder == false); + io::Metadata metadata; + io::Data data; + EXPECT_THROWS_AS(encode(encoder, metadata, data), eckit::AssertionFailed); + } + + SECTION("Encoder via reference") { + io::Encoder encoder; + auto ops = std::make_shared(); + + EncodableType encodable("string", ops); + EXPECT_EQ(ops->size(), 1); + EXPECT_EQ(ops->back(), "constructor"); + + io::ref(encodable); + EXPECT_EQ(ops->size(), 1); + EXPECT_EQ(ops->back(), "constructor"); + + encoder = io::Encoder{io::ref(encodable)}; + EXPECT_EQ(ops->size(), 2); + EXPECT_EQ(ops->back(), "encode_metadata"); + + io::Metadata metadata; + io::Data data; + encode(encoder, metadata, data); + EXPECT_EQ(ops->size(), 3); + EXPECT_EQ(ops->back(), "encode_data"); + } + + SECTION("Encoder via copy") { + io::Encoder encoder; + auto ops = std::make_shared(); + + EncodableType encodable("string", ops); + EXPECT_EQ(ops->size(), 1); + EXPECT_EQ(ops->back(), "constructor"); + + + encoder = io::Encoder{io::copy(encodable)}; + EXPECT_EQ(ops->size(), 3); + EXPECT_EQ(ops->at(1), "encode_metadata"); + EXPECT_EQ(ops->at(2), "encode_data"); + + io::Metadata metadata; + io::Data data; + encode(encoder, metadata, data); + EXPECT_EQ(ops->size(), 3); + EXPECT_EQ(ops->at(2), "encode_data"); + } + + SECTION("Encoder via move") { + io::Encoder encoder; + auto ops = std::make_shared(); + + EncodableType encodable("string", ops); + EXPECT_EQ(ops->size(), 1); + EXPECT_EQ(ops->back(), "constructor"); + + encoder = io::Encoder{std::move(encodable)}; + EXPECT_EQ(ops->size(), 3); + EXPECT_EQ(ops->at(1), "move constructor"); + EXPECT_EQ(ops->at(2), "encode_metadata"); + + io::Metadata metadata; + io::Data data; + encode(encoder, metadata, data); + EXPECT_EQ(ops->size(), 4); + EXPECT_EQ(ops->at(3), "encode_data"); + } +} + +// ------------------------------------------------------------------------------------------------------- + +CASE("Encoder for std::vector") { + SECTION("ref") { + using T = double; + std::vector v{1, 2, 3, 4, 5, 6, 7, 8}; + + io::Encoder encoder(io::ref(v)); + + // We can only encode with reference to original vector (no copies were made) + io::Metadata metadata; + io::Data data; + encode(encoder, metadata, data); + EXPECT(data.size() == v.size() * sizeof(T)); + EXPECT(::memcmp(data, v.data(), data.size()) == 0); + } + + SECTION("copy") { + using T = double; + std::vector v{1, 2, 3, 4, 5, 6, 7, 8}; + + io::Encoder encoder; + { + std::vector scoped = v; + encoder = io::Encoder(io::copy(scoped)); + scoped.assign(scoped.size(), 0); // zero out before destruction + } + + // We can now encode with scoped vector destroyed + io::Metadata metadata; + io::Data data; + encode(encoder, metadata, data); + EXPECT_EQ(data.size(), v.size() * sizeof(T)); + EXPECT(::memcmp(data, v.data(), data.size()) == 0); + } +} + +// ------------------------------------------------------------------------------------------------------- + +CASE("Encoder for std::array") { + SECTION("ref") { + using T = double; + std::array v{1, 2, 3, 4, 5, 6, 7, 8}; + + io::Encoder encoder(io::ref(v)); + + // We can only encode with reference to original vector (no copies were made) + io::Metadata metadata; + io::Data data; + encode(encoder, metadata, data); + EXPECT(data.size() == v.size() * sizeof(T)); + EXPECT(::memcmp(data, v.data(), data.size()) == 0); + } + + SECTION("copy") { + using T = double; + std::array v{1, 2, 3, 4, 5, 6, 7, 8}; + + io::Encoder encoder; + { + std::array scoped = v; + encoder = io::Encoder(io::copy(scoped)); + std::fill(std::begin(scoped), std::end(scoped), 0); // zero out before destruction + } + + // We can now encode with scoped vector destroyed + io::Metadata metadata; + io::Data data; + encode(encoder, metadata, data); + EXPECT_EQ(data.size(), v.size() * sizeof(T)); + EXPECT(::memcmp(data, v.data(), data.size()) == 0); + } +} + +// ------------------------------------------------------------------------------------------------------- + +CASE("Encoder of encoder") { + using T = double; + std::vector v{1, 2, 3, 4, 5, 6, 7, 8}; + + io::Encoder encoder(io::ref(v)); + io::Encoder encoder_of_encoder(io::ref(encoder)); + + io::Metadata metadata; + io::Data data; + encode(encoder_of_encoder, metadata, data); + EXPECT_EQ(data.size(), v.size() * sizeof(T)); + EXPECT(::memcmp(data, v.data(), data.size()) == 0); +} + +// ------------------------------------------------------------------------------------------------------- + +/// Helper class to be used in testing decoding of arrays. +template +struct EncodedArray { + atlas::io::Data data; + atlas::io::Metadata metadata; + + EncodedArray(): in{1, 2, 3, 4, 5, 6, 7, 8} { encode(in, metadata, data); } + + friend bool operator==(const std::vector& lhs, const EncodedArray& rhs) { + if (lhs.size() != rhs.in.size()) { + return false; + } + return ::memcmp(lhs.data(), rhs.in.data(), rhs.in.size() * sizeof(T)) == 0; + } + friend bool operator==(const std::array& lhs, const EncodedArray& rhs) { + if (lhs.size() != rhs.in.size()) { + return false; + } + return ::memcmp(lhs.data(), rhs.in.data(), rhs.in.size() * sizeof(T)) == 0; + } + +private: + std::vector in; +}; + +template <> +struct EncodedArray { + using T = std::byte; + atlas::io::Data data; + atlas::io::Metadata metadata; + + EncodedArray() { + std::bitset<8> bits; + in.resize(5); + size_t n{0}; + for (auto& byte : in) { + bits.set(n++, true); + byte = *reinterpret_cast(&bits); + } + encode(in, metadata, data); + } + + friend bool operator==(const std::vector& lhs, const EncodedArray& rhs) { + if (lhs.size() != rhs.in.size()) { + return false; + } + return ::memcmp(lhs.data(), rhs.in.data(), rhs.in.size() * sizeof(T)) == 0; + } + +private: + std::vector in; +}; + + +// ------------------------------------------------------------------------------------------------------- + +CASE("Decoding to std::vector") { + using T = double; + EncodedArray encoded; + std::vector out; + + SECTION("decode std::vector directly") { + EXPECT_NO_THROW(decode(encoded.metadata, encoded.data, out)); + EXPECT(out == encoded); + } + + SECTION("decode using rvalue io::Decoder (type erasure)") { + EXPECT_NO_THROW(decode(encoded.metadata, encoded.data, io::Decoder(out))); + EXPECT(out == encoded); + } + + SECTION("decode using lvalue io::Decoder (type erasure)") { + io::Decoder decoder(out); + EXPECT_NO_THROW(decode(encoded.metadata, encoded.data, io::Decoder(out))); + EXPECT(out == encoded); + } + + SECTION("decode using decoder of decoder") { + io::Decoder decoder(out); + EXPECT_NO_THROW(decode(encoded.metadata, encoded.data, io::Decoder(decoder))); + EXPECT(out == encoded); + } +} + +// ------------------------------------------------------------------------------------------------------- + +CASE("Decoding to std::array") { + using T = double; + EncodedArray encoded; + std::array out; + + SECTION("decode std::vector directly") { + EXPECT_NO_THROW(decode(encoded.metadata, encoded.data, out)); + EXPECT(out == encoded); + } + + SECTION("decode using rvalue io::Decoder (type erasure)") { + EXPECT_NO_THROW(decode(encoded.metadata, encoded.data, io::Decoder(out))); + EXPECT(out == encoded); + } + + SECTION("decode using lvalue io::Decoder (type erasure)") { + io::Decoder decoder(out); + EXPECT_NO_THROW(decode(encoded.metadata, encoded.data, io::Decoder(out))); + EXPECT(out == encoded); + } + + SECTION("decode using decoder of decoder") { + io::Decoder decoder(out); + EXPECT_NO_THROW(decode(encoded.metadata, encoded.data, io::Decoder(decoder))); + EXPECT(out == encoded); + } +} + +// ------------------------------------------------------------------------------------------------------- + +CASE("Encode/Decode byte array") { + using T = std::byte; + EncodedArray encoded; + std::vector out; + + auto validate = [&]() { + EXPECT(out == encoded); + + auto str = [](std::byte byte) { + std::bitset<8> bitset(reinterpret_cast(byte)); + return bitset.to_string(); + }; + EXPECT_EQ(str(out[0]), "00000001"); + EXPECT_EQ(str(out[1]), "00000011"); + EXPECT_EQ(str(out[2]), "00000111"); + EXPECT_EQ(str(out[3]), "00001111"); + EXPECT_EQ(str(out[4]), "00011111"); + }; + + + SECTION("decode directly") { + EXPECT_NO_THROW(decode(encoded.metadata, encoded.data, out)); + validate(); + } + + SECTION("decode using rvalue io::Decoder (type erasure)") { + EXPECT_NO_THROW(decode(encoded.metadata, encoded.data, io::Decoder(out))); + validate(); + } + + SECTION("decode using lvalue io::Decoder (type erasure)") { + io::Decoder decoder(out); + EXPECT_NO_THROW(decode(encoded.metadata, encoded.data, io::Decoder(out))); + validate(); + } + + SECTION("decode using decoder of decoder") { + io::Decoder decoder(out); + EXPECT_NO_THROW(decode(encoded.metadata, encoded.data, io::Decoder(decoder))); + validate(); + } +} + +// ------------------------------------------------------------------------------------------------------- + +CASE("Encode/Decode string") { + std::string in{"short string"}; + io::Metadata metadata; + io::Data data; + encode(in, metadata, data); + EXPECT_EQ(data.size(), 0); + + std::string out; + decode(metadata, data, out); + EXPECT_EQ(out, in); +} + +// ------------------------------------------------------------------------------------------------------- + +template +void test_encode_decode_scalar() { + T in{std::numeric_limits::max()}, out; + io::Metadata metadata; + io::Data data; + encode(in, metadata, data); + EXPECT_EQ(data.size(), 0); + + decode(metadata, data, out); + EXPECT_EQ(out, in); +} + +CASE("Encode/Decode scalar") { + // bit identical encoding via Base64 string within the metadata! + SECTION("int32") { test_encode_decode_scalar(); } + SECTION("int64") { test_encode_decode_scalar(); } + SECTION("real32") { test_encode_decode_scalar(); } + SECTION("real64") { test_encode_decode_scalar(); } + SECTION("uint64") { test_encode_decode_scalar(); } +} + +// ------------------------------------------------------------------------------------------------------- + +} // namespace test +} // namespace atlas + +int main(int argc, char** argv) { + return atlas::test::run(argc, argv); +} diff --git a/atlas-io/tests/test_io_record.cc b/atlas-io/tests/test_io_record.cc new file mode 100644 index 000000000..a1e057cfa --- /dev/null +++ b/atlas-io/tests/test_io_record.cc @@ -0,0 +1,596 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#include +#include +#include + +#include "eckit/io/MemoryHandle.h" + +#include "TestEnvironment.h" + +namespace atlas { +namespace test { + +template +struct Matrix { + std::vector data_; + size_t rows_; + size_t cols_; + size_t size() const { return rows_ * cols_; } + T* data() { return data_.data(); } + const T* data() const { return data_.data(); } + atlas::io::DataType datatype() const { return atlas::io::make_datatype(); } + Matrix(size_t rows, size_t cols) { resize(rows, cols); } + void resize(size_t rows, size_t cols) { + rows_ = rows; + cols_ = cols; + data_.resize(size()); + } + void assign(std::initializer_list list) { data_.assign(list); } + void assign(T value) { data_.assign(size(), value); } +}; + +template +void interprete(const Matrix& in, atlas::io::ArrayReference& out) { + out = io::ArrayReference(in.data(), in.datatype(), atlas::io::ArrayShape{in.rows_, in.cols_}); +} + +template +void decode(const atlas::io::Metadata& metadata, const atlas::io::Data& data, Matrix& out) { + atlas::io::ArrayMetadata array(metadata); + out.resize(array.shape(0), array.shape(1)); + ::memcpy(out.data(), data, data.size()); +} + + +struct Arrays { + std::vector v1; + std::vector v2; + Matrix v3{0, 0}; + bool operator==(const Arrays& other) const { + return v1 == other.v1 && ::memcmp(v2.data(), other.v2.data(), v2.size() * sizeof(float)) == 0 && + ::memcmp(v3.data(), other.v3.data(), v3.size() * v3.datatype().size()) == 0; + } + bool operator!=(const Arrays& other) const { return not operator==(other); } +}; + +//----------------------------------------------------------------------------- + +static eckit::LocalConfiguration no_compression = [] { + eckit::LocalConfiguration c; + c.set("compression", "none"); + return c; +}(); + +//----------------------------------------------------------------------------- + +std::string suffix() { + static std::string suffix = eckit::Resource("--suffix", ""); + return suffix; +} + +//----------------------------------------------------------------------------- + +namespace globals { +struct TestRecord { + Arrays data; + TestRecord() = default; + TestRecord(const std::function& initializer) { initializer(data); } +}; + +static TestRecord record1{[](Arrays& data) { + data.v1 = {0, 1, 2, 3, 4}; + data.v2 = {3, 2, 1}; + data.v3.resize(3, 2); + data.v3.assign({11, 12, 21, 22, 31, 32}); +}}; + +static TestRecord record2{[](Arrays& data) { + data.v1 = {0, 10, 20, 30, 40, 50}; + data.v2 = {30, 20, 10, 40}; + data.v3.resize(2, 3); + data.v3.assign({11, 12, 13, 21, 22, 23}); +}}; + +static TestRecord record3{[](Arrays& data) { + data.v1.assign(1024 / 8 - 1, 2.); + data.v2.assign(1023 * 1024 / 4 + 512 / 4, 1.); + data.v3.resize(1024, 1024); + data.v3.assign(3); +}}; + +std::vector records; + +} // namespace globals + +//----------------------------------------------------------------------------- + +template +void write_length(Length length, const std::string& path) { + std::ofstream file(path); + file << length; + file.close(); +} + +//-----------------------------------------------------------------------------// +// // +// Writing records // +// // +//-----------------------------------------------------------------------------// + + +CASE("Write records, each in separate file (offset=0)") { + auto write_record = [&](const Arrays& data, const eckit::PathName& path) { + io::RecordWriter record; + record.set("v1", io::ref(data.v1), no_compression); + record.set("v2", io::ref(data.v2), no_compression); + record.set("v3", io::ref(data.v3)); + auto length = record.write(path); + write_length(length, path + ".length"); + }; + + SECTION("record1.atlas" + suffix()) { write_record(globals::record1.data, "record1.atlas" + suffix()); } + SECTION("record2.atlas" + suffix()) { write_record(globals::record2.data, "record2.atlas" + suffix()); } + SECTION("record3.atlas" + suffix()) { write_record(globals::record3.data, "record3.atlas" + suffix()); } +} + +//----------------------------------------------------------------------------- + +CASE("Write records to same file using record.write(path,io::Mode)") { + // This will reopen files upon every append, bad for performance + + static std::vector lengths; + static std::vector offsets{0}; + + auto write_record = [&](Arrays& data, const eckit::PathName& path, io::Mode mode) { + io::RecordWriter record; + record.set("v1", io::ref(data.v1), no_compression); + record.set("v2", io::ref(data.v2), no_compression); + record.set("v3", io::ref(data.v3)); + + globals::records.emplace_back(io::Record::URI{path, offsets.back()}); + lengths.emplace_back(record.write(path, mode)); + offsets.emplace_back(offsets.back() + lengths.back()); + }; + + SECTION("record1 -> records.atlas" + suffix()) { + write_record(globals::record1.data, "records.atlas" + suffix(), io::Mode::write); + } + SECTION("record2 -> records.atlas" + suffix()) { + write_record(globals::record2.data, "records.atlas" + suffix(), io::Mode::append); + } + SECTION("record3 -> records.atlas" + suffix()) { + write_record(globals::record3.data, "records.atlas" + suffix(), io::Mode::append); + } +} + +//----------------------------------------------------------------------------- + +CASE("Write records to same file using record.write(Stream) keeping Stream open") { + // This should give exactly the same output file as previous, except no + + auto write_record = [&](const Arrays& data, io::Stream stream) { + io::RecordWriter record; + record.set("v1", io::ref(data.v1), no_compression); + record.set("v2", io::ref(data.v2), no_compression); + record.set("v3", io::ref(data.v3)); + + record.write(stream); + }; + + static io::OutputFileStream stream("records.atlas" + suffix() + ".duplicate"); + + SECTION("record1 -> records.atlas" + suffix() + ".duplicate") { + EXPECT_EQ(stream.position(), globals::records[0].offset); + write_record(globals::record1.data, stream); + } + SECTION("record2 -> records.atlas" + suffix() + ".duplicate") { + EXPECT_EQ(stream.position(), globals::records[1].offset); + write_record(globals::record2.data, stream); + } + SECTION("record3 -> records.atlas" + suffix() + ".duplicate") { + EXPECT_EQ(stream.position(), globals::records[2].offset); + write_record(globals::record3.data, stream); + } + SECTION("close stream") { + stream.close(); // required because stream is a static variable + } +} + +//----------------------------------------------------------------------------- + +CASE("Write master record referencing record1 and record2 and record3") { + io::RecordWriter record; + record.set("v1", io::link("file:record1.atlas" + suffix() + "?key=v1")); + record.set("v2", io::link("file:record1.atlas" + suffix() + "?key=v2")); + record.set("v3", io::link("file:record1.atlas" + suffix() + "?key=v3")); + record.set("v4", io::link("file:record2.atlas" + suffix() + "?key=v1")); + record.set("v5", io::link("file:record2.atlas" + suffix() + "?key=v2")); + record.set("v6", io::link("file:record2.atlas" + suffix() + "?key=v3")); + record.set("v7", io::link("file:record3.atlas" + suffix() + "?key=v1")); + record.set("v8", io::link("file:record3.atlas" + suffix() + "?key=v2")); + record.set("v9", io::link("file:record3.atlas" + suffix() + "?key=v3")); + record.write("record.atlas" + suffix()); +} + +//----------------------------------------------------------------------------- + +CASE("Write records in nested subdirectories") { + auto reference_path = eckit::PathName{"atlas_test_io_refpath"}; + { + eckit::PathName{reference_path / "links" / "1"}.mkdir(); + + io::RecordWriter record; + record.set("v1", io::ref(globals::record1.data.v1)); + record.set("v2", io::ref(globals::record1.data.v2)); + record.set("v3", io::ref(globals::record1.data.v3)); + record.set("s1", std::string("short string")); + record.set("s2", double(1. / 3.)); + record.write(reference_path / "links" / "1" / "record.atlas" + suffix()); + } + { + eckit::PathName{reference_path / "links" / "2"}.mkdir(); + + io::RecordWriter record; + record.set("v1", io::ref(globals::record2.data.v1)); + record.set("v2", io::ref(globals::record2.data.v2)); + record.set("v3", io::ref(globals::record2.data.v3)); + record.set("s1", size_t(10000000000)); + record.write(reference_path / "links" / "2" / "record.atlas" + suffix()); + } + { + io::RecordWriter record; + record.set("l1", io::link("file:1/record.atlas" + suffix() + "?key=v1")); + record.set("l2", io::link("file:1/record.atlas" + suffix() + "?key=v2")); + record.set("l3", io::link("file:1/record.atlas" + suffix() + "?key=v3")); + record.set("l4", io::link("file:2/record.atlas" + suffix() + "?key=v1")); + record.set("l5", io::link("file:2/record.atlas" + suffix() + "?key=v2")); + record.set("l6", io::link("file:2/record.atlas" + suffix() + "?key=v3")); + record.set("l7", io::link("file:1/record.atlas" + suffix() + "?key=s1")); + record.set("l8", io::link("file:1/record.atlas" + suffix() + "?key=s2")); + record.set("l9", io::link("file:2/record.atlas" + suffix() + "?key=s1")); + record.write(reference_path / "links" / "record.atlas" + suffix()); + } + { + io::RecordWriter record; + record.set("l1", io::link("file:links/record.atlas" + suffix() + "?key=l1")); + record.set("l2", io::link("file:links/record.atlas" + suffix() + "?key=l2")); + record.set("l3", io::link("file:links/record.atlas" + suffix() + "?key=l3")); + record.set("l4", io::link("file:links/record.atlas" + suffix() + "?key=l4")); + record.set("l5", io::link("file:links/record.atlas" + suffix() + "?key=l5")); + record.set("l6", io::link("file:links/record.atlas" + suffix() + "?key=l6")); + record.set("l7", io::link("file:links/record.atlas" + suffix() + "?key=l7")); + record.set("l8", io::link("file:links/record.atlas" + suffix() + "?key=l8")); + record.set("l9", io::link("file:links/record.atlas" + suffix() + "?key=l9")); + record.write(reference_path / "record.atlas" + suffix()); + } +} + +//-----------------------------------------------------------------------------// +// // +// Reading tests // +// // +//-----------------------------------------------------------------------------// + +CASE("Test RecordItemReader") { + SECTION("file:record1.atlas" + suffix() + "?key=v2") { + io::RecordItemReader reader{"file:record1.atlas" + suffix() + "?key=v2"}; + { + // When we only want to read metadata + io::Metadata metadata; + reader.read(metadata); + EXPECT(metadata.link() == false); + EXPECT_EQ(metadata.type(), "array"); + EXPECT(metadata.data.compressed() == false); + EXPECT_EQ(metadata.data.compression(), "none"); + EXPECT_EQ(metadata.data.size(), globals::record1.data.v2.size() * sizeof(float)); + } + { + // When we want to read both metadata and data + io::Metadata metadata; + io::Data data; + reader.read(metadata, data); + EXPECT(metadata.data.compressed() == false); + EXPECT_EQ(metadata.data.compression(), "none"); + EXPECT_EQ(metadata.data.size(), globals::record1.data.v2.size() * sizeof(float)); + EXPECT_EQ(data.size(), metadata.data.size()); + EXPECT_EQ(data.size(), metadata.data.compressed_size()); + EXPECT(::memcmp(data, globals::record1.data.v2.data(), data.size()) == 0); + } + } + + SECTION("file:record.atlas" + suffix() + "?key=v9") { + io::RecordItemReader reader{"file:record.atlas" + suffix() + "?key=v9"}; + { + // When we only want to read metadata + io::Metadata metadata; + reader.read(metadata); + EXPECT(metadata.link() == true); + EXPECT_EQ(metadata.link().str(), "file:record3.atlas" + suffix() + "?key=v3"); + EXPECT_EQ(metadata.type(), "array"); + } + { + // When we want to read both metadata and data + io::Metadata metadata; + io::Data data; + reader.read(metadata, data); + EXPECT(metadata.data.compressed() == (io::defaults::compression_algorithm() != "none")); + EXPECT_EQ(metadata.data.compression(), io::defaults::compression_algorithm()); + EXPECT_EQ(metadata.data.size(), globals::record3.data.v3.size() * sizeof(int)); + EXPECT_EQ(data.size(), metadata.data.compressed_size()); + } + } +} + +//----------------------------------------------------------------------------- + +CASE("Read records from different files") { + Arrays data1, data2, data3; + + auto read_record = [&](const eckit::PathName& path, Arrays& data) { + io::RecordReader record(path); + record.read("v1", data.v1).wait(); + record.read("v2", data.v2).wait(); + record.read("v3", data.v3).wait(); + }; + + read_record("record1.atlas" + suffix(), data1); + read_record("record2.atlas" + suffix(), data2); + read_record("record3.atlas" + suffix(), data3); + + EXPECT(data1 == globals::record1.data); + EXPECT(data2 == globals::record2.data); + EXPECT(data3 == globals::record3.data); +} + +//----------------------------------------------------------------------------- + +CASE("Read multiple records from same file") { + Arrays data1, data2; + io::RecordReader record1(globals::records[0]); + io::RecordReader record2(globals::records[1]); + + record1.read("v1", data1.v1).wait(); + record1.read("v2", data1.v2).wait(); + record1.read("v3", data1.v3).wait(); + + record2.read("v1", data2.v1).wait(); + record2.read("v2", data2.v2).wait(); + record2.read("v3", data2.v3).wait(); + + EXPECT(data1 == globals::record1.data); + EXPECT(data2 == globals::record2.data); +} + +//----------------------------------------------------------------------------- + +CASE("Write master record referencing record1 and record2") { + io::RecordWriter record; + record.set("v1", io::link("file:record1.atlas" + suffix() + "?key=v1")); + record.set("v2", io::link("file:record1.atlas" + suffix() + "?key=v2")); + record.set("v3", io::link("file:record1.atlas" + suffix() + "?key=v3")); + record.set("v4", io::link("file:record2.atlas" + suffix() + "?key=v1")); + record.set("v5", io::link("file:record2.atlas" + suffix() + "?key=v2")); + record.set("v6", io::link("file:record2.atlas" + suffix() + "?key=v3")); + record.write("record.atlas" + suffix()); +} + + +//----------------------------------------------------------------------------- + +CASE("Read master record") { + Arrays data1, data2; + io::RecordReader record("record.atlas" + suffix()); + + eckit::Log::info() << "record.metadata(\"v1\"): " << record.metadata("v1") << std::endl; + + + record.read("v1", data1.v1).wait(); + record.read("v2", data1.v2).wait(); + record.read("v3", data1.v3).wait(); + record.read("v4", data2.v1).wait(); + record.read("v5", data2.v2).wait(); + record.read("v6", data2.v3).wait(); + + EXPECT(data1 == globals::record1.data); + EXPECT(data2 == globals::record2.data); +} + +//----------------------------------------------------------------------------- + +CASE("Async read") { + Arrays data1, data2; + io::RecordReader record("record.atlas" + suffix()); + + // Request reads + record.read("v1", data1.v1); + record.read("v2", data1.v2); + record.read("v3", data1.v3); + record.read("v4", data2.v1); + record.read("v5", data2.v2); + record.read("v6", data2.v3); + + // Wait for specific requests + record.wait("v4"); + record.wait("v5"); + record.wait("v6"); + + // Should have completed + EXPECT(data2 == globals::record2.data); + + // Should not be complete yet + EXPECT(data1 != globals::record1.data); + + // Wait for all requests; + record.wait(); + + // Should have completed + EXPECT(data1 == globals::record1.data); +} + +//----------------------------------------------------------------------------- + +CASE("Recursive Write/read records in nested subdirectories") { + auto reference_path = eckit::PathName{"atlas_test_io_refpath"}; + + // Read + + Arrays data1, data2; + io::RecordReader record(reference_path / "record.atlas" + suffix()); + + record.read("l1", data1.v1).wait(); + record.read("l2", data1.v2).wait(); + record.read("l3", data1.v3).wait(); + + record.read("l4", data2.v1).wait(); + record.read("l5", data2.v2).wait(); + record.read("l6", data2.v3).wait(); + + std::string l7; + double l8; + size_t l9; + record.read("l7", l7).wait(); + record.read("l8", l8).wait(); + record.read("l9", l9).wait(); + + EXPECT(data1 == globals::record1.data); + EXPECT(data2 == globals::record2.data); + EXPECT_EQ(l7, "short string"); + EXPECT_EQ(l8, 1. / 3.); + EXPECT_EQ(l9, 10000000000ul); +} + +//----------------------------------------------------------------------------- + +CASE("Write record to memory") { + const auto& data_write = globals::record3.data; + const auto& v1 = globals::record3.data.v1; + const auto& v2 = globals::record3.data.v2; + const auto& v3 = globals::record3.data.v3; + + eckit::Buffer memory; + + // write + { + ATLAS_IO_TRACE("write"); + io::RecordWriter record; + record.compression(false); + record.checksum(false); + record.set("v1", io::ref(v1)); + record.set("v2", io::ref(v2)); + record.set("v3", io::ref(v3)); + + memory.resize(record.estimateMaximumSize()); + + eckit::Log::info() << "memory.size() : " << memory.size() << std::endl; + ; + + eckit::MemoryHandle datahandle_out{memory}; + datahandle_out.openForWrite(0); + auto record_length = record.write(datahandle_out); + datahandle_out.close(); + + // Without compression, this should be exact + EXPECT_EQ(memory.size(), record_length); + } + + // read with individual RecordItemReader + { + ATLAS_IO_TRACE("read with RecordItemReader"); + + io::Session session; + + eckit::MemoryHandle datahandle_in{memory}; + datahandle_in.openForRead(); + + { + io::RecordItemReader reader(datahandle_in, "v1"); + io::Metadata metadata; + io::Data data; + reader.read(metadata, data); + EXPECT(::memcmp(data, data_write.v1.data(), data.size()) == 0); + } + { + io::RecordItemReader reader(datahandle_in, "v2"); + io::Metadata metadata; + io::Data data; + reader.read(metadata, data); + EXPECT(::memcmp(data, data_write.v2.data(), data.size()) == 0); + } + { + io::RecordItemReader reader(datahandle_in, "v3"); + io::Metadata metadata; + io::Data data; + reader.read(metadata, data); + EXPECT(::memcmp(data, data_write.v3.data(), data.size()) == 0); + } + datahandle_in.close(); + } + + // read with RecordReader + { + ATLAS_IO_TRACE("read with RecordReader"); + Arrays data_read; + + eckit::MemoryHandle datahandle_in{memory}; + datahandle_in.openForRead(); + + io::RecordReader reader(datahandle_in); + reader.read("v1", data_read.v1); + reader.read("v2", data_read.v2); + reader.read("v3", data_read.v3); + reader.wait(); + + datahandle_in.close(); + + EXPECT(data_read == data_write); + } +} + +//-----------------------------------------------------------------------------// +// // +// Reading tests // +// // +//-----------------------------------------------------------------------------// + +CASE("RecordPrinter") { + SECTION("table") { + eckit::LocalConfiguration table_with_details; + table_with_details.set("format", "table"); + table_with_details.set("details", true); + + io::RecordPrinter record{eckit::PathName("record1.atlas" + suffix()), table_with_details}; + std::stringstream out; + EXPECT_NO_THROW(out << record); + eckit::Log::debug() << out.str(); + } + + SECTION("yaml") { + eckit::LocalConfiguration yaml_with_details; + yaml_with_details.set("format", "yaml"); + yaml_with_details.set("details", true); + + io::RecordPrinter record{eckit::PathName("record1.atlas" + suffix()), yaml_with_details}; + std::stringstream out; + EXPECT_NO_THROW(out << record); + eckit::Log::debug() << out.str(); + } +} + +//----------------------------------------------------------------------------- + +} // namespace test +} // namespace atlas + + +int main(int argc, char** argv) { + return atlas::test::run(argc, argv); +} diff --git a/atlas-io/tests/test_io_stream.cc b/atlas-io/tests/test_io_stream.cc new file mode 100644 index 000000000..46ed49874 --- /dev/null +++ b/atlas-io/tests/test_io_stream.cc @@ -0,0 +1,211 @@ +/* + * (C) Copyright 2013 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#include "atlas_io/FileStream.h" +#include "atlas_io/Session.h" + +#include "eckit/io/FileHandle.h" +#include "eckit/io/PooledHandle.h" + +#include "TestEnvironment.h" + + +namespace atlas { +namespace test { + +CASE("Stream interoperability with eckit::DataHandle") { + SECTION("own pointer") { + io::Stream s; + { + eckit::DataHandle* datahandle = new eckit::FileHandle("test_io_session.data"); + datahandle->openForWrite(0); + s = io::Stream{datahandle}; + } + s.datahandle().close(); + } + SECTION("shared pointer") { + io::Stream s; + { + std::shared_ptr datahandle = std::make_shared("test_io_session.data"); + datahandle->openForWrite(0); + s = io::Stream{datahandle}; + } + s.datahandle().close(); + } + SECTION("reference") { + io::Stream s; + eckit::FileHandle datahandle("test_io_session.data"); + datahandle.openForWrite(0); + s = io::Stream{datahandle}; + s.datahandle().close(); + } +} + +CASE("Test seek-for-write works when opening OutputFileStream for append") { + std::string s1("write \n"); + std::string s2("append \n"); + std::string s3("overwrite\n"); + { + ATLAS_IO_TRACE("write"); + io::Stream f = io::OutputFileStream("append-test"); + f.write(s1.c_str(), s1.size()); + } + { + ATLAS_IO_TRACE("append"); + io::Stream f = io::OutputFileStream("append-test", io::Mode::append); + auto offset = f.position(); + f.write(s2.c_str(), s2.size()); + + // Rewind to beginning of append + f.seek(offset); + f.write(s3.c_str(), s3.size()); + } + { + ATLAS_IO_TRACE("read"); + io::Stream f = io::InputFileStream("append-test"); + std::string expected = s1 + s3; + std::string read(expected.size(), ' '); + f.read(const_cast(read.data()), read.size()); + EXPECT_EQ(read, expected); + } +} + + +CASE("Opening same file in same scope") { + // Opening same file within same scope will avoid opening it multiple times, good for perfmance + + // write a file + { + io::OutputFileStream out("test_io_session.data"); + out.write("line1\n", 6); + out.write("line2\n", 6); + out.write("line3\n", 6); + } + + std::string l1(5, ' '), l2(5, ' '), l3(5, ' '); + + io::Stream f1 = io::InputFileStream{"test_io_session.data"}; + f1.seek(0 * 6); + f1.read(const_cast(l1.data()), 5); + + io::Stream f2 = io::InputFileStream{"test_io_session.data"}; + f2.seek(1 * 6); + f2.read(const_cast(l2.data()), 5); + + io::Stream f3 = io::InputFileStream{"test_io_session.data"}; + f3.seek(2 * 6); + f3.read(const_cast(l3.data()), 5); + + EXPECT_EQ(l1, "line1"); + EXPECT_EQ(l2, "line2"); + EXPECT_EQ(l3, "line3"); + + auto& pooled_handle = dynamic_cast(f1.datahandle()); + EXPECT_EQ(pooled_handle.nbOpens(), 1); + EXPECT_EQ(pooled_handle.nbSeeks(), 3); + EXPECT_EQ(pooled_handle.nbReads(), 3); +} + +CASE("Opening same file in parallel scopes") { + // Files are opened and closed within each scope, bad for performance + + // write a file + { + io::OutputFileStream out("test_io_session.data"); + out.write("line1\n", 6); + out.write("line2\n", 6); + out.write("line3\n", 6); + } + + std::string l1(5, ' '), l2(5, ' '), l3(5, ' '); + + { + io::Stream f1 = io::InputFileStream{"test_io_session.data"}; + f1.seek(0 * 6); + f1.read(const_cast(l1.data()), 5); + auto& pooled_handle = dynamic_cast(f1.datahandle()); + EXPECT_EQ(pooled_handle.nbOpens(), 1); + EXPECT_EQ(pooled_handle.nbSeeks(), 1); + EXPECT_EQ(pooled_handle.nbReads(), 1); + } + { + io::Stream f2 = io::InputFileStream{"test_io_session.data"}; + f2.seek(1 * 6); + f2.read(const_cast(l2.data()), 5); + auto& pooled_handle = dynamic_cast(f2.datahandle()); + EXPECT_EQ(pooled_handle.nbOpens(), 1); + EXPECT_EQ(pooled_handle.nbSeeks(), 1); + EXPECT_EQ(pooled_handle.nbReads(), 1); + } + { + io::Stream f3 = io::InputFileStream{"test_io_session.data"}; + f3.seek(2 * 6); + f3.read(const_cast(l3.data()), 5); + auto& pooled_handle = dynamic_cast(f3.datahandle()); + EXPECT_EQ(pooled_handle.nbOpens(), 1); + EXPECT_EQ(pooled_handle.nbSeeks(), 1); + EXPECT_EQ(pooled_handle.nbReads(), 1); + } +} + +CASE("Opening same file in parallel scopes with Session") { + // Declaring this in an outer scope will keep storage of InputFileStream + // within nested scopes, so that files will not be opened/closed repeatedly + + io::Session session; + + // write a file + { + io::OutputFileStream out("test_io_session.data"); + out.write("line1\n", 6); + out.write("line2\n", 6); + out.write("line3\n", 6); + } + + + std::string l1(5, ' '), l2(5, ' '), l3(5, ' '); + + { + io::Stream f1 = io::InputFileStream{"test_io_session.data"}; + f1.seek(0 * 6); + f1.read(const_cast(l1.data()), 5); + auto& pooled_handle = dynamic_cast(f1.datahandle()); + EXPECT_EQ(pooled_handle.nbOpens(), 1); + EXPECT_EQ(pooled_handle.nbSeeks(), 1); + EXPECT_EQ(pooled_handle.nbReads(), 1); + } + { + io::Stream f2 = io::InputFileStream{"test_io_session.data"}; + f2.seek(1 * 6); + f2.read(const_cast(l2.data()), 5); + auto& pooled_handle = dynamic_cast(f2.datahandle()); + EXPECT_EQ(pooled_handle.nbOpens(), 1); + EXPECT_EQ(pooled_handle.nbSeeks(), 2); + EXPECT_EQ(pooled_handle.nbReads(), 2); + } + { + io::Stream f3 = io::InputFileStream{"test_io_session.data"}; + f3.seek(2 * 6); + f3.read(const_cast(l3.data()), 5); + auto& pooled_handle = dynamic_cast(f3.datahandle()); + EXPECT_EQ(pooled_handle.nbOpens(), 1); + EXPECT_EQ(pooled_handle.nbSeeks(), 3); + EXPECT_EQ(pooled_handle.nbReads(), 3); + } +} + + +} // namespace test +} // namespace atlas + + +int main(int argc, char** argv) { + return atlas::test::run(argc, argv); +} diff --git a/cmake/features/CXX17.cmake b/cmake/features/CXX17.cmake new file mode 100644 index 000000000..7c6864031 --- /dev/null +++ b/cmake/features/CXX17.cmake @@ -0,0 +1,5 @@ +### C++17 ... + +ecbuild_add_option( FEATURE CXX17 + DESCRIPTION "Use C++17 standard" + DEFAULT OFF ) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 88dc8c11e..74c80f31f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -69,19 +69,6 @@ if( CMAKE_BUILD_TYPE MATCHES "Debug" ) set( atlas_BUILD_TYPE_DEBUG 1 ) endif() -check_cxx_source_compiles( "#include \n int main() { char * type; int status; char * r = abi::__cxa_demangle(type, 0, 0, &status); }" - ATLAS_HAVE_CXXABI_H ) - -test_big_endian( _BIG_ENDIAN ) - -if( _BIG_ENDIAN ) - set( ATLAS_BIG_ENDIAN 1 ) - set( ATLAS_LITTLE_ENDIAN 0 ) -else() - set( ATLAS_BIG_ENDIAN 0 ) - set( ATLAS_LITTLE_ENDIAN 1 ) -endif() - ecbuild_parse_version( ${eckit_VERSION} PREFIX ATLAS_ECKIT ) math( EXPR ATLAS_ECKIT_VERSION_INT "( 10000 * ${ATLAS_ECKIT_VERSION_MAJOR} ) + ( 100 * ${ATLAS_ECKIT_VERSION_MINOR} ) + ${ATLAS_ECKIT_VERSION_PATCH}" ) diff --git a/src/apps/CMakeLists.txt b/src/apps/CMakeLists.txt index 0a7d06658..b7cebeec8 100644 --- a/src/apps/CMakeLists.txt +++ b/src/apps/CMakeLists.txt @@ -6,10 +6,6 @@ # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. -if( ECKIT_INCLUDE_DIRS ) # eckit not yet ported to CMake3 - include_directories( ${ECKIT_INCLUDE_DIRS} ) -endif() - ecbuild_add_executable( TARGET atlas-main OUTPUT_NAME atlas @@ -30,9 +26,3 @@ ecbuild_add_executable( TARGET atlas-gaussian-latitudes SOURCES atlas-gaussian-latitudes.cc LIBS atlas ) - -ecbuild_add_executable( - TARGET atlas-io-list - SOURCES atlas-io-list.cc - LIBS atlas ) - diff --git a/src/apps/atlas-io-list.cc b/src/apps/atlas-io-list.cc deleted file mode 100644 index ff080aba8..000000000 --- a/src/apps/atlas-io-list.cc +++ /dev/null @@ -1,108 +0,0 @@ -/* - * (C) Copyright 2013 ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation - * nor does it submit to any jurisdiction. - */ - -#include -#include - - -#include "eckit/filesystem/PathName.h" - -#include "atlas/io/Exceptions.h" -#include "atlas/io/RecordPrinter.h" -#include "atlas/io/print/Bytes.h" -#include "atlas/runtime/AtlasTool.h" - -namespace atlas { - - -//---------------------------------------------------------------------------------------------------------------------- - -struct AtlasIOList : public atlas::AtlasTool { - bool serial() override { return true; } - int execute(const Args& args) override; - std::string briefDescription() override { return "Inspection of atlas-io files"; } - std::string usage() override { return name() + " [OPTION]... [--help,-h]"; } - std::string longDescription() override { - return "Inspection of atlas-io files\n" - "\n" - " : path to atlas-io file"; - } - - AtlasIOList(int argc, char** argv): AtlasTool(argc, argv) { - add_option(new SimpleOption("format", "Output format")); - add_option(new SimpleOption("version", "Print version of records")); - add_option(new SimpleOption("details", "Print detailed information")); - } -}; - -//------------------------------------------------------------------------------------------------------ - -int AtlasIOList::execute(const Args& args) { - auto return_code = success(); - - using namespace atlas; - - // User sanity checks - if (args.count() == 0) { - Log::error() << "No file specified." << std::endl; - help(std::cout); - return failed(); - } - - // Configuration - util::Config config; - config.set("format", args.getString("format", "table")); - config.set("details", args.getBool("details", false)); - - // Loop over files - for (size_t f = 0; f < args.count(); ++f) { - eckit::PathName file(args(f)); - if (!file.exists()) { - Log::error() << "File does not exist: " << file << std::endl; - return failed(); - } - auto filesize = size_t(file.size()); - - io::Session session; - - std::uint64_t pos = 0; - try { - while (pos < filesize) { - auto uri = io::Record::URI{file, pos}; - auto record = io::RecordPrinter{uri, config}; - - std::stringstream out; - out << "\n# " << uri.path << " [" << uri.offset << "] " - << "{ size: " << atlas::io::Bytes{record.size()}.str(0) << ", version: " << record.version() - << ", created: " << record.time() << " }"; - out << '\n' << (config.getString("format") == "table" ? "" : "---") << '\n'; - out << record << std::endl; - - std::cout << out.str(); - - pos += record.size(); - } - } - catch (const io::Exception& e) { - Log::error() << " ATLAS-IO-ERROR: " << e.what() << std::endl; - return_code = failed(); - } - } - return return_code; -} - -} // namespace atlas - -//------------------------------------------------------------------------------------------------------ - -int main(int argc, char** argv) { - atlas::AtlasIOList tool(argc, argv); - return tool.start(); -} diff --git a/src/atlas/CMakeLists.txt b/src/atlas/CMakeLists.txt index 14850fd4e..258c14ec8 100644 --- a/src/atlas/CMakeLists.txt +++ b/src/atlas/CMakeLists.txt @@ -790,9 +790,12 @@ list( APPEND atlas_io_srcs io/detail/Checksum.h io/detail/Checksum.cc io/detail/DataInfo.h + io/detail/DataType.cc + io/detail/DataType.h io/detail/Decoder.cc io/detail/Decoder.h io/detail/Defaults.h + io/detail/Encoder.cc io/detail/Encoder.h io/detail/Endian.h io/detail/Link.cc @@ -844,15 +847,20 @@ list( APPEND atlas_io_srcs io/types/array/ArrayMetadata.h io/types/array/ArrayReference.cc io/types/array/ArrayReference.h - io/types/array/adaptors/ArrayAdaptor.cc - io/types/array/adaptors/ArrayAdaptor.h io/types/array/adaptors/StdVectorAdaptor.h - io/types/array/adaptors/VectorAdaptor.h io/types/string.h io/types/scalar.h io/types/scalar.cc ) + +list( APPEND atlas_io_adaptor_srcs + io/ArrayAdaptor.cc + io/ArrayAdaptor.h + io/VectorAdaptor.h +) + + ### atlas c++ library if( NOT atlas_HAVE_TRANS ) @@ -885,7 +893,7 @@ list( APPEND source_list ${atlas_numerics_srcs} ${atlas_output_srcs} ${atlas_util_srcs} - ${atlas_io_srcs} + ${atlas_io_adaptor_srcs} ${atlas_internals_srcs} ${CMAKE_CURRENT_BINARY_DIR}/library/git_sha1.h ${CMAKE_CURRENT_BINARY_DIR}/library/defines.h @@ -906,6 +914,20 @@ atlas_host_device( source_list mesh/Connectivity.cc ) +#ecbuild_add_library( TARGET atlas_io + +# INSTALL_HEADERS ALL +# HEADER_DESTINATION include/atlas_io +# SOURCES ${atlas_io_srcs} +# PUBLIC_LIBS eckit +# PUBLIC_INCLUDES +# $ +# $ + + +#) + + ecbuild_add_library( TARGET atlas AUTO_VERSION @@ -932,6 +954,7 @@ ecbuild_add_library( TARGET atlas eckit_maths eckit_mpi eckit_option + atlas_io $<${atlas_HAVE_EIGEN}:Eigen3::Eigen> $<${atlas_HAVE_OMP_CXX}:OpenMP::OpenMP_CXX> $<${atlas_HAVE_GRIDTOOLS_STORAGE}:GridTools::gridtools> diff --git a/src/atlas/io/types/array/adaptors/ArrayAdaptor.cc b/src/atlas/io/ArrayAdaptor.cc similarity index 81% rename from src/atlas/io/types/array/adaptors/ArrayAdaptor.cc rename to src/atlas/io/ArrayAdaptor.cc index 324239eac..c8aafbe80 100644 --- a/src/atlas/io/types/array/adaptors/ArrayAdaptor.cc +++ b/src/atlas/io/ArrayAdaptor.cc @@ -14,11 +14,12 @@ #include #include "atlas/array/Array.h" -#include "atlas/io/Data.h" -#include "atlas/io/Exceptions.h" -#include "atlas/io/Metadata.h" -#include "atlas/io/types/array/ArrayReference.h" -#include "atlas/runtime/Exception.h" +#include "atlas_io/Data.h" +#include "atlas_io/Exceptions.h" +#include "atlas_io/Metadata.h" +#include "atlas_io/atlas_compat.h" +#include "atlas_io/detail/Assert.h" +#include "atlas_io/types/array/ArrayReference.h" namespace atlas { namespace array { @@ -26,7 +27,7 @@ namespace array { //--------------------------------------------------------------------------------------------------------------------- void interprete(const atlas::array::Array& a, atlas::io::ArrayReference& out) { - out = io::ArrayReference(a.data(), a.datatype(), a.shape()); + out = io::ArrayReference(a.data(), atlas::io::DataType(a.datatype().str()), a.shape()); } //--------------------------------------------------------------------------------------------------------------------- @@ -34,7 +35,7 @@ void interprete(const atlas::array::Array& a, atlas::io::ArrayReference& out) { void decode(const atlas::io::Metadata& metadata, const atlas::io::Data& data, atlas::array::Array& out) { atlas::io::ArrayMetadata array(metadata); - if (array.datatype() != out.datatype()) { + if (array.datatype().str() != out.datatype().str()) { std::stringstream err; err << "Could not decode " << metadata.json() << " into Array with datatype " << out.datatype().str() << "." << "Incompatible datatype!"; @@ -49,7 +50,7 @@ void decode(const atlas::io::Metadata& metadata, const atlas::io::Data& data, at out.resize(array.shape()); - ATLAS_ASSERT(out.contiguous()); + ATLAS_IO_ASSERT(out.contiguous()); ::memcpy(out.data(), data, data.size()); } diff --git a/src/atlas/io/types/array/adaptors/ArrayAdaptor.h b/src/atlas/io/ArrayAdaptor.h similarity index 100% rename from src/atlas/io/types/array/adaptors/ArrayAdaptor.h rename to src/atlas/io/ArrayAdaptor.h diff --git a/src/atlas/io/Metadata.h b/src/atlas/io/Metadata.h deleted file mode 100644 index 0cb22b26c..000000000 --- a/src/atlas/io/Metadata.h +++ /dev/null @@ -1,61 +0,0 @@ -/* - * (C) Copyright 2020 ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation - * nor does it submit to any jurisdiction. - */ - -#pragma once - -#include - -#include "atlas/io/Stream.h" -#include "atlas/io/detail/Checksum.h" -#include "atlas/io/detail/DataInfo.h" -#include "atlas/io/detail/Endian.h" -#include "atlas/io/detail/Link.h" -#include "atlas/io/detail/RecordInfo.h" -#include "atlas/io/detail/Type.h" -#include "atlas/util/Config.h" - -namespace atlas { -namespace io { - -class Metadata; -class Stream; - -//--------------------------------------------------------------------------------------------------------------------- - -size_t uncompressed_size(const atlas::io::Metadata& m); - -//--------------------------------------------------------------------------------------------------------------------- - -class Metadata : public util::Config { -public: - using util::Config::Config; - - Link link() const { return Link{getString("link", "")}; } - - Type type() const { return Type{getString("type", "")}; } - - void link(atlas::io::Metadata&&); - - std::string json() const; - - DataInfo data; - RecordInfo record; -}; - -//--------------------------------------------------------------------------------------------------------------------- - -void write(const atlas::io::Metadata&, std::ostream& out); - -void write(const atlas::io::Metadata&, atlas::io::Stream& out); - -//--------------------------------------------------------------------------------------------------------------------- - -} // namespace io -} // namespace atlas diff --git a/src/atlas/io/types/array/adaptors/VectorAdaptor.h b/src/atlas/io/VectorAdaptor.h similarity index 93% rename from src/atlas/io/types/array/adaptors/VectorAdaptor.h rename to src/atlas/io/VectorAdaptor.h index 57b7dc1df..633a1798d 100644 --- a/src/atlas/io/types/array/adaptors/VectorAdaptor.h +++ b/src/atlas/io/VectorAdaptor.h @@ -12,9 +12,9 @@ #include "atlas/util/vector.h" -#include "atlas/io/Data.h" -#include "atlas/io/Metadata.h" -#include "atlas/io/types/array/ArrayReference.h" +#include "atlas_io/Data.h" +#include "atlas_io/Metadata.h" +#include "atlas_io/types/array/ArrayReference.h" namespace atlas { diff --git a/src/atlas/io/atlas-io.h b/src/atlas/io/atlas-io.h index 196ef2728..5ee5d2a79 100644 --- a/src/atlas/io/atlas-io.h +++ b/src/atlas/io/atlas-io.h @@ -10,117 +10,7 @@ #pragma once -#include -#include -#include +#include "atlas_io/atlas-io.h" -#include "atlas/io/detail/Link.h" -#include "atlas/io/detail/Reference.h" -#include "atlas/io/detail/StaticAssert.h" -#include "atlas/io/detail/sfinae.h" - -#include "atlas/io/Exceptions.h" -#include "atlas/io/FileStream.h" -#include "atlas/io/Record.h" -#include "atlas/io/RecordItemReader.h" -#include "atlas/io/RecordPrinter.h" -#include "atlas/io/RecordReader.h" -#include "atlas/io/RecordWriter.h" -#include "atlas/io/Session.h" -#include "atlas/io/Stream.h" - -#include "atlas/io/types/array.h" -#include "atlas/io/types/scalar.h" -#include "atlas/io/types/string.h" - - -namespace atlas { -namespace io { - -//--------------------------------------------------------------------------------------------------------------------- - -inline Link link(const std::string& uri) { - return Link{uri}; -} - -//--------------------------------------------------------------------------------------------------------------------- - -template = 0> -Reference ref(const T& x, tag::enable_static_assert = tag::enable_static_assert()) { - static_assert(is_encodable(), - "in atlas::io::ref(const Value&)" - "\n" - "\n Static assertion failed" - "\n -----------------------" - "\n" - "\n Cannot encode values of referenced type." - "\n" - "\n Implement the functions" - "\n" - "\n void encode_data(const Value& in, atlas::io::Data& out);" - "\n size_t encode_metadata(const Value& value, atlas::io::Metadata& metadata);" - "\n" - "\n or alternatively a conversion function to atlas::io::types::ArrayView" - "\n" - "\n void interprete(const Value& in, atlas::io::types::ArrayView& out)" - "\n" - "\n Rules of argument-dependent-lookup apply." - "\n --> Functions need to be declared in namespace of any of the arguments." - "\n" - "\n Note, turn this into a runtime exception by calling this function instead:" - "\n" - "\n atlas::io::ref(const T&, atlas::io::no_static_assert() )" - "\n"); - return Reference(x); -} - - -template = 0> -Reference ref(const T& x, tag::disable_static_assert) { - if (not is_encodable()) { - throw NotEncodable(x); - } - return Reference(x); -} - - -template = 0> -ArrayReference ref(const T& x, tag::enable_static_assert = tag::enable_static_assert()) { - ArrayReference w; - interprete(x, w); - return w; -} - -//--------------------------------------------------------------------------------------------------------------------- - -template -RecordItem copy(T&& value, tag::disable_static_assert) { - return RecordItem(std::forward(value), tag::disable_static_assert()); -} - -template -RecordItem copy(T&& value) { - return RecordItem(std::forward(value)); -} - -//--------------------------------------------------------------------------------------------------------------------- - -template -void encode(const T& in, atlas::io::Metadata& metadata, atlas::io::Data& data, - tag::enable_static_assert = tag::enable_static_assert()) { - auto referenced = ref(in, tag::enable_static_assert()); - sfinae::encode_metadata(referenced, metadata); - sfinae::encode_data(referenced, data); -} - -template -void encode(const T& in, atlas::io::Metadata& metadata, atlas::io::Data& data, tag::disable_static_assert) { - auto referenced = ref(in, tag::disable_static_assert()); - sfinae::encode_metadata(referenced, metadata); - sfinae::encode_data(referenced, data); -} - -//--------------------------------------------------------------------------------------------------------------------- - -} // namespace io -} // namespace atlas +#include "atlas/io/ArrayAdaptor.h" +#include "atlas/io/VectorAdaptor.h" diff --git a/src/atlas/library/Library.cc b/src/atlas/library/Library.cc index 1b58a2b1e..cfc70d41f 100644 --- a/src/atlas/library/Library.cc +++ b/src/atlas/library/Library.cc @@ -43,6 +43,8 @@ static bool feature_MKL() { } // namespace #endif +#include "atlas_io/Trace.h" + #include "atlas/library/FloatingPointExceptions.h" #include "atlas/library/Plugin.h" #include "atlas/library/config.h" @@ -296,6 +298,15 @@ void Library::initialise(const eckit::Parametrisation& config) { init_data_paths(data_paths_); } + atlas::io::TraceHookRegistry::add([](const eckit::CodeLocation& loc, const std::string& title) { + struct Adaptor : public atlas::io::TraceHook { + Adaptor(const eckit::CodeLocation& loc, const std::string& title): trace{loc, title} {} + atlas::Trace trace; + }; + return std::unique_ptr(new Adaptor{loc, title}); + }); + + // Summary if (getEnv("ATLAS_LOG_RANK", 0) == int(mpi::rank())) { out << "Executable [" << Main::instance().name() << "]\n"; diff --git a/src/atlas/library/defines.h.in b/src/atlas/library/defines.h.in index 926ad64f0..f59630451 100644 --- a/src/atlas/library/defines.h.in +++ b/src/atlas/library/defines.h.in @@ -39,10 +39,6 @@ #define ATLAS_BUILD_TYPE_RELEASE @atlas_BUILD_TYPE_RELEASE@ #define ATLAS_ECKIT_VERSION_INT @ATLAS_ECKIT_VERSION_INT@ -#cmakedefine01 ATLAS_HAVE_CXXABI_H -#cmakedefine01 ATLAS_LITTLE_ENDIAN -#cmakedefine01 ATLAS_BIG_ENDIAN - #ifdef __CUDACC__ #define ATLAS_HOST_DEVICE __host__ __device__ #define ATLAS_DEVICE __device__ diff --git a/src/atlas/runtime/trace/Timings.cc b/src/atlas/runtime/trace/Timings.cc index 3f7610284..40c8f4841 100644 --- a/src/atlas/runtime/trace/Timings.cc +++ b/src/atlas/runtime/trace/Timings.cc @@ -232,7 +232,7 @@ void TimingsRegistry::report(std::ostream& out, const eckit::Configuration& conf auto digits = [](long x) -> long { return std::floor(std::log10(std::max(1l, x))) + 1l; }; std::vector excluded_timers_vector; - for (auto label : labels_) { + for (auto& label : labels_) { auto name = label.first; if (excluded_labels.count(name)) { auto timers = label.second; @@ -428,7 +428,7 @@ void TimingsRegistry::report(std::ostream& out, const eckit::Configuration& conf out << std::left << std::setw(40) << "Timers accumulated by label" << sep << std::left << std::setw(5) << "count" << sep << "time" << std::endl; out << std::left << box_horizontal(40) << seph << box_horizontal(5) << seph << box_horizontal(12) << "\n"; - for (auto label : labels_) { + for (auto& label : labels_) { auto name = label.first; auto timers = label.second; double tot(0); @@ -444,18 +444,15 @@ void TimingsRegistry::report(std::ostream& out, const eckit::Configuration& conf } std::string TimingsRegistry::filter_filepath(const std::string& filepath) const { - std::regex filepath_re("(.*)?/atlas/src/(.*)"); std::smatch matches; - std::string filtered(""); - if (std::regex_search(filepath, matches, filepath_re)) { - // filtered = matches[2]; - filtered = "[atlas] "; + std::string basename = eckit::PathName(filepath).baseName(); + if (std::regex_search(filepath, matches, std::regex{"(.*)?/atlas/src/(.*)"})) { + return "[atlas] " + basename; } - filtered += eckit::PathName(filepath).baseName(); - return filtered; - // - // return filtered; - // return filepath; + if (std::regex_search(filepath, matches, std::regex{"(.*)?/atlas-io/src/(.*)"})) { + return "[atlas-io] " + basename; + } + return basename; } Timings::Identifier Timings::add(const CodeLocation& loc, const CallStack& stack, const std::string& title, diff --git a/src/tests/AtlasTestEnvironment.h b/src/tests/AtlasTestEnvironment.h index d656f9feb..503fa43e4 100644 --- a/src/tests/AtlasTestEnvironment.h +++ b/src/tests/AtlasTestEnvironment.h @@ -350,8 +350,6 @@ void setEnv(const std::string& env, bool value) { } // namespace struct AtlasTestEnvironment { - using Config = util::Config; - AtlasTestEnvironment(int argc, char* argv[]) { eckit::Main::initialise(argc, argv); eckit::Main::instance().taskID(eckit::mpi::comm("world").rank()); @@ -415,6 +413,7 @@ struct AtlasTestEnvironment { //---------------------------------------------------------------------------------------------------------------------- + template int run(int argc, char* argv[]) { Environment env(argc, argv); diff --git a/src/tests/io/test_io_encoding.cc b/src/tests/io/test_io_encoding.cc index 13e274a57..2f9137b5e 100644 --- a/src/tests/io/test_io_encoding.cc +++ b/src/tests/io/test_io_encoding.cc @@ -15,10 +15,8 @@ #include "atlas/array/ArrayView.h" #include "atlas/io/atlas-io.h" -#include "atlas/io/types/array/ArrayReference.h" -#include "tests/AtlasTestEnvironment.h" -#include "atlas/io/types/array/adaptors/ArrayAdaptor.h" +#include "tests/AtlasTestEnvironment.h" namespace atlas { namespace test { @@ -348,7 +346,7 @@ CASE("encoding atlas::array::Array") { { auto interpreted = io::interprete(in); - EXPECT(interpreted.datatype() == in.datatype()); + EXPECT_EQ(interpreted.datatype().str(), in.datatype().str()); EXPECT_EQ(interpreted.rank(), 2); EXPECT_EQ(interpreted.shape(0), 4); EXPECT_EQ(interpreted.shape(1), 2); @@ -360,7 +358,7 @@ CASE("encoding atlas::array::Array") { EXPECT_NO_THROW(encode(in, metadata, data)); io::ArrayMetadata array_metadata{metadata}; - EXPECT(array_metadata.datatype() == in.datatype()); + EXPECT_EQ(array_metadata.datatype().str(), in.datatype().str()); EXPECT_EQ(array_metadata.rank(), 2); EXPECT_EQ(array_metadata.shape(0), 4); EXPECT_EQ(array_metadata.shape(1), 2); @@ -384,6 +382,8 @@ CASE("test Encoder") { EXPECT_THROWS_AS(encode(encoder, metadata, data), eckit::AssertionFailed); } + Log::info() << "here" << std::endl; + SECTION("Encoder via reference") { io::Encoder encoder; auto ops = std::make_shared(); diff --git a/src/tests/io/test_io_record.cc b/src/tests/io/test_io_record.cc index a1ef9e1c9..5b821eb41 100644 --- a/src/tests/io/test_io_record.cc +++ b/src/tests/io/test_io_record.cc @@ -20,6 +20,10 @@ #include "tests/AtlasTestEnvironment.h" + +#include "atlas/io/atlas-io.h" + + namespace atlas { namespace test { diff --git a/src/tests/io/test_io_stream.cc b/src/tests/io/test_io_stream.cc index 8c3c6d47d..abb57e07a 100644 --- a/src/tests/io/test_io_stream.cc +++ b/src/tests/io/test_io_stream.cc @@ -8,8 +8,9 @@ * nor does it submit to any jurisdiction. */ -#include "atlas/io/FileStream.h" -#include "atlas/io/Session.h" +#include "atlas_io/FileStream.h" +#include "atlas_io/Session.h" + #include "eckit/io/FileHandle.h" #include "eckit/io/PooledHandle.h" diff --git a/tools/apply-clang-format.sh b/tools/apply-clang-format.sh index 29b6c9f3a..9d7b24f1f 100755 --- a/tools/apply-clang-format.sh +++ b/tools/apply-clang-format.sh @@ -93,11 +93,17 @@ if ! [[ $(clang-format --version) =~ ${_REQUIRED_CLANG_VERSION} ]]; then fi if [[ $all =~ "yes" ]]; then + echo "Applying $(clang-format --version) to all files ..." SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd $SCRIPTDIR/../src - - echo "Applying $(clang-format --version) to all files ..." + if [[ $dryrun =~ "yes" ]]; then + echo "+ find . -iname *.h -o -iname *.cc | xargs clang-format -i -style=file" + else + find . -iname *.h -o -iname *.cc | xargs clang-format -i -style=file + fi + + cd $SCRIPTDIR/../atlas-io if [[ $dryrun =~ "yes" ]]; then echo "+ find . -iname *.h -o -iname *.cc | xargs clang-format -i -style=file" else From 1872fbc1d6ddd47d5d3b0f90d62092a1b30ff72a Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 18 Jul 2022 17:51:42 +0000 Subject: [PATCH 47/53] ATLAS-364 Improve performance and memory usage of ATLAS_TRACE --- src/atlas/runtime/trace/CallStack.cc | 14 +++++++++----- src/atlas/runtime/trace/CallStack.h | 26 ++++++++++++-------------- src/atlas/runtime/trace/Nesting.h | 4 ++-- src/atlas/runtime/trace/Timings.cc | 9 ++++----- 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/src/atlas/runtime/trace/CallStack.cc b/src/atlas/runtime/trace/CallStack.cc index d3379884b..8813f429d 100644 --- a/src/atlas/runtime/trace/CallStack.cc +++ b/src/atlas/runtime/trace/CallStack.cc @@ -18,19 +18,23 @@ namespace atlas { namespace runtime { namespace trace { -void CallStack::push_front(const CodeLocation& loc, const std::string& id) { - stack_.push_front(std::hash{}(loc.asString() + id)); +void CallStack::push(const CodeLocation& loc, const std::string& id) { + if (stack_.size() == size_) { + stack_.resize(2 * size_); + } + stack_[size_++] = std::hash{}(loc.asString() + id); } -void CallStack::pop_front() { - stack_.pop_front(); +void CallStack::pop() { + --size_; } size_t CallStack::hash() const { if (hash_) { return hash_; } - for (auto h : stack_) { + for (long i = size_ - 1; i >= 0; --i) { + auto h = stack_[i]; hash_ ^= (h << 1); } return hash_; diff --git a/src/atlas/runtime/trace/CallStack.h b/src/atlas/runtime/trace/CallStack.h index cb76a0235..b1d091000 100644 --- a/src/atlas/runtime/trace/CallStack.h +++ b/src/atlas/runtime/trace/CallStack.h @@ -11,8 +11,8 @@ #pragma once #include -#include #include +#include namespace atlas { class CodeLocation; @@ -26,35 +26,33 @@ namespace trace { /// Instances of CallStack can keep track of nested CodeLocations class CallStack { public: - using const_iterator = std::list::const_iterator; - using const_reverse_iterator = std::list::const_reverse_iterator; + using const_iterator = std::vector::const_iterator; public: - void push_front(const CodeLocation&, const std::string& id = ""); - void pop_front(); + void push(const CodeLocation&, const std::string& id = ""); + void pop(); const_iterator begin() const { return stack_.begin(); } - const_iterator end() const { return stack_.end(); } - - const_reverse_iterator rbegin() const { return stack_.rbegin(); } - const_reverse_iterator rend() const { return stack_.rend(); } + const_iterator end() const { return stack_.begin() + size_; } size_t hash() const; - size_t size() const { return stack_.size(); } + size_t size() const { return size_; } - operator bool() const { return not stack_.empty(); } + operator bool() const { return size_ > 0; } public: - CallStack() = default; - CallStack(const CallStack& other): stack_(other.stack_) {} + CallStack(): stack_(64){}; + CallStack(const CallStack& other): stack_(other.stack_), size_(other.size_) {} CallStack& operator=(const CallStack& other) { stack_ = other.stack_; + size_ = other.size_; hash_ = 0; return *this; } private: - std::list stack_; + std::vector stack_; + size_t size_{0}; mutable size_t hash_{0}; }; diff --git a/src/atlas/runtime/trace/Nesting.h b/src/atlas/runtime/trace/Nesting.h index 300d76054..109ef043f 100644 --- a/src/atlas/runtime/trace/Nesting.h +++ b/src/atlas/runtime/trace/Nesting.h @@ -35,12 +35,12 @@ class CurrentCallStack { operator CallStack() const { return stack_; } CallStack& push(const CodeLocation& loc, const std::string& id) { if (Control::enabled()) - stack_.push_front(loc, id); + stack_.push(loc, id); return stack_; } void pop() { if (Control::enabled()) - stack_.pop_front(); + stack_.pop(); } }; diff --git a/src/atlas/runtime/trace/Timings.cc b/src/atlas/runtime/trace/Timings.cc index 40c8f4841..a477969d6 100644 --- a/src/atlas/runtime/trace/Timings.cc +++ b/src/atlas/runtime/trace/Timings.cc @@ -82,7 +82,7 @@ struct Node { auto this_stack_hash = TimingsRegistry::instance().stack_[index].hash(); auto is_child = [&](size_t i) -> bool { CallStack child_stack = TimingsRegistry::instance().stack_[i]; - child_stack.pop_front(); + child_stack.pop(); auto child_stack_hash = child_stack.hash(); return child_stack_hash == this_stack_hash; }; @@ -353,10 +353,9 @@ void TimingsRegistry::report(std::ostream& out, const eckit::Configuration& conf } const CallStack& next_stack = *next_stack_ptr; - auto this_it = this_stack.rbegin(); - auto next_it = next_stack.rbegin(); - for (size_t i = 0; this_it != this_stack.rend() && next_it != next_stack.rend(); - ++i, ++this_it, ++next_it) { + auto this_it = this_stack.begin(); + auto next_it = next_stack.begin(); + for (size_t i = 0; this_it != this_stack.end() && next_it != next_stack.end(); ++i, ++this_it, ++next_it) { if (*this_it == *next_it) { active[i] = active[i] or false; } From dd36a7518b1c26bc43bcd343ab62ccc6189df16e Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 16 Aug 2022 13:29:39 +0200 Subject: [PATCH 48/53] ATLAS-368 Fix installation of atlas_io --- CMakeLists.txt | 4 ++-- {atlas-io => atlas_io}/AUTHORS | 0 {atlas-io => atlas_io}/CHANGELOG.md | 0 {atlas-io => atlas_io}/CMakeLists.txt | 4 ++-- {atlas-io => atlas_io}/LICENSE | 0 {atlas-io => atlas_io}/README.md | 0 {atlas-io => atlas_io}/cmake/atlas-io-import.cmake.in | 0 {atlas-io => atlas_io}/src/CMakeLists.txt | 0 {atlas-io => atlas_io}/src/atlas_io/CMakeLists.txt | 5 +++++ {atlas-io => atlas_io}/src/atlas_io/Data.cc | 0 {atlas-io => atlas_io}/src/atlas_io/Data.h | 0 {atlas-io => atlas_io}/src/atlas_io/Exceptions.cc | 0 {atlas-io => atlas_io}/src/atlas_io/Exceptions.h | 0 {atlas-io => atlas_io}/src/atlas_io/FileStream.cc | 0 {atlas-io => atlas_io}/src/atlas_io/FileStream.h | 0 {atlas-io => atlas_io}/src/atlas_io/Metadata.cc | 0 {atlas-io => atlas_io}/src/atlas_io/Metadata.h | 0 {atlas-io => atlas_io}/src/atlas_io/ReadRequest.cc | 0 {atlas-io => atlas_io}/src/atlas_io/ReadRequest.h | 0 {atlas-io => atlas_io}/src/atlas_io/Record.cc | 0 {atlas-io => atlas_io}/src/atlas_io/Record.h | 0 {atlas-io => atlas_io}/src/atlas_io/RecordItem.cc | 0 {atlas-io => atlas_io}/src/atlas_io/RecordItem.h | 0 {atlas-io => atlas_io}/src/atlas_io/RecordItemReader.cc | 0 {atlas-io => atlas_io}/src/atlas_io/RecordItemReader.h | 0 {atlas-io => atlas_io}/src/atlas_io/RecordPrinter.cc | 0 {atlas-io => atlas_io}/src/atlas_io/RecordPrinter.h | 0 {atlas-io => atlas_io}/src/atlas_io/RecordReader.cc | 0 {atlas-io => atlas_io}/src/atlas_io/RecordReader.h | 0 {atlas-io => atlas_io}/src/atlas_io/RecordWriter.cc | 0 {atlas-io => atlas_io}/src/atlas_io/RecordWriter.h | 0 {atlas-io => atlas_io}/src/atlas_io/Session.cc | 0 {atlas-io => atlas_io}/src/atlas_io/Session.h | 0 {atlas-io => atlas_io}/src/atlas_io/Stream.cc | 0 {atlas-io => atlas_io}/src/atlas_io/Stream.h | 0 {atlas-io => atlas_io}/src/atlas_io/Trace.cc | 0 {atlas-io => atlas_io}/src/atlas_io/Trace.h | 0 {atlas-io => atlas_io}/src/atlas_io/atlas-io.h | 0 {atlas-io => atlas_io}/src/atlas_io/atlas_compat.h | 0 {atlas-io => atlas_io}/src/atlas_io/detail/Assert.h | 0 {atlas-io => atlas_io}/src/atlas_io/detail/Base64.cc | 0 {atlas-io => atlas_io}/src/atlas_io/detail/Base64.h | 0 {atlas-io => atlas_io}/src/atlas_io/detail/BlackMagic.h | 0 {atlas-io => atlas_io}/src/atlas_io/detail/Checksum.cc | 0 {atlas-io => atlas_io}/src/atlas_io/detail/Checksum.h | 0 {atlas-io => atlas_io}/src/atlas_io/detail/DataInfo.h | 0 {atlas-io => atlas_io}/src/atlas_io/detail/DataType.cc | 0 {atlas-io => atlas_io}/src/atlas_io/detail/DataType.h | 0 {atlas-io => atlas_io}/src/atlas_io/detail/Decoder.cc | 0 {atlas-io => atlas_io}/src/atlas_io/detail/Decoder.h | 0 {atlas-io => atlas_io}/src/atlas_io/detail/Defaults.h | 0 {atlas-io => atlas_io}/src/atlas_io/detail/Encoder.cc | 0 {atlas-io => atlas_io}/src/atlas_io/detail/Encoder.h | 0 {atlas-io => atlas_io}/src/atlas_io/detail/Endian.h | 0 {atlas-io => atlas_io}/src/atlas_io/detail/Link.cc | 0 {atlas-io => atlas_io}/src/atlas_io/detail/Link.h | 0 {atlas-io => atlas_io}/src/atlas_io/detail/NoConfig.h | 0 {atlas-io => atlas_io}/src/atlas_io/detail/ParsedRecord.h | 0 {atlas-io => atlas_io}/src/atlas_io/detail/RecordInfo.h | 0 {atlas-io => atlas_io}/src/atlas_io/detail/RecordSections.h | 0 {atlas-io => atlas_io}/src/atlas_io/detail/Reference.h | 0 {atlas-io => atlas_io}/src/atlas_io/detail/StaticAssert.h | 0 {atlas-io => atlas_io}/src/atlas_io/detail/Time.cc | 0 {atlas-io => atlas_io}/src/atlas_io/detail/Time.h | 0 {atlas-io => atlas_io}/src/atlas_io/detail/Type.h | 0 {atlas-io => atlas_io}/src/atlas_io/detail/TypeTraits.h | 0 {atlas-io => atlas_io}/src/atlas_io/detail/Version.h | 0 {atlas-io => atlas_io}/src/atlas_io/detail/defines.h.in | 0 {atlas-io => atlas_io}/src/atlas_io/detail/sfinae.h | 0 {atlas-io => atlas_io}/src/atlas_io/detail/tag.h | 0 {atlas-io => atlas_io}/src/atlas_io/print/Bytes.cc | 0 {atlas-io => atlas_io}/src/atlas_io/print/Bytes.h | 0 {atlas-io => atlas_io}/src/atlas_io/print/JSONFormat.cc | 0 {atlas-io => atlas_io}/src/atlas_io/print/JSONFormat.h | 0 {atlas-io => atlas_io}/src/atlas_io/print/TableFormat.cc | 0 {atlas-io => atlas_io}/src/atlas_io/print/TableFormat.h | 0 {atlas-io => atlas_io}/src/atlas_io/types/array.h | 0 .../src/atlas_io/types/array/ArrayMetadata.cc | 0 .../src/atlas_io/types/array/ArrayMetadata.h | 0 .../src/atlas_io/types/array/ArrayReference.cc | 0 .../src/atlas_io/types/array/ArrayReference.h | 0 .../src/atlas_io/types/array/adaptors/StdArrayAdaptor.h | 0 .../src/atlas_io/types/array/adaptors/StdVectorAdaptor.h | 0 {atlas-io => atlas_io}/src/atlas_io/types/scalar.cc | 0 {atlas-io => atlas_io}/src/atlas_io/types/scalar.h | 0 {atlas-io => atlas_io}/src/atlas_io/types/string.h | 0 {atlas-io => atlas_io}/src/tools/CMakeLists.txt | 0 {atlas-io => atlas_io}/src/tools/atlas-io-list.cc | 0 {atlas-io => atlas_io}/tests/CMakeLists.txt | 0 {atlas-io => atlas_io}/tests/TestEnvironment.h | 0 {atlas-io => atlas_io}/tests/test_io_encoding.cc | 0 {atlas-io => atlas_io}/tests/test_io_record.cc | 0 {atlas-io => atlas_io}/tests/test_io_stream.cc | 0 cmake/atlas-import.cmake.in | 2 ++ 94 files changed, 11 insertions(+), 4 deletions(-) rename {atlas-io => atlas_io}/AUTHORS (100%) rename {atlas-io => atlas_io}/CHANGELOG.md (100%) rename {atlas-io => atlas_io}/CMakeLists.txt (96%) rename {atlas-io => atlas_io}/LICENSE (100%) rename {atlas-io => atlas_io}/README.md (100%) rename {atlas-io => atlas_io}/cmake/atlas-io-import.cmake.in (100%) rename {atlas-io => atlas_io}/src/CMakeLists.txt (100%) rename {atlas-io => atlas_io}/src/atlas_io/CMakeLists.txt (95%) rename {atlas-io => atlas_io}/src/atlas_io/Data.cc (100%) rename {atlas-io => atlas_io}/src/atlas_io/Data.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/Exceptions.cc (100%) rename {atlas-io => atlas_io}/src/atlas_io/Exceptions.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/FileStream.cc (100%) rename {atlas-io => atlas_io}/src/atlas_io/FileStream.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/Metadata.cc (100%) rename {atlas-io => atlas_io}/src/atlas_io/Metadata.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/ReadRequest.cc (100%) rename {atlas-io => atlas_io}/src/atlas_io/ReadRequest.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/Record.cc (100%) rename {atlas-io => atlas_io}/src/atlas_io/Record.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/RecordItem.cc (100%) rename {atlas-io => atlas_io}/src/atlas_io/RecordItem.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/RecordItemReader.cc (100%) rename {atlas-io => atlas_io}/src/atlas_io/RecordItemReader.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/RecordPrinter.cc (100%) rename {atlas-io => atlas_io}/src/atlas_io/RecordPrinter.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/RecordReader.cc (100%) rename {atlas-io => atlas_io}/src/atlas_io/RecordReader.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/RecordWriter.cc (100%) rename {atlas-io => atlas_io}/src/atlas_io/RecordWriter.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/Session.cc (100%) rename {atlas-io => atlas_io}/src/atlas_io/Session.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/Stream.cc (100%) rename {atlas-io => atlas_io}/src/atlas_io/Stream.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/Trace.cc (100%) rename {atlas-io => atlas_io}/src/atlas_io/Trace.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/atlas-io.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/atlas_compat.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/Assert.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/Base64.cc (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/Base64.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/BlackMagic.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/Checksum.cc (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/Checksum.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/DataInfo.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/DataType.cc (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/DataType.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/Decoder.cc (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/Decoder.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/Defaults.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/Encoder.cc (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/Encoder.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/Endian.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/Link.cc (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/Link.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/NoConfig.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/ParsedRecord.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/RecordInfo.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/RecordSections.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/Reference.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/StaticAssert.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/Time.cc (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/Time.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/Type.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/TypeTraits.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/Version.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/defines.h.in (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/sfinae.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/detail/tag.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/print/Bytes.cc (100%) rename {atlas-io => atlas_io}/src/atlas_io/print/Bytes.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/print/JSONFormat.cc (100%) rename {atlas-io => atlas_io}/src/atlas_io/print/JSONFormat.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/print/TableFormat.cc (100%) rename {atlas-io => atlas_io}/src/atlas_io/print/TableFormat.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/types/array.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/types/array/ArrayMetadata.cc (100%) rename {atlas-io => atlas_io}/src/atlas_io/types/array/ArrayMetadata.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/types/array/ArrayReference.cc (100%) rename {atlas-io => atlas_io}/src/atlas_io/types/array/ArrayReference.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/types/array/adaptors/StdArrayAdaptor.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/types/array/adaptors/StdVectorAdaptor.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/types/scalar.cc (100%) rename {atlas-io => atlas_io}/src/atlas_io/types/scalar.h (100%) rename {atlas-io => atlas_io}/src/atlas_io/types/string.h (100%) rename {atlas-io => atlas_io}/src/tools/CMakeLists.txt (100%) rename {atlas-io => atlas_io}/src/tools/atlas-io-list.cc (100%) rename {atlas-io => atlas_io}/tests/CMakeLists.txt (100%) rename {atlas-io => atlas_io}/tests/TestEnvironment.h (100%) rename {atlas-io => atlas_io}/tests/test_io_encoding.cc (100%) rename {atlas-io => atlas_io}/tests/test_io_record.cc (100%) rename {atlas-io => atlas_io}/tests/test_io_stream.cc (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 86a8ffc4d..bbb27b8a8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,8 +32,8 @@ endif() ecbuild_debug( " eckit_FEATURES : [${eckit_FEATURES}]" ) -add_subdirectory(atlas-io) -find_package(atlas-io) +add_subdirectory(atlas_io) +find_package(atlas_io) ################################################################################ # Features that can be enabled / disabled with -DENABLE_ diff --git a/atlas-io/AUTHORS b/atlas_io/AUTHORS similarity index 100% rename from atlas-io/AUTHORS rename to atlas_io/AUTHORS diff --git a/atlas-io/CHANGELOG.md b/atlas_io/CHANGELOG.md similarity index 100% rename from atlas-io/CHANGELOG.md rename to atlas_io/CHANGELOG.md diff --git a/atlas-io/CMakeLists.txt b/atlas_io/CMakeLists.txt similarity index 96% rename from atlas-io/CMakeLists.txt rename to atlas_io/CMakeLists.txt index b01ca4a0a..cf29b9a4d 100644 --- a/atlas-io/CMakeLists.txt +++ b/atlas_io/CMakeLists.txt @@ -16,7 +16,7 @@ find_package( ecbuild 3.4 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CUR ################################################################################ # Initialise project Atlas -project( atlas-io VERSION ${atlas_VERSION} LANGUAGES CXX ) +project( atlas_io VERSION ${atlas_VERSION} LANGUAGES CXX ) ################################################################################ # Required packages @@ -59,7 +59,7 @@ add_subdirectory( tests ) # Export and summarize ecbuild_add_resources( - TARGET atlas-io-others + TARGET atlas_io-others SOURCES_PACK README.md CHANGELOG.md diff --git a/atlas-io/LICENSE b/atlas_io/LICENSE similarity index 100% rename from atlas-io/LICENSE rename to atlas_io/LICENSE diff --git a/atlas-io/README.md b/atlas_io/README.md similarity index 100% rename from atlas-io/README.md rename to atlas_io/README.md diff --git a/atlas-io/cmake/atlas-io-import.cmake.in b/atlas_io/cmake/atlas-io-import.cmake.in similarity index 100% rename from atlas-io/cmake/atlas-io-import.cmake.in rename to atlas_io/cmake/atlas-io-import.cmake.in diff --git a/atlas-io/src/CMakeLists.txt b/atlas_io/src/CMakeLists.txt similarity index 100% rename from atlas-io/src/CMakeLists.txt rename to atlas_io/src/CMakeLists.txt diff --git a/atlas-io/src/atlas_io/CMakeLists.txt b/atlas_io/src/atlas_io/CMakeLists.txt similarity index 95% rename from atlas-io/src/atlas_io/CMakeLists.txt rename to atlas_io/src/atlas_io/CMakeLists.txt index 623ecb177..97140b00a 100644 --- a/atlas-io/src/atlas_io/CMakeLists.txt +++ b/atlas_io/src/atlas_io/CMakeLists.txt @@ -1,4 +1,9 @@ configure_file( detail/defines.h.in detail/defines.h ) +install( FILES + ${CMAKE_CURRENT_BINARY_DIR}/detail/defines.h + DESTINATION + ${INSTALL_INCLUDE_DIR}/atlas_io/detail +) ecbuild_add_library( TARGET atlas_io diff --git a/atlas-io/src/atlas_io/Data.cc b/atlas_io/src/atlas_io/Data.cc similarity index 100% rename from atlas-io/src/atlas_io/Data.cc rename to atlas_io/src/atlas_io/Data.cc diff --git a/atlas-io/src/atlas_io/Data.h b/atlas_io/src/atlas_io/Data.h similarity index 100% rename from atlas-io/src/atlas_io/Data.h rename to atlas_io/src/atlas_io/Data.h diff --git a/atlas-io/src/atlas_io/Exceptions.cc b/atlas_io/src/atlas_io/Exceptions.cc similarity index 100% rename from atlas-io/src/atlas_io/Exceptions.cc rename to atlas_io/src/atlas_io/Exceptions.cc diff --git a/atlas-io/src/atlas_io/Exceptions.h b/atlas_io/src/atlas_io/Exceptions.h similarity index 100% rename from atlas-io/src/atlas_io/Exceptions.h rename to atlas_io/src/atlas_io/Exceptions.h diff --git a/atlas-io/src/atlas_io/FileStream.cc b/atlas_io/src/atlas_io/FileStream.cc similarity index 100% rename from atlas-io/src/atlas_io/FileStream.cc rename to atlas_io/src/atlas_io/FileStream.cc diff --git a/atlas-io/src/atlas_io/FileStream.h b/atlas_io/src/atlas_io/FileStream.h similarity index 100% rename from atlas-io/src/atlas_io/FileStream.h rename to atlas_io/src/atlas_io/FileStream.h diff --git a/atlas-io/src/atlas_io/Metadata.cc b/atlas_io/src/atlas_io/Metadata.cc similarity index 100% rename from atlas-io/src/atlas_io/Metadata.cc rename to atlas_io/src/atlas_io/Metadata.cc diff --git a/atlas-io/src/atlas_io/Metadata.h b/atlas_io/src/atlas_io/Metadata.h similarity index 100% rename from atlas-io/src/atlas_io/Metadata.h rename to atlas_io/src/atlas_io/Metadata.h diff --git a/atlas-io/src/atlas_io/ReadRequest.cc b/atlas_io/src/atlas_io/ReadRequest.cc similarity index 100% rename from atlas-io/src/atlas_io/ReadRequest.cc rename to atlas_io/src/atlas_io/ReadRequest.cc diff --git a/atlas-io/src/atlas_io/ReadRequest.h b/atlas_io/src/atlas_io/ReadRequest.h similarity index 100% rename from atlas-io/src/atlas_io/ReadRequest.h rename to atlas_io/src/atlas_io/ReadRequest.h diff --git a/atlas-io/src/atlas_io/Record.cc b/atlas_io/src/atlas_io/Record.cc similarity index 100% rename from atlas-io/src/atlas_io/Record.cc rename to atlas_io/src/atlas_io/Record.cc diff --git a/atlas-io/src/atlas_io/Record.h b/atlas_io/src/atlas_io/Record.h similarity index 100% rename from atlas-io/src/atlas_io/Record.h rename to atlas_io/src/atlas_io/Record.h diff --git a/atlas-io/src/atlas_io/RecordItem.cc b/atlas_io/src/atlas_io/RecordItem.cc similarity index 100% rename from atlas-io/src/atlas_io/RecordItem.cc rename to atlas_io/src/atlas_io/RecordItem.cc diff --git a/atlas-io/src/atlas_io/RecordItem.h b/atlas_io/src/atlas_io/RecordItem.h similarity index 100% rename from atlas-io/src/atlas_io/RecordItem.h rename to atlas_io/src/atlas_io/RecordItem.h diff --git a/atlas-io/src/atlas_io/RecordItemReader.cc b/atlas_io/src/atlas_io/RecordItemReader.cc similarity index 100% rename from atlas-io/src/atlas_io/RecordItemReader.cc rename to atlas_io/src/atlas_io/RecordItemReader.cc diff --git a/atlas-io/src/atlas_io/RecordItemReader.h b/atlas_io/src/atlas_io/RecordItemReader.h similarity index 100% rename from atlas-io/src/atlas_io/RecordItemReader.h rename to atlas_io/src/atlas_io/RecordItemReader.h diff --git a/atlas-io/src/atlas_io/RecordPrinter.cc b/atlas_io/src/atlas_io/RecordPrinter.cc similarity index 100% rename from atlas-io/src/atlas_io/RecordPrinter.cc rename to atlas_io/src/atlas_io/RecordPrinter.cc diff --git a/atlas-io/src/atlas_io/RecordPrinter.h b/atlas_io/src/atlas_io/RecordPrinter.h similarity index 100% rename from atlas-io/src/atlas_io/RecordPrinter.h rename to atlas_io/src/atlas_io/RecordPrinter.h diff --git a/atlas-io/src/atlas_io/RecordReader.cc b/atlas_io/src/atlas_io/RecordReader.cc similarity index 100% rename from atlas-io/src/atlas_io/RecordReader.cc rename to atlas_io/src/atlas_io/RecordReader.cc diff --git a/atlas-io/src/atlas_io/RecordReader.h b/atlas_io/src/atlas_io/RecordReader.h similarity index 100% rename from atlas-io/src/atlas_io/RecordReader.h rename to atlas_io/src/atlas_io/RecordReader.h diff --git a/atlas-io/src/atlas_io/RecordWriter.cc b/atlas_io/src/atlas_io/RecordWriter.cc similarity index 100% rename from atlas-io/src/atlas_io/RecordWriter.cc rename to atlas_io/src/atlas_io/RecordWriter.cc diff --git a/atlas-io/src/atlas_io/RecordWriter.h b/atlas_io/src/atlas_io/RecordWriter.h similarity index 100% rename from atlas-io/src/atlas_io/RecordWriter.h rename to atlas_io/src/atlas_io/RecordWriter.h diff --git a/atlas-io/src/atlas_io/Session.cc b/atlas_io/src/atlas_io/Session.cc similarity index 100% rename from atlas-io/src/atlas_io/Session.cc rename to atlas_io/src/atlas_io/Session.cc diff --git a/atlas-io/src/atlas_io/Session.h b/atlas_io/src/atlas_io/Session.h similarity index 100% rename from atlas-io/src/atlas_io/Session.h rename to atlas_io/src/atlas_io/Session.h diff --git a/atlas-io/src/atlas_io/Stream.cc b/atlas_io/src/atlas_io/Stream.cc similarity index 100% rename from atlas-io/src/atlas_io/Stream.cc rename to atlas_io/src/atlas_io/Stream.cc diff --git a/atlas-io/src/atlas_io/Stream.h b/atlas_io/src/atlas_io/Stream.h similarity index 100% rename from atlas-io/src/atlas_io/Stream.h rename to atlas_io/src/atlas_io/Stream.h diff --git a/atlas-io/src/atlas_io/Trace.cc b/atlas_io/src/atlas_io/Trace.cc similarity index 100% rename from atlas-io/src/atlas_io/Trace.cc rename to atlas_io/src/atlas_io/Trace.cc diff --git a/atlas-io/src/atlas_io/Trace.h b/atlas_io/src/atlas_io/Trace.h similarity index 100% rename from atlas-io/src/atlas_io/Trace.h rename to atlas_io/src/atlas_io/Trace.h diff --git a/atlas-io/src/atlas_io/atlas-io.h b/atlas_io/src/atlas_io/atlas-io.h similarity index 100% rename from atlas-io/src/atlas_io/atlas-io.h rename to atlas_io/src/atlas_io/atlas-io.h diff --git a/atlas-io/src/atlas_io/atlas_compat.h b/atlas_io/src/atlas_io/atlas_compat.h similarity index 100% rename from atlas-io/src/atlas_io/atlas_compat.h rename to atlas_io/src/atlas_io/atlas_compat.h diff --git a/atlas-io/src/atlas_io/detail/Assert.h b/atlas_io/src/atlas_io/detail/Assert.h similarity index 100% rename from atlas-io/src/atlas_io/detail/Assert.h rename to atlas_io/src/atlas_io/detail/Assert.h diff --git a/atlas-io/src/atlas_io/detail/Base64.cc b/atlas_io/src/atlas_io/detail/Base64.cc similarity index 100% rename from atlas-io/src/atlas_io/detail/Base64.cc rename to atlas_io/src/atlas_io/detail/Base64.cc diff --git a/atlas-io/src/atlas_io/detail/Base64.h b/atlas_io/src/atlas_io/detail/Base64.h similarity index 100% rename from atlas-io/src/atlas_io/detail/Base64.h rename to atlas_io/src/atlas_io/detail/Base64.h diff --git a/atlas-io/src/atlas_io/detail/BlackMagic.h b/atlas_io/src/atlas_io/detail/BlackMagic.h similarity index 100% rename from atlas-io/src/atlas_io/detail/BlackMagic.h rename to atlas_io/src/atlas_io/detail/BlackMagic.h diff --git a/atlas-io/src/atlas_io/detail/Checksum.cc b/atlas_io/src/atlas_io/detail/Checksum.cc similarity index 100% rename from atlas-io/src/atlas_io/detail/Checksum.cc rename to atlas_io/src/atlas_io/detail/Checksum.cc diff --git a/atlas-io/src/atlas_io/detail/Checksum.h b/atlas_io/src/atlas_io/detail/Checksum.h similarity index 100% rename from atlas-io/src/atlas_io/detail/Checksum.h rename to atlas_io/src/atlas_io/detail/Checksum.h diff --git a/atlas-io/src/atlas_io/detail/DataInfo.h b/atlas_io/src/atlas_io/detail/DataInfo.h similarity index 100% rename from atlas-io/src/atlas_io/detail/DataInfo.h rename to atlas_io/src/atlas_io/detail/DataInfo.h diff --git a/atlas-io/src/atlas_io/detail/DataType.cc b/atlas_io/src/atlas_io/detail/DataType.cc similarity index 100% rename from atlas-io/src/atlas_io/detail/DataType.cc rename to atlas_io/src/atlas_io/detail/DataType.cc diff --git a/atlas-io/src/atlas_io/detail/DataType.h b/atlas_io/src/atlas_io/detail/DataType.h similarity index 100% rename from atlas-io/src/atlas_io/detail/DataType.h rename to atlas_io/src/atlas_io/detail/DataType.h diff --git a/atlas-io/src/atlas_io/detail/Decoder.cc b/atlas_io/src/atlas_io/detail/Decoder.cc similarity index 100% rename from atlas-io/src/atlas_io/detail/Decoder.cc rename to atlas_io/src/atlas_io/detail/Decoder.cc diff --git a/atlas-io/src/atlas_io/detail/Decoder.h b/atlas_io/src/atlas_io/detail/Decoder.h similarity index 100% rename from atlas-io/src/atlas_io/detail/Decoder.h rename to atlas_io/src/atlas_io/detail/Decoder.h diff --git a/atlas-io/src/atlas_io/detail/Defaults.h b/atlas_io/src/atlas_io/detail/Defaults.h similarity index 100% rename from atlas-io/src/atlas_io/detail/Defaults.h rename to atlas_io/src/atlas_io/detail/Defaults.h diff --git a/atlas-io/src/atlas_io/detail/Encoder.cc b/atlas_io/src/atlas_io/detail/Encoder.cc similarity index 100% rename from atlas-io/src/atlas_io/detail/Encoder.cc rename to atlas_io/src/atlas_io/detail/Encoder.cc diff --git a/atlas-io/src/atlas_io/detail/Encoder.h b/atlas_io/src/atlas_io/detail/Encoder.h similarity index 100% rename from atlas-io/src/atlas_io/detail/Encoder.h rename to atlas_io/src/atlas_io/detail/Encoder.h diff --git a/atlas-io/src/atlas_io/detail/Endian.h b/atlas_io/src/atlas_io/detail/Endian.h similarity index 100% rename from atlas-io/src/atlas_io/detail/Endian.h rename to atlas_io/src/atlas_io/detail/Endian.h diff --git a/atlas-io/src/atlas_io/detail/Link.cc b/atlas_io/src/atlas_io/detail/Link.cc similarity index 100% rename from atlas-io/src/atlas_io/detail/Link.cc rename to atlas_io/src/atlas_io/detail/Link.cc diff --git a/atlas-io/src/atlas_io/detail/Link.h b/atlas_io/src/atlas_io/detail/Link.h similarity index 100% rename from atlas-io/src/atlas_io/detail/Link.h rename to atlas_io/src/atlas_io/detail/Link.h diff --git a/atlas-io/src/atlas_io/detail/NoConfig.h b/atlas_io/src/atlas_io/detail/NoConfig.h similarity index 100% rename from atlas-io/src/atlas_io/detail/NoConfig.h rename to atlas_io/src/atlas_io/detail/NoConfig.h diff --git a/atlas-io/src/atlas_io/detail/ParsedRecord.h b/atlas_io/src/atlas_io/detail/ParsedRecord.h similarity index 100% rename from atlas-io/src/atlas_io/detail/ParsedRecord.h rename to atlas_io/src/atlas_io/detail/ParsedRecord.h diff --git a/atlas-io/src/atlas_io/detail/RecordInfo.h b/atlas_io/src/atlas_io/detail/RecordInfo.h similarity index 100% rename from atlas-io/src/atlas_io/detail/RecordInfo.h rename to atlas_io/src/atlas_io/detail/RecordInfo.h diff --git a/atlas-io/src/atlas_io/detail/RecordSections.h b/atlas_io/src/atlas_io/detail/RecordSections.h similarity index 100% rename from atlas-io/src/atlas_io/detail/RecordSections.h rename to atlas_io/src/atlas_io/detail/RecordSections.h diff --git a/atlas-io/src/atlas_io/detail/Reference.h b/atlas_io/src/atlas_io/detail/Reference.h similarity index 100% rename from atlas-io/src/atlas_io/detail/Reference.h rename to atlas_io/src/atlas_io/detail/Reference.h diff --git a/atlas-io/src/atlas_io/detail/StaticAssert.h b/atlas_io/src/atlas_io/detail/StaticAssert.h similarity index 100% rename from atlas-io/src/atlas_io/detail/StaticAssert.h rename to atlas_io/src/atlas_io/detail/StaticAssert.h diff --git a/atlas-io/src/atlas_io/detail/Time.cc b/atlas_io/src/atlas_io/detail/Time.cc similarity index 100% rename from atlas-io/src/atlas_io/detail/Time.cc rename to atlas_io/src/atlas_io/detail/Time.cc diff --git a/atlas-io/src/atlas_io/detail/Time.h b/atlas_io/src/atlas_io/detail/Time.h similarity index 100% rename from atlas-io/src/atlas_io/detail/Time.h rename to atlas_io/src/atlas_io/detail/Time.h diff --git a/atlas-io/src/atlas_io/detail/Type.h b/atlas_io/src/atlas_io/detail/Type.h similarity index 100% rename from atlas-io/src/atlas_io/detail/Type.h rename to atlas_io/src/atlas_io/detail/Type.h diff --git a/atlas-io/src/atlas_io/detail/TypeTraits.h b/atlas_io/src/atlas_io/detail/TypeTraits.h similarity index 100% rename from atlas-io/src/atlas_io/detail/TypeTraits.h rename to atlas_io/src/atlas_io/detail/TypeTraits.h diff --git a/atlas-io/src/atlas_io/detail/Version.h b/atlas_io/src/atlas_io/detail/Version.h similarity index 100% rename from atlas-io/src/atlas_io/detail/Version.h rename to atlas_io/src/atlas_io/detail/Version.h diff --git a/atlas-io/src/atlas_io/detail/defines.h.in b/atlas_io/src/atlas_io/detail/defines.h.in similarity index 100% rename from atlas-io/src/atlas_io/detail/defines.h.in rename to atlas_io/src/atlas_io/detail/defines.h.in diff --git a/atlas-io/src/atlas_io/detail/sfinae.h b/atlas_io/src/atlas_io/detail/sfinae.h similarity index 100% rename from atlas-io/src/atlas_io/detail/sfinae.h rename to atlas_io/src/atlas_io/detail/sfinae.h diff --git a/atlas-io/src/atlas_io/detail/tag.h b/atlas_io/src/atlas_io/detail/tag.h similarity index 100% rename from atlas-io/src/atlas_io/detail/tag.h rename to atlas_io/src/atlas_io/detail/tag.h diff --git a/atlas-io/src/atlas_io/print/Bytes.cc b/atlas_io/src/atlas_io/print/Bytes.cc similarity index 100% rename from atlas-io/src/atlas_io/print/Bytes.cc rename to atlas_io/src/atlas_io/print/Bytes.cc diff --git a/atlas-io/src/atlas_io/print/Bytes.h b/atlas_io/src/atlas_io/print/Bytes.h similarity index 100% rename from atlas-io/src/atlas_io/print/Bytes.h rename to atlas_io/src/atlas_io/print/Bytes.h diff --git a/atlas-io/src/atlas_io/print/JSONFormat.cc b/atlas_io/src/atlas_io/print/JSONFormat.cc similarity index 100% rename from atlas-io/src/atlas_io/print/JSONFormat.cc rename to atlas_io/src/atlas_io/print/JSONFormat.cc diff --git a/atlas-io/src/atlas_io/print/JSONFormat.h b/atlas_io/src/atlas_io/print/JSONFormat.h similarity index 100% rename from atlas-io/src/atlas_io/print/JSONFormat.h rename to atlas_io/src/atlas_io/print/JSONFormat.h diff --git a/atlas-io/src/atlas_io/print/TableFormat.cc b/atlas_io/src/atlas_io/print/TableFormat.cc similarity index 100% rename from atlas-io/src/atlas_io/print/TableFormat.cc rename to atlas_io/src/atlas_io/print/TableFormat.cc diff --git a/atlas-io/src/atlas_io/print/TableFormat.h b/atlas_io/src/atlas_io/print/TableFormat.h similarity index 100% rename from atlas-io/src/atlas_io/print/TableFormat.h rename to atlas_io/src/atlas_io/print/TableFormat.h diff --git a/atlas-io/src/atlas_io/types/array.h b/atlas_io/src/atlas_io/types/array.h similarity index 100% rename from atlas-io/src/atlas_io/types/array.h rename to atlas_io/src/atlas_io/types/array.h diff --git a/atlas-io/src/atlas_io/types/array/ArrayMetadata.cc b/atlas_io/src/atlas_io/types/array/ArrayMetadata.cc similarity index 100% rename from atlas-io/src/atlas_io/types/array/ArrayMetadata.cc rename to atlas_io/src/atlas_io/types/array/ArrayMetadata.cc diff --git a/atlas-io/src/atlas_io/types/array/ArrayMetadata.h b/atlas_io/src/atlas_io/types/array/ArrayMetadata.h similarity index 100% rename from atlas-io/src/atlas_io/types/array/ArrayMetadata.h rename to atlas_io/src/atlas_io/types/array/ArrayMetadata.h diff --git a/atlas-io/src/atlas_io/types/array/ArrayReference.cc b/atlas_io/src/atlas_io/types/array/ArrayReference.cc similarity index 100% rename from atlas-io/src/atlas_io/types/array/ArrayReference.cc rename to atlas_io/src/atlas_io/types/array/ArrayReference.cc diff --git a/atlas-io/src/atlas_io/types/array/ArrayReference.h b/atlas_io/src/atlas_io/types/array/ArrayReference.h similarity index 100% rename from atlas-io/src/atlas_io/types/array/ArrayReference.h rename to atlas_io/src/atlas_io/types/array/ArrayReference.h diff --git a/atlas-io/src/atlas_io/types/array/adaptors/StdArrayAdaptor.h b/atlas_io/src/atlas_io/types/array/adaptors/StdArrayAdaptor.h similarity index 100% rename from atlas-io/src/atlas_io/types/array/adaptors/StdArrayAdaptor.h rename to atlas_io/src/atlas_io/types/array/adaptors/StdArrayAdaptor.h diff --git a/atlas-io/src/atlas_io/types/array/adaptors/StdVectorAdaptor.h b/atlas_io/src/atlas_io/types/array/adaptors/StdVectorAdaptor.h similarity index 100% rename from atlas-io/src/atlas_io/types/array/adaptors/StdVectorAdaptor.h rename to atlas_io/src/atlas_io/types/array/adaptors/StdVectorAdaptor.h diff --git a/atlas-io/src/atlas_io/types/scalar.cc b/atlas_io/src/atlas_io/types/scalar.cc similarity index 100% rename from atlas-io/src/atlas_io/types/scalar.cc rename to atlas_io/src/atlas_io/types/scalar.cc diff --git a/atlas-io/src/atlas_io/types/scalar.h b/atlas_io/src/atlas_io/types/scalar.h similarity index 100% rename from atlas-io/src/atlas_io/types/scalar.h rename to atlas_io/src/atlas_io/types/scalar.h diff --git a/atlas-io/src/atlas_io/types/string.h b/atlas_io/src/atlas_io/types/string.h similarity index 100% rename from atlas-io/src/atlas_io/types/string.h rename to atlas_io/src/atlas_io/types/string.h diff --git a/atlas-io/src/tools/CMakeLists.txt b/atlas_io/src/tools/CMakeLists.txt similarity index 100% rename from atlas-io/src/tools/CMakeLists.txt rename to atlas_io/src/tools/CMakeLists.txt diff --git a/atlas-io/src/tools/atlas-io-list.cc b/atlas_io/src/tools/atlas-io-list.cc similarity index 100% rename from atlas-io/src/tools/atlas-io-list.cc rename to atlas_io/src/tools/atlas-io-list.cc diff --git a/atlas-io/tests/CMakeLists.txt b/atlas_io/tests/CMakeLists.txt similarity index 100% rename from atlas-io/tests/CMakeLists.txt rename to atlas_io/tests/CMakeLists.txt diff --git a/atlas-io/tests/TestEnvironment.h b/atlas_io/tests/TestEnvironment.h similarity index 100% rename from atlas-io/tests/TestEnvironment.h rename to atlas_io/tests/TestEnvironment.h diff --git a/atlas-io/tests/test_io_encoding.cc b/atlas_io/tests/test_io_encoding.cc similarity index 100% rename from atlas-io/tests/test_io_encoding.cc rename to atlas_io/tests/test_io_encoding.cc diff --git a/atlas-io/tests/test_io_record.cc b/atlas_io/tests/test_io_record.cc similarity index 100% rename from atlas-io/tests/test_io_record.cc rename to atlas_io/tests/test_io_record.cc diff --git a/atlas-io/tests/test_io_stream.cc b/atlas_io/tests/test_io_stream.cc similarity index 100% rename from atlas-io/tests/test_io_stream.cc rename to atlas_io/tests/test_io_stream.cc diff --git a/cmake/atlas-import.cmake.in b/cmake/atlas-import.cmake.in index 6c31779df..d8af94223 100644 --- a/cmake/atlas-import.cmake.in +++ b/cmake/atlas-import.cmake.in @@ -22,6 +22,8 @@ if( atlas_HAVE_FORTRAN ) find_dependency( fckit HINTS ${CMAKE_CURRENT_LIST_DIR}/../fckit @fckit_DIR@ @fckit_BINARY_DIR@ ) endif() +find_dependency( atlas_io HINTS ${CMAKE_CURRENT_LIST_DIR}/../atlas_io @atlas_io_DIR@ @atlas_io_BINARY_DIR@ ) + ## Eigen3 set( Eigen3_HINT @Eigen3_DIR@ ) if( atlas_HAVE_EIGEN AND Eigen3_HINT ) From aeb417ff2967e62e8dcdf7a06a743041093180f2 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 22 Aug 2022 13:43:38 +0200 Subject: [PATCH 49/53] ATLAS-369 Support 'variables' option in functionspace::PointCloud::createField --- src/atlas/functionspace/PointCloud.cc | 80 ++++++++++++++++--- src/atlas/functionspace/PointCloud.h | 13 +++ src/tests/functionspace/CMakeLists.txt | 9 ++- .../functionspace/fctest_functionspace.F90 | 56 +++++++++++++ src/tests/functionspace/test_pointcloud.cc | 6 ++ 5 files changed, 151 insertions(+), 13 deletions(-) diff --git a/src/atlas/functionspace/PointCloud.cc b/src/atlas/functionspace/PointCloud.cc index 0217d7458..c9cd23a95 100644 --- a/src/atlas/functionspace/PointCloud.cc +++ b/src/atlas/functionspace/PointCloud.cc @@ -17,6 +17,7 @@ #include "atlas/option/Options.h" #include "atlas/runtime/Exception.h" #include "atlas/util/CoordinateEnums.h" +#include "atlas/util/Metadata.h" #if ATLAS_HAVE_FORTRAN #define REMOTE_IDX_BASE 1 @@ -76,32 +77,87 @@ Field PointCloud::ghost() const { return ghost_; } -Field PointCloud::createField(const eckit::Configuration& config) const { +array::ArrayShape PointCloud::config_shape(const eckit::Configuration& config) const { + array::ArrayShape shape; + + shape.emplace_back(size()); + + idx_t levels(levels_); + config.get("levels", levels); + if (levels > 0) { + shape.emplace_back(levels); + } + + idx_t variables(0); + config.get("variables", variables); + if (variables > 0) { + shape.emplace_back(variables); + } + + return shape; +} + +array::ArrayAlignment PointCloud::config_alignment(const eckit::Configuration& config) const { + int alignment(1); + config.get("alignment", alignment); + return alignment; +} + +array::ArraySpec PointCloud::config_spec(const eckit::Configuration& config) const { + return array::ArraySpec(config_shape(config), config_alignment(config)); +} + +array::DataType PointCloud::config_datatype(const eckit::Configuration& config) const { array::DataType::kind_t kind; if (!config.get("datatype", kind)) { throw_Exception("datatype missing", Here()); } - auto datatype = array::DataType(kind); + return array::DataType(kind); +} +std::string PointCloud::config_name(const eckit::Configuration& config) const { std::string name; config.get("name", name); - idx_t levels = levels_; - config.get("levels", levels); - Field field; - if (levels) { - field = Field(name, datatype, array::make_shape(size(), levels)); - field.set_levels(levels); + return name; +} + +void PointCloud::set_field_metadata(const eckit::Configuration& config, Field& field) const { + field.set_functionspace(this); + + bool global(false); + if (config.get("global", global)) { + if (global) { + idx_t owner(0); + config.get("owner", owner); + field.metadata().set("owner", owner); + } } - else { - field = Field(name, datatype, array::make_shape(size())); + field.metadata().set("global", global); + + idx_t levels(levels_); + config.get("levels", levels); + field.set_levels(levels); + + idx_t variables(0); + config.get("variables", variables); + field.set_variables(variables); + + if (config.has("type")) { + field.metadata().set("type", config.getString("type")); } - field.set_functionspace(this); +} + + +Field PointCloud::createField(const eckit::Configuration& options) const { + Field field(config_name(options), config_datatype(options), config_spec(options)); + set_field_metadata(options, field); return field; } Field PointCloud::createField(const Field& other, const eckit::Configuration& config) const { return createField(option::datatype(other.datatype()) | option::levels(other.levels()) | - option::variables(other.variables()) | config); + option::variables(other.variables()) | + option::type(other.metadata().getString("type", "scalar")) | config); } std::string PointCloud::distribution() const { diff --git a/src/atlas/functionspace/PointCloud.h b/src/atlas/functionspace/PointCloud.h index 8d8b5fa94..e70c2f3b1 100644 --- a/src/atlas/functionspace/PointCloud.h +++ b/src/atlas/functionspace/PointCloud.h @@ -116,6 +116,19 @@ class PointCloud : public functionspace::FunctionSpaceImpl { Iterate iterate() const { return Iterate(*this); } +private: + array::ArrayShape config_shape(const eckit::Configuration& config) const; + + array::ArrayAlignment config_alignment(const eckit::Configuration& config) const; + + array::ArraySpec config_spec(const eckit::Configuration& config) const; + + array::DataType config_datatype(const eckit::Configuration& config) const; + + std::string config_name(const eckit::Configuration& config) const; + + void set_field_metadata(const eckit::Configuration& config, Field& field) const; + private: Field lonlat_; Field vertical_; diff --git a/src/tests/functionspace/CMakeLists.txt b/src/tests/functionspace/CMakeLists.txt index bf7d5d378..53ec29d3a 100644 --- a/src/tests/functionspace/CMakeLists.txt +++ b/src/tests/functionspace/CMakeLists.txt @@ -6,15 +6,22 @@ # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. +ecbuild_warn_var( HAVE_FCTEST ) if( HAVE_FCTEST ) if( NOT HAVE_TRANS ) set( transi_HAVE_MPI 1 ) + set( ectrans_HAVE_MPI 1 ) endif() + ecbuild_warn_var( HAVE_TRANS ) + ecbuild_warn_var( eckit_HAVE_MPI ) + ecbuild_warn_var( transi_HAVE_MPI ) + + add_fctest( TARGET atlas_fctest_functionspace MPI 4 - CONDITION eckit_HAVE_MPI AND transi_HAVE_MPI + CONDITION eckit_HAVE_MPI AND (transi_HAVE_MPI OR ectrans_HAVE_MPI) LINKER_LANGUAGE Fortran SOURCES fctest_functionspace.F90 LIBS atlas_f diff --git a/src/tests/functionspace/fctest_functionspace.F90 b/src/tests/functionspace/fctest_functionspace.F90 index 26bbbae08..95789b8cf 100644 --- a/src/tests/functionspace/fctest_functionspace.F90 +++ b/src/tests/functionspace/fctest_functionspace.F90 @@ -585,6 +585,62 @@ module fcta_FunctionSpace_fxt #endif END_TEST +TEST( test_pointcloud ) +#if 1 +type(atlas_StructuredGrid) :: grid +type(atlas_functionspace_PointCloud) :: fs +type(atlas_functionspace) :: fs_base +character(len=10) str + +type(atlas_Field) :: field, field2 +type(atlas_Field) :: field_lonlat +real(8), pointer :: lonlat(:,:), x(:) + +grid = atlas_Grid("O8") +fs = atlas_functionspace_PointCloud(grid) + +field = fs%create_field(name="field",kind=atlas_real(8)) +FCTEST_CHECK_EQUAL( field%owners(), 1 ) +FCTEST_CHECK_EQUAL( field%levels(), 0 ) +field_lonlat = fs%lonlat() +FCTEST_CHECK_EQUAL( field_lonlat%owners(), 2 ) +call field%data(x) +call field_lonlat%data(lonlat) + +FCTEST_CHECK_EQUAL( field_lonlat%owners(), 2 ) +fs = atlas_functionspace_PointCloud(grid) + +field = fs%create_field(atlas_real(c_float), levels=5) +FCTEST_CHECK_EQUAL( field%rank() , 2 ) +FCTEST_CHECK_EQUAL( field%name() , "" ) +FCTEST_CHECK_EQUAL( field%kind() , atlas_real(c_float) ) + + +write(0,*) "before: name = ", fs%name() +write(0,*) "before: owners = ", fs%owners() +fs_base = field%functionspace() +write(0,*) "after: name = " , fs%name() +write(0,*) "after: owners = " , fs%owners() + +field2 = fs%create_field(name="field2",kind=atlas_real(8),levels=3,variables=2) + +FCTEST_CHECK_EQUAL( field%shape(), ([5,grid%size()]) ) +FCTEST_CHECK_EQUAL( field2%shape(), ([2,3,grid%size()]) ) + + +FCTEST_CHECK_EQUAL( field%owners(), 1 ) +call field%final() +FCTEST_CHECK_EQUAL( field_lonlat%owners(), 1 ) +call field_lonlat%final() +call fs%final() +call fs_base%final() +call grid%final() +#else +#warning test test_pointcloud disabled +#endif +END_TEST + + ! ----------------------------------------------------------------------------- diff --git a/src/tests/functionspace/test_pointcloud.cc b/src/tests/functionspace/test_pointcloud.cc index 8d994b642..336be54ba 100644 --- a/src/tests/functionspace/test_pointcloud.cc +++ b/src/tests/functionspace/test_pointcloud.cc @@ -147,6 +147,12 @@ CASE("test_createField") { EXPECT_EQ(f2.levels(), 3); EXPECT_EQ(f2.shape(0), 4); EXPECT_EQ(f2.shape(1), 3); + + Field f3 = p2.createField(f1, option::variables(5)); + EXPECT_EQ(f3.levels(), 3); + EXPECT_EQ(f3.shape(0), 4); + EXPECT_EQ(f3.shape(1), 3); + EXPECT_EQ(f3.shape(2), 5); } From 2036543b551228cbf7a015da6bc388d94bd62eb3 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 22 Aug 2022 15:06:31 +0200 Subject: [PATCH 50/53] ATLAS-369 Fixup running test on Cray --- src/tests/functionspace/fctest_functionspace.F90 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/tests/functionspace/fctest_functionspace.F90 b/src/tests/functionspace/fctest_functionspace.F90 index 95789b8cf..6eb49b05c 100644 --- a/src/tests/functionspace/fctest_functionspace.F90 +++ b/src/tests/functionspace/fctest_functionspace.F90 @@ -627,10 +627,13 @@ module fcta_FunctionSpace_fxt FCTEST_CHECK_EQUAL( field%shape(), ([5,grid%size()]) ) FCTEST_CHECK_EQUAL( field2%shape(), ([2,3,grid%size()]) ) - +#ifndef _CRAYFTN FCTEST_CHECK_EQUAL( field%owners(), 1 ) +#endif call field%final() +#ifndef _CRAYFTN FCTEST_CHECK_EQUAL( field_lonlat%owners(), 1 ) +#endif call field_lonlat%final() call fs%final() call fs_base%final() From 52315f92b79271377a8b52e1fa98b9165366b902 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 22 Aug 2022 15:35:19 +0200 Subject: [PATCH 51/53] ATLAS-368 Fix apply-clang-format.sh --- tools/apply-clang-format.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/apply-clang-format.sh b/tools/apply-clang-format.sh index 9d7b24f1f..ed44034ba 100755 --- a/tools/apply-clang-format.sh +++ b/tools/apply-clang-format.sh @@ -103,7 +103,7 @@ if [[ $all =~ "yes" ]]; then find . -iname *.h -o -iname *.cc | xargs clang-format -i -style=file fi - cd $SCRIPTDIR/../atlas-io + cd $SCRIPTDIR/../atlas_io if [[ $dryrun =~ "yes" ]]; then echo "+ find . -iname *.h -o -iname *.cc | xargs clang-format -i -style=file" else From 74e40b9fe075db29f127b4f4aa9f6c21bbb35d82 Mon Sep 17 00:00:00 2001 From: Slavko Brdar Date: Fri, 19 Aug 2022 13:28:02 +0200 Subject: [PATCH 52/53] ATLAS-358 add support for pentagon elements in accumulating facets --- src/atlas/mesh/detail/AccumulateFacets.cc | 36 ++++++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/src/atlas/mesh/detail/AccumulateFacets.cc b/src/atlas/mesh/detail/AccumulateFacets.cc index b13cf7386..dad219d70 100644 --- a/src/atlas/mesh/detail/AccumulateFacets.cc +++ b/src/atlas/mesh/detail/AccumulateFacets.cc @@ -52,7 +52,21 @@ void accumulate_facets(const mesh::HybridElements& cells, const mesh::Nodes& nod std::vector> facet_node_numbering; idx_t nb_facets_in_elem; - if (elements.name() == "Quadrilateral") { + if (elements.name() == "Pentagon") { + nb_facets_in_elem = 5; + facet_node_numbering.resize(nb_facets_in_elem, std::vector(nb_nodes_in_facet)); + facet_node_numbering[0][0] = 0; + facet_node_numbering[0][1] = 1; + facet_node_numbering[1][0] = 1; + facet_node_numbering[1][1] = 2; + facet_node_numbering[2][0] = 2; + facet_node_numbering[2][1] = 3; + facet_node_numbering[3][0] = 3; + facet_node_numbering[3][1] = 4; + facet_node_numbering[4][0] = 4; + facet_node_numbering[4][1] = 0; + } + else if (elements.name() == "Quadrilateral") { nb_facets_in_elem = 4; facet_node_numbering.resize(nb_facets_in_elem, std::vector(nb_nodes_in_facet)); facet_node_numbering[0][0] = 0; @@ -75,7 +89,7 @@ void accumulate_facets(const mesh::HybridElements& cells, const mesh::Nodes& nod facet_node_numbering[2][1] = 0; } else { - throw_Exception(elements.name() + " is not \"Quadrilateral\" or \"Triangle\"", Here()); + throw_Exception(elements.name() + " is not \"Pentagon\", \"Quadrilateral\", or \"Triangle\"", Here()); } std::vector facet_nodes(nb_nodes_in_facet); @@ -156,7 +170,21 @@ void accumulate_facets_in_range(std::vector& range, const mesh::Hy std::vector> facet_node_numbering; idx_t nb_facets_in_elem; - if (elements.name() == "Quadrilateral") { + if (elements.name() == "Pentagon") { + nb_facets_in_elem = 5; + facet_node_numbering.resize(nb_facets_in_elem, std::vector(nb_nodes_in_facet)); + facet_node_numbering[0][0] = 0; + facet_node_numbering[0][1] = 1; + facet_node_numbering[1][0] = 1; + facet_node_numbering[1][1] = 2; + facet_node_numbering[2][0] = 2; + facet_node_numbering[2][1] = 3; + facet_node_numbering[3][0] = 3; + facet_node_numbering[3][1] = 4; + facet_node_numbering[4][0] = 4; + facet_node_numbering[4][1] = 0; + } + else if (elements.name() == "Quadrilateral") { nb_facets_in_elem = 4; facet_node_numbering.resize(nb_facets_in_elem, std::vector(nb_nodes_in_facet)); facet_node_numbering[0][0] = 0; @@ -179,7 +207,7 @@ void accumulate_facets_in_range(std::vector& range, const mesh::Hy facet_node_numbering[2][1] = 0; } else { - throw_Exception(elements.name() + " is not \"Quadrilateral\" or \"Triangle\"", Here()); + throw_Exception(elements.name() + " is not \"Pentagon\", \"Quadrilateral\", or \"Triangle\"", Here()); } std::vector facet_nodes(nb_nodes_in_facet); From c43a0fc0a004609be3b69af3c2c99b7d0cb825c3 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 15 Aug 2022 17:49:19 +0200 Subject: [PATCH 53/53] Version 0.30.0 --- CHANGELOG.md | 21 +++++++++++++++++++++ VERSION | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a89a3b52..6fc68a51c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,26 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html ## [Unreleased] +## [0.30.0] - 2022-08-22 +### Added +- Fortran API for Interpolation::execute_adjoint() +- Fortran API for FieldSet::name() +- Fortran API for MeshGenerator::generate(grid,partitioner) +- Pentagon element type +- SphericalHarmonic function +- New interpolation method: ConservativeSphericalPolygonInterpolation +- Support 'variables' option in functionspace::PointCloud::createField + +### Changed +- Atlas-IO is now standalone project, still embedded but only depending on eckit +- Deprecate Trans naming of 'ifs' or 'trans' in favour of 'ectrans' +- Default StructuredMeshGenerator partitioner is equal_regions instead of trans/ectrans + +### Fixed +- Fix global numbering HEALPix grid to standard +- Fix NodeColumns remote_index for parallel orca grids +- Fix use of ATLAS_LINALG_DENSE_BACKEND environment variable + ## [0.29.0] - 2022-04-21 ### Added - MatchingMeshPartitioner "cubedsphere" @@ -366,6 +386,7 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html ## 0.13.0 - 2018-02-16 [Unreleased]: https://github.com/ecmwf/atlas/compare/master...develop +[0.30.0]: https://github.com/ecmwf/atlas/compare/0.29.0...0.30.0 [0.29.0]: https://github.com/ecmwf/atlas/compare/0.28.1...0.29.0 [0.28.1]: https://github.com/ecmwf/atlas/compare/0.28.0...0.28.1 [0.28.0]: https://github.com/ecmwf/atlas/compare/0.27.0...0.28.0 diff --git a/VERSION b/VERSION index ae6dd4e20..c25c8e5b7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.29.0 +0.30.0