From fa4f950b90efbcf8675d47d5bfb719b718cfb4c0 Mon Sep 17 00:00:00 2001
From: Markus Wittmann <markus.wittmann@fau.de>
Date: Fri, 28 Apr 2023 11:44:49 +0200
Subject: [PATCH] fix overlapping send and recv buffers

---
 cilk/basic/make_local_matrix.hpp                 | 3 ++-
 cilk/src/make_local_matrix.hpp                   | 3 ++-
 cuda/basic/make_local_matrix.hpp                 | 3 ++-
 cuda/src/make_local_matrix.hpp                   | 3 ++-
 kokkos/src/make_local_matrix.hpp                 | 3 ++-
 mkl/basic/make_local_matrix.hpp                  | 3 ++-
 mkl/src/make_local_matrix.hpp                    | 3 ++-
 nvamg/basic/make_local_matrix.hpp                | 3 ++-
 nvamg/src/make_local_matrix.hpp                  | 3 ++-
 openmp-opt-knl-memkind/src/make_local_matrix.hpp | 3 ++-
 openmp-opt-knl/basic/make_local_matrix.hpp       | 3 ++-
 openmp-opt-knl/src/make_local_matrix.hpp         | 3 ++-
 openmp-opt/basic/make_local_matrix.hpp           | 3 ++-
 openmp-opt/src/make_local_matrix.hpp             | 3 ++-
 openmp/basic/make_local_matrix.hpp               | 3 ++-
 openmp/src/make_local_matrix.hpp                 | 3 ++-
 openmp45-opt/src/make_local_matrix.hpp           | 3 ++-
 openmp45/basic/make_local_matrix.hpp             | 3 ++-
 openmp45/src/make_local_matrix.hpp               | 3 ++-
 qthreads/basic/make_local_matrix.hpp             | 3 ++-
 qthreads/src/make_local_matrix.hpp               | 3 ++-
 ref/basic/make_local_matrix.hpp                  | 3 ++-
 ref/src/make_local_matrix.hpp                    | 3 ++-
 23 files changed, 46 insertions(+), 23 deletions(-)

diff --git a/cilk/basic/make_local_matrix.hpp b/cilk/basic/make_local_matrix.hpp
index 99c2cf7..1cade1e 100644
--- a/cilk/basic/make_local_matrix.hpp
+++ b/cilk/basic/make_local_matrix.hpp
@@ -251,9 +251,10 @@ make_local_matrix(MatrixType& A)
   //
   int MPI_MY_TAG = 99;
 
+  std::vector<GlobalOrdinal> tmp_recv_buffer(num_send_neighbors);
   std::vector<MPI_Request> request(num_send_neighbors);
   for(int i=0; i<num_send_neighbors; ++i) {
-    MPI_Irecv(&tmp_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
+    MPI_Irecv(&tmp_recv_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
               MPI_COMM_WORLD, &request[i]);
   }
 
diff --git a/cilk/src/make_local_matrix.hpp b/cilk/src/make_local_matrix.hpp
index 8823119..da85e0e 100644
--- a/cilk/src/make_local_matrix.hpp
+++ b/cilk/src/make_local_matrix.hpp
@@ -256,9 +256,10 @@ make_local_matrix(MatrixType& A)
   //
   int MPI_MY_TAG = 99;
 
+  std::vector<GlobalOrdinal> tmp_recv_buffer(num_send_neighbors);
   std::vector<MPI_Request> request(num_send_neighbors);
   for(int i=0; i<num_send_neighbors; ++i) {
-    MPI_Irecv(&tmp_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
+    MPI_Irecv(&tmp_recv_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
               MPI_COMM_WORLD, &request[i]);
   }
 
diff --git a/cuda/basic/make_local_matrix.hpp b/cuda/basic/make_local_matrix.hpp
index 99c2cf7..1cade1e 100644
--- a/cuda/basic/make_local_matrix.hpp
+++ b/cuda/basic/make_local_matrix.hpp
@@ -251,9 +251,10 @@ make_local_matrix(MatrixType& A)
   //
   int MPI_MY_TAG = 99;
 
+  std::vector<GlobalOrdinal> tmp_recv_buffer(num_send_neighbors);
   std::vector<MPI_Request> request(num_send_neighbors);
   for(int i=0; i<num_send_neighbors; ++i) {
-    MPI_Irecv(&tmp_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
+    MPI_Irecv(&tmp_recv_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
               MPI_COMM_WORLD, &request[i]);
   }
 
diff --git a/cuda/src/make_local_matrix.hpp b/cuda/src/make_local_matrix.hpp
index 67ee4e2..cc43963 100644
--- a/cuda/src/make_local_matrix.hpp
+++ b/cuda/src/make_local_matrix.hpp
@@ -487,9 +487,10 @@ make_local_matrix(MatrixType& A)
   int MPI_MY_TAG = 99;
 
   nvtxRangeId_t r4=nvtxRangeStartA("MPI Communication");
+  std::vector<GlobalOrdinal> tmp_recv_buffer(num_send_neighbors);
   std::vector<MPI_Request> request(num_send_neighbors);
   for(int i=0; i<num_send_neighbors; ++i) {
-    MPI_Irecv(&tmp_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
+    MPI_Irecv(&tmp_recv_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
               MPI_COMM_WORLD, &request[i]);
   }
 
diff --git a/kokkos/src/make_local_matrix.hpp b/kokkos/src/make_local_matrix.hpp
index 5d91b3b..734cb1f 100644
--- a/kokkos/src/make_local_matrix.hpp
+++ b/kokkos/src/make_local_matrix.hpp
@@ -252,9 +252,10 @@ make_local_matrix(MatrixType& A)
   // wait call below.
   //
   int MPI_MY_TAG = 99;
+  std::vector<GlobalOrdinal> tmp_recv_buffer(num_send_neighbors);
   std::vector<MPI_Request> request(num_send_neighbors);
   for(int i=0; i<num_send_neighbors; ++i) {
-    MPI_Irecv(&tmp_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
+    MPI_Irecv(&tmp_recv_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
               MPI_COMM_WORLD, &request[i]);
   }
 
diff --git a/mkl/basic/make_local_matrix.hpp b/mkl/basic/make_local_matrix.hpp
index 99c2cf7..1cade1e 100644
--- a/mkl/basic/make_local_matrix.hpp
+++ b/mkl/basic/make_local_matrix.hpp
@@ -251,9 +251,10 @@ make_local_matrix(MatrixType& A)
   //
   int MPI_MY_TAG = 99;
 
+  std::vector<GlobalOrdinal> tmp_recv_buffer(num_send_neighbors);
   std::vector<MPI_Request> request(num_send_neighbors);
   for(int i=0; i<num_send_neighbors; ++i) {
-    MPI_Irecv(&tmp_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
+    MPI_Irecv(&tmp_recv_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
               MPI_COMM_WORLD, &request[i]);
   }
 
diff --git a/mkl/src/make_local_matrix.hpp b/mkl/src/make_local_matrix.hpp
index 8823119..da85e0e 100644
--- a/mkl/src/make_local_matrix.hpp
+++ b/mkl/src/make_local_matrix.hpp
@@ -256,9 +256,10 @@ make_local_matrix(MatrixType& A)
   //
   int MPI_MY_TAG = 99;
 
+  std::vector<GlobalOrdinal> tmp_recv_buffer(num_send_neighbors);
   std::vector<MPI_Request> request(num_send_neighbors);
   for(int i=0; i<num_send_neighbors; ++i) {
-    MPI_Irecv(&tmp_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
+    MPI_Irecv(&tmp_recv_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
               MPI_COMM_WORLD, &request[i]);
   }
 
diff --git a/nvamg/basic/make_local_matrix.hpp b/nvamg/basic/make_local_matrix.hpp
index 99c2cf7..1cade1e 100644
--- a/nvamg/basic/make_local_matrix.hpp
+++ b/nvamg/basic/make_local_matrix.hpp
@@ -251,9 +251,10 @@ make_local_matrix(MatrixType& A)
   //
   int MPI_MY_TAG = 99;
 
+  std::vector<GlobalOrdinal> tmp_recv_buffer(num_send_neighbors);
   std::vector<MPI_Request> request(num_send_neighbors);
   for(int i=0; i<num_send_neighbors; ++i) {
-    MPI_Irecv(&tmp_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
+    MPI_Irecv(&tmp_recv_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
               MPI_COMM_WORLD, &request[i]);
   }
 
diff --git a/nvamg/src/make_local_matrix.hpp b/nvamg/src/make_local_matrix.hpp
index 76970d0..b53df72 100644
--- a/nvamg/src/make_local_matrix.hpp
+++ b/nvamg/src/make_local_matrix.hpp
@@ -254,9 +254,10 @@ make_local_matrix(MatrixType& A)
   //
   int MPI_MY_TAG = 99;
 
+  std::vector<GlobalOrdinal> tmp_recv_buffer(num_send_neighbors);
   std::vector<MPI_Request> request(num_send_neighbors);
   for(int i=0; i<num_send_neighbors; ++i) {
-    MPI_Irecv(&tmp_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
+    MPI_Irecv(&tmp_recv_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
               MPI_COMM_WORLD, &request[i]);
   }
 
diff --git a/openmp-opt-knl-memkind/src/make_local_matrix.hpp b/openmp-opt-knl-memkind/src/make_local_matrix.hpp
index 76970d0..b53df72 100644
--- a/openmp-opt-knl-memkind/src/make_local_matrix.hpp
+++ b/openmp-opt-knl-memkind/src/make_local_matrix.hpp
@@ -254,9 +254,10 @@ make_local_matrix(MatrixType& A)
   //
   int MPI_MY_TAG = 99;
 
+  std::vector<GlobalOrdinal> tmp_recv_buffer(num_send_neighbors);
   std::vector<MPI_Request> request(num_send_neighbors);
   for(int i=0; i<num_send_neighbors; ++i) {
-    MPI_Irecv(&tmp_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
+    MPI_Irecv(&tmp_recv_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
               MPI_COMM_WORLD, &request[i]);
   }
 
diff --git a/openmp-opt-knl/basic/make_local_matrix.hpp b/openmp-opt-knl/basic/make_local_matrix.hpp
index 99c2cf7..1cade1e 100644
--- a/openmp-opt-knl/basic/make_local_matrix.hpp
+++ b/openmp-opt-knl/basic/make_local_matrix.hpp
@@ -251,9 +251,10 @@ make_local_matrix(MatrixType& A)
   //
   int MPI_MY_TAG = 99;
 
+  std::vector<GlobalOrdinal> tmp_recv_buffer(num_send_neighbors);
   std::vector<MPI_Request> request(num_send_neighbors);
   for(int i=0; i<num_send_neighbors; ++i) {
-    MPI_Irecv(&tmp_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
+    MPI_Irecv(&tmp_recv_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
               MPI_COMM_WORLD, &request[i]);
   }
 
diff --git a/openmp-opt-knl/src/make_local_matrix.hpp b/openmp-opt-knl/src/make_local_matrix.hpp
index 76970d0..b53df72 100644
--- a/openmp-opt-knl/src/make_local_matrix.hpp
+++ b/openmp-opt-knl/src/make_local_matrix.hpp
@@ -254,9 +254,10 @@ make_local_matrix(MatrixType& A)
   //
   int MPI_MY_TAG = 99;
 
+  std::vector<GlobalOrdinal> tmp_recv_buffer(num_send_neighbors);
   std::vector<MPI_Request> request(num_send_neighbors);
   for(int i=0; i<num_send_neighbors; ++i) {
-    MPI_Irecv(&tmp_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
+    MPI_Irecv(&tmp_recv_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
               MPI_COMM_WORLD, &request[i]);
   }
 
diff --git a/openmp-opt/basic/make_local_matrix.hpp b/openmp-opt/basic/make_local_matrix.hpp
index 99c2cf7..1cade1e 100644
--- a/openmp-opt/basic/make_local_matrix.hpp
+++ b/openmp-opt/basic/make_local_matrix.hpp
@@ -251,9 +251,10 @@ make_local_matrix(MatrixType& A)
   //
   int MPI_MY_TAG = 99;
 
+  std::vector<GlobalOrdinal> tmp_recv_buffer(num_send_neighbors);
   std::vector<MPI_Request> request(num_send_neighbors);
   for(int i=0; i<num_send_neighbors; ++i) {
-    MPI_Irecv(&tmp_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
+    MPI_Irecv(&tmp_recv_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
               MPI_COMM_WORLD, &request[i]);
   }
 
diff --git a/openmp-opt/src/make_local_matrix.hpp b/openmp-opt/src/make_local_matrix.hpp
index 76970d0..b53df72 100644
--- a/openmp-opt/src/make_local_matrix.hpp
+++ b/openmp-opt/src/make_local_matrix.hpp
@@ -254,9 +254,10 @@ make_local_matrix(MatrixType& A)
   //
   int MPI_MY_TAG = 99;
 
+  std::vector<GlobalOrdinal> tmp_recv_buffer(num_send_neighbors);
   std::vector<MPI_Request> request(num_send_neighbors);
   for(int i=0; i<num_send_neighbors; ++i) {
-    MPI_Irecv(&tmp_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
+    MPI_Irecv(&tmp_recv_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
               MPI_COMM_WORLD, &request[i]);
   }
 
diff --git a/openmp/basic/make_local_matrix.hpp b/openmp/basic/make_local_matrix.hpp
index 99c2cf7..1cade1e 100644
--- a/openmp/basic/make_local_matrix.hpp
+++ b/openmp/basic/make_local_matrix.hpp
@@ -251,9 +251,10 @@ make_local_matrix(MatrixType& A)
   //
   int MPI_MY_TAG = 99;
 
+  std::vector<GlobalOrdinal> tmp_recv_buffer(num_send_neighbors);
   std::vector<MPI_Request> request(num_send_neighbors);
   for(int i=0; i<num_send_neighbors; ++i) {
-    MPI_Irecv(&tmp_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
+    MPI_Irecv(&tmp_recv_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
               MPI_COMM_WORLD, &request[i]);
   }
 
diff --git a/openmp/src/make_local_matrix.hpp b/openmp/src/make_local_matrix.hpp
index 76970d0..b53df72 100644
--- a/openmp/src/make_local_matrix.hpp
+++ b/openmp/src/make_local_matrix.hpp
@@ -254,9 +254,10 @@ make_local_matrix(MatrixType& A)
   //
   int MPI_MY_TAG = 99;
 
+  std::vector<GlobalOrdinal> tmp_recv_buffer(num_send_neighbors);
   std::vector<MPI_Request> request(num_send_neighbors);
   for(int i=0; i<num_send_neighbors; ++i) {
-    MPI_Irecv(&tmp_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
+    MPI_Irecv(&tmp_recv_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
               MPI_COMM_WORLD, &request[i]);
   }
 
diff --git a/openmp45-opt/src/make_local_matrix.hpp b/openmp45-opt/src/make_local_matrix.hpp
index 76970d0..b53df72 100644
--- a/openmp45-opt/src/make_local_matrix.hpp
+++ b/openmp45-opt/src/make_local_matrix.hpp
@@ -254,9 +254,10 @@ make_local_matrix(MatrixType& A)
   //
   int MPI_MY_TAG = 99;
 
+  std::vector<GlobalOrdinal> tmp_recv_buffer(num_send_neighbors);
   std::vector<MPI_Request> request(num_send_neighbors);
   for(int i=0; i<num_send_neighbors; ++i) {
-    MPI_Irecv(&tmp_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
+    MPI_Irecv(&tmp_recv_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
               MPI_COMM_WORLD, &request[i]);
   }
 
diff --git a/openmp45/basic/make_local_matrix.hpp b/openmp45/basic/make_local_matrix.hpp
index 99c2cf7..1cade1e 100644
--- a/openmp45/basic/make_local_matrix.hpp
+++ b/openmp45/basic/make_local_matrix.hpp
@@ -251,9 +251,10 @@ make_local_matrix(MatrixType& A)
   //
   int MPI_MY_TAG = 99;
 
+  std::vector<GlobalOrdinal> tmp_recv_buffer(num_send_neighbors);
   std::vector<MPI_Request> request(num_send_neighbors);
   for(int i=0; i<num_send_neighbors; ++i) {
-    MPI_Irecv(&tmp_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
+    MPI_Irecv(&tmp_recv_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
               MPI_COMM_WORLD, &request[i]);
   }
 
diff --git a/openmp45/src/make_local_matrix.hpp b/openmp45/src/make_local_matrix.hpp
index 76970d0..b53df72 100644
--- a/openmp45/src/make_local_matrix.hpp
+++ b/openmp45/src/make_local_matrix.hpp
@@ -254,9 +254,10 @@ make_local_matrix(MatrixType& A)
   //
   int MPI_MY_TAG = 99;
 
+  std::vector<GlobalOrdinal> tmp_recv_buffer(num_send_neighbors);
   std::vector<MPI_Request> request(num_send_neighbors);
   for(int i=0; i<num_send_neighbors; ++i) {
-    MPI_Irecv(&tmp_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
+    MPI_Irecv(&tmp_recv_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
               MPI_COMM_WORLD, &request[i]);
   }
 
diff --git a/qthreads/basic/make_local_matrix.hpp b/qthreads/basic/make_local_matrix.hpp
index 99c2cf7..1cade1e 100644
--- a/qthreads/basic/make_local_matrix.hpp
+++ b/qthreads/basic/make_local_matrix.hpp
@@ -251,9 +251,10 @@ make_local_matrix(MatrixType& A)
   //
   int MPI_MY_TAG = 99;
 
+  std::vector<GlobalOrdinal> tmp_recv_buffer(num_send_neighbors);
   std::vector<MPI_Request> request(num_send_neighbors);
   for(int i=0; i<num_send_neighbors; ++i) {
-    MPI_Irecv(&tmp_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
+    MPI_Irecv(&tmp_recv_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
               MPI_COMM_WORLD, &request[i]);
   }
 
diff --git a/qthreads/src/make_local_matrix.hpp b/qthreads/src/make_local_matrix.hpp
index 8823119..da85e0e 100644
--- a/qthreads/src/make_local_matrix.hpp
+++ b/qthreads/src/make_local_matrix.hpp
@@ -256,9 +256,10 @@ make_local_matrix(MatrixType& A)
   //
   int MPI_MY_TAG = 99;
 
+  std::vector<GlobalOrdinal> tmp_recv_buffer(num_send_neighbors);
   std::vector<MPI_Request> request(num_send_neighbors);
   for(int i=0; i<num_send_neighbors; ++i) {
-    MPI_Irecv(&tmp_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
+    MPI_Irecv(&tmp_recv_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
               MPI_COMM_WORLD, &request[i]);
   }
 
diff --git a/ref/basic/make_local_matrix.hpp b/ref/basic/make_local_matrix.hpp
index 99c2cf7..1cade1e 100644
--- a/ref/basic/make_local_matrix.hpp
+++ b/ref/basic/make_local_matrix.hpp
@@ -251,9 +251,10 @@ make_local_matrix(MatrixType& A)
   //
   int MPI_MY_TAG = 99;
 
+  std::vector<GlobalOrdinal> tmp_recv_buffer(num_send_neighbors);
   std::vector<MPI_Request> request(num_send_neighbors);
   for(int i=0; i<num_send_neighbors; ++i) {
-    MPI_Irecv(&tmp_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
+    MPI_Irecv(&tmp_recv_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
               MPI_COMM_WORLD, &request[i]);
   }
 
diff --git a/ref/src/make_local_matrix.hpp b/ref/src/make_local_matrix.hpp
index 6dfdfbc..a2653cc 100644
--- a/ref/src/make_local_matrix.hpp
+++ b/ref/src/make_local_matrix.hpp
@@ -254,9 +254,10 @@ make_local_matrix(MatrixType& A)
   //
   int MPI_MY_TAG = 99;
 
+  std::vector<GlobalOrdinal> tmp_recv_buffer(num_send_neighbors);
   std::vector<MPI_Request> request(num_send_neighbors);
   for(int i=0; i<num_send_neighbors; ++i) {
-    MPI_Irecv(&tmp_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
+    MPI_Irecv(&tmp_recv_buffer[i], 1, mpi_dtype, MPI_ANY_SOURCE, MPI_MY_TAG,
               MPI_COMM_WORLD, &request[i]);
   }