From 6c5be0dfe9f0ade0330577ae4066f107095ecc7e Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Thu, 26 Oct 2017 12:38:05 +0200
Subject: [PATCH] Ensure that the limit will not be included in the interval --
Add an assertion if a dead lock will appear
---
bfps/cpp/full_code/NSVEparticlesP2P.cpp | 2 +-
bfps/cpp/particles/p2p_distr_mpi.hpp | 46 +++++++++++++++++++++++--
2 files changed, 44 insertions(+), 4 deletions(-)
diff --git a/bfps/cpp/full_code/NSVEparticlesP2P.cpp b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
index 11b400d1..e8065e2c 100644
--- a/bfps/cpp/full_code/NSVEparticlesP2P.cpp
+++ b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
@@ -16,7 +16,7 @@ int NSVEparticlesP2P<rnumber>::initialize(void)
particles_inner_computer<double, long long int> current_particles_inner_computer(inner_v0);
current_particles_inner_computer.setEnable(enable_inner);
- this->cutoff = 5.0;
+ this->cutoff = 1.0;
this->ps = particles_system_builder_with_p2p(
this->fs->cvelocity, // (field object)
diff --git a/bfps/cpp/particles/p2p_distr_mpi.hpp b/bfps/cpp/particles/p2p_distr_mpi.hpp
index d607666a..ac69822a 100644
--- a/bfps/cpp/particles/p2p_distr_mpi.hpp
+++ b/bfps/cpp/particles/p2p_distr_mpi.hpp
@@ -193,8 +193,13 @@ public:
long int last_cell_level_proc(const int dest_proc) const{
const real_number field_section_width_z = spatial_box_width[IDX_Z]/real_number(field_grid_dim[IDX_Z]);
- return static_cast<long int>((field_section_width_z*real_number(partition_interval_offset_per_proc[dest_proc+1])
+ const long int limite = static_cast<long int>((field_section_width_z*real_number(partition_interval_offset_per_proc[dest_proc+1])
- std::numeric_limits<real_number>::epsilon())/cutoff_radius);
+ if(static_cast<real_number>(limite)*cutoff_radius
+ == field_section_width_z*real_number(partition_interval_offset_per_proc[dest_proc+1])){
+ return limite-1;
+ }
+ return limite;
}
real_number apply_pbc(real_number pos, IDXS_3D dim) const{
@@ -421,7 +426,10 @@ public:
assert(whatNext.size() == 0);
assert(mpiRequests.size() == 0);
-
+#ifndef NDEBUG // Just for assertion
+ std::vector<int> willsend(nb_processes_involved, 0);
+ std::vector<int> willrecv(nb_processes_involved, 0);
+#endif
for(int idxDescr = 0 ; idxDescr < int(neigDescriptors.size()) ; ++idxDescr){
NeighborDescriptor& descriptor = neigDescriptors[idxDescr];
@@ -433,7 +441,9 @@ public:
1, particles_utils::GetMpiType(partsize_t()),
descriptor.destProc, TAG_NB_PARTICLES,
current_com, &mpiRequests.back()));
-
+#ifndef NDEBUG // Just for assertion
+ willsend[descriptor.destProc] += 1;
+#endif
if(descriptor.nbParticlesToExchange){
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
@@ -454,6 +464,9 @@ public:
}
}
else{
+#ifndef NDEBUG // Just for assertion
+ willrecv[descriptor.destProc] += 1;
+#endif
whatNext.emplace_back(std::pair<Action,int>{RECV_PARTICLES, idxDescr});
mpiRequests.emplace_back();
AssertMpi(MPI_Irecv(&descriptor.nbParticlesToExchange,
@@ -462,6 +475,33 @@ public:
}
}
+#ifndef NDEBUG // Just for assertion
+ {
+ if(myrank == 0){
+ std::vector<int> willsendall(nb_processes_involved*nb_processes_involved, 0);// TODO debug
+ std::vector<int> willrecvall(nb_processes_involved*nb_processes_involved, 0);// TODO debug
+
+ MPI_Gather(willrecv.data(), nb_processes_involved, MPI_INT, willrecvall.data(),
+ nb_processes_involved, MPI_INT, 0, MPI_COMM_WORLD);
+ MPI_Gather(willsend.data(), nb_processes_involved, MPI_INT, willsendall.data(),
+ nb_processes_involved, MPI_INT, 0, MPI_COMM_WORLD);
+
+ for(int idxproc = 0 ; idxproc < nb_processes_involved ; ++idxproc){
+ for(int idxtest = 0 ; idxtest < nb_processes_involved ; ++idxtest){
+ assert(willsendall[idxproc*nb_processes_involved + idxtest]
+ == willrecvall[idxtest*nb_processes_involved + idxproc]);
+ }
+ }
+ }
+ else{
+ MPI_Gather(willrecv.data(), nb_processes_involved, MPI_INT, nullptr,
+ 0, MPI_INT, 0, MPI_COMM_WORLD);
+ MPI_Gather(willsend.data(), nb_processes_involved, MPI_INT, nullptr,
+ 0, MPI_INT, 0, MPI_COMM_WORLD);
+ }
+ }
+#endif
+
lock_free_bool_array cells_locker(512);
TIMEZONE_OMP_INIT_PREPARALLEL(omp_get_max_threads())
--
GitLab