Skip to content
Snippets Groups Projects
Commit b44c214d authored by Berenger Bramas's avatar Berenger Bramas
Browse files

Start tags from a higher value -- shift the tag used at each call

parent 0caebd29
No related branches found
No related tags found
No related merge requests found
Pipeline #46707 failed
......@@ -45,7 +45,7 @@ protected:
static const int MaxNbRhs = 10;
enum MpiTag{
TAG_LOW_UP_NB_PARTICLES,
TAG_LOW_UP_NB_PARTICLES = 999,
TAG_UP_LOW_NB_PARTICLES,
TAG_LOW_UP_PARTICLES,
TAG_UP_LOW_PARTICLES,
......@@ -65,6 +65,8 @@ protected:
TAG_UP_LOW_MOVED_PARTICLES_RHS = TAG_LOW_UP_MOVED_PARTICLES_RHS_MAX,
TAG_UP_LOW_MOVED_PARTICLES_RHS_MAX = TAG_UP_LOW_MOVED_PARTICLES_RHS+MaxNbRhs,
TAG_SHIFT_OFFSET
};
struct NeighborDescriptor{
......@@ -114,6 +116,8 @@ protected:
std::vector<MPI_Request> mpiRequests;
std::vector<NeighborDescriptor> neigDescriptors;
int counter_shift_tags;
public:
////////////////////////////////////////////////////////////////////////////
......@@ -124,7 +128,8 @@ public:
my_rank(-1), nb_processes(-1),nb_processes_involved(-1),
current_partition_interval(in_current_partitions),
current_partition_size(current_partition_interval.second-current_partition_interval.first),
field_grid_dim(in_field_grid_dim){
field_grid_dim(in_field_grid_dim),
counter_shift_tags(0){
AssertMpi(MPI_Comm_rank(current_com, &my_rank));
AssertMpi(MPI_Comm_size(current_com, &nb_processes));
......@@ -271,14 +276,14 @@ public:
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
AssertMpi(MPI_Isend(const_cast<partsize_t*>(&descriptor.nbParticlesToSend), 1, particles_utils::GetMpiType(partsize_t()),
descriptor.destProc, TAG_LOW_UP_NB_PARTICLES,
descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_NB_PARTICLES,
current_com, &mpiRequests.back()));
if(descriptor.nbParticlesToSend){
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
assert(descriptor.nbParticlesToSend*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(const_cast<real_number*>(&particles_positions[0]), int(descriptor.nbParticlesToSend*size_particle_positions), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_LOW_UP_PARTICLES,
AssertMpi(MPI_Isend(const_cast<real_number*>(&particles_positions[0]), int(descriptor.nbParticlesToSend*size_particle_positions), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_PARTICLES,
current_com, &mpiRequests.back()));
assert(descriptor.toRecvAndMerge == nullptr);
......@@ -286,7 +291,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{MERGE_PARTICLES, idxDescr});
mpiRequests.emplace_back();
assert(descriptor.nbParticlesToSend*size_particle_rhs < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(descriptor.toRecvAndMerge.get(), int(descriptor.nbParticlesToSend*size_particle_rhs), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_UP_LOW_RESULTS,
AssertMpi(MPI_Irecv(descriptor.toRecvAndMerge.get(), int(descriptor.nbParticlesToSend*size_particle_rhs), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_RESULTS,
current_com, &mpiRequests.back()));
}
}
......@@ -295,7 +300,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{RECV_PARTICLES, idxDescr});
mpiRequests.emplace_back();
AssertMpi(MPI_Irecv(&descriptor.nbParticlesToRecv,
1, particles_utils::GetMpiType(partsize_t()), descriptor.destProc, TAG_UP_LOW_NB_PARTICLES,
1, particles_utils::GetMpiType(partsize_t()), descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_NB_PARTICLES,
current_com, &mpiRequests.back()));
}
else{
......@@ -303,7 +308,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
AssertMpi(MPI_Isend(const_cast<partsize_t*>(&descriptor.nbParticlesToSend), 1, particles_utils::GetMpiType(partsize_t()),
descriptor.destProc, TAG_UP_LOW_NB_PARTICLES,
descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_NB_PARTICLES,
current_com, &mpiRequests.back()));
if(descriptor.nbParticlesToSend){
......@@ -312,7 +317,7 @@ public:
assert(descriptor.nbParticlesToSend*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(const_cast<real_number*>(&particles_positions[(current_offset_particles_for_partition[current_partition_size-descriptor.nbPartitionsToSend])*size_particle_positions]),
int(descriptor.nbParticlesToSend*size_particle_positions), particles_utils::GetMpiType(real_number()),
descriptor.destProc, TAG_UP_LOW_PARTICLES,
descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_PARTICLES,
current_com, &mpiRequests.back()));
assert(descriptor.toRecvAndMerge == nullptr);
......@@ -320,7 +325,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{MERGE_PARTICLES, idxDescr});
mpiRequests.emplace_back();
assert(descriptor.nbParticlesToSend*size_particle_rhs < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(descriptor.toRecvAndMerge.get(), int(descriptor.nbParticlesToSend*size_particle_rhs), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_LOW_UP_RESULTS,
AssertMpi(MPI_Irecv(descriptor.toRecvAndMerge.get(), int(descriptor.nbParticlesToSend*size_particle_rhs), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_RESULTS,
current_com, &mpiRequests.back()));
}
......@@ -328,15 +333,15 @@ public:
whatNext.emplace_back(std::pair<Action,int>{RECV_PARTICLES, idxDescr});
mpiRequests.emplace_back();
AssertMpi(MPI_Irecv(&descriptor.nbParticlesToRecv,
1, particles_utils::GetMpiType(partsize_t()), descriptor.destProc, TAG_LOW_UP_NB_PARTICLES,
1, particles_utils::GetMpiType(partsize_t()), descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_NB_PARTICLES,
current_com, &mpiRequests.back()));
}
}
}
const bool more_than_one_thread = (omp_get_max_threads() > 1);
MPI_Barrier(MPI_COMM_WORLD);
DEBUG_MSG_WAIT(MPI_COMM_WORLD, "line 338 of particles_distr_mpi.hpp\n");
/// MPI_Barrier(MPI_COMM_WORLD);
/// DEBUG_MSG_WAIT(MPI_COMM_WORLD, "line 338 of particles_distr_mpi.hpp\n");
TIMEZONE_OMP_INIT_PREPARALLEL(omp_get_max_threads())
#pragma omp parallel default(shared)
......@@ -376,7 +381,7 @@ public:
mpiRequests.emplace_back();
assert(NbParticlesToReceive*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(descriptor.toCompute.get(), int(NbParticlesToReceive*size_particle_positions),
particles_utils::GetMpiType(real_number()), destProc, TAG_UP_LOW_PARTICLES,
particles_utils::GetMpiType(real_number()), destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_PARTICLES,
current_com, &mpiRequests.back()));
}
}
......@@ -393,7 +398,7 @@ public:
mpiRequests.emplace_back();
assert(NbParticlesToReceive*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(descriptor.toCompute.get(), int(NbParticlesToReceive*size_particle_positions),
particles_utils::GetMpiType(real_number()), destProc, TAG_LOW_UP_PARTICLES,
particles_utils::GetMpiType(real_number()), destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_PARTICLES,
current_com, &mpiRequests.back()));
}
}
......@@ -434,7 +439,7 @@ public:
const int destProc = descriptor.destProc;
whatNext.emplace_back(std::pair<Action,int>{RELEASE_BUFFER_PARTICLES, releasedAction.second});
mpiRequests.emplace_back();
const int tag = descriptor.isLower? TAG_LOW_UP_RESULTS : TAG_UP_LOW_RESULTS;
const int tag = descriptor.isLower? TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_RESULTS : TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_RESULTS;
assert(NbParticlesToReceive*size_particle_rhs < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(descriptor.results.get(), int(NbParticlesToReceive*size_particle_rhs), particles_utils::GetMpiType(real_number()), destProc, tag,
current_com, &mpiRequests.back()));
......@@ -526,6 +531,8 @@ public:
assert(whatNext.size() == 0);
assert(mpiRequests.size() == 0);
counter_shift_tags += 1;
}
......@@ -635,35 +642,35 @@ public:
whatNext.emplace_back(std::pair<Action,int>{RECV_MOVE_NB_LOW, -1});
mpiRequests.emplace_back();
AssertMpi(MPI_Irecv(&nbNewFromLow, 1, particles_utils::GetMpiType(partsize_t()),
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_UP_LOW_MOVED_NB_PARTICLES,
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_NB_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
eventsBeforeWaitall += 1;
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
AssertMpi(MPI_Isend(const_cast<partsize_t*>(&nbOutLower), 1, particles_utils::GetMpiType(partsize_t()),
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_LOW_UP_MOVED_NB_PARTICLES,
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_NB_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
if(nbOutLower){
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
assert(nbOutLower*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(&(*inout_positions_particles)[0], int(nbOutLower*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES,
AssertMpi(MPI_Isend(&(*inout_positions_particles)[0], int(nbOutLower*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
assert(nbOutLower*size_particle_index < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(&(*inout_index_particles)[0], int(nbOutLower*size_particle_index), particles_utils::GetMpiType(partsize_t()),
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES_INDEXES,
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES_INDEXES,
MPI_COMM_WORLD, &mpiRequests.back()));
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
assert(nbOutLower*size_particle_rhs < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(&inout_rhs_particles[idx_rhs][0], int(nbOutLower*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES_RHS+idx_rhs,
AssertMpi(MPI_Isend(&inout_rhs_particles[idx_rhs][0], int(nbOutLower*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES_RHS+idx_rhs,
MPI_COMM_WORLD, &mpiRequests.back()));
}
}
......@@ -671,14 +678,14 @@ public:
whatNext.emplace_back(std::pair<Action,int>{RECV_MOVE_NB_UP, -1});
mpiRequests.emplace_back();
AssertMpi(MPI_Irecv(&nbNewFromUp, 1, particles_utils::GetMpiType(partsize_t()), (my_rank+1)%nb_processes_involved,
TAG_LOW_UP_MOVED_NB_PARTICLES,
TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_NB_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
eventsBeforeWaitall += 1;
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
AssertMpi(MPI_Isend(const_cast<partsize_t*>(&nbOutUpper), 1, particles_utils::GetMpiType(partsize_t()),
(my_rank+1)%nb_processes_involved, TAG_UP_LOW_MOVED_NB_PARTICLES,
(my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_NB_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
if(nbOutUpper){
......@@ -686,14 +693,14 @@ public:
mpiRequests.emplace_back();
assert(nbOutUpper*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(&(*inout_positions_particles)[(myTotalNbParticles-nbOutUpper)*size_particle_positions],
int(nbOutUpper*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES,
int(nbOutUpper*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
assert(nbOutUpper*size_particle_index < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(&(*inout_index_particles)[(myTotalNbParticles-nbOutUpper)*size_particle_index], int(nbOutUpper*size_particle_index),
particles_utils::GetMpiType(partsize_t()), (my_rank+1)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES_INDEXES,
particles_utils::GetMpiType(partsize_t()), (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES_INDEXES,
MPI_COMM_WORLD, &mpiRequests.back()));
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
......@@ -701,7 +708,7 @@ public:
mpiRequests.emplace_back();
assert(nbOutUpper*size_particle_rhs < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(&inout_rhs_particles[idx_rhs][(myTotalNbParticles-nbOutUpper)*size_particle_rhs],
int(nbOutUpper*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES_RHS+idx_rhs,
int(nbOutUpper*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES_RHS+idx_rhs,
MPI_COMM_WORLD, &mpiRequests.back()));
}
}
......@@ -726,7 +733,7 @@ public:
mpiRequests.emplace_back();
assert(nbNewFromLow*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(&newParticlesLow[0], int(nbNewFromLow*size_particle_positions), particles_utils::GetMpiType(real_number()),
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES,
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
newParticlesLowIndexes.reset(new partsize_t[nbNewFromLow*size_particle_index]);
......@@ -735,7 +742,7 @@ public:
assert(nbNewFromLow*size_particle_index < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(&newParticlesLowIndexes[0], int(nbNewFromLow*size_particle_index),
particles_utils::GetMpiType(partsize_t()),
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES_INDEXES,
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES_INDEXES,
MPI_COMM_WORLD, &mpiRequests.back()));
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
......@@ -743,7 +750,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
assert(nbNewFromLow*size_particle_rhs < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(&newParticlesLowRhs[idx_rhs][0], int(nbNewFromLow*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES_RHS+idx_rhs,
AssertMpi(MPI_Irecv(&newParticlesLowRhs[idx_rhs][0], int(nbNewFromLow*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES_RHS+idx_rhs,
MPI_COMM_WORLD, &mpiRequests.back()));
}
}
......@@ -756,7 +763,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{RECV_MOVE_UP, -1});
mpiRequests.emplace_back();
assert(nbNewFromUp*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(&newParticlesUp[0], int(nbNewFromUp*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES,
AssertMpi(MPI_Irecv(&newParticlesUp[0], int(nbNewFromUp*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
newParticlesUpIndexes.reset(new partsize_t[nbNewFromUp*size_particle_index]);
......@@ -765,7 +772,7 @@ public:
assert(nbNewFromUp*size_particle_index < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(&newParticlesUpIndexes[0], int(nbNewFromUp*size_particle_index),
particles_utils::GetMpiType(partsize_t()),
(my_rank+1)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES_INDEXES,
(my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES_INDEXES,
MPI_COMM_WORLD, &mpiRequests.back()));
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
......@@ -773,7 +780,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
assert(nbNewFromUp*size_particle_rhs < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(&newParticlesUpRhs[idx_rhs][0], int(nbNewFromUp*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES_RHS+idx_rhs,
AssertMpi(MPI_Irecv(&newParticlesUpRhs[idx_rhs][0], int(nbNewFromUp*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES_RHS+idx_rhs,
MPI_COMM_WORLD, &mpiRequests.back()));
}
}
......@@ -880,6 +887,8 @@ public:
(*nb_particles) = myTotalNbParticles;
assert(mpiRequests.size() == 0);
counter_shift_tags += 1;
}
};
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment