Skip to content
GitLab
Commit c14be9dc authored by Cristian Lalescu's avatar Cristian Lalescu
Browse files

Merge branch 'bugfix/particle_distribution' into develop

parents 407d3044 415b6b89
Pipeline #64807 passed with stage
in 4 minutes and 46 seconds
Hide whitespace changes
Inline Side-by-side
TurTLE/test/test_particle_clouds.py
View file @ c14be9dc
......@@ -33,22 +33,22 @@ import sys
import TurTLE
from TurTLE import DNS
def main():
def basic_test():
nclouds = 10
nparticles_per_cloud = 1000
nparticles = nclouds*nparticles_per_cloud
niterations = 32
c = DNS()
c.dns_type = 'NSVEparticles'
c.parameters['nparticles'] = nparticles
c.parameters['tracers1_integration_steps'] = 4
c.generate_tracer_state(rseed = 2, species = 1)
del c.parameters['nparticles']
del c.parameters['tracers1_integration_steps']
c.simname = 'basic_cloud_test'
f0 = h5py.File(
os.path.join(
os.path.join(TurTLE.lib_dir, 'test'),
'B32p1e4_checkpoint_0.h5'),
'r')
ic_file = h5py.File(c.get_checkpoint_0_fname(), 'a')
ic_file['tracers0/state/0'] = ic_file['tracers1/state/0'][...].reshape(nclouds, nparticles_per_cloud, 3)
ic_file['tracers0/rhs/0'] = ic_file['tracers1/rhs/0'][...].reshape(4, nclouds, nparticles_per_cloud, 3)
ic_file['tracers0/state/0'] = f0['tracers0/state/0'][...].reshape(nclouds, nparticles_per_cloud, 3)
ic_file['tracers0/rhs/0'] = f0['tracers0/rhs/0'][...].reshape(4, nclouds, nparticles_per_cloud, 3)
ic_file.close()
c.launch(
['NSVEparticles',
......@@ -57,12 +57,14 @@ def main():
'--forcing_type', 'linear',
'--src-wd', TurTLE.lib_dir + '/test',
'--src-iteration', '0',
'--simname', c.simname,
'--np', '4',
'--ntpp', '1',
'--fftw_plan_rigor', 'FFTW_PATIENT',
'--niter_todo', '{0}'.format(niterations),
'--niter_out', '{0}'.format(niterations),
'--niter_stat', '1',
'--checkpoints_per_file', '{0}'.format(3),
'--nparticles', '{0}'.format(nparticles),
'--njobs', '2',
'--wd', './'])
......@@ -79,6 +81,7 @@ def main():
x0 = f0['tracers0/state/{0}'.format(iteration)][...]
x1 = f1['tracers0/state/{0}'.format(iteration)][...].reshape(x0.shape)
traj_error = np.max(np.abs(x0 - x1))
print(traj_error)
y0 = f0['tracers0/rhs/{0}'.format(iteration)][...]
y1 = f1['tracers0/rhs/{0}'.format(iteration)][...].reshape(y0.shape)
rhs_error = np.max(np.abs(y0 - y1))
......@@ -88,6 +91,53 @@ def main():
print('SUCCESS! Basic test passed.')
return None
def nasty_test():
nclouds = 10
nparticles_per_cloud = 1000000
nparticles = nclouds*nparticles_per_cloud
niterations = 8
c = DNS()
c.dns_type = 'NSVEparticles'
c.simname = 'nasty_cloud_test'
c.parameters['nparticles'] = nparticles
c.parameters['tracers1_integration_steps'] = 4
c.generate_tracer_state(rseed = 2, species = 1)
del c.parameters['nparticles']
del c.parameters['tracers1_integration_steps']
ic_file = h5py.File(c.get_checkpoint_0_fname(), 'a')
ic_file['tracers0/state/0'] = ic_file['tracers1/state/0'][...].reshape(nclouds, nparticles_per_cloud, 3)
ic_file['tracers0/rhs/0'] = ic_file['tracers1/rhs/0'][...].reshape(4, nclouds, nparticles_per_cloud, 3)
# put all particles in single z cell
ic_file['tracers0/state/0'][..., 2] = 0.0001
# put one cloud on another process
ic_file['tracers0/state/0'][1, :, 2] = np.pi + 0.0001
ic_file.close()
c.launch(
['NSVEparticles',
'-n', '8',
'--src-simname', 'B32p1e4',
'--simname', c.simname,
'--forcing_type', 'linear',
'--src-wd', TurTLE.lib_dir + '/test',
'--src-iteration', '0',
'--np', '4',
'--ntpp', '2',
'--fftw_plan_rigor', 'FFTW_PATIENT',
'--niter_todo', '{0}'.format(niterations),
'--niter_out', '{0}'.format(niterations),
'--niter_stat', '1',
'--nparticles', '{0}'.format(nparticles),
'--njobs', '1',
'--wd', './'])
print('SUCCESS! Nasty test passed.')
return None
def main():
basic_test()
nasty_test()
return None
if __name__ == '__main__':
main()
cpp/particles/particles_distr_mpi.hpp
View file @ c14be9dc
......@@ -45,7 +45,7 @@ protected:
static const int MaxNbRhs = 10;
enum MpiTag{
TAG_LOW_UP_NB_PARTICLES,
TAG_LOW_UP_NB_PARTICLES = 999,
TAG_UP_LOW_NB_PARTICLES,
TAG_LOW_UP_PARTICLES,
TAG_UP_LOW_PARTICLES,
......@@ -65,6 +65,8 @@ protected:
TAG_UP_LOW_MOVED_PARTICLES_RHS = TAG_LOW_UP_MOVED_PARTICLES_RHS_MAX,
TAG_UP_LOW_MOVED_PARTICLES_RHS_MAX = TAG_UP_LOW_MOVED_PARTICLES_RHS+MaxNbRhs,
TAG_SHIFT_OFFSET
};
struct NeighborDescriptor{
......@@ -114,6 +116,8 @@ protected:
std::vector<MPI_Request> mpiRequests;
std::vector<NeighborDescriptor> neigDescriptors;
int counter_shift_tags;
public:
////////////////////////////////////////////////////////////////////////////
......@@ -124,7 +128,8 @@ public:
my_rank(-1), nb_processes(-1),nb_processes_involved(-1),
current_partition_interval(in_current_partitions),
current_partition_size(current_partition_interval.second-current_partition_interval.first),
field_grid_dim(in_field_grid_dim){
field_grid_dim(in_field_grid_dim),
counter_shift_tags(0){
AssertMpi(MPI_Comm_rank(current_com, &my_rank));
AssertMpi(MPI_Comm_size(current_com, &nb_processes));
......@@ -271,14 +276,14 @@ public:
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
AssertMpi(MPI_Isend(const_cast<partsize_t*>(&descriptor.nbParticlesToSend), 1, particles_utils::GetMpiType(partsize_t()),
descriptor.destProc, TAG_LOW_UP_NB_PARTICLES,
descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_NB_PARTICLES,
current_com, &mpiRequests.back()));
if(descriptor.nbParticlesToSend){
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
assert(descriptor.nbParticlesToSend*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(const_cast<real_number*>(&particles_positions[0]), int(descriptor.nbParticlesToSend*size_particle_positions), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_LOW_UP_PARTICLES,
AssertMpi(MPI_Isend(const_cast<real_number*>(&particles_positions[0]), int(descriptor.nbParticlesToSend*size_particle_positions), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_PARTICLES,
current_com, &mpiRequests.back()));
assert(descriptor.toRecvAndMerge == nullptr);
......@@ -286,7 +291,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{MERGE_PARTICLES, idxDescr});
mpiRequests.emplace_back();
assert(descriptor.nbParticlesToSend*size_particle_rhs < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(descriptor.toRecvAndMerge.get(), int(descriptor.nbParticlesToSend*size_particle_rhs), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_UP_LOW_RESULTS,
AssertMpi(MPI_Irecv(descriptor.toRecvAndMerge.get(), int(descriptor.nbParticlesToSend*size_particle_rhs), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_RESULTS,
current_com, &mpiRequests.back()));
}
}
......@@ -295,7 +300,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{RECV_PARTICLES, idxDescr});
mpiRequests.emplace_back();
AssertMpi(MPI_Irecv(&descriptor.nbParticlesToRecv,
1, particles_utils::GetMpiType(partsize_t()), descriptor.destProc, TAG_UP_LOW_NB_PARTICLES,
1, particles_utils::GetMpiType(partsize_t()), descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_NB_PARTICLES,
current_com, &mpiRequests.back()));
}
else{
......@@ -303,7 +308,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
AssertMpi(MPI_Isend(const_cast<partsize_t*>(&descriptor.nbParticlesToSend), 1, particles_utils::GetMpiType(partsize_t()),
descriptor.destProc, TAG_UP_LOW_NB_PARTICLES,
descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_NB_PARTICLES,
current_com, &mpiRequests.back()));
if(descriptor.nbParticlesToSend){
......@@ -312,7 +317,7 @@ public:
assert(descriptor.nbParticlesToSend*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(const_cast<real_number*>(&particles_positions[(current_offset_particles_for_partition[current_partition_size-descriptor.nbPartitionsToSend])*size_particle_positions]),
int(descriptor.nbParticlesToSend*size_particle_positions), particles_utils::GetMpiType(real_number()),
descriptor.destProc, TAG_UP_LOW_PARTICLES,
descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_PARTICLES,
current_com, &mpiRequests.back()));
assert(descriptor.toRecvAndMerge == nullptr);
......@@ -320,7 +325,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{MERGE_PARTICLES, idxDescr});
mpiRequests.emplace_back();
assert(descriptor.nbParticlesToSend*size_particle_rhs < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(descriptor.toRecvAndMerge.get(), int(descriptor.nbParticlesToSend*size_particle_rhs), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_LOW_UP_RESULTS,
AssertMpi(MPI_Irecv(descriptor.toRecvAndMerge.get(), int(descriptor.nbParticlesToSend*size_particle_rhs), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_RESULTS,
current_com, &mpiRequests.back()));
}
......@@ -328,15 +333,15 @@ public:
whatNext.emplace_back(std::pair<Action,int>{RECV_PARTICLES, idxDescr});
mpiRequests.emplace_back();
AssertMpi(MPI_Irecv(&descriptor.nbParticlesToRecv,
1, particles_utils::GetMpiType(partsize_t()), descriptor.destProc, TAG_LOW_UP_NB_PARTICLES,
1, particles_utils::GetMpiType(partsize_t()), descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_NB_PARTICLES,
current_com, &mpiRequests.back()));
}
}
}
const bool more_than_one_thread = (omp_get_max_threads() > 1);
MPI_Barrier(MPI_COMM_WORLD);
//DEBUG_MSG_WAIT(MPI_COMM_WORLD, "line 338 of particles_distr_mpi.hpp\n");
/// MPI_Barrier(MPI_COMM_WORLD);
/// DEBUG_MSG_WAIT(MPI_COMM_WORLD, "line 338 of particles_distr_mpi.hpp\n");
TIMEZONE_OMP_INIT_PREPARALLEL(omp_get_max_threads())
#pragma omp parallel default(shared)
......@@ -376,7 +381,7 @@ public:
mpiRequests.emplace_back();
assert(NbParticlesToReceive*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(descriptor.toCompute.get(), int(NbParticlesToReceive*size_particle_positions),
particles_utils::GetMpiType(real_number()), destProc, TAG_UP_LOW_PARTICLES,
particles_utils::GetMpiType(real_number()), destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_PARTICLES,
current_com, &mpiRequests.back()));
}
}
......@@ -393,7 +398,7 @@ public:
mpiRequests.emplace_back();
assert(NbParticlesToReceive*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(descriptor.toCompute.get(), int(NbParticlesToReceive*size_particle_positions),
particles_utils::GetMpiType(real_number()), destProc, TAG_LOW_UP_PARTICLES,
particles_utils::GetMpiType(real_number()), destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_PARTICLES,
current_com, &mpiRequests.back()));
}
}
......@@ -434,7 +439,7 @@ public:
const int destProc = descriptor.destProc;
whatNext.emplace_back(std::pair<Action,int>{RELEASE_BUFFER_PARTICLES, releasedAction.second});
mpiRequests.emplace_back();
const int tag = descriptor.isLower? TAG_LOW_UP_RESULTS : TAG_UP_LOW_RESULTS;
const int tag = descriptor.isLower? TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_RESULTS : TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_RESULTS;
assert(NbParticlesToReceive*size_particle_rhs < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(descriptor.results.get(), int(NbParticlesToReceive*size_particle_rhs), particles_utils::GetMpiType(real_number()), destProc, tag,
current_com, &mpiRequests.back()));
......@@ -526,6 +531,8 @@ public:
assert(whatNext.size() == 0);
assert(mpiRequests.size() == 0);
counter_shift_tags += 1;
}
......@@ -635,35 +642,35 @@ public:
whatNext.emplace_back(std::pair<Action,int>{RECV_MOVE_NB_LOW, -1});
mpiRequests.emplace_back();
AssertMpi(MPI_Irecv(&nbNewFromLow, 1, particles_utils::GetMpiType(partsize_t()),
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_UP_LOW_MOVED_NB_PARTICLES,
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_NB_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
eventsBeforeWaitall += 1;
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
AssertMpi(MPI_Isend(const_cast<partsize_t*>(&nbOutLower), 1, particles_utils::GetMpiType(partsize_t()),
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_LOW_UP_MOVED_NB_PARTICLES,
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_NB_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
if(nbOutLower){
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
assert(nbOutLower*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(&(*inout_positions_particles)[0], int(nbOutLower*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES,
AssertMpi(MPI_Isend(&(*inout_positions_particles)[0], int(nbOutLower*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
assert(nbOutLower*size_particle_index < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(&(*inout_index_particles)[0], int(nbOutLower*size_particle_index), particles_utils::GetMpiType(partsize_t()),
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES_INDEXES,
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES_INDEXES,
MPI_COMM_WORLD, &mpiRequests.back()));
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
assert(nbOutLower*size_particle_rhs < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(&inout_rhs_particles[idx_rhs][0], int(nbOutLower*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES_RHS+idx_rhs,
AssertMpi(MPI_Isend(&inout_rhs_particles[idx_rhs][0], int(nbOutLower*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES_RHS+idx_rhs,
MPI_COMM_WORLD, &mpiRequests.back()));
}
}
......@@ -671,14 +678,14 @@ public:
whatNext.emplace_back(std::pair<Action,int>{RECV_MOVE_NB_UP, -1});
mpiRequests.emplace_back();
AssertMpi(MPI_Irecv(&nbNewFromUp, 1, particles_utils::GetMpiType(partsize_t()), (my_rank+1)%nb_processes_involved,
TAG_LOW_UP_MOVED_NB_PARTICLES,
TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_NB_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
eventsBeforeWaitall += 1;
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
AssertMpi(MPI_Isend(const_cast<partsize_t*>(&nbOutUpper), 1, particles_utils::GetMpiType(partsize_t()),
(my_rank+1)%nb_processes_involved, TAG_UP_LOW_MOVED_NB_PARTICLES,
(my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_NB_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
if(nbOutUpper){
......@@ -686,14 +693,14 @@ public:
mpiRequests.emplace_back();
assert(nbOutUpper*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(&(*inout_positions_particles)[(myTotalNbParticles-nbOutUpper)*size_particle_positions],
int(nbOutUpper*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES,
int(nbOutUpper*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
assert(nbOutUpper*size_particle_index < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(&(*inout_index_particles)[(myTotalNbParticles-nbOutUpper)*size_particle_index], int(nbOutUpper*size_particle_index),
particles_utils::GetMpiType(partsize_t()), (my_rank+1)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES_INDEXES,
particles_utils::GetMpiType(partsize_t()), (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES_INDEXES,
MPI_COMM_WORLD, &mpiRequests.back()));
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
......@@ -701,7 +708,7 @@ public:
mpiRequests.emplace_back();
assert(nbOutUpper*size_particle_rhs < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(&inout_rhs_particles[idx_rhs][(myTotalNbParticles-nbOutUpper)*size_particle_rhs],
int(nbOutUpper*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES_RHS+idx_rhs,
int(nbOutUpper*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES_RHS+idx_rhs,
MPI_COMM_WORLD, &mpiRequests.back()));
}
}
......@@ -726,7 +733,7 @@ public:
mpiRequests.emplace_back();
assert(nbNewFromLow*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(&newParticlesLow[0], int(nbNewFromLow*size_particle_positions), particles_utils::GetMpiType(real_number()),
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES,
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
newParticlesLowIndexes.reset(new partsize_t[nbNewFromLow*size_particle_index]);
......@@ -735,7 +742,7 @@ public:
assert(nbNewFromLow*size_particle_index < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(&newParticlesLowIndexes[0], int(nbNewFromLow*size_particle_index),
particles_utils::GetMpiType(partsize_t()),
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES_INDEXES,
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES_INDEXES,
MPI_COMM_WORLD, &mpiRequests.back()));
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
......@@ -743,7 +750,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
assert(nbNewFromLow*size_particle_rhs < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(&newParticlesLowRhs[idx_rhs][0], int(nbNewFromLow*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES_RHS+idx_rhs,
AssertMpi(MPI_Irecv(&newParticlesLowRhs[idx_rhs][0], int(nbNewFromLow*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES_RHS+idx_rhs,
MPI_COMM_WORLD, &mpiRequests.back()));
}
}
......@@ -756,7 +763,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{RECV_MOVE_UP, -1});
mpiRequests.emplace_back();
assert(nbNewFromUp*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(&newParticlesUp[0], int(nbNewFromUp*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES,
AssertMpi(MPI_Irecv(&newParticlesUp[0], int(nbNewFromUp*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
newParticlesUpIndexes.reset(new partsize_t[nbNewFromUp*size_particle_index]);
......@@ -765,7 +772,7 @@ public:
assert(nbNewFromUp*size_particle_index < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(&newParticlesUpIndexes[0], int(nbNewFromUp*size_particle_index),
particles_utils::GetMpiType(partsize_t()),
(my_rank+1)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES_INDEXES,
(my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES_INDEXES,
MPI_COMM_WORLD, &mpiRequests.back()));
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
......@@ -773,7 +780,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
assert(nbNewFromUp*size_particle_rhs < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(&newParticlesUpRhs[idx_rhs][0], int(nbNewFromUp*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES_RHS+idx_rhs,
AssertMpi(MPI_Irecv(&newParticlesUpRhs[idx_rhs][0], int(nbNewFromUp*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES_RHS+idx_rhs,
MPI_COMM_WORLD, &mpiRequests.back()));
}
}
......@@ -880,6 +887,8 @@ public:
(*nb_particles) = myTotalNbParticles;
assert(mpiRequests.size() == 0);
counter_shift_tags += 1;
}
};
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment