Skip to content
Snippets Groups Projects
Commit c14be9dc authored by Cristian Lalescu's avatar Cristian Lalescu
Browse files

Merge branch 'bugfix/particle_distribution' into develop

parents 407d3044 415b6b89
Branches
Tags
No related merge requests found
Pipeline #64807 passed
Stage: build
Hide whitespace changes
Inline Side-by-side
TurTLE/test/test_particle_clouds.py
+ 59
− 9
View file @ c14be9dc
...@@ -33,22 +33,22 @@ import sys ...@@ -33,22 +33,22 @@ import sys
import TurTLE import TurTLE
from TurTLE import DNS from TurTLE import DNS
def basic_test():
def main():
nclouds = 10 nclouds = 10
nparticles_per_cloud = 1000 nparticles_per_cloud = 1000
nparticles = nclouds*nparticles_per_cloud nparticles = nclouds*nparticles_per_cloud
niterations = 32 niterations = 32
c = DNS() c = DNS()
c.dns_type = 'NSVEparticles' c.dns_type = 'NSVEparticles'
c.parameters['nparticles'] = nparticles c.simname = 'basic_cloud_test'
c.parameters['tracers1_integration_steps'] = 4 f0 = h5py.File(
c.generate_tracer_state(rseed = 2, species = 1) os.path.join(
del c.parameters['nparticles'] os.path.join(TurTLE.lib_dir, 'test'),
del c.parameters['tracers1_integration_steps'] 'B32p1e4_checkpoint_0.h5'),
'r')
ic_file = h5py.File(c.get_checkpoint_0_fname(), 'a') ic_file = h5py.File(c.get_checkpoint_0_fname(), 'a')
ic_file['tracers0/state/0'] = ic_file['tracers1/state/0'][...].reshape(nclouds, nparticles_per_cloud, 3) ic_file['tracers0/state/0'] = f0['tracers0/state/0'][...].reshape(nclouds, nparticles_per_cloud, 3)
ic_file['tracers0/rhs/0'] = ic_file['tracers1/rhs/0'][...].reshape(4, nclouds, nparticles_per_cloud, 3) ic_file['tracers0/rhs/0'] = f0['tracers0/rhs/0'][...].reshape(4, nclouds, nparticles_per_cloud, 3)
ic_file.close() ic_file.close()
c.launch( c.launch(
['NSVEparticles', ['NSVEparticles',
...@@ -57,12 +57,14 @@ def main(): ...@@ -57,12 +57,14 @@ def main():
'--forcing_type', 'linear', '--forcing_type', 'linear',
'--src-wd', TurTLE.lib_dir + '/test', '--src-wd', TurTLE.lib_dir + '/test',
'--src-iteration', '0', '--src-iteration', '0',
'--simname', c.simname,
'--np', '4', '--np', '4',
'--ntpp', '1', '--ntpp', '1',
'--fftw_plan_rigor', 'FFTW_PATIENT', '--fftw_plan_rigor', 'FFTW_PATIENT',
'--niter_todo', '{0}'.format(niterations), '--niter_todo', '{0}'.format(niterations),
'--niter_out', '{0}'.format(niterations), '--niter_out', '{0}'.format(niterations),
'--niter_stat', '1', '--niter_stat', '1',
'--checkpoints_per_file', '{0}'.format(3),
'--nparticles', '{0}'.format(nparticles), '--nparticles', '{0}'.format(nparticles),
'--njobs', '2', '--njobs', '2',
'--wd', './']) '--wd', './'])
...@@ -79,6 +81,7 @@ def main(): ...@@ -79,6 +81,7 @@ def main():
x0 = f0['tracers0/state/{0}'.format(iteration)][...] x0 = f0['tracers0/state/{0}'.format(iteration)][...]
x1 = f1['tracers0/state/{0}'.format(iteration)][...].reshape(x0.shape) x1 = f1['tracers0/state/{0}'.format(iteration)][...].reshape(x0.shape)
traj_error = np.max(np.abs(x0 - x1)) traj_error = np.max(np.abs(x0 - x1))
print(traj_error)
y0 = f0['tracers0/rhs/{0}'.format(iteration)][...] y0 = f0['tracers0/rhs/{0}'.format(iteration)][...]
y1 = f1['tracers0/rhs/{0}'.format(iteration)][...].reshape(y0.shape) y1 = f1['tracers0/rhs/{0}'.format(iteration)][...].reshape(y0.shape)
rhs_error = np.max(np.abs(y0 - y1)) rhs_error = np.max(np.abs(y0 - y1))
...@@ -88,6 +91,53 @@ def main(): ...@@ -88,6 +91,53 @@ def main():
print('SUCCESS! Basic test passed.') print('SUCCESS! Basic test passed.')
return None return None
def nasty_test():
nclouds = 10
nparticles_per_cloud = 1000000
nparticles = nclouds*nparticles_per_cloud
niterations = 8
c = DNS()
c.dns_type = 'NSVEparticles'
c.simname = 'nasty_cloud_test'
c.parameters['nparticles'] = nparticles
c.parameters['tracers1_integration_steps'] = 4
c.generate_tracer_state(rseed = 2, species = 1)
del c.parameters['nparticles']
del c.parameters['tracers1_integration_steps']
ic_file = h5py.File(c.get_checkpoint_0_fname(), 'a')
ic_file['tracers0/state/0'] = ic_file['tracers1/state/0'][...].reshape(nclouds, nparticles_per_cloud, 3)
ic_file['tracers0/rhs/0'] = ic_file['tracers1/rhs/0'][...].reshape(4, nclouds, nparticles_per_cloud, 3)
# put all particles in single z cell
ic_file['tracers0/state/0'][..., 2] = 0.0001
# put one cloud on another process
ic_file['tracers0/state/0'][1, :, 2] = np.pi + 0.0001
ic_file.close()
c.launch(
['NSVEparticles',
'-n', '8',
'--src-simname', 'B32p1e4',
'--simname', c.simname,
'--forcing_type', 'linear',
'--src-wd', TurTLE.lib_dir + '/test',
'--src-iteration', '0',
'--np', '4',
'--ntpp', '2',
'--fftw_plan_rigor', 'FFTW_PATIENT',
'--niter_todo', '{0}'.format(niterations),
'--niter_out', '{0}'.format(niterations),
'--niter_stat', '1',
'--nparticles', '{0}'.format(nparticles),
'--njobs', '1',
'--wd', './'])
print('SUCCESS! Nasty test passed.')
return None
def main():
basic_test()
nasty_test()
return None
if __name__ == '__main__': if __name__ == '__main__':
main() main()
cpp/particles/particles_distr_mpi.hpp
+ 40
− 31
View file @ c14be9dc
...@@ -45,7 +45,7 @@ protected: ...@@ -45,7 +45,7 @@ protected:
static const int MaxNbRhs = 10; static const int MaxNbRhs = 10;
enum MpiTag{ enum MpiTag{
TAG_LOW_UP_NB_PARTICLES, TAG_LOW_UP_NB_PARTICLES = 999,
TAG_UP_LOW_NB_PARTICLES, TAG_UP_LOW_NB_PARTICLES,
TAG_LOW_UP_PARTICLES, TAG_LOW_UP_PARTICLES,
TAG_UP_LOW_PARTICLES, TAG_UP_LOW_PARTICLES,
...@@ -65,6 +65,8 @@ protected: ...@@ -65,6 +65,8 @@ protected:
TAG_UP_LOW_MOVED_PARTICLES_RHS = TAG_LOW_UP_MOVED_PARTICLES_RHS_MAX, TAG_UP_LOW_MOVED_PARTICLES_RHS = TAG_LOW_UP_MOVED_PARTICLES_RHS_MAX,
TAG_UP_LOW_MOVED_PARTICLES_RHS_MAX = TAG_UP_LOW_MOVED_PARTICLES_RHS+MaxNbRhs, TAG_UP_LOW_MOVED_PARTICLES_RHS_MAX = TAG_UP_LOW_MOVED_PARTICLES_RHS+MaxNbRhs,
TAG_SHIFT_OFFSET
}; };
struct NeighborDescriptor{ struct NeighborDescriptor{
...@@ -114,6 +116,8 @@ protected: ...@@ -114,6 +116,8 @@ protected:
std::vector<MPI_Request> mpiRequests; std::vector<MPI_Request> mpiRequests;
std::vector<NeighborDescriptor> neigDescriptors; std::vector<NeighborDescriptor> neigDescriptors;
int counter_shift_tags;
public: public:
//////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////
...@@ -124,7 +128,8 @@ public: ...@@ -124,7 +128,8 @@ public:
my_rank(-1), nb_processes(-1),nb_processes_involved(-1), my_rank(-1), nb_processes(-1),nb_processes_involved(-1),
current_partition_interval(in_current_partitions), current_partition_interval(in_current_partitions),
current_partition_size(current_partition_interval.second-current_partition_interval.first), current_partition_size(current_partition_interval.second-current_partition_interval.first),
field_grid_dim(in_field_grid_dim){ field_grid_dim(in_field_grid_dim),
counter_shift_tags(0){
AssertMpi(MPI_Comm_rank(current_com, &my_rank)); AssertMpi(MPI_Comm_rank(current_com, &my_rank));
AssertMpi(MPI_Comm_size(current_com, &nb_processes)); AssertMpi(MPI_Comm_size(current_com, &nb_processes));
...@@ -271,14 +276,14 @@ public: ...@@ -271,14 +276,14 @@ public:
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1}); whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back(); mpiRequests.emplace_back();
AssertMpi(MPI_Isend(const_cast<partsize_t*>(&descriptor.nbParticlesToSend), 1, particles_utils::GetMpiType(partsize_t()), AssertMpi(MPI_Isend(const_cast<partsize_t*>(&descriptor.nbParticlesToSend), 1, particles_utils::GetMpiType(partsize_t()),
descriptor.destProc, TAG_LOW_UP_NB_PARTICLES, descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_NB_PARTICLES,
current_com, &mpiRequests.back())); current_com, &mpiRequests.back()));
if(descriptor.nbParticlesToSend){ if(descriptor.nbParticlesToSend){
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1}); whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back(); mpiRequests.emplace_back();
assert(descriptor.nbParticlesToSend*size_particle_positions < std::numeric_limits<int>::max()); assert(descriptor.nbParticlesToSend*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(const_cast<real_number*>(&particles_positions[0]), int(descriptor.nbParticlesToSend*size_particle_positions), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_LOW_UP_PARTICLES, AssertMpi(MPI_Isend(const_cast<real_number*>(&particles_positions[0]), int(descriptor.nbParticlesToSend*size_particle_positions), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_PARTICLES,
current_com, &mpiRequests.back())); current_com, &mpiRequests.back()));
assert(descriptor.toRecvAndMerge == nullptr); assert(descriptor.toRecvAndMerge == nullptr);
...@@ -286,7 +291,7 @@ public: ...@@ -286,7 +291,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{MERGE_PARTICLES, idxDescr}); whatNext.emplace_back(std::pair<Action,int>{MERGE_PARTICLES, idxDescr});
mpiRequests.emplace_back(); mpiRequests.emplace_back();
assert(descriptor.nbParticlesToSend*size_particle_rhs < std::numeric_limits<int>::max()); assert(descriptor.nbParticlesToSend*size_particle_rhs < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(descriptor.toRecvAndMerge.get(), int(descriptor.nbParticlesToSend*size_particle_rhs), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_UP_LOW_RESULTS, AssertMpi(MPI_Irecv(descriptor.toRecvAndMerge.get(), int(descriptor.nbParticlesToSend*size_particle_rhs), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_RESULTS,
current_com, &mpiRequests.back())); current_com, &mpiRequests.back()));
} }
} }
...@@ -295,7 +300,7 @@ public: ...@@ -295,7 +300,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{RECV_PARTICLES, idxDescr}); whatNext.emplace_back(std::pair<Action,int>{RECV_PARTICLES, idxDescr});
mpiRequests.emplace_back(); mpiRequests.emplace_back();
AssertMpi(MPI_Irecv(&descriptor.nbParticlesToRecv, AssertMpi(MPI_Irecv(&descriptor.nbParticlesToRecv,
1, particles_utils::GetMpiType(partsize_t()), descriptor.destProc, TAG_UP_LOW_NB_PARTICLES, 1, particles_utils::GetMpiType(partsize_t()), descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_NB_PARTICLES,
current_com, &mpiRequests.back())); current_com, &mpiRequests.back()));
} }
else{ else{
...@@ -303,7 +308,7 @@ public: ...@@ -303,7 +308,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1}); whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back(); mpiRequests.emplace_back();
AssertMpi(MPI_Isend(const_cast<partsize_t*>(&descriptor.nbParticlesToSend), 1, particles_utils::GetMpiType(partsize_t()), AssertMpi(MPI_Isend(const_cast<partsize_t*>(&descriptor.nbParticlesToSend), 1, particles_utils::GetMpiType(partsize_t()),
descriptor.destProc, TAG_UP_LOW_NB_PARTICLES, descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_NB_PARTICLES,
current_com, &mpiRequests.back())); current_com, &mpiRequests.back()));
if(descriptor.nbParticlesToSend){ if(descriptor.nbParticlesToSend){
...@@ -312,7 +317,7 @@ public: ...@@ -312,7 +317,7 @@ public:
assert(descriptor.nbParticlesToSend*size_particle_positions < std::numeric_limits<int>::max()); assert(descriptor.nbParticlesToSend*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(const_cast<real_number*>(&particles_positions[(current_offset_particles_for_partition[current_partition_size-descriptor.nbPartitionsToSend])*size_particle_positions]), AssertMpi(MPI_Isend(const_cast<real_number*>(&particles_positions[(current_offset_particles_for_partition[current_partition_size-descriptor.nbPartitionsToSend])*size_particle_positions]),
int(descriptor.nbParticlesToSend*size_particle_positions), particles_utils::GetMpiType(real_number()), int(descriptor.nbParticlesToSend*size_particle_positions), particles_utils::GetMpiType(real_number()),
descriptor.destProc, TAG_UP_LOW_PARTICLES, descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_PARTICLES,
current_com, &mpiRequests.back())); current_com, &mpiRequests.back()));
assert(descriptor.toRecvAndMerge == nullptr); assert(descriptor.toRecvAndMerge == nullptr);
...@@ -320,7 +325,7 @@ public: ...@@ -320,7 +325,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{MERGE_PARTICLES, idxDescr}); whatNext.emplace_back(std::pair<Action,int>{MERGE_PARTICLES, idxDescr});
mpiRequests.emplace_back(); mpiRequests.emplace_back();
assert(descriptor.nbParticlesToSend*size_particle_rhs < std::numeric_limits<int>::max()); assert(descriptor.nbParticlesToSend*size_particle_rhs < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(descriptor.toRecvAndMerge.get(), int(descriptor.nbParticlesToSend*size_particle_rhs), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_LOW_UP_RESULTS, AssertMpi(MPI_Irecv(descriptor.toRecvAndMerge.get(), int(descriptor.nbParticlesToSend*size_particle_rhs), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_RESULTS,
current_com, &mpiRequests.back())); current_com, &mpiRequests.back()));
} }
...@@ -328,15 +333,15 @@ public: ...@@ -328,15 +333,15 @@ public:
whatNext.emplace_back(std::pair<Action,int>{RECV_PARTICLES, idxDescr}); whatNext.emplace_back(std::pair<Action,int>{RECV_PARTICLES, idxDescr});
mpiRequests.emplace_back(); mpiRequests.emplace_back();
AssertMpi(MPI_Irecv(&descriptor.nbParticlesToRecv, AssertMpi(MPI_Irecv(&descriptor.nbParticlesToRecv,
1, particles_utils::GetMpiType(partsize_t()), descriptor.destProc, TAG_LOW_UP_NB_PARTICLES, 1, particles_utils::GetMpiType(partsize_t()), descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_NB_PARTICLES,
current_com, &mpiRequests.back())); current_com, &mpiRequests.back()));
} }
} }
} }
const bool more_than_one_thread = (omp_get_max_threads() > 1); const bool more_than_one_thread = (omp_get_max_threads() > 1);
MPI_Barrier(MPI_COMM_WORLD); /// MPI_Barrier(MPI_COMM_WORLD);
//DEBUG_MSG_WAIT(MPI_COMM_WORLD, "line 338 of particles_distr_mpi.hpp\n"); /// DEBUG_MSG_WAIT(MPI_COMM_WORLD, "line 338 of particles_distr_mpi.hpp\n");
TIMEZONE_OMP_INIT_PREPARALLEL(omp_get_max_threads()) TIMEZONE_OMP_INIT_PREPARALLEL(omp_get_max_threads())
#pragma omp parallel default(shared) #pragma omp parallel default(shared)
...@@ -376,7 +381,7 @@ public: ...@@ -376,7 +381,7 @@ public:
mpiRequests.emplace_back(); mpiRequests.emplace_back();
assert(NbParticlesToReceive*size_particle_positions < std::numeric_limits<int>::max()); assert(NbParticlesToReceive*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(descriptor.toCompute.get(), int(NbParticlesToReceive*size_particle_positions), AssertMpi(MPI_Irecv(descriptor.toCompute.get(), int(NbParticlesToReceive*size_particle_positions),
particles_utils::GetMpiType(real_number()), destProc, TAG_UP_LOW_PARTICLES, particles_utils::GetMpiType(real_number()), destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_PARTICLES,
current_com, &mpiRequests.back())); current_com, &mpiRequests.back()));
} }
} }
...@@ -393,7 +398,7 @@ public: ...@@ -393,7 +398,7 @@ public:
mpiRequests.emplace_back(); mpiRequests.emplace_back();
assert(NbParticlesToReceive*size_particle_positions < std::numeric_limits<int>::max()); assert(NbParticlesToReceive*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(descriptor.toCompute.get(), int(NbParticlesToReceive*size_particle_positions), AssertMpi(MPI_Irecv(descriptor.toCompute.get(), int(NbParticlesToReceive*size_particle_positions),
particles_utils::GetMpiType(real_number()), destProc, TAG_LOW_UP_PARTICLES, particles_utils::GetMpiType(real_number()), destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_PARTICLES,
current_com, &mpiRequests.back())); current_com, &mpiRequests.back()));
} }
} }
...@@ -434,7 +439,7 @@ public: ...@@ -434,7 +439,7 @@ public:
const int destProc = descriptor.destProc; const int destProc = descriptor.destProc;
whatNext.emplace_back(std::pair<Action,int>{RELEASE_BUFFER_PARTICLES, releasedAction.second}); whatNext.emplace_back(std::pair<Action,int>{RELEASE_BUFFER_PARTICLES, releasedAction.second});
mpiRequests.emplace_back(); mpiRequests.emplace_back();
const int tag = descriptor.isLower? TAG_LOW_UP_RESULTS : TAG_UP_LOW_RESULTS; const int tag = descriptor.isLower? TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_RESULTS : TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_RESULTS;
assert(NbParticlesToReceive*size_particle_rhs < std::numeric_limits<int>::max()); assert(NbParticlesToReceive*size_particle_rhs < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(descriptor.results.get(), int(NbParticlesToReceive*size_particle_rhs), particles_utils::GetMpiType(real_number()), destProc, tag, AssertMpi(MPI_Isend(descriptor.results.get(), int(NbParticlesToReceive*size_particle_rhs), particles_utils::GetMpiType(real_number()), destProc, tag,
current_com, &mpiRequests.back())); current_com, &mpiRequests.back()));
...@@ -526,6 +531,8 @@ public: ...@@ -526,6 +531,8 @@ public:
assert(whatNext.size() == 0); assert(whatNext.size() == 0);
assert(mpiRequests.size() == 0); assert(mpiRequests.size() == 0);
counter_shift_tags += 1;
} }
...@@ -635,35 +642,35 @@ public: ...@@ -635,35 +642,35 @@ public:
whatNext.emplace_back(std::pair<Action,int>{RECV_MOVE_NB_LOW, -1}); whatNext.emplace_back(std::pair<Action,int>{RECV_MOVE_NB_LOW, -1});
mpiRequests.emplace_back(); mpiRequests.emplace_back();
AssertMpi(MPI_Irecv(&nbNewFromLow, 1, particles_utils::GetMpiType(partsize_t()), AssertMpi(MPI_Irecv(&nbNewFromLow, 1, particles_utils::GetMpiType(partsize_t()),
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_UP_LOW_MOVED_NB_PARTICLES, (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_NB_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back())); MPI_COMM_WORLD, &mpiRequests.back()));
eventsBeforeWaitall += 1; eventsBeforeWaitall += 1;
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1}); whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back(); mpiRequests.emplace_back();
AssertMpi(MPI_Isend(const_cast<partsize_t*>(&nbOutLower), 1, particles_utils::GetMpiType(partsize_t()), AssertMpi(MPI_Isend(const_cast<partsize_t*>(&nbOutLower), 1, particles_utils::GetMpiType(partsize_t()),
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_LOW_UP_MOVED_NB_PARTICLES, (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_NB_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back())); MPI_COMM_WORLD, &mpiRequests.back()));
if(nbOutLower){ if(nbOutLower){
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1}); whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back(); mpiRequests.emplace_back();
assert(nbOutLower*size_particle_positions < std::numeric_limits<int>::max()); assert(nbOutLower*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(&(*inout_positions_particles)[0], int(nbOutLower*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES, AssertMpi(MPI_Isend(&(*inout_positions_particles)[0], int(nbOutLower*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back())); MPI_COMM_WORLD, &mpiRequests.back()));
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1}); whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back(); mpiRequests.emplace_back();
assert(nbOutLower*size_particle_index < std::numeric_limits<int>::max()); assert(nbOutLower*size_particle_index < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(&(*inout_index_particles)[0], int(nbOutLower*size_particle_index), particles_utils::GetMpiType(partsize_t()), AssertMpi(MPI_Isend(&(*inout_index_particles)[0], int(nbOutLower*size_particle_index), particles_utils::GetMpiType(partsize_t()),
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES_INDEXES, (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES_INDEXES,
MPI_COMM_WORLD, &mpiRequests.back())); MPI_COMM_WORLD, &mpiRequests.back()));
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){ for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1}); whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back(); mpiRequests.emplace_back();
assert(nbOutLower*size_particle_rhs < std::numeric_limits<int>::max()); assert(nbOutLower*size_particle_rhs < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(&inout_rhs_particles[idx_rhs][0], int(nbOutLower*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES_RHS+idx_rhs, AssertMpi(MPI_Isend(&inout_rhs_particles[idx_rhs][0], int(nbOutLower*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES_RHS+idx_rhs,
MPI_COMM_WORLD, &mpiRequests.back())); MPI_COMM_WORLD, &mpiRequests.back()));
} }
} }
...@@ -671,14 +678,14 @@ public: ...@@ -671,14 +678,14 @@ public:
whatNext.emplace_back(std::pair<Action,int>{RECV_MOVE_NB_UP, -1}); whatNext.emplace_back(std::pair<Action,int>{RECV_MOVE_NB_UP, -1});
mpiRequests.emplace_back(); mpiRequests.emplace_back();
AssertMpi(MPI_Irecv(&nbNewFromUp, 1, particles_utils::GetMpiType(partsize_t()), (my_rank+1)%nb_processes_involved, AssertMpi(MPI_Irecv(&nbNewFromUp, 1, particles_utils::GetMpiType(partsize_t()), (my_rank+1)%nb_processes_involved,
TAG_LOW_UP_MOVED_NB_PARTICLES, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_NB_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back())); MPI_COMM_WORLD, &mpiRequests.back()));
eventsBeforeWaitall += 1; eventsBeforeWaitall += 1;
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1}); whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back(); mpiRequests.emplace_back();
AssertMpi(MPI_Isend(const_cast<partsize_t*>(&nbOutUpper), 1, particles_utils::GetMpiType(partsize_t()), AssertMpi(MPI_Isend(const_cast<partsize_t*>(&nbOutUpper), 1, particles_utils::GetMpiType(partsize_t()),
(my_rank+1)%nb_processes_involved, TAG_UP_LOW_MOVED_NB_PARTICLES, (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_NB_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back())); MPI_COMM_WORLD, &mpiRequests.back()));
if(nbOutUpper){ if(nbOutUpper){
...@@ -686,14 +693,14 @@ public: ...@@ -686,14 +693,14 @@ public:
mpiRequests.emplace_back(); mpiRequests.emplace_back();
assert(nbOutUpper*size_particle_positions < std::numeric_limits<int>::max()); assert(nbOutUpper*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(&(*inout_positions_particles)[(myTotalNbParticles-nbOutUpper)*size_particle_positions], AssertMpi(MPI_Isend(&(*inout_positions_particles)[(myTotalNbParticles-nbOutUpper)*size_particle_positions],
int(nbOutUpper*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES, int(nbOutUpper*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back())); MPI_COMM_WORLD, &mpiRequests.back()));
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1}); whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back(); mpiRequests.emplace_back();
assert(nbOutUpper*size_particle_index < std::numeric_limits<int>::max()); assert(nbOutUpper*size_particle_index < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(&(*inout_index_particles)[(myTotalNbParticles-nbOutUpper)*size_particle_index], int(nbOutUpper*size_particle_index), AssertMpi(MPI_Isend(&(*inout_index_particles)[(myTotalNbParticles-nbOutUpper)*size_particle_index], int(nbOutUpper*size_particle_index),
particles_utils::GetMpiType(partsize_t()), (my_rank+1)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES_INDEXES, particles_utils::GetMpiType(partsize_t()), (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES_INDEXES,
MPI_COMM_WORLD, &mpiRequests.back())); MPI_COMM_WORLD, &mpiRequests.back()));
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){ for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
...@@ -701,7 +708,7 @@ public: ...@@ -701,7 +708,7 @@ public:
mpiRequests.emplace_back(); mpiRequests.emplace_back();
assert(nbOutUpper*size_particle_rhs < std::numeric_limits<int>::max()); assert(nbOutUpper*size_particle_rhs < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(&inout_rhs_particles[idx_rhs][(myTotalNbParticles-nbOutUpper)*size_particle_rhs], AssertMpi(MPI_Isend(&inout_rhs_particles[idx_rhs][(myTotalNbParticles-nbOutUpper)*size_particle_rhs],
int(nbOutUpper*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES_RHS+idx_rhs, int(nbOutUpper*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES_RHS+idx_rhs,
MPI_COMM_WORLD, &mpiRequests.back())); MPI_COMM_WORLD, &mpiRequests.back()));
} }
} }
...@@ -726,7 +733,7 @@ public: ...@@ -726,7 +733,7 @@ public:
mpiRequests.emplace_back(); mpiRequests.emplace_back();
assert(nbNewFromLow*size_particle_positions < std::numeric_limits<int>::max()); assert(nbNewFromLow*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(&newParticlesLow[0], int(nbNewFromLow*size_particle_positions), particles_utils::GetMpiType(real_number()), AssertMpi(MPI_Irecv(&newParticlesLow[0], int(nbNewFromLow*size_particle_positions), particles_utils::GetMpiType(real_number()),
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES, (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back())); MPI_COMM_WORLD, &mpiRequests.back()));
newParticlesLowIndexes.reset(new partsize_t[nbNewFromLow*size_particle_index]); newParticlesLowIndexes.reset(new partsize_t[nbNewFromLow*size_particle_index]);
...@@ -735,7 +742,7 @@ public: ...@@ -735,7 +742,7 @@ public:
assert(nbNewFromLow*size_particle_index < std::numeric_limits<int>::max()); assert(nbNewFromLow*size_particle_index < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(&newParticlesLowIndexes[0], int(nbNewFromLow*size_particle_index), AssertMpi(MPI_Irecv(&newParticlesLowIndexes[0], int(nbNewFromLow*size_particle_index),
particles_utils::GetMpiType(partsize_t()), particles_utils::GetMpiType(partsize_t()),
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES_INDEXES, (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES_INDEXES,
MPI_COMM_WORLD, &mpiRequests.back())); MPI_COMM_WORLD, &mpiRequests.back()));
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){ for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
...@@ -743,7 +750,7 @@ public: ...@@ -743,7 +750,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1}); whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back(); mpiRequests.emplace_back();
assert(nbNewFromLow*size_particle_rhs < std::numeric_limits<int>::max()); assert(nbNewFromLow*size_particle_rhs < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(&newParticlesLowRhs[idx_rhs][0], int(nbNewFromLow*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES_RHS+idx_rhs, AssertMpi(MPI_Irecv(&newParticlesLowRhs[idx_rhs][0], int(nbNewFromLow*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES_RHS+idx_rhs,
MPI_COMM_WORLD, &mpiRequests.back())); MPI_COMM_WORLD, &mpiRequests.back()));
} }
} }
...@@ -756,7 +763,7 @@ public: ...@@ -756,7 +763,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{RECV_MOVE_UP, -1}); whatNext.emplace_back(std::pair<Action,int>{RECV_MOVE_UP, -1});
mpiRequests.emplace_back(); mpiRequests.emplace_back();
assert(nbNewFromUp*size_particle_positions < std::numeric_limits<int>::max()); assert(nbNewFromUp*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(&newParticlesUp[0], int(nbNewFromUp*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES, AssertMpi(MPI_Irecv(&newParticlesUp[0], int(nbNewFromUp*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back())); MPI_COMM_WORLD, &mpiRequests.back()));
newParticlesUpIndexes.reset(new partsize_t[nbNewFromUp*size_particle_index]); newParticlesUpIndexes.reset(new partsize_t[nbNewFromUp*size_particle_index]);
...@@ -765,7 +772,7 @@ public: ...@@ -765,7 +772,7 @@ public:
assert(nbNewFromUp*size_particle_index < std::numeric_limits<int>::max()); assert(nbNewFromUp*size_particle_index < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(&newParticlesUpIndexes[0], int(nbNewFromUp*size_particle_index), AssertMpi(MPI_Irecv(&newParticlesUpIndexes[0], int(nbNewFromUp*size_particle_index),
particles_utils::GetMpiType(partsize_t()), particles_utils::GetMpiType(partsize_t()),
(my_rank+1)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES_INDEXES, (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES_INDEXES,
MPI_COMM_WORLD, &mpiRequests.back())); MPI_COMM_WORLD, &mpiRequests.back()));
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){ for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
...@@ -773,7 +780,7 @@ public: ...@@ -773,7 +780,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1}); whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back(); mpiRequests.emplace_back();
assert(nbNewFromUp*size_particle_rhs < std::numeric_limits<int>::max()); assert(nbNewFromUp*size_particle_rhs < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(&newParticlesUpRhs[idx_rhs][0], int(nbNewFromUp*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES_RHS+idx_rhs, AssertMpi(MPI_Irecv(&newParticlesUpRhs[idx_rhs][0], int(nbNewFromUp*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES_RHS+idx_rhs,
MPI_COMM_WORLD, &mpiRequests.back())); MPI_COMM_WORLD, &mpiRequests.back()));
} }
} }
...@@ -880,6 +887,8 @@ public: ...@@ -880,6 +887,8 @@ public:
(*nb_particles) = myTotalNbParticles; (*nb_particles) = myTotalNbParticles;
assert(mpiRequests.size() == 0); assert(mpiRequests.size() == 0);
counter_shift_tags += 1;
} }
}; };
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment