diff --git a/TurTLE/test/test_particle_clouds.py b/TurTLE/test/test_particle_clouds.py
index f72071143778bb79cbe2ea1128769c9c32004f5e..b6c401d18039d2b5b02991bd5a1cf1aae8152a50 100644
--- a/TurTLE/test/test_particle_clouds.py
+++ b/TurTLE/test/test_particle_clouds.py
@@ -33,22 +33,22 @@ import sys
import TurTLE
from TurTLE import DNS
-
-def main():
+def basic_test():
nclouds = 10
nparticles_per_cloud = 1000
nparticles = nclouds*nparticles_per_cloud
niterations = 32
c = DNS()
c.dns_type = 'NSVEparticles'
- c.parameters['nparticles'] = nparticles
- c.parameters['tracers1_integration_steps'] = 4
- c.generate_tracer_state(rseed = 2, species = 1)
- del c.parameters['nparticles']
- del c.parameters['tracers1_integration_steps']
+ c.simname = 'basic_cloud_test'
+ f0 = h5py.File(
+ os.path.join(
+ os.path.join(TurTLE.lib_dir, 'test'),
+ 'B32p1e4_checkpoint_0.h5'),
+ 'r')
ic_file = h5py.File(c.get_checkpoint_0_fname(), 'a')
- ic_file['tracers0/state/0'] = ic_file['tracers1/state/0'][...].reshape(nclouds, nparticles_per_cloud, 3)
- ic_file['tracers0/rhs/0'] = ic_file['tracers1/rhs/0'][...].reshape(4, nclouds, nparticles_per_cloud, 3)
+ ic_file['tracers0/state/0'] = f0['tracers0/state/0'][...].reshape(nclouds, nparticles_per_cloud, 3)
+ ic_file['tracers0/rhs/0'] = f0['tracers0/rhs/0'][...].reshape(4, nclouds, nparticles_per_cloud, 3)
ic_file.close()
c.launch(
['NSVEparticles',
@@ -57,12 +57,14 @@ def main():
'--forcing_type', 'linear',
'--src-wd', TurTLE.lib_dir + '/test',
'--src-iteration', '0',
+ '--simname', c.simname,
'--np', '4',
'--ntpp', '1',
'--fftw_plan_rigor', 'FFTW_PATIENT',
'--niter_todo', '{0}'.format(niterations),
'--niter_out', '{0}'.format(niterations),
'--niter_stat', '1',
+ '--checkpoints_per_file', '{0}'.format(3),
'--nparticles', '{0}'.format(nparticles),
'--njobs', '2',
'--wd', './'])
@@ -79,6 +81,7 @@ def main():
x0 = f0['tracers0/state/{0}'.format(iteration)][...]
x1 = f1['tracers0/state/{0}'.format(iteration)][...].reshape(x0.shape)
traj_error = np.max(np.abs(x0 - x1))
+ print(traj_error)
y0 = f0['tracers0/rhs/{0}'.format(iteration)][...]
y1 = f1['tracers0/rhs/{0}'.format(iteration)][...].reshape(y0.shape)
rhs_error = np.max(np.abs(y0 - y1))
@@ -88,6 +91,53 @@ def main():
print('SUCCESS! Basic test passed.')
return None
+def nasty_test():
+ nclouds = 10
+ nparticles_per_cloud = 1000000
+ nparticles = nclouds*nparticles_per_cloud
+ niterations = 8
+ c = DNS()
+ c.dns_type = 'NSVEparticles'
+ c.simname = 'nasty_cloud_test'
+ c.parameters['nparticles'] = nparticles
+ c.parameters['tracers1_integration_steps'] = 4
+ c.generate_tracer_state(rseed = 2, species = 1)
+ del c.parameters['nparticles']
+ del c.parameters['tracers1_integration_steps']
+ ic_file = h5py.File(c.get_checkpoint_0_fname(), 'a')
+ ic_file['tracers0/state/0'] = ic_file['tracers1/state/0'][...].reshape(nclouds, nparticles_per_cloud, 3)
+ ic_file['tracers0/rhs/0'] = ic_file['tracers1/rhs/0'][...].reshape(4, nclouds, nparticles_per_cloud, 3)
+ # put all particles in single z cell
+ ic_file['tracers0/state/0'][..., 2] = 0.0001
+ # put one cloud on another process
+ ic_file['tracers0/state/0'][1, :, 2] = np.pi + 0.0001
+ ic_file.close()
+ c.launch(
+ ['NSVEparticles',
+ '-n', '8',
+ '--src-simname', 'B32p1e4',
+ '--simname', c.simname,
+ '--forcing_type', 'linear',
+ '--src-wd', TurTLE.lib_dir + '/test',
+ '--src-iteration', '0',
+ '--np', '4',
+ '--ntpp', '2',
+ '--fftw_plan_rigor', 'FFTW_PATIENT',
+ '--niter_todo', '{0}'.format(niterations),
+ '--niter_out', '{0}'.format(niterations),
+ '--niter_stat', '1',
+ '--nparticles', '{0}'.format(nparticles),
+ '--njobs', '1',
+ '--wd', './'])
+ print('SUCCESS! Nasty test passed.')
+ return None
+
+
+def main():
+ basic_test()
+ nasty_test()
+ return None
+
if __name__ == '__main__':
main()
diff --git a/cpp/particles/particles_distr_mpi.hpp b/cpp/particles/particles_distr_mpi.hpp
index 57d8067059a1408afb90c53e32c83c0731030bd2..0d56871b7ef46d5cad0643ddf1ff0acb2c3cf385 100644
--- a/cpp/particles/particles_distr_mpi.hpp
+++ b/cpp/particles/particles_distr_mpi.hpp
@@ -45,7 +45,7 @@ protected:
static const int MaxNbRhs = 10;
enum MpiTag{
- TAG_LOW_UP_NB_PARTICLES,
+ TAG_LOW_UP_NB_PARTICLES = 999,
TAG_UP_LOW_NB_PARTICLES,
TAG_LOW_UP_PARTICLES,
TAG_UP_LOW_PARTICLES,
@@ -65,6 +65,8 @@ protected:
TAG_UP_LOW_MOVED_PARTICLES_RHS = TAG_LOW_UP_MOVED_PARTICLES_RHS_MAX,
TAG_UP_LOW_MOVED_PARTICLES_RHS_MAX = TAG_UP_LOW_MOVED_PARTICLES_RHS+MaxNbRhs,
+
+ TAG_SHIFT_OFFSET
};
struct NeighborDescriptor{
@@ -114,6 +116,8 @@ protected:
std::vector<MPI_Request> mpiRequests;
std::vector<NeighborDescriptor> neigDescriptors;
+ int counter_shift_tags;
+
public:
////////////////////////////////////////////////////////////////////////////
@@ -124,7 +128,8 @@ public:
my_rank(-1), nb_processes(-1),nb_processes_involved(-1),
current_partition_interval(in_current_partitions),
current_partition_size(current_partition_interval.second-current_partition_interval.first),
- field_grid_dim(in_field_grid_dim){
+ field_grid_dim(in_field_grid_dim),
+ counter_shift_tags(0){
AssertMpi(MPI_Comm_rank(current_com, &my_rank));
AssertMpi(MPI_Comm_size(current_com, &nb_processes));
@@ -271,14 +276,14 @@ public:
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
AssertMpi(MPI_Isend(const_cast<partsize_t*>(&descriptor.nbParticlesToSend), 1, particles_utils::GetMpiType(partsize_t()),
- descriptor.destProc, TAG_LOW_UP_NB_PARTICLES,
+ descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_NB_PARTICLES,
current_com, &mpiRequests.back()));
if(descriptor.nbParticlesToSend){
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
assert(descriptor.nbParticlesToSend*size_particle_positions < std::numeric_limits<int>::max());
- AssertMpi(MPI_Isend(const_cast<real_number*>(&particles_positions[0]), int(descriptor.nbParticlesToSend*size_particle_positions), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_LOW_UP_PARTICLES,
+ AssertMpi(MPI_Isend(const_cast<real_number*>(&particles_positions[0]), int(descriptor.nbParticlesToSend*size_particle_positions), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_PARTICLES,
current_com, &mpiRequests.back()));
assert(descriptor.toRecvAndMerge == nullptr);
@@ -286,7 +291,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{MERGE_PARTICLES, idxDescr});
mpiRequests.emplace_back();
assert(descriptor.nbParticlesToSend*size_particle_rhs < std::numeric_limits<int>::max());
- AssertMpi(MPI_Irecv(descriptor.toRecvAndMerge.get(), int(descriptor.nbParticlesToSend*size_particle_rhs), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_UP_LOW_RESULTS,
+ AssertMpi(MPI_Irecv(descriptor.toRecvAndMerge.get(), int(descriptor.nbParticlesToSend*size_particle_rhs), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_RESULTS,
current_com, &mpiRequests.back()));
}
}
@@ -295,7 +300,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{RECV_PARTICLES, idxDescr});
mpiRequests.emplace_back();
AssertMpi(MPI_Irecv(&descriptor.nbParticlesToRecv,
- 1, particles_utils::GetMpiType(partsize_t()), descriptor.destProc, TAG_UP_LOW_NB_PARTICLES,
+ 1, particles_utils::GetMpiType(partsize_t()), descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_NB_PARTICLES,
current_com, &mpiRequests.back()));
}
else{
@@ -303,7 +308,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
AssertMpi(MPI_Isend(const_cast<partsize_t*>(&descriptor.nbParticlesToSend), 1, particles_utils::GetMpiType(partsize_t()),
- descriptor.destProc, TAG_UP_LOW_NB_PARTICLES,
+ descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_NB_PARTICLES,
current_com, &mpiRequests.back()));
if(descriptor.nbParticlesToSend){
@@ -312,7 +317,7 @@ public:
assert(descriptor.nbParticlesToSend*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(const_cast<real_number*>(&particles_positions[(current_offset_particles_for_partition[current_partition_size-descriptor.nbPartitionsToSend])*size_particle_positions]),
int(descriptor.nbParticlesToSend*size_particle_positions), particles_utils::GetMpiType(real_number()),
- descriptor.destProc, TAG_UP_LOW_PARTICLES,
+ descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_PARTICLES,
current_com, &mpiRequests.back()));
assert(descriptor.toRecvAndMerge == nullptr);
@@ -320,7 +325,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{MERGE_PARTICLES, idxDescr});
mpiRequests.emplace_back();
assert(descriptor.nbParticlesToSend*size_particle_rhs < std::numeric_limits<int>::max());
- AssertMpi(MPI_Irecv(descriptor.toRecvAndMerge.get(), int(descriptor.nbParticlesToSend*size_particle_rhs), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_LOW_UP_RESULTS,
+ AssertMpi(MPI_Irecv(descriptor.toRecvAndMerge.get(), int(descriptor.nbParticlesToSend*size_particle_rhs), particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_RESULTS,
current_com, &mpiRequests.back()));
}
@@ -328,15 +333,15 @@ public:
whatNext.emplace_back(std::pair<Action,int>{RECV_PARTICLES, idxDescr});
mpiRequests.emplace_back();
AssertMpi(MPI_Irecv(&descriptor.nbParticlesToRecv,
- 1, particles_utils::GetMpiType(partsize_t()), descriptor.destProc, TAG_LOW_UP_NB_PARTICLES,
+ 1, particles_utils::GetMpiType(partsize_t()), descriptor.destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_NB_PARTICLES,
current_com, &mpiRequests.back()));
}
}
}
const bool more_than_one_thread = (omp_get_max_threads() > 1);
- MPI_Barrier(MPI_COMM_WORLD);
- //DEBUG_MSG_WAIT(MPI_COMM_WORLD, "line 338 of particles_distr_mpi.hpp\n");
+ /// MPI_Barrier(MPI_COMM_WORLD);
+ /// DEBUG_MSG_WAIT(MPI_COMM_WORLD, "line 338 of particles_distr_mpi.hpp\n");
TIMEZONE_OMP_INIT_PREPARALLEL(omp_get_max_threads())
#pragma omp parallel default(shared)
@@ -376,7 +381,7 @@ public:
mpiRequests.emplace_back();
assert(NbParticlesToReceive*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(descriptor.toCompute.get(), int(NbParticlesToReceive*size_particle_positions),
- particles_utils::GetMpiType(real_number()), destProc, TAG_UP_LOW_PARTICLES,
+ particles_utils::GetMpiType(real_number()), destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_PARTICLES,
current_com, &mpiRequests.back()));
}
}
@@ -393,7 +398,7 @@ public:
mpiRequests.emplace_back();
assert(NbParticlesToReceive*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(descriptor.toCompute.get(), int(NbParticlesToReceive*size_particle_positions),
- particles_utils::GetMpiType(real_number()), destProc, TAG_LOW_UP_PARTICLES,
+ particles_utils::GetMpiType(real_number()), destProc, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_PARTICLES,
current_com, &mpiRequests.back()));
}
}
@@ -434,7 +439,7 @@ public:
const int destProc = descriptor.destProc;
whatNext.emplace_back(std::pair<Action,int>{RELEASE_BUFFER_PARTICLES, releasedAction.second});
mpiRequests.emplace_back();
- const int tag = descriptor.isLower? TAG_LOW_UP_RESULTS : TAG_UP_LOW_RESULTS;
+ const int tag = descriptor.isLower? TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_RESULTS : TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_RESULTS;
assert(NbParticlesToReceive*size_particle_rhs < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(descriptor.results.get(), int(NbParticlesToReceive*size_particle_rhs), particles_utils::GetMpiType(real_number()), destProc, tag,
current_com, &mpiRequests.back()));
@@ -526,6 +531,8 @@ public:
assert(whatNext.size() == 0);
assert(mpiRequests.size() == 0);
+
+ counter_shift_tags += 1;
}
@@ -635,35 +642,35 @@ public:
whatNext.emplace_back(std::pair<Action,int>{RECV_MOVE_NB_LOW, -1});
mpiRequests.emplace_back();
AssertMpi(MPI_Irecv(&nbNewFromLow, 1, particles_utils::GetMpiType(partsize_t()),
- (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_UP_LOW_MOVED_NB_PARTICLES,
+ (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_NB_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
eventsBeforeWaitall += 1;
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
AssertMpi(MPI_Isend(const_cast<partsize_t*>(&nbOutLower), 1, particles_utils::GetMpiType(partsize_t()),
- (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_LOW_UP_MOVED_NB_PARTICLES,
+ (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_NB_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
if(nbOutLower){
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
assert(nbOutLower*size_particle_positions < std::numeric_limits<int>::max());
- AssertMpi(MPI_Isend(&(*inout_positions_particles)[0], int(nbOutLower*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES,
+ AssertMpi(MPI_Isend(&(*inout_positions_particles)[0], int(nbOutLower*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
assert(nbOutLower*size_particle_index < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(&(*inout_index_particles)[0], int(nbOutLower*size_particle_index), particles_utils::GetMpiType(partsize_t()),
- (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES_INDEXES,
+ (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES_INDEXES,
MPI_COMM_WORLD, &mpiRequests.back()));
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
assert(nbOutLower*size_particle_rhs < std::numeric_limits<int>::max());
- AssertMpi(MPI_Isend(&inout_rhs_particles[idx_rhs][0], int(nbOutLower*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES_RHS+idx_rhs,
+ AssertMpi(MPI_Isend(&inout_rhs_particles[idx_rhs][0], int(nbOutLower*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES_RHS+idx_rhs,
MPI_COMM_WORLD, &mpiRequests.back()));
}
}
@@ -671,14 +678,14 @@ public:
whatNext.emplace_back(std::pair<Action,int>{RECV_MOVE_NB_UP, -1});
mpiRequests.emplace_back();
AssertMpi(MPI_Irecv(&nbNewFromUp, 1, particles_utils::GetMpiType(partsize_t()), (my_rank+1)%nb_processes_involved,
- TAG_LOW_UP_MOVED_NB_PARTICLES,
+ TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_NB_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
eventsBeforeWaitall += 1;
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
AssertMpi(MPI_Isend(const_cast<partsize_t*>(&nbOutUpper), 1, particles_utils::GetMpiType(partsize_t()),
- (my_rank+1)%nb_processes_involved, TAG_UP_LOW_MOVED_NB_PARTICLES,
+ (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_NB_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
if(nbOutUpper){
@@ -686,14 +693,14 @@ public:
mpiRequests.emplace_back();
assert(nbOutUpper*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(&(*inout_positions_particles)[(myTotalNbParticles-nbOutUpper)*size_particle_positions],
- int(nbOutUpper*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES,
+ int(nbOutUpper*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
assert(nbOutUpper*size_particle_index < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(&(*inout_index_particles)[(myTotalNbParticles-nbOutUpper)*size_particle_index], int(nbOutUpper*size_particle_index),
- particles_utils::GetMpiType(partsize_t()), (my_rank+1)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES_INDEXES,
+ particles_utils::GetMpiType(partsize_t()), (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES_INDEXES,
MPI_COMM_WORLD, &mpiRequests.back()));
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
@@ -701,7 +708,7 @@ public:
mpiRequests.emplace_back();
assert(nbOutUpper*size_particle_rhs < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(&inout_rhs_particles[idx_rhs][(myTotalNbParticles-nbOutUpper)*size_particle_rhs],
- int(nbOutUpper*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES_RHS+idx_rhs,
+ int(nbOutUpper*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES_RHS+idx_rhs,
MPI_COMM_WORLD, &mpiRequests.back()));
}
}
@@ -726,7 +733,7 @@ public:
mpiRequests.emplace_back();
assert(nbNewFromLow*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(&newParticlesLow[0], int(nbNewFromLow*size_particle_positions), particles_utils::GetMpiType(real_number()),
- (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES,
+ (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
newParticlesLowIndexes.reset(new partsize_t[nbNewFromLow*size_particle_index]);
@@ -735,7 +742,7 @@ public:
assert(nbNewFromLow*size_particle_index < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(&newParticlesLowIndexes[0], int(nbNewFromLow*size_particle_index),
particles_utils::GetMpiType(partsize_t()),
- (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES_INDEXES,
+ (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES_INDEXES,
MPI_COMM_WORLD, &mpiRequests.back()));
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
@@ -743,7 +750,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
assert(nbNewFromLow*size_particle_rhs < std::numeric_limits<int>::max());
- AssertMpi(MPI_Irecv(&newParticlesLowRhs[idx_rhs][0], int(nbNewFromLow*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES_RHS+idx_rhs,
+ AssertMpi(MPI_Irecv(&newParticlesLowRhs[idx_rhs][0], int(nbNewFromLow*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_UP_LOW_MOVED_PARTICLES_RHS+idx_rhs,
MPI_COMM_WORLD, &mpiRequests.back()));
}
}
@@ -756,7 +763,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{RECV_MOVE_UP, -1});
mpiRequests.emplace_back();
assert(nbNewFromUp*size_particle_positions < std::numeric_limits<int>::max());
- AssertMpi(MPI_Irecv(&newParticlesUp[0], int(nbNewFromUp*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES,
+ AssertMpi(MPI_Irecv(&newParticlesUp[0], int(nbNewFromUp*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
newParticlesUpIndexes.reset(new partsize_t[nbNewFromUp*size_particle_index]);
@@ -765,7 +772,7 @@ public:
assert(nbNewFromUp*size_particle_index < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(&newParticlesUpIndexes[0], int(nbNewFromUp*size_particle_index),
particles_utils::GetMpiType(partsize_t()),
- (my_rank+1)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES_INDEXES,
+ (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES_INDEXES,
MPI_COMM_WORLD, &mpiRequests.back()));
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
@@ -773,7 +780,7 @@ public:
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
assert(nbNewFromUp*size_particle_rhs < std::numeric_limits<int>::max());
- AssertMpi(MPI_Irecv(&newParticlesUpRhs[idx_rhs][0], int(nbNewFromUp*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES_RHS+idx_rhs,
+ AssertMpi(MPI_Irecv(&newParticlesUpRhs[idx_rhs][0], int(nbNewFromUp*size_particle_rhs), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_SHIFT_OFFSET*counter_shift_tags + TAG_LOW_UP_MOVED_PARTICLES_RHS+idx_rhs,
MPI_COMM_WORLD, &mpiRequests.back()));
}
}
@@ -880,6 +887,8 @@ public:
(*nb_particles) = myTotalNbParticles;
assert(mpiRequests.size() == 0);
+
+ counter_shift_tags += 1;
}
};