Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
TurTLE
TurTLE
Commits
c14be9dc
Commit
c14be9dc
authored
Dec 02, 2019
by
Cristian Lalescu
Browse files
Merge branch 'bugfix/particle_distribution' into develop
parents
407d3044
415b6b89
Pipeline
#64807
passed with stage
in 4 minutes and 46 seconds
Changes
2
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
TurTLE/test/test_particle_clouds.py
View file @
c14be9dc
...
...
@@ -33,22 +33,22 @@ import sys
import
TurTLE
from
TurTLE
import
DNS
def
main
():
def
basic_test
():
nclouds
=
10
nparticles_per_cloud
=
1000
nparticles
=
nclouds
*
nparticles_per_cloud
niterations
=
32
c
=
DNS
()
c
.
dns_type
=
'NSVEparticles'
c
.
parameters
[
'nparticles'
]
=
nparticles
c
.
parameters
[
'tracers1_integration_steps'
]
=
4
c
.
generate_tracer_state
(
rseed
=
2
,
species
=
1
)
del
c
.
parameters
[
'nparticles'
]
del
c
.
parameters
[
'tracers1_integration_steps'
]
c
.
simname
=
'basic_cloud_test'
f0
=
h5py
.
File
(
os
.
path
.
join
(
os
.
path
.
join
(
TurTLE
.
lib_dir
,
'test'
),
'B32p1e4_checkpoint_0.h5'
),
'r'
)
ic_file
=
h5py
.
File
(
c
.
get_checkpoint_0_fname
(),
'a'
)
ic_file
[
'tracers0/state/0'
]
=
ic_file
[
'tracers
1
/state/0'
][...].
reshape
(
nclouds
,
nparticles_per_cloud
,
3
)
ic_file
[
'tracers0/rhs/0'
]
=
ic_file
[
'tracers
1
/rhs/0'
][...].
reshape
(
4
,
nclouds
,
nparticles_per_cloud
,
3
)
ic_file
[
'tracers0/state/0'
]
=
f0
[
'tracers
0
/state/0'
][...].
reshape
(
nclouds
,
nparticles_per_cloud
,
3
)
ic_file
[
'tracers0/rhs/0'
]
=
f0
[
'tracers
0
/rhs/0'
][...].
reshape
(
4
,
nclouds
,
nparticles_per_cloud
,
3
)
ic_file
.
close
()
c
.
launch
(
[
'NSVEparticles'
,
...
...
@@ -57,12 +57,14 @@ def main():
'--forcing_type'
,
'linear'
,
'--src-wd'
,
TurTLE
.
lib_dir
+
'/test'
,
'--src-iteration'
,
'0'
,
'--simname'
,
c
.
simname
,
'--np'
,
'4'
,
'--ntpp'
,
'1'
,
'--fftw_plan_rigor'
,
'FFTW_PATIENT'
,
'--niter_todo'
,
'{0}'
.
format
(
niterations
),
'--niter_out'
,
'{0}'
.
format
(
niterations
),
'--niter_stat'
,
'1'
,
'--checkpoints_per_file'
,
'{0}'
.
format
(
3
),
'--nparticles'
,
'{0}'
.
format
(
nparticles
),
'--njobs'
,
'2'
,
'--wd'
,
'./'
])
...
...
@@ -79,6 +81,7 @@ def main():
x0
=
f0
[
'tracers0/state/{0}'
.
format
(
iteration
)][...]
x1
=
f1
[
'tracers0/state/{0}'
.
format
(
iteration
)][...].
reshape
(
x0
.
shape
)
traj_error
=
np
.
max
(
np
.
abs
(
x0
-
x1
))
print
(
traj_error
)
y0
=
f0
[
'tracers0/rhs/{0}'
.
format
(
iteration
)][...]
y1
=
f1
[
'tracers0/rhs/{0}'
.
format
(
iteration
)][...].
reshape
(
y0
.
shape
)
rhs_error
=
np
.
max
(
np
.
abs
(
y0
-
y1
))
...
...
@@ -88,6 +91,53 @@ def main():
print
(
'SUCCESS! Basic test passed.'
)
return
None
def
nasty_test
():
nclouds
=
10
nparticles_per_cloud
=
1000000
nparticles
=
nclouds
*
nparticles_per_cloud
niterations
=
8
c
=
DNS
()
c
.
dns_type
=
'NSVEparticles'
c
.
simname
=
'nasty_cloud_test'
c
.
parameters
[
'nparticles'
]
=
nparticles
c
.
parameters
[
'tracers1_integration_steps'
]
=
4
c
.
generate_tracer_state
(
rseed
=
2
,
species
=
1
)
del
c
.
parameters
[
'nparticles'
]
del
c
.
parameters
[
'tracers1_integration_steps'
]
ic_file
=
h5py
.
File
(
c
.
get_checkpoint_0_fname
(),
'a'
)
ic_file
[
'tracers0/state/0'
]
=
ic_file
[
'tracers1/state/0'
][...].
reshape
(
nclouds
,
nparticles_per_cloud
,
3
)
ic_file
[
'tracers0/rhs/0'
]
=
ic_file
[
'tracers1/rhs/0'
][...].
reshape
(
4
,
nclouds
,
nparticles_per_cloud
,
3
)
# put all particles in single z cell
ic_file
[
'tracers0/state/0'
][...,
2
]
=
0.0001
# put one cloud on another process
ic_file
[
'tracers0/state/0'
][
1
,
:,
2
]
=
np
.
pi
+
0.0001
ic_file
.
close
()
c
.
launch
(
[
'NSVEparticles'
,
'-n'
,
'8'
,
'--src-simname'
,
'B32p1e4'
,
'--simname'
,
c
.
simname
,
'--forcing_type'
,
'linear'
,
'--src-wd'
,
TurTLE
.
lib_dir
+
'/test'
,
'--src-iteration'
,
'0'
,
'--np'
,
'4'
,
'--ntpp'
,
'2'
,
'--fftw_plan_rigor'
,
'FFTW_PATIENT'
,
'--niter_todo'
,
'{0}'
.
format
(
niterations
),
'--niter_out'
,
'{0}'
.
format
(
niterations
),
'--niter_stat'
,
'1'
,
'--nparticles'
,
'{0}'
.
format
(
nparticles
),
'--njobs'
,
'1'
,
'--wd'
,
'./'
])
print
(
'SUCCESS! Nasty test passed.'
)
return
None
def
main
():
basic_test
()
nasty_test
()
return
None
if
__name__
==
'__main__'
:
main
()
cpp/particles/particles_distr_mpi.hpp
View file @
c14be9dc
...
...
@@ -45,7 +45,7 @@ protected:
static
const
int
MaxNbRhs
=
10
;
enum
MpiTag
{
TAG_LOW_UP_NB_PARTICLES
,
TAG_LOW_UP_NB_PARTICLES
=
999
,
TAG_UP_LOW_NB_PARTICLES
,
TAG_LOW_UP_PARTICLES
,
TAG_UP_LOW_PARTICLES
,
...
...
@@ -65,6 +65,8 @@ protected:
TAG_UP_LOW_MOVED_PARTICLES_RHS
=
TAG_LOW_UP_MOVED_PARTICLES_RHS_MAX
,
TAG_UP_LOW_MOVED_PARTICLES_RHS_MAX
=
TAG_UP_LOW_MOVED_PARTICLES_RHS
+
MaxNbRhs
,
TAG_SHIFT_OFFSET
};
struct
NeighborDescriptor
{
...
...
@@ -114,6 +116,8 @@ protected:
std
::
vector
<
MPI_Request
>
mpiRequests
;
std
::
vector
<
NeighborDescriptor
>
neigDescriptors
;
int
counter_shift_tags
;
public:
////////////////////////////////////////////////////////////////////////////
...
...
@@ -124,7 +128,8 @@ public:
my_rank
(
-
1
),
nb_processes
(
-
1
),
nb_processes_involved
(
-
1
),
current_partition_interval
(
in_current_partitions
),
current_partition_size
(
current_partition_interval
.
second
-
current_partition_interval
.
first
),
field_grid_dim
(
in_field_grid_dim
){
field_grid_dim
(
in_field_grid_dim
),
counter_shift_tags
(
0
){
AssertMpi
(
MPI_Comm_rank
(
current_com
,
&
my_rank
));
AssertMpi
(
MPI_Comm_size
(
current_com
,
&
nb_processes
));
...
...
@@ -271,14 +276,14 @@ public:
whatNext
.
emplace_back
(
std
::
pair
<
Action
,
int
>
{
NOTHING_TODO
,
-
1
});
mpiRequests
.
emplace_back
();
AssertMpi
(
MPI_Isend
(
const_cast
<
partsize_t
*>
(
&
descriptor
.
nbParticlesToSend
),
1
,
particles_utils
::
GetMpiType
(
partsize_t
()),
descriptor
.
destProc
,
TAG_LOW_UP_NB_PARTICLES
,
descriptor
.
destProc
,
TAG_SHIFT_OFFSET
*
counter_shift_tags
+
TAG_LOW_UP_NB_PARTICLES
,
current_com
,
&
mpiRequests
.
back
()));
if
(
descriptor
.
nbParticlesToSend
){
whatNext
.
emplace_back
(
std
::
pair
<
Action
,
int
>
{
NOTHING_TODO
,
-
1
});
mpiRequests
.
emplace_back
();
assert
(
descriptor
.
nbParticlesToSend
*
size_particle_positions
<
std
::
numeric_limits
<
int
>::
max
());
AssertMpi
(
MPI_Isend
(
const_cast
<
real_number
*>
(
&
particles_positions
[
0
]),
int
(
descriptor
.
nbParticlesToSend
*
size_particle_positions
),
particles_utils
::
GetMpiType
(
real_number
()),
descriptor
.
destProc
,
TAG_LOW_UP_PARTICLES
,
AssertMpi
(
MPI_Isend
(
const_cast
<
real_number
*>
(
&
particles_positions
[
0
]),
int
(
descriptor
.
nbParticlesToSend
*
size_particle_positions
),
particles_utils
::
GetMpiType
(
real_number
()),
descriptor
.
destProc
,
TAG_SHIFT_OFFSET
*
counter_shift_tags
+
TAG_LOW_UP_PARTICLES
,
current_com
,
&
mpiRequests
.
back
()));
assert
(
descriptor
.
toRecvAndMerge
==
nullptr
);
...
...
@@ -286,7 +291,7 @@ public:
whatNext
.
emplace_back
(
std
::
pair
<
Action
,
int
>
{
MERGE_PARTICLES
,
idxDescr
});
mpiRequests
.
emplace_back
();
assert
(
descriptor
.
nbParticlesToSend
*
size_particle_rhs
<
std
::
numeric_limits
<
int
>::
max
());
AssertMpi
(
MPI_Irecv
(
descriptor
.
toRecvAndMerge
.
get
(),
int
(
descriptor
.
nbParticlesToSend
*
size_particle_rhs
),
particles_utils
::
GetMpiType
(
real_number
()),
descriptor
.
destProc
,
TAG_UP_LOW_RESULTS
,
AssertMpi
(
MPI_Irecv
(
descriptor
.
toRecvAndMerge
.
get
(),
int
(
descriptor
.
nbParticlesToSend
*
size_particle_rhs
),
particles_utils
::
GetMpiType
(
real_number
()),
descriptor
.
destProc
,
TAG_SHIFT_OFFSET
*
counter_shift_tags
+
TAG_UP_LOW_RESULTS
,
current_com
,
&
mpiRequests
.
back
()));
}
}
...
...
@@ -295,7 +300,7 @@ public:
whatNext
.
emplace_back
(
std
::
pair
<
Action
,
int
>
{
RECV_PARTICLES
,
idxDescr
});
mpiRequests
.
emplace_back
();
AssertMpi
(
MPI_Irecv
(
&
descriptor
.
nbParticlesToRecv
,
1
,
particles_utils
::
GetMpiType
(
partsize_t
()),
descriptor
.
destProc
,
TAG_UP_LOW_NB_PARTICLES
,
1
,
particles_utils
::
GetMpiType
(
partsize_t
()),
descriptor
.
destProc
,
TAG_SHIFT_OFFSET
*
counter_shift_tags
+
TAG_UP_LOW_NB_PARTICLES
,
current_com
,
&
mpiRequests
.
back
()));
}
else
{
...
...
@@ -303,7 +308,7 @@ public:
whatNext
.
emplace_back
(
std
::
pair
<
Action
,
int
>
{
NOTHING_TODO
,
-
1
});
mpiRequests
.
emplace_back
();
AssertMpi
(
MPI_Isend
(
const_cast
<
partsize_t
*>
(
&
descriptor
.
nbParticlesToSend
),
1
,
particles_utils
::
GetMpiType
(
partsize_t
()),
descriptor
.
destProc
,
TAG_UP_LOW_NB_PARTICLES
,
descriptor
.
destProc
,
TAG_SHIFT_OFFSET
*
counter_shift_tags
+
TAG_UP_LOW_NB_PARTICLES
,
current_com
,
&
mpiRequests
.
back
()));
if
(
descriptor
.
nbParticlesToSend
){
...
...
@@ -312,7 +317,7 @@ public:
assert
(
descriptor
.
nbParticlesToSend
*
size_particle_positions
<
std
::
numeric_limits
<
int
>::
max
());
AssertMpi
(
MPI_Isend
(
const_cast
<
real_number
*>
(
&
particles_positions
[(
current_offset_particles_for_partition
[
current_partition_size
-
descriptor
.
nbPartitionsToSend
])
*
size_particle_positions
]),
int
(
descriptor
.
nbParticlesToSend
*
size_particle_positions
),
particles_utils
::
GetMpiType
(
real_number
()),
descriptor
.
destProc
,
TAG_UP_LOW_PARTICLES
,
descriptor
.
destProc
,
TAG_SHIFT_OFFSET
*
counter_shift_tags
+
TAG_UP_LOW_PARTICLES
,
current_com
,
&
mpiRequests
.
back
()));
assert
(
descriptor
.
toRecvAndMerge
==
nullptr
);
...
...
@@ -320,7 +325,7 @@ public:
whatNext
.
emplace_back
(
std
::
pair
<
Action
,
int
>
{
MERGE_PARTICLES
,
idxDescr
});
mpiRequests
.
emplace_back
();
assert
(
descriptor
.
nbParticlesToSend
*
size_particle_rhs
<
std
::
numeric_limits
<
int
>::
max
());
AssertMpi
(
MPI_Irecv
(
descriptor
.
toRecvAndMerge
.
get
(),
int
(
descriptor
.
nbParticlesToSend
*
size_particle_rhs
),
particles_utils
::
GetMpiType
(
real_number
()),
descriptor
.
destProc
,
TAG_LOW_UP_RESULTS
,
AssertMpi
(
MPI_Irecv
(
descriptor
.
toRecvAndMerge
.
get
(),
int
(
descriptor
.
nbParticlesToSend
*
size_particle_rhs
),
particles_utils
::
GetMpiType
(
real_number
()),
descriptor
.
destProc
,
TAG_SHIFT_OFFSET
*
counter_shift_tags
+
TAG_LOW_UP_RESULTS
,
current_com
,
&
mpiRequests
.
back
()));
}
...
...
@@ -328,15 +333,15 @@ public:
whatNext
.
emplace_back
(
std
::
pair
<
Action
,
int
>
{
RECV_PARTICLES
,
idxDescr
});
mpiRequests
.
emplace_back
();
AssertMpi
(
MPI_Irecv
(
&
descriptor
.
nbParticlesToRecv
,
1
,
particles_utils
::
GetMpiType
(
partsize_t
()),
descriptor
.
destProc
,
TAG_LOW_UP_NB_PARTICLES
,
1
,
particles_utils
::
GetMpiType
(
partsize_t
()),
descriptor
.
destProc
,
TAG_SHIFT_OFFSET
*
counter_shift_tags
+
TAG_LOW_UP_NB_PARTICLES
,
current_com
,
&
mpiRequests
.
back
()));
}
}
}
const
bool
more_than_one_thread
=
(
omp_get_max_threads
()
>
1
);
MPI_Barrier
(
MPI_COMM_WORLD
);
//DEBUG_MSG_WAIT(MPI_COMM_WORLD, "line 338 of particles_distr_mpi.hpp\n");
///
MPI_Barrier(MPI_COMM_WORLD);
//
/
DEBUG_MSG_WAIT(MPI_COMM_WORLD, "line 338 of particles_distr_mpi.hpp\n");
TIMEZONE_OMP_INIT_PREPARALLEL
(
omp_get_max_threads
())
#pragma omp parallel default(shared)
...
...
@@ -376,7 +381,7 @@ public:
mpiRequests
.
emplace_back
();
assert
(
NbParticlesToReceive
*
size_particle_positions
<
std
::
numeric_limits
<
int
>::
max
());
AssertMpi
(
MPI_Irecv
(
descriptor
.
toCompute
.
get
(),
int
(
NbParticlesToReceive
*
size_particle_positions
),
particles_utils
::
GetMpiType
(
real_number
()),
destProc
,
TAG_UP_LOW_PARTICLES
,
particles_utils
::
GetMpiType
(
real_number
()),
destProc
,
TAG_SHIFT_OFFSET
*
counter_shift_tags
+
TAG_UP_LOW_PARTICLES
,
current_com
,
&
mpiRequests
.
back
()));
}
}
...
...
@@ -393,7 +398,7 @@ public:
mpiRequests
.
emplace_back
();
assert
(
NbParticlesToReceive
*
size_particle_positions
<
std
::
numeric_limits
<
int
>::
max
());
AssertMpi
(
MPI_Irecv
(
descriptor
.
toCompute
.
get
(),
int
(
NbParticlesToReceive
*
size_particle_positions
),
particles_utils
::
GetMpiType
(
real_number
()),
destProc
,
TAG_LOW_UP_PARTICLES
,
particles_utils
::
GetMpiType
(
real_number
()),
destProc
,
TAG_SHIFT_OFFSET
*
counter_shift_tags
+
TAG_LOW_UP_PARTICLES
,
current_com
,
&
mpiRequests
.
back
()));
}
}
...
...
@@ -434,7 +439,7 @@ public:
const
int
destProc
=
descriptor
.
destProc
;
whatNext
.
emplace_back
(
std
::
pair
<
Action
,
int
>
{
RELEASE_BUFFER_PARTICLES
,
releasedAction
.
second
});
mpiRequests
.
emplace_back
();
const
int
tag
=
descriptor
.
isLower
?
TAG_
LOW_UP_RESULTS
:
TAG_UP_LOW_RESULTS
;
const
int
tag
=
descriptor
.
isLower
?
TAG_
SHIFT_OFFSET
*
counter_shift_tags
+
TAG_LOW_UP_RESULTS
:
TAG_SHIFT_OFFSET
*
counter_shift_tags
+
TAG_UP_LOW_RESULTS
;
assert
(
NbParticlesToReceive
*
size_particle_rhs
<
std
::
numeric_limits
<
int
>::
max
());
AssertMpi
(
MPI_Isend
(
descriptor
.
results
.
get
(),
int
(
NbParticlesToReceive
*
size_particle_rhs
),
particles_utils
::
GetMpiType
(
real_number
()),
destProc
,
tag
,
current_com
,
&
mpiRequests
.
back
()));
...
...
@@ -526,6 +531,8 @@ public:
assert
(
whatNext
.
size
()
==
0
);
assert
(
mpiRequests
.
size
()
==
0
);
counter_shift_tags
+=
1
;
}
...
...
@@ -635,35 +642,35 @@ public:
whatNext
.
emplace_back
(
std
::
pair
<
Action
,
int
>
{
RECV_MOVE_NB_LOW
,
-
1
});
mpiRequests
.
emplace_back
();
AssertMpi
(
MPI_Irecv
(
&
nbNewFromLow
,
1
,
particles_utils
::
GetMpiType
(
partsize_t
()),
(
my_rank
-
1
+
nb_processes_involved
)
%
nb_processes_involved
,
TAG_UP_LOW_MOVED_NB_PARTICLES
,
(
my_rank
-
1
+
nb_processes_involved
)
%
nb_processes_involved
,
TAG_SHIFT_OFFSET
*
counter_shift_tags
+
TAG_UP_LOW_MOVED_NB_PARTICLES
,
MPI_COMM_WORLD
,
&
mpiRequests
.
back
()));
eventsBeforeWaitall
+=
1
;
whatNext
.
emplace_back
(
std
::
pair
<
Action
,
int
>
{
NOTHING_TODO
,
-
1
});
mpiRequests
.
emplace_back
();
AssertMpi
(
MPI_Isend
(
const_cast
<
partsize_t
*>
(
&
nbOutLower
),
1
,
particles_utils
::
GetMpiType
(
partsize_t
()),
(
my_rank
-
1
+
nb_processes_involved
)
%
nb_processes_involved
,
TAG_LOW_UP_MOVED_NB_PARTICLES
,
(
my_rank
-
1
+
nb_processes_involved
)
%
nb_processes_involved
,
TAG_SHIFT_OFFSET
*
counter_shift_tags
+
TAG_LOW_UP_MOVED_NB_PARTICLES
,
MPI_COMM_WORLD
,
&
mpiRequests
.
back
()));
if
(
nbOutLower
){
whatNext
.
emplace_back
(
std
::
pair
<
Action
,
int
>
{
NOTHING_TODO
,
-
1
});
mpiRequests
.
emplace_back
();
assert
(
nbOutLower
*
size_particle_positions
<
std
::
numeric_limits
<
int
>::
max
());
AssertMpi
(
MPI_Isend
(
&
(
*
inout_positions_particles
)[
0
],
int
(
nbOutLower
*
size_particle_positions
),
particles_utils
::
GetMpiType
(
real_number
()),
(
my_rank
-
1
+
nb_processes_involved
)
%
nb_processes_involved
,
TAG_LOW_UP_MOVED_PARTICLES
,
AssertMpi
(
MPI_Isend
(
&
(
*
inout_positions_particles
)[
0
],
int
(
nbOutLower
*
size_particle_positions
),
particles_utils
::
GetMpiType
(
real_number
()),
(
my_rank
-
1
+
nb_processes_involved
)
%
nb_processes_involved
,
TAG_SHIFT_OFFSET
*
counter_shift_tags
+
TAG_LOW_UP_MOVED_PARTICLES
,
MPI_COMM_WORLD
,
&
mpiRequests
.
back
()));
whatNext
.
emplace_back
(
std
::
pair
<
Action
,
int
>
{
NOTHING_TODO
,
-
1
});
mpiRequests
.
emplace_back
();
assert
(
nbOutLower
*
size_particle_index
<
std
::
numeric_limits
<
int
>::
max
());
AssertMpi
(
MPI_Isend
(
&
(
*
inout_index_particles
)[
0
],
int
(
nbOutLower
*
size_particle_index
),
particles_utils
::
GetMpiType
(
partsize_t
()),
(
my_rank
-
1
+
nb_processes_involved
)
%
nb_processes_involved
,
TAG_LOW_UP_MOVED_PARTICLES_INDEXES
,
(
my_rank
-
1
+
nb_processes_involved
)
%
nb_processes_involved
,
TAG_SHIFT_OFFSET
*
counter_shift_tags
+
TAG_LOW_UP_MOVED_PARTICLES_INDEXES
,
MPI_COMM_WORLD
,
&
mpiRequests
.
back
()));
for
(
int
idx_rhs
=
0
;
idx_rhs
<
in_nb_rhs
;
++
idx_rhs
){
whatNext
.
emplace_back
(
std
::
pair
<
Action
,
int
>
{
NOTHING_TODO
,
-
1
});
mpiRequests
.
emplace_back
();
assert
(
nbOutLower
*
size_particle_rhs
<
std
::
numeric_limits
<
int
>::
max
());
AssertMpi
(
MPI_Isend
(
&
inout_rhs_particles
[
idx_rhs
][
0
],
int
(
nbOutLower
*
size_particle_rhs
),
particles_utils
::
GetMpiType
(
real_number
()),
(
my_rank
-
1
+
nb_processes_involved
)
%
nb_processes_involved
,
TAG_LOW_UP_MOVED_PARTICLES_RHS
+
idx_rhs
,
AssertMpi
(
MPI_Isend
(
&
inout_rhs_particles
[
idx_rhs
][
0
],
int
(
nbOutLower
*
size_particle_rhs
),
particles_utils
::
GetMpiType
(
real_number
()),
(
my_rank
-
1
+
nb_processes_involved
)
%
nb_processes_involved
,
TAG_SHIFT_OFFSET
*
counter_shift_tags
+
TAG_LOW_UP_MOVED_PARTICLES_RHS
+
idx_rhs
,
MPI_COMM_WORLD
,
&
mpiRequests
.
back
()));
}
}
...
...
@@ -671,14 +678,14 @@ public:
whatNext
.
emplace_back
(
std
::
pair
<
Action
,
int
>
{
RECV_MOVE_NB_UP
,
-
1
});
mpiRequests
.
emplace_back
();
AssertMpi
(
MPI_Irecv
(
&
nbNewFromUp
,
1
,
particles_utils
::
GetMpiType
(
partsize_t
()),
(
my_rank
+
1
)
%
nb_processes_involved
,
TAG_LOW_UP_MOVED_NB_PARTICLES
,
TAG_SHIFT_OFFSET
*
counter_shift_tags
+
TAG_LOW_UP_MOVED_NB_PARTICLES
,
MPI_COMM_WORLD
,
&
mpiRequests
.
back
()));
eventsBeforeWaitall
+=
1
;
whatNext
.
emplace_back
(
std
::
pair
<
Action
,
int
>
{
NOTHING_TODO
,
-
1
});
mpiRequests
.
emplace_back
();
AssertMpi
(
MPI_Isend
(
const_cast
<
partsize_t
*>
(
&
nbOutUpper
),
1
,
particles_utils
::
GetMpiType
(
partsize_t
()),
(
my_rank
+
1
)
%
nb_processes_involved
,
TAG_UP_LOW_MOVED_NB_PARTICLES
,
(
my_rank
+
1
)
%
nb_processes_involved
,
TAG_SHIFT_OFFSET
*
counter_shift_tags
+
TAG_UP_LOW_MOVED_NB_PARTICLES
,
MPI_COMM_WORLD
,
&
mpiRequests
.
back
()));
if
(
nbOutUpper
){
...
...
@@ -686,14 +693,14 @@ public:
mpiRequests
.
emplace_back
();
assert
(
nbOutUpper
*
size_particle_positions
<
std
::
numeric_limits
<
int
>::
max
());
AssertMpi
(
MPI_Isend
(
&
(
*
inout_positions_particles
)[(
myTotalNbParticles
-
nbOutUpper
)
*
size_particle_positions
],
int
(
nbOutUpper
*
size_particle_positions
),
particles_utils
::
GetMpiType
(
real_number
()),
(
my_rank
+
1
)
%
nb_processes_involved
,
TAG_UP_LOW_MOVED_PARTICLES
,
int
(
nbOutUpper
*
size_particle_positions
),
particles_utils
::
GetMpiType
(
real_number
()),
(
my_rank
+
1
)
%
nb_processes_involved
,
TAG_SHIFT_OFFSET
*
counter_shift_tags
+
TAG_UP_LOW_MOVED_PARTICLES
,
MPI_COMM_WORLD
,
&
mpiRequests
.
back
()));
whatNext
.
emplace_back
(
std
::
pair
<
Action
,
int
>
{
NOTHING_TODO
,
-
1
});
mpiRequests
.
emplace_back
();
assert
(
nbOutUpper
*
size_particle_index
<
std
::
numeric_limits
<
int
>::
max
());
AssertMpi
(
MPI_Isend
(
&
(
*
inout_index_particles
)[(
myTotalNbParticles
-
nbOutUpper
)
*
size_particle_index
],
int
(
nbOutUpper
*
size_particle_index
),
particles_utils
::
GetMpiType
(
partsize_t
()),
(
my_rank
+
1
)
%
nb_processes_involved
,
TAG_UP_LOW_MOVED_PARTICLES_INDEXES
,
particles_utils
::
GetMpiType
(
partsize_t
()),
(
my_rank
+
1
)
%
nb_processes_involved
,
TAG_SHIFT_OFFSET
*
counter_shift_tags
+
TAG_UP_LOW_MOVED_PARTICLES_INDEXES
,
MPI_COMM_WORLD
,
&
mpiRequests
.
back
()));
for
(
int
idx_rhs
=
0
;
idx_rhs
<
in_nb_rhs
;
++
idx_rhs
){
...
...
@@ -701,7 +708,7 @@ public:
mpiRequests
.
emplace_back
();
assert
(
nbOutUpper
*
size_particle_rhs
<
std
::
numeric_limits
<
int
>::
max
());
AssertMpi
(
MPI_Isend
(
&
inout_rhs_particles
[
idx_rhs
][(
myTotalNbParticles
-
nbOutUpper
)
*
size_particle_rhs
],
int
(
nbOutUpper
*
size_particle_rhs
),
particles_utils
::
GetMpiType
(
real_number
()),
(
my_rank
+
1
)
%
nb_processes_involved
,
TAG_UP_LOW_MOVED_PARTICLES_RHS
+
idx_rhs
,
int
(
nbOutUpper
*
size_particle_rhs
),
particles_utils
::
GetMpiType
(
real_number
()),
(
my_rank
+
1
)
%
nb_processes_involved
,
TAG_SHIFT_OFFSET
*
counter_shift_tags
+
TAG_UP_LOW_MOVED_PARTICLES_RHS
+
idx_rhs
,
MPI_COMM_WORLD
,
&
mpiRequests
.
back
()));
}
}
...
...
@@ -726,7 +733,7 @@ public:
mpiRequests
.
emplace_back
();
assert
(
nbNewFromLow
*
size_particle_positions
<
std
::
numeric_limits
<
int
>::
max
());
AssertMpi
(
MPI_Irecv
(
&
newParticlesLow
[
0
],
int
(
nbNewFromLow
*
size_particle_positions
),
particles_utils
::
GetMpiType
(
real_number
()),
(
my_rank
-
1
+
nb_processes_involved
)
%
nb_processes_involved
,
TAG_UP_LOW_MOVED_PARTICLES
,
(
my_rank
-
1
+
nb_processes_involved
)
%
nb_processes_involved
,
TAG_SHIFT_OFFSET
*
counter_shift_tags
+
TAG_UP_LOW_MOVED_PARTICLES
,
MPI_COMM_WORLD
,
&
mpiRequests
.
back
()));
newParticlesLowIndexes
.
reset
(
new
partsize_t
[
nbNewFromLow
*
size_particle_index
]);
...
...
@@ -735,7 +742,7 @@ public:
assert
(
nbNewFromLow
*
size_particle_index
<
std
::
numeric_limits
<
int
>::
max
());
AssertMpi
(
MPI_Irecv
(
&
newParticlesLowIndexes
[
0
],
int
(
nbNewFromLow
*
size_particle_index
),
particles_utils
::
GetMpiType
(
partsize_t
()),
(
my_rank
-
1
+
nb_processes_involved
)
%
nb_processes_involved
,
TAG_UP_LOW_MOVED_PARTICLES_INDEXES
,
(
my_rank
-
1
+
nb_processes_involved
)
%
nb_processes_involved
,
TAG_SHIFT_OFFSET
*
counter_shift_tags
+
TAG_UP_LOW_MOVED_PARTICLES_INDEXES
,
MPI_COMM_WORLD
,
&
mpiRequests
.
back
()));
for
(
int
idx_rhs
=
0
;
idx_rhs
<
in_nb_rhs
;
++
idx_rhs
){
...
...
@@ -743,7 +750,7 @@ public:
whatNext
.
emplace_back
(
std
::
pair
<
Action
,
int
>
{
NOTHING_TODO
,
-
1
});
mpiRequests
.
emplace_back
();
assert
(
nbNewFromLow
*
size_particle_rhs
<
std
::
numeric_limits
<
int
>::
max
());
AssertMpi
(
MPI_Irecv
(
&
newParticlesLowRhs
[
idx_rhs
][
0
],
int
(
nbNewFromLow
*
size_particle_rhs
),
particles_utils
::
GetMpiType
(
real_number
()),
(
my_rank
-
1
+
nb_processes_involved
)
%
nb_processes_involved
,
TAG_UP_LOW_MOVED_PARTICLES_RHS
+
idx_rhs
,
AssertMpi
(
MPI_Irecv
(
&
newParticlesLowRhs
[
idx_rhs
][
0
],
int
(
nbNewFromLow
*
size_particle_rhs
),
particles_utils
::
GetMpiType
(
real_number
()),
(
my_rank
-
1
+
nb_processes_involved
)
%
nb_processes_involved
,
TAG_SHIFT_OFFSET
*
counter_shift_tags
+
TAG_UP_LOW_MOVED_PARTICLES_RHS
+
idx_rhs
,
MPI_COMM_WORLD
,
&
mpiRequests
.
back
()));
}
}
...
...
@@ -756,7 +763,7 @@ public:
whatNext
.
emplace_back
(
std
::
pair
<
Action
,
int
>
{
RECV_MOVE_UP
,
-
1
});
mpiRequests
.
emplace_back
();
assert
(
nbNewFromUp
*
size_particle_positions
<
std
::
numeric_limits
<
int
>::
max
());
AssertMpi
(
MPI_Irecv
(
&
newParticlesUp
[
0
],
int
(
nbNewFromUp
*
size_particle_positions
),
particles_utils
::
GetMpiType
(
real_number
()),
(
my_rank
+
1
)
%
nb_processes_involved
,
TAG_LOW_UP_MOVED_PARTICLES
,
AssertMpi
(
MPI_Irecv
(
&
newParticlesUp
[
0
],
int
(
nbNewFromUp
*
size_particle_positions
),
particles_utils
::
GetMpiType
(
real_number
()),
(
my_rank
+
1
)
%
nb_processes_involved
,
TAG_SHIFT_OFFSET
*
counter_shift_tags
+
TAG_LOW_UP_MOVED_PARTICLES
,
MPI_COMM_WORLD
,
&
mpiRequests
.
back
()));
newParticlesUpIndexes
.
reset
(
new
partsize_t
[
nbNewFromUp
*
size_particle_index
]);
...
...
@@ -765,7 +772,7 @@ public:
assert
(
nbNewFromUp
*
size_particle_index
<
std
::
numeric_limits
<
int
>::
max
());
AssertMpi
(
MPI_Irecv
(
&
newParticlesUpIndexes
[
0
],
int
(
nbNewFromUp
*
size_particle_index
),
particles_utils
::
GetMpiType
(
partsize_t
()),
(
my_rank
+
1
)
%
nb_processes_involved
,
TAG_LOW_UP_MOVED_PARTICLES_INDEXES
,
(
my_rank
+
1
)
%
nb_processes_involved
,
TAG_SHIFT_OFFSET
*
counter_shift_tags
+
TAG_LOW_UP_MOVED_PARTICLES_INDEXES
,
MPI_COMM_WORLD
,
&
mpiRequests
.
back
()));
for
(
int
idx_rhs
=
0
;
idx_rhs
<
in_nb_rhs
;
++
idx_rhs
){
...
...
@@ -773,7 +780,7 @@ public:
whatNext
.
emplace_back
(
std
::
pair
<
Action
,
int
>
{
NOTHING_TODO
,
-
1
});
mpiRequests
.
emplace_back
();
assert
(
nbNewFromUp
*
size_particle_rhs
<
std
::
numeric_limits
<
int
>::
max
());
AssertMpi
(
MPI_Irecv
(
&
newParticlesUpRhs
[
idx_rhs
][
0
],
int
(
nbNewFromUp
*
size_particle_rhs
),
particles_utils
::
GetMpiType
(
real_number
()),
(
my_rank
+
1
)
%
nb_processes_involved
,
TAG_LOW_UP_MOVED_PARTICLES_RHS
+
idx_rhs
,
AssertMpi
(
MPI_Irecv
(
&
newParticlesUpRhs
[
idx_rhs
][
0
],
int
(
nbNewFromUp
*
size_particle_rhs
),
particles_utils
::
GetMpiType
(
real_number
()),
(
my_rank
+
1
)
%
nb_processes_involved
,
TAG_SHIFT_OFFSET
*
counter_shift_tags
+
TAG_LOW_UP_MOVED_PARTICLES_RHS
+
idx_rhs
,
MPI_COMM_WORLD
,
&
mpiRequests
.
back
()));
}
}
...
...
@@ -880,6 +887,8 @@ public:
(
*
nb_particles
)
=
myTotalNbParticles
;
assert
(
mpiRequests
.
size
()
==
0
);
counter_shift_tags
+=
1
;
}
};
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment