From d405d39319951d78a0015494621e133f16055934 Mon Sep 17 00:00:00 2001
From: Cristian Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 8 Aug 2017 13:42:11 +0200
Subject: [PATCH 001/342] fix calls to assert
---
bfps/_code.py | 30 +++++++++++++++++-------------
1 file changed, 17 insertions(+), 13 deletions(-)
diff --git a/bfps/_code.py b/bfps/_code.py
index 22bcd910..7fc4642d 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -246,7 +246,7 @@ class _code(_base):
need_to_compile = (datetime.fromtimestamp(os.path.getctime(os.path.join(self.work_dir, self.name))) <
bfps.install_info['install_date'])
if need_to_compile:
- assert(self.compile_code() == 0)
+ assert self.compile_code() == 0
if self.work_dir != os.path.realpath(os.getcwd()):
shutil.copy(self.name, self.work_dir)
if 'niter_todo' not in self.parameters.keys():
@@ -386,13 +386,14 @@ class _code(_base):
script_file.write('# @ notify_user = $(user)@rzg.mpg.de\n')
nb_cpus_per_node = self.host_info['deltanprocs']
- assert(isinstance(nb_cpus_per_node, int) and nb_cpus_per_node >= 1,
- 'nb_cpus_per_node is {}'.format(nb_cpus_per_node))
+ assert isinstance(nb_cpus_per_node, int) and \
+ nb_cpus_per_node >= 1, \
+ 'nb_cpus_per_node is {}'.format(nb_cpus_per_node)
# No more threads than the number of cores
- assert(nb_threads_per_process <= nb_cpus_per_node,
+ assert nb_threads_per_process <= nb_cpus_per_node, \
"Cannot use more threads ({} asked) than the number of cores ({})".format(
- nb_threads_per_process, nb_cpus_per_node))
+ nb_threads_per_process, nb_cpus_per_node)
# Warn if some core will not be ued
if nb_cpus_per_node%nb_threads_per_process != 0:
warnings.warn("The number of threads is smaller than the number of cores (machine will be underused)",
@@ -479,12 +480,14 @@ class _code(_base):
script_file.write('#\n')
nb_cpus_per_node = self.host_info['deltanprocs']
- assert(isinstance(nb_cpus_per_node, int) and nb_cpus_per_node >= 1, 'nb_cpus_per_node is {}'.format(nb_cpus_per_node))
+ assert isinstance(nb_cpus_per_node, int) and \
+ nb_cpus_per_node >= 1, \
+ 'nb_cpus_per_node is {}'.format(nb_cpus_per_node)
# No more threads than the number of cores
- assert(nb_threads_per_process <= nb_cpus_per_node,
+ assert nb_threads_per_process <= nb_cpus_per_node, \
"Cannot use more threads ({} asked) than the number of cores ({})".format(
- nb_threads_per_process, nb_cpus_per_node))
+ nb_threads_per_process, nb_cpus_per_node)
# Warn if some core will not be ued
if nb_cpus_per_node%nb_threads_per_process != 0:
warnings.warn("The number of threads is smaller than the number of cores (machine will be underused)",
@@ -507,7 +510,7 @@ class _code(_base):
script_file.write('# @ resources = ConsumableCpus({})\n'.format(nb_threads_per_process))
script_file.write('# @ network.MPI = sn_all,not_shared,us\n')
script_file.write('# @ wall_clock_limit = {0}:{1:0>2d}:00\n'.format(hours, minutes))
- assert(type(self.host_info['environment']) != type(None))
+ assert type(self.host_info['environment']) != type(None)
script_file.write('# @ node = {0}\n'.format(nb_nodes))
script_file.write('# @ tasks_per_node = {0}\n'.format(nb_processes_per_node))
if (first_node_tasks > 0):
@@ -611,13 +614,14 @@ class _code(_base):
self.host_info['environment']))
nb_cpus_per_node = self.host_info['deltanprocs']
- assert(isinstance(nb_cpus_per_node, int) and nb_cpus_per_node >= 1,
- 'nb_cpus_per_node is {}'.format(nb_cpus_per_node))
+ assert isinstance(nb_cpus_per_node, int) \
+ and nb_cpus_per_node >= 1, \
+ 'nb_cpus_per_node is {}'.format(nb_cpus_per_node)
# No more threads than the number of cores
- assert(nb_threads_per_process <= nb_cpus_per_node,
+ assert nb_threads_per_process <= nb_cpus_per_node, \
"Cannot use more threads ({} asked) than the number of cores ({})".format(
- nb_threads_per_process, nb_cpus_per_node))
+ nb_threads_per_process, nb_cpus_per_node)
# Warn if some core will not be ued
if nb_cpus_per_node%nb_threads_per_process != 0:
warnings.warn(
--
GitLab
From e6b94e6c56f94981ff155fdb11be0e8509f93dec Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 8 Aug 2017 16:48:19 +0200
Subject: [PATCH 002/342] consider negative values in joint PDF of scalars
this is a significant change of the behavior of joint_rspace_PDF,
because I didn't realize I was doing something stupid...
---
bfps/cpp/field.cpp | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/bfps/cpp/field.cpp b/bfps/cpp/field.cpp
index 197ccb5d..6f2ff938 100644
--- a/bfps/cpp/field.cpp
+++ b/bfps/cpp/field.cpp
@@ -1235,8 +1235,8 @@ int joint_rspace_PDF(
{
for (unsigned int i=0; i<4; i++)
{
- bin1size[i] = max_f1_estimate[0] / nbins;
- bin2size[i] = max_f2_estimate[0] / nbins;
+ bin1size[i] = 2*max_f1_estimate[0] / nbins;
+ bin2size[i] = 2*max_f2_estimate[0] / nbins;
}
}
@@ -1279,8 +1279,8 @@ int joint_rspace_PDF(
}
else if (fc == ONE)
{
- bin1 = int(floor(f1->rval(rindex)/bin1size[3]));
- bin2 = int(floor(f2->rval(rindex)/bin2size[3]));
+ bin1 = int(floor((f1->rval(rindex) + max_f1_estimate[0])/bin1size[3]));
+ bin2 = int(floor((f2->rval(rindex) + max_f2_estimate[0])/bin2size[3]));
}
if ((bin1 >= 0 && bin1 < nbins) &&
(bin2 >= 0 && bin2 < nbins))
--
GitLab
From 32e6ba50026a9114076cc2e525df8f40df2f83ac Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Sun, 13 Aug 2017 12:14:13 +0200
Subject: [PATCH 003/342] more control over particle_output_hdf5
I need to use it for many species of particles, so I added direct
control over the particle species name.
---
bfps/cpp/particles/particles_output_hdf5.hpp | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/bfps/cpp/particles/particles_output_hdf5.hpp b/bfps/cpp/particles/particles_output_hdf5.hpp
index bc0a0369..647103ca 100644
--- a/bfps/cpp/particles/particles_output_hdf5.hpp
+++ b/bfps/cpp/particles/particles_output_hdf5.hpp
@@ -21,7 +21,7 @@ class particles_output_hdf5 : public abstract_particles_output<partsize_t,
size_particle_positions,
size_particle_rhs>;
- const std::string particle_species_name;
+ std::string particle_species_name;
hid_t file_id;
const partsize_t total_nb_particles;
@@ -90,6 +90,12 @@ public:
~particles_output_hdf5(){}
+ void update_particle_species_name(
+ const std::string new_name)
+ {
+ this->particle_species_name.assign(new_name);
+ }
+
int close_file(void){
if(Parent::isInvolved()){
TIMEZONE("particles_output_hdf5::close_file");
--
GitLab
From 98c3eb2f2e9b12ab4f1dddef90de3bae61065ae1 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 14 Aug 2017 16:54:48 +0200
Subject: [PATCH 004/342] make the DNS class more general
---
bfps/DNS.py | 33 ++++++++++++++++++----------
bfps/cpp/full_code/NSVEparticles.cpp | 2 +-
2 files changed, 22 insertions(+), 13 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 4f26b86c..57dc879a 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -78,7 +78,8 @@ class DNS(_code):
self.C_field_dtype = 'double'
self.fluid_precision = 'double'
return None
- def write_src(self):
+ def write_src(
+ self):
self.version_message = (
'/***********************************************************************\n' +
'* this code automatically generated by bfps\n' +
@@ -623,7 +624,8 @@ class DNS(_code):
return None
def prepare_launch(
self,
- args = []):
+ args = [],
+ extra_parameters = None):
"""Set up reasonable parameters.
With the default Lundgren forcing applied in the band [2, 4],
@@ -657,6 +659,10 @@ class DNS(_code):
if self.dns_type in ['NSVEparticles', 'NSVEparticles_no_output']:
for k in self.NSVEp_extra_parameters.keys():
self.parameters[k] = self.NSVEp_extra_parameters[k]
+ if type(extra_parameters) != type(None):
+ if self.dns_type in extra_parameters.keys():
+ for k in extra_parameters[self.dns_type].keys():
+ self.parameters[k] = extra_parameters[self.dns_type][k]
self.parameters['nu'] = (opt.kMeta * 2 / opt.n)**(4./3)
self.parameters['dt'] = (opt.dtfactor / opt.n)
# custom famplitude for 288 and 576
@@ -837,12 +843,10 @@ class DNS(_code):
for kz in range(src_file[src_dset_name].shape[0]):
dst_file[dst_dset_name][kz] = src_file[src_dset_name][kz]
else:
- print('aloha')
min_shape = (min(dst_shape[0], src_file[src_dset_name].shape[0]),
min(dst_shape[1], src_file[src_dset_name].shape[1]),
min(dst_shape[2], src_file[src_dset_name].shape[2]),
3)
- print(self.ctype)
dst_file.create_dataset(
dst_dset_name,
shape = dst_shape,
@@ -852,6 +856,18 @@ class DNS(_code):
dst_file[dst_dset_name][kz,:min_shape[1], :min_shape[2]] = \
src_file[src_dset_name][kz, :min_shape[1], :min_shape[2]]
return None
+ def generate_particle_data(
+ self,
+ opt = None):
+ if self.parameters['nparticles'] > 0:
+ self.generate_tracer_state(
+ species = 0,
+ rseed = opt.particle_rand_seed)
+ if not os.path.exists(self.get_particle_file_name()):
+ with h5py.File(self.get_particle_file_name(), 'w') as particle_file:
+ particle_file.create_group('tracers0/velocity')
+ particle_file.create_group('tracers0/acceleration')
+ return None
def launch_jobs(
self,
opt = None,
@@ -911,14 +927,7 @@ class DNS(_code):
self.write_par(
particle_ic = None)
if self.dns_type in ['NSVEparticles', 'NSVEparticles_no_output']:
- if self.parameters['nparticles'] > 0:
- self.generate_tracer_state(
- species = 0,
- rseed = opt.particle_rand_seed)
- if not os.path.exists(self.get_particle_file_name()):
- with h5py.File(self.get_particle_file_name(), 'w') as particle_file:
- particle_file.create_group('tracers0/velocity')
- particle_file.create_group('tracers0/acceleration')
+ self.generate_particle_data(opt = opt)
self.run(
nb_processes = opt.nb_processes,
nb_threads_per_process = opt.nb_threads_per_process,
diff --git a/bfps/cpp/full_code/NSVEparticles.cpp b/bfps/cpp/full_code/NSVEparticles.cpp
index ba84b394..90b948b6 100644
--- a/bfps/cpp/full_code/NSVEparticles.cpp
+++ b/bfps/cpp/full_code/NSVEparticles.cpp
@@ -58,9 +58,9 @@ int NSVEparticles<rnumber>::write_checkpoint(void)
template <typename rnumber>
int NSVEparticles<rnumber>::finalize(void)
{
- this->NSVE<rnumber>::finalize();
this->ps.release();
delete this->particles_output_writer_mpi;
+ this->NSVE<rnumber>::finalize();
return EXIT_SUCCESS;
}
--
GitLab
From 9b5e5698bd017d2e0cc94e0e4b52c7b8c21cb143 Mon Sep 17 00:00:00 2001
From: Cristian Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 14 Aug 2017 21:29:08 +0200
Subject: [PATCH 005/342] add required header
---
bfps/cpp/particles/particles_system_builder.hpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/bfps/cpp/particles/particles_system_builder.hpp b/bfps/cpp/particles/particles_system_builder.hpp
index 7a2d49c0..a3bc689d 100644
--- a/bfps/cpp/particles/particles_system_builder.hpp
+++ b/bfps/cpp/particles/particles_system_builder.hpp
@@ -3,6 +3,7 @@
#include <string>
+#include <cmath>
#include "abstract_particles_system.hpp"
#include "particles_system.hpp"
#include "particles_input_hdf5.hpp"
--
GitLab
From 6391eed63b4ee118b1e5556869f7923963b6a671 Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Mon, 14 Aug 2017 22:58:57 +0200
Subject: [PATCH 006/342] get rid of assert that breaks interpolation
---
bfps/cpp/field.hpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bfps/cpp/field.hpp b/bfps/cpp/field.hpp
index 52a93632..9a5ab1be 100644
--- a/bfps/cpp/field.hpp
+++ b/bfps/cpp/field.hpp
@@ -154,7 +154,7 @@ class field
inline const rnumber& rval(ptrdiff_t rindex, unsigned int component = 0) const
{
- assert(fc == ONE || fc == THREE);
+ //assert(fc == ONE || fc == THREE);
assert(component >= 0 && component < ncomp(fc));
return *(this->data + rindex*ncomp(fc) + component);
}
--
GitLab
From cf6ee0724aedf259bc8cdc9477c68ca14037bc79 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 17 Aug 2017 17:41:11 +0200
Subject: [PATCH 007/342] code seems to compile
---
bfps/DNS.py | 1 +
bfps/cpp/full_code/NSVEparticles.cpp | 8 ++++++
bfps/cpp/particles/particles_sampling.hpp | 34 ++++++++++++++++++++++-
3 files changed, 42 insertions(+), 1 deletion(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 57dc879a..faf756c7 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -865,6 +865,7 @@ class DNS(_code):
rseed = opt.particle_rand_seed)
if not os.path.exists(self.get_particle_file_name()):
with h5py.File(self.get_particle_file_name(), 'w') as particle_file:
+ particle_file.create_group('tracers0/position')
particle_file.create_group('tracers0/velocity')
particle_file.create_group('tracers0/acceleration')
return None
diff --git a/bfps/cpp/full_code/NSVEparticles.cpp b/bfps/cpp/full_code/NSVEparticles.cpp
index 90b948b6..2384e3f1 100644
--- a/bfps/cpp/full_code/NSVEparticles.cpp
+++ b/bfps/cpp/full_code/NSVEparticles.cpp
@@ -77,6 +77,14 @@ int NSVEparticles<rnumber>::do_stats()
if (!(this->iteration % this->niter_part == 0))
return EXIT_SUCCESS;
+ /// sample position
+ sample_particles_system_position(
+ this->ps,
+ (this->simname + "_particles.h5"), // filename
+ "tracers0", // hdf5 parent group
+ "position" // dataset basename TODO
+ );
+
/// sample velocity
sample_from_particles_system(*this->tmp_vec_field, // field to save
this->ps,
diff --git a/bfps/cpp/particles/particles_sampling.hpp b/bfps/cpp/particles/particles_sampling.hpp
index 3adc2553..c6b7e295 100644
--- a/bfps/cpp/particles/particles_sampling.hpp
+++ b/bfps/cpp/particles/particles_sampling.hpp
@@ -48,5 +48,37 @@ void sample_from_particles_system(const field<rnumber, be, fc>& in_field, // a p
ps->get_step_idx());
}
-#endif
+template <class partsize_t, class particles_rnumber>
+void sample_particles_system_position(
+ std::unique_ptr<abstract_particles_system<partsize_t, particles_rnumber>>& ps, // a pointer to an particles_system<double>
+ const std::string& filename,
+ const std::string& parent_groupname,
+ const std::string& fname){
+ const std::string datasetname = fname + std::string("/") + std::to_string(ps->get_step_idx());
+
+ // Stop here if already exists
+ if(particles_output_sampling_hdf5<partsize_t, particles_rnumber, 3, 3>::DatasetExistsCol(MPI_COMM_WORLD,
+ filename,
+ parent_groupname,
+ datasetname)){
+ return;
+ }
+
+ const partsize_t nb_particles = ps->getLocalNbParticles();
+ std::unique_ptr<particles_rnumber[]> sample_rhs(new particles_rnumber[3*nb_particles]);
+ std::copy(ps->getParticlesPositions(), ps->getParticlesPositions() + 3*nb_particles, sample_rhs.get());
+
+ particles_output_sampling_hdf5<partsize_t, particles_rnumber, 3, 3> outputclass(MPI_COMM_WORLD,
+ ps->getGlobalNbParticles(),
+ filename,
+ parent_groupname,
+ datasetname);
+ outputclass.save(ps->getParticlesPositions(),
+ &sample_rhs,
+ ps->getParticlesIndexes(),
+ ps->getLocalNbParticles(),
+ ps->get_step_idx());
+}
+
+#endif//PARTICLES_SAMPLING_HPP
--
GitLab
From 119717307ebcdbbde29b8e7f134b8f99daf45ce0 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 7 Sep 2017 13:29:47 +0200
Subject: [PATCH 008/342] fix implicit resize for non-cubic domains
---
bfps/DNS.py | 21 ++++++++++++++++++---
1 file changed, 18 insertions(+), 3 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index faf756c7..e15e3ee8 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -852,9 +852,24 @@ class DNS(_code):
shape = dst_shape,
dtype = np.dtype(self.ctype),
fillvalue = complex(0))
- for kz in range(min_shape[0]):
- dst_file[dst_dset_name][kz,:min_shape[1], :min_shape[2]] = \
- src_file[src_dset_name][kz, :min_shape[1], :min_shape[2]]
+ for kz in range(min_shape[0]//2):
+ dst_file[dst_dset_name][kz,:min_shape[1]//2, :min_shape[2]] = \
+ src_file[src_dset_name][kz, :min_shape[1]//2, :min_shape[2]]
+ dst_file[dst_dset_name][kz,
+ dst_shape[1] - min_shape[1]//2+1:,
+ :min_shape[2]] = \
+ src_file[src_dset_name][kz,
+ src_shape[1] - min_shape[1]//2+1,
+ :min_shape[2]]
+ if kz > 0:
+ dst_file[dst_dset_name][-kz,:min_shape[1]//2, :min_shape[2]] = \
+ src_file[src_dset_name][-kz, :min_shape[1]//2, :min_shape[2]]
+ dst_file[dst_dset_name][-kz,
+ dst_shape[1] - min_shape[1]//2+1:,
+ :min_shape[2]] = \
+ src_file[src_dset_name][-kz,
+ src_shape[1] - min_shape[1]//2+1,
+ :min_shape[2]]
return None
def generate_particle_data(
self,
--
GitLab
From 01075923adccd738d1ef6ca92615f2bdec67875a Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 7 Sep 2017 13:33:19 +0200
Subject: [PATCH 009/342] bugfix: fix the previous fix
---
bfps/DNS.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index e15e3ee8..8cbe8d9c 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -847,6 +847,7 @@ class DNS(_code):
min(dst_shape[1], src_file[src_dset_name].shape[1]),
min(dst_shape[2], src_file[src_dset_name].shape[2]),
3)
+ src_shape = src_file[src_dset_name].shape
dst_file.create_dataset(
dst_dset_name,
shape = dst_shape,
--
GitLab
From 9abc3b7bdb49bc277597c8f3ffb2bfb051cdabdc Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 12 Sep 2017 15:30:45 +0200
Subject: [PATCH 010/342] bugfix: get_rfields should use _post file
---
bfps/PP.py | 9 +++++----
bfps/cpp/full_code/get_rfields.cpp | 9 ++++++++-
2 files changed, 13 insertions(+), 5 deletions(-)
diff --git a/bfps/PP.py b/bfps/PP.py
index 6e02f2ae..705c363f 100644
--- a/bfps/PP.py
+++ b/bfps/PP.py
@@ -674,10 +674,11 @@ class PP(_code):
group = self.dns_type + '/parameters',
parameters = self.pp_parameters,
file_name = os.path.join(self.work_dir, self.simname + '_post.h5'))
- histogram_bins = opt.histogram_bins
- if (type(histogram_bins) == type(None) and
- 'histogram_bins' in self.pp_parameters.keys()):
- histogram_bins = self.pp_parameters['histogram_bins']
+ if 'histogram_bins' in opt.__dict__.keys():
+ histogram_bins = opt.histogram_bins
+ if (type(histogram_bins) == type(None) and
+ 'histogram_bins' in self.pp_parameters.keys()):
+ histogram_bins = self.pp_parameters['histogram_bins']
with h5py.File(os.path.join(self.work_dir, self.simname + '_post.h5'), 'r+') as ofile:
group = ofile[self.dns_type]
group.require_group('histograms')
diff --git a/bfps/cpp/full_code/get_rfields.cpp b/bfps/cpp/full_code/get_rfields.cpp
index 0df8b564..0a751f70 100644
--- a/bfps/cpp/full_code/get_rfields.cpp
+++ b/bfps/cpp/full_code/get_rfields.cpp
@@ -8,6 +8,7 @@ template <typename rnumber>
int get_rfields<rnumber>::initialize(void)
{
this->NSVE_field_stats<rnumber>::initialize();
+ DEBUG_MSG("after NSVE_field_stats::initialize\n");
this->kk = new kspace<FFTW, SMOOTH>(
this->vorticity->clayout, this->dkx, this->dky, this->dkz);
hid_t parameter_file = H5Fopen(
@@ -25,9 +26,15 @@ int get_rfields<rnumber>::initialize(void)
}
else
this->checkpoints_per_file = 1;
+ H5Fclose(parameter_file);
+ parameter_file = H5Fopen(
+ (this->simname + std::string("_post.h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ DEBUG_MSG("before read_vector\n");
this->iteration_list = hdf5_tools::read_vector<int>(
parameter_file,
- "/get_rfields/iteration_list");
+ "/get_rfields/parameters/iteration_list");
H5Fclose(parameter_file);
return EXIT_SUCCESS;
}
--
GitLab
From 0ef93327054f4577aa1a8e3324ad38e2bcf2a0dd Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 14 Sep 2017 16:55:41 +0200
Subject: [PATCH 011/342] save rspace vorticity as well
---
bfps/cpp/full_code/get_rfields.cpp | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/bfps/cpp/full_code/get_rfields.cpp b/bfps/cpp/full_code/get_rfields.cpp
index 0a751f70..3986b5f9 100644
--- a/bfps/cpp/full_code/get_rfields.cpp
+++ b/bfps/cpp/full_code/get_rfields.cpp
@@ -84,6 +84,13 @@ int get_rfields<rnumber>::work_on_current_iteration(void)
false);
delete vel;
+
+ this->vorticity->ift();
+ this->vorticity->io(
+ fname,
+ "vorticity",
+ this->iteration,
+ false);
return EXIT_SUCCESS;
}
--
GitLab
From 0ce7d47f5ecb7349f035aed2a675949ae1748854 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 18 Sep 2017 14:58:31 +0200
Subject: [PATCH 012/342] fix checkpoint postprocessing issue
---
bfps/PP.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/bfps/PP.py b/bfps/PP.py
index 705c363f..c93c9551 100644
--- a/bfps/PP.py
+++ b/bfps/PP.py
@@ -773,7 +773,7 @@ class PP(_code):
dtype = np.float64)
df.close()
return None
- def prepare_field_file(self):
+ def prepare_field_file(self, iter0 = 0):
df = self.get_data_file()
if 'field_dtype' in df.keys():
# we don't need to do anything, raw binary files are used
@@ -784,7 +784,7 @@ class PP(_code):
with h5py.File(os.path.join(self.work_dir, self.simname + '_fields.h5'), 'a') as ff:
ff.require_group('vorticity')
ff.require_group('vorticity/complex')
- checkpoint = 0
+ checkpoint = (iter0 // niter_out) // cppf
while True:
cpf_name = os.path.join(
self.work_dir,
@@ -805,7 +805,7 @@ class PP(_code):
opt = None,
particle_initial_condition = None):
self.prepare_post_file(opt)
- self.prepare_field_file()
+ self.prepare_field_file(iter0 = opt.iter0)
self.run(
nb_processes = opt.nb_processes,
nb_threads_per_process = opt.nb_threads_per_process,
--
GitLab
From eb1b7d361a66433d7423f124f14a2924ce311248 Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Wed, 20 Sep 2017 15:14:20 +0200
Subject: [PATCH 013/342] Add first version of particle particle interactions
---
bfps/cpp/particles/.tocompile | 2 +
bfps/cpp/particles/p2p_computer.hpp | 32 ++
bfps/cpp/particles/p2p_distr_mpi.hpp | 688 +++++++++++++++++++++++++++
bfps/cpp/particles/p2p_tree.hpp | 109 +++++
4 files changed, 831 insertions(+)
create mode 100644 bfps/cpp/particles/.tocompile
create mode 100644 bfps/cpp/particles/p2p_computer.hpp
create mode 100644 bfps/cpp/particles/p2p_distr_mpi.hpp
create mode 100644 bfps/cpp/particles/p2p_tree.hpp
diff --git a/bfps/cpp/particles/.tocompile b/bfps/cpp/particles/.tocompile
new file mode 100644
index 00000000..02874ed7
--- /dev/null
+++ b/bfps/cpp/particles/.tocompile
@@ -0,0 +1,2 @@
+mpicxx -g main_tocompile.cpp -o /tmp/main_test_part.exe -I/home/bbramas/Projects/bfps/bfps/cpp/ -I/home/bbramas/Downloads/hdf5install/include -L/home/bbramas/Downloads/hdf5install/lib -lhdf5 -lsz -lz
+mpicxx -fPIC -rdynamic -g NSVE-v2.0.1-single.cpp -o /tmp/NSVE-v2.0.1-single.exe -I/home/bbramas/Projects/bfps/bfps/cpp/ -I/home/bbramas/Downloads/hdf5install/include -I/home/bbramas/Downloads/fftw-3.3.4/install/include/ -L/home/bbramas/Downloads/hdf5install/lib -lhdf5 -lsz -lz -L/home/bbramas/.local/lib/python2.7/site-packages/bfps-2.0.1.post31+g12693ea-py2.7.egg/bfps/ -lbfps -fopenmp -lgomp -L/home/bbramas/Downloads/fftw-3.3.4/install/lib/ -lfftw3_mpi -lfftw3f_mpi -lfftw3_omp -lfftw3f_omp -lfftw3 -lfftw3f
diff --git a/bfps/cpp/particles/p2p_computer.hpp b/bfps/cpp/particles/p2p_computer.hpp
new file mode 100644
index 00000000..efe0e5e3
--- /dev/null
+++ b/bfps/cpp/particles/p2p_computer.hpp
@@ -0,0 +1,32 @@
+#ifndef P2P_COMPUTER_HPP
+#define P2P_COMPUTER_HPP
+
+#include <cstring>
+
+template <class real_number, class partsize_t>
+class p2p_computer{
+public:
+ template <int size_particle_rhs>
+ void init_result_array(real_number rhs[], const partsize_t nbParticles) const{
+ memset(rhs, 0, sizeof(real_number)*nbParticles);
+ }
+
+ template <int size_particle_rhs>
+ void reduce_particles_rhs(real_number rhs_dst[], const real_number rhs_src[], const partsize_t nbParticles) const{
+ for(int idx_part = 0 ; idx_part < nbParticles ; ++idx_part){
+ for(int idx_rhs = 0 ; idx_rhs < size_particle_rhs ; ++idx_rhs){
+ rhs_dst[idx_part*size_particle_rhs+idx_rhs] += rhs_src[idx_part*size_particle_rhs+idx_rhs];
+ }
+ }
+ }
+
+ template <int size_particle_positions, int size_particle_rhs>
+ void compute_interaction(const real_number pos_part1[], real_number rhs_part1[],
+ const real_number pos_part2[], real_number rhs_part2[],
+ const real_number dist_pow2) const{
+ rhs_part1[0] += 1;
+ rhs_part2[0] += 1;
+ }
+};
+
+#endif
diff --git a/bfps/cpp/particles/p2p_distr_mpi.hpp b/bfps/cpp/particles/p2p_distr_mpi.hpp
new file mode 100644
index 00000000..7f610be1
--- /dev/null
+++ b/bfps/cpp/particles/p2p_distr_mpi.hpp
@@ -0,0 +1,688 @@
+#ifndef P2P_DISTR_MPI_HPP
+#define P2P_DISTR_MPI_HPP
+
+#include <mpi.h>
+
+#include <vector>
+#include <memory>
+#include <cassert>
+
+#include <type_traits>
+#include <omp.h>
+#include <algorithm>
+
+#include "scope_timer.hpp"
+#include "particles_utils.hpp"
+#include "p2p_tree.hpp"
+
+/*
+- method to reorder each particle section following the cutoff grid (will permite index too)
+- exchange particles (with upper only) and receive particle (from lower only)
+- 1 message at a time! so need the offset of each cell of the cutoff grid
+- iterate on what has been received with my own particles, fill both rhs
+- send back the rhs
+- merge rhs
+- update particles property
+ */
+
+template <class partsize_t, class real_number>
+class p2p_distr_mpi {
+protected:
+ static const int MaxNbRhs = 100;
+
+ enum MpiTag{
+ TAG_NB_PARTICLES,
+ TAG_POSITION_PARTICLES,
+ TAG_RESULT_PARTICLES,
+ };
+
+ struct NeighborDescriptor{
+ partsize_t nbParticlesToExchange;
+ int destProc;
+ int nbLevelsToExchange;
+ bool isRecv;
+
+ std::unique_ptr<real_number[]> toRecvAndMerge;
+ std::unique_ptr<real_number[]> toCompute;
+ std::unique_ptr<real_number[]> results;
+ };
+
+ enum Action{
+ NOTHING_TODO,
+ RECV_PARTICLES,
+ COMPUTE_PARTICLES,
+ RELEASE_BUFFER_PARTICLES,
+ MERGE_PARTICLES,
+
+ RECV_MOVE_NB_LOW,
+ RECV_MOVE_NB_UP,
+ RECV_MOVE_LOW,
+ RECV_MOVE_UP
+ };
+
+ MPI_Comm current_com;
+
+ int my_rank;
+ int nb_processes;
+ int nb_processes_involved;
+
+ const std::pair<int,int> current_partition_interval;
+ const int current_partition_size;
+ const std::array<size_t,3> field_grid_dim;
+
+ std::unique_ptr<int[]> partition_interval_size_per_proc;
+ std::unique_ptr<int[]> partition_interval_offset_per_proc;
+
+ std::unique_ptr<partsize_t[]> current_offset_particles_for_partition;
+
+ std::vector<std::pair<Action,int>> whatNext;
+ std::vector<MPI_Request> mpiRequests;
+ std::vector<NeighborDescriptor> neigDescriptors;
+
+ std::array<real_number,3> spatial_box_width;
+ std::array<real_number,3> spatial_box_offset;
+
+ const real_number cutoff_radius;
+ std::array<long int,3> nb_cell_levels;
+
+public:
+ ////////////////////////////////////////////////////////////////////////////
+
+ p2p_distr_mpi(MPI_Comm in_current_com,
+ const std::pair<int,int>& in_current_partitions,
+ const std::array<size_t,3>& in_field_grid_dim,
+ const std::array<real_number,3>& in_spatial_box_width,
+ const std::array<real_number,3>& in_spatial_box_offset,
+ const real_number in_cutoff_radius)
+ : current_com(in_current_com),
+ my_rank(-1), nb_processes(-1),nb_processes_involved(-1),
+ current_partition_interval(in_current_partitions),
+ current_partition_size(current_partition_interval.second-current_partition_interval.first),
+ field_grid_dim(in_field_grid_dim),
+ spatial_box_width(in_spatial_box_width), spatial_box_offset(in_spatial_box_offset),
+ cutoff_radius(in_cutoff_radius){
+
+ AssertMpi(MPI_Comm_rank(current_com, &my_rank));
+ AssertMpi(MPI_Comm_size(current_com, &nb_processes));
+
+ partition_interval_size_per_proc.reset(new int[nb_processes]);
+ AssertMpi( MPI_Allgather( const_cast<int*>(¤t_partition_size), 1, MPI_INT,
+ partition_interval_size_per_proc.get(), 1, MPI_INT,
+ current_com) );
+ assert(partition_interval_size_per_proc[my_rank] == current_partition_size);
+
+ partition_interval_offset_per_proc.reset(new int[nb_processes+1]);
+ partition_interval_offset_per_proc[0] = 0;
+ for(int idxProc = 0 ; idxProc < nb_processes ; ++idxProc){
+ partition_interval_offset_per_proc[idxProc+1] = partition_interval_offset_per_proc[idxProc] + partition_interval_size_per_proc[idxProc];
+ }
+
+ current_offset_particles_for_partition.reset(new partsize_t[current_partition_size+1]);
+
+ nb_processes_involved = nb_processes;
+ while(nb_processes_involved != 0 && partition_interval_size_per_proc[nb_processes_involved-1] == 0){
+ nb_processes_involved -= 1;
+ }
+ assert(nb_processes_involved != 0);
+ for(int idx_proc_involved = 0 ; idx_proc_involved < nb_processes_involved ; ++idx_proc_involved){
+ assert(partition_interval_size_per_proc[idx_proc_involved] != 0);
+ }
+
+ assert(int(field_grid_dim[IDX_Z]) == partition_interval_offset_per_proc[nb_processes_involved]);
+
+ nb_cell_levels[IDX_X] = spatial_box_width[IDX_X]/cutoff_radius;
+ nb_cell_levels[IDX_Y] = spatial_box_width[IDX_Y]/cutoff_radius;
+ nb_cell_levels[IDX_Z] = spatial_box_width[IDX_Z]/cutoff_radius;
+ }
+
+ virtual ~p2p_distr_mpi(){}
+
+ ////////////////////////////////////////////////////////////////////////////
+
+ long int get_cell_coord_x_from_index(const long int index) const{
+ return index % nb_cell_levels[IDX_X];
+ }
+
+ long int get_cell_coord_y_from_index(const long int index) const{
+ return (index - get_cell_coord_z_from_index(index)*(nb_cell_levels[IDX_X]*nb_cell_levels[IDX_Y]))
+ / nb_cell_levels[IDX_X];
+ }
+
+ long int get_cell_coord_z_from_index(const long int index) const{
+ return index / (nb_cell_levels[IDX_X]*nb_cell_levels[IDX_Y]);
+ }
+
+ long int first_cell_level_proc(const int dest_proc) const{
+ const real_number field_section_width_z = spatial_box_width[IDX_Z]/real_number(field_grid_dim[IDX_Z]);
+ return static_cast<long int>((field_section_width_z*real_number(partition_interval_offset_per_proc[dest_proc]))/cutoff_radius);
+ }
+
+ long int last_cell_level_proc(const int dest_proc) const{
+ const real_number field_section_width_z = spatial_box_width[IDX_Z]/real_number(field_grid_dim[IDX_Z]);
+ return static_cast<long int>((field_section_width_z*real_number(partition_interval_offset_per_proc[dest_proc+1])
+ - std::numeric_limits<real_number>::epsilon())/cutoff_radius);
+ }
+
+ std::array<long int,3> get_cell_coordinate(const real_number pos_x, const real_number pos_y,
+ const real_number pos_z) const {
+ const real_number diff_x = pos_x - spatial_box_offset[IDX_X];
+ const real_number diff_y = pos_y - spatial_box_offset[IDX_Y];
+ const real_number diff_z = pos_z - spatial_box_offset[IDX_Z];
+ std::array<long int,3> coord;
+ coord[IDX_X] = static_cast<long int>(diff_x/cutoff_radius);
+ coord[IDX_Y] = static_cast<long int>(diff_y/cutoff_radius);
+ coord[IDX_Z] = static_cast<long int>(diff_z/cutoff_radius);
+ return coord;
+ }
+
+ long int get_cell_idx(const real_number pos_x, const real_number pos_y,
+ const real_number pos_z) const {
+ std::array<long int,3> coord = get_cell_coordinate(pos_x, pos_y, pos_z);
+ return ((coord[IDX_Z]*nb_cell_levels[IDX_Y])+coord[IDX_Y])*nb_cell_levels[IDX_X]+coord[IDX_X];
+ }
+
+ real_number compute_distance_r2(const real_number x1, const real_number y1, const real_number z1,
+ const real_number x2, const real_number y2, const real_number z2) const {
+ return ((x1-x2)*(x1-x2)) + ((y1-y2)*(y1-y2)) + ((z1-z2)*(z1-z2));
+ }
+
+
+ template <int size_particle_positions, int size_particle_rhs>
+ struct ParticleView{
+ partsize_t p_index;
+ real_number* ptr_particles_positions;
+ real_number* ptr_particles_current_rhs;
+ partsize_t* ptr_global_idx;
+ long int* ptr_cell_idx;
+
+ void swap(ParticleView& p1, ParticleView& p2){
+ for(int idx_pos = 0 ; idx_pos < size_particle_positions ; ++idx_pos){
+ std::swap(p1.ptr_particles_positions[p1.p_index*size_particle_positions+idx_pos],
+ p2.ptr_particles_positions[p2.p_index*size_particle_positions+idx_pos]);
+ }
+ for(int idx_rhs = 0 ; idx_rhs < size_particle_rhs ; ++idx_rhs){
+ std::swap(p1.ptr_particles_current_rhs[p1.p_index*size_particle_rhs+idx_rhs],
+ p2.ptr_particles_current_rhs[p2.p_index*size_particle_rhs+idx_rhs]);
+ }
+ std::swap(p1.ptr_cell_idx[p1.p_index],p2.ptr_cell_idx[p2.p_index]);
+ std::swap(p1.ptr_global_idx[p1.p_index],p2.ptr_global_idx[p2.p_index]);
+ std::swap(p1.p_index,p2.p_index);
+ }
+ };
+
+ template <class computer_class, int size_particle_positions, int size_particle_rhs>
+ void compute_distr(computer_class& in_computer,
+ const partsize_t current_my_nb_particles_per_partition[],
+ real_number particles_positions[],
+ real_number particles_current_rhs[],
+ partsize_t inout_index_particles[]){
+ TIMEZONE("compute_distr");
+
+ // Some processes might not be involved
+ if(nb_processes_involved <= my_rank){
+ return;
+ }
+
+ const long int my_top_z_cell_level = last_cell_level_proc(my_rank);
+ const long int my_down_z_cell_level = first_cell_level_proc(my_rank);
+ const long int my_nb_cell_levels = 1+my_top_z_cell_level-my_down_z_cell_level;
+
+ current_offset_particles_for_partition[0] = 0;
+ partsize_t myTotalNbParticles = 0;
+ for(int idxPartition = 0 ; idxPartition < current_partition_size ; ++idxPartition){
+ myTotalNbParticles += current_my_nb_particles_per_partition[idxPartition];
+ current_offset_particles_for_partition[idxPartition+1] = current_offset_particles_for_partition[idxPartition] + current_my_nb_particles_per_partition[idxPartition];
+ }
+
+ // Compute box idx for each particle
+ std::unique_ptr<long int[]> particles_coord(new long int[current_offset_particles_for_partition[current_partition_size]]);
+
+ {
+ for(int idxPartition = 0 ; idxPartition < current_partition_size ; ++idxPartition){
+ #pragma omp parallel for schedule(static)
+ for(partsize_t idxPart = current_offset_particles_for_partition[idxPartition] ; idxPart < current_offset_particles_for_partition[idxPartition+1] ; ++idxPart ){
+ particles_coord[idxPart] = get_cell_idx(particles_positions[(idxPart)*size_particle_positions + IDX_X],
+ particles_positions[(idxPart)*size_particle_positions + IDX_Y],
+ particles_positions[(idxPart)*size_particle_positions + IDX_Z]);
+ assert(my_down_z_cell_level <= get_cell_coord_z_from_index(particles_coord[idxPart]));
+ assert(get_cell_coord_z_from_index(particles_coord[idxPart]) <= my_top_z_cell_level);
+ }
+ }
+
+ std::vector<ParticleView<size_particle_positions,size_particle_rhs>> part_to_sort;
+
+ // Sort each partition in cells
+ for(int idxPartition = 0 ; idxPartition < current_partition_size ; ++idxPartition){
+ part_to_sort.clear();
+
+ for(partsize_t idxPart = current_offset_particles_for_partition[idxPartition] ; idxPart < current_offset_particles_for_partition[idxPartition+1] ; ++idxPart ){
+ part_to_sort.emplace_back();
+ part_to_sort.back().p_index = idxPart;
+ part_to_sort.back().ptr_particles_positions = particles_positions;
+ part_to_sort.back().ptr_particles_current_rhs = particles_current_rhs;
+ part_to_sort.back().ptr_global_idx = inout_index_particles;
+ part_to_sort.back().ptr_cell_idx = particles_coord.get();
+ }
+
+ assert(part_to_sort.size() == (current_offset_particles_for_partition[idxPartition+1]-current_offset_particles_for_partition[idxPartition]));
+
+ std::sort(part_to_sort.begin(), part_to_sort.end(),
+ [](const ParticleView<size_particle_positions,size_particle_rhs>& p1,
+ const ParticleView<size_particle_positions,size_particle_rhs>& p2){
+ return p1.ptr_cell_idx[p1.p_index] < p2.ptr_cell_idx[p2.p_index];
+ });
+ }
+ }
+
+ // Build the tree
+ p2p_tree<std::vector<std::pair<partsize_t,partsize_t>>> my_tree(nb_cell_levels);
+
+ for(int idxPartition = 0 ; idxPartition < current_partition_size ; ++idxPartition){
+ long int current_cell_idx = -1;
+ partsize_t current_nb_particles_in_cell = 0;
+ partsize_t current_cell_offset = 0;
+
+ for(partsize_t idx_part = current_offset_particles_for_partition[idxPartition] ;
+ idx_part != current_offset_particles_for_partition[idxPartition+1]; ++idx_part){
+ if(particles_coord[idx_part] != current_cell_idx){
+ if(current_nb_particles_in_cell){
+ my_tree.getCell(current_cell_idx).emplace_back(current_cell_offset,current_nb_particles_in_cell);
+ }
+ current_cell_idx = particles_coord[idx_part];
+ current_nb_particles_in_cell = 1;
+ current_cell_offset = idx_part;
+ }
+ }
+ if(current_nb_particles_in_cell){
+ my_tree.getCell(current_cell_idx).emplace_back(current_cell_offset,current_nb_particles_in_cell);
+
+ }
+ }
+
+ printf("[%d] go from cutoff level %ld to %ld\n",
+ my_rank, my_down_z_cell_level, my_top_z_cell_level); // TODO remove
+ fflush(stdout); // TODO
+
+ // Offset per cell layers
+ std::unique_ptr<partsize_t[]> particles_offset_layers(new partsize_t[my_nb_cell_levels+1]());
+ for(int idxPartition = 0 ; idxPartition < current_partition_size ; ++idxPartition){
+ for(partsize_t idx_part = current_offset_particles_for_partition[idxPartition] ;
+ idx_part != current_offset_particles_for_partition[idxPartition+1]; ++idx_part){
+ assert(my_down_z_cell_level <= get_cell_coord_z_from_index(particles_coord[idx_part]));
+ assert(get_cell_coord_z_from_index(particles_coord[idx_part]) <= my_top_z_cell_level);
+ particles_offset_layers[get_cell_coord_z_from_index(particles_coord[idx_part])+1-my_down_z_cell_level] += 1;
+ }
+ }
+ for(size_t idx_layer = 0 ; idx_layer < my_nb_cell_levels ; ++idx_layer){
+ printf("[%d] nb particles in cutoff level %llu are %ld\n",
+ my_rank, idx_layer, particles_offset_layers[idx_layer+1]); // TODO remove
+ fflush(stdout); // TODO
+ particles_offset_layers[idx_layer+1] += particles_offset_layers[idx_layer];
+ }
+
+ // Reset vectors
+ assert(whatNext.size() == 0);
+ assert(mpiRequests.size() == 0);
+ neigDescriptors.clear();
+
+ // Find process with at least one neighbor
+ {
+ std::cout << my_rank << " my_top_z_cell_level " << my_top_z_cell_level << std::endl;
+ std::cout << my_rank << " my_down_z_cell_level " << my_down_z_cell_level << std::endl;
+ std::cout.flush();// TODO
+
+ int dest_proc = (my_rank+1)%nb_processes_involved;
+ while(dest_proc != my_rank
+ && (my_top_z_cell_level == first_cell_level_proc(dest_proc)
+ || (my_top_z_cell_level+1)%nb_cell_levels[IDX_Z] == first_cell_level_proc(dest_proc))){
+ // Find if we have to send 1 or 2 cell levels
+ int nb_levels_to_send = 1;
+ if(my_nb_cell_levels > 1 // I have more than one level
+ && (my_top_z_cell_level-1+2)%nb_cell_levels[IDX_Z] <= last_cell_level_proc(dest_proc)){
+ nb_levels_to_send += 1;
+ }
+
+ std::cout << my_rank << " dest_proc " << dest_proc << std::endl;
+ std::cout << my_rank << " first_cell_level_proc(dest_proc) " << first_cell_level_proc(dest_proc) << std::endl;
+ std::cout << my_rank << " last_cell_level_proc(dest_proc) " << last_cell_level_proc(dest_proc) << std::endl;
+ std::cout.flush();// TODO
+
+ NeighborDescriptor descriptor;
+ descriptor.destProc = dest_proc;
+ descriptor.nbLevelsToExchange = nb_levels_to_send;
+ descriptor.nbParticlesToExchange = particles_offset_layers[my_nb_cell_levels] - particles_offset_layers[my_nb_cell_levels-nb_levels_to_send];
+ descriptor.isRecv = false;
+
+ std::cout << my_rank << "SEND" << std::endl;
+ std::cout << "descriptor.destProc " << descriptor.destProc << std::endl;
+ std::cout << "descriptor.nbLevelsToExchange " << descriptor.nbLevelsToExchange << std::endl;
+ std::cout << "descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
+ std::cout << "descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
+ std::cout << "descriptor.isRecv " << descriptor.isRecv << std::endl;
+ std::cout << "neigDescriptors.size() " << neigDescriptors.size() << std::endl;
+ std::cout.flush();// TODO
+
+ neigDescriptors.emplace_back(std::move(descriptor));
+
+ dest_proc = (dest_proc+1)%nb_processes_involved;
+ }
+ std::cout << my_rank << " NO dest_proc " << dest_proc << std::endl;
+ std::cout << my_rank << " NO first_cell_level_proc(dest_proc) " << first_cell_level_proc(dest_proc) << std::endl;
+ std::cout.flush();// TODO
+
+ int src_proc = (my_rank-1+nb_processes_involved)%nb_processes_involved;
+ while(src_proc != my_rank
+ && (last_cell_level_proc(src_proc) == my_down_z_cell_level
+ || (last_cell_level_proc(src_proc)+1)%nb_cell_levels[IDX_Z] == my_down_z_cell_level)){
+ // Find if we have to send 1 or 2 cell levels
+ int nb_levels_to_recv = 1;
+ if(my_nb_cell_levels > 1 // I have more than one level
+ && first_cell_level_proc(src_proc) <= (my_down_z_cell_level-1+2)%nb_cell_levels[IDX_Z]){
+ nb_levels_to_recv += 1;
+ }
+
+ std::cout << my_rank << " src_proc " << src_proc << std::endl;
+ std::cout << my_rank << " first_cell_level_proc(src_proc) " << first_cell_level_proc(src_proc) << std::endl;
+ std::cout.flush();// TODO
+
+ NeighborDescriptor descriptor;
+ descriptor.destProc = src_proc;
+ descriptor.nbLevelsToExchange = nb_levels_to_recv;
+ descriptor.nbParticlesToExchange = -1;
+ descriptor.isRecv = true;
+
+ neigDescriptors.emplace_back(std::move(descriptor));
+
+ std::cout << my_rank << "] RECV" << std::endl;
+ std::cout << "descriptor.destProc " << descriptor.destProc << std::endl;
+ std::cout << "descriptor.nbLevelsToExchange " << descriptor.nbLevelsToExchange << std::endl;
+ std::cout << "descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
+ std::cout << "descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
+ std::cout << "descriptor.isRecv " << descriptor.isRecv << std::endl;
+ std::cout << "neigDescriptors.size() " << neigDescriptors.size() << std::endl;
+ std::cout.flush();// TODO
+
+ src_proc = (src_proc-1+nb_processes_involved)%nb_processes_involved;
+ }
+ std::cout << my_rank << " NO src_proc " << src_proc << std::endl;
+ std::cout << my_rank << " NO first_cell_level_proc(src_proc) " << first_cell_level_proc(src_proc) << std::endl;
+ std::cout.flush();// TODO
+ }
+
+ //////////////////////////////////////////////////////////////////////
+ /// Exchange the number of particles in each partition
+ /// Could involve only here but I do not think it will be a problem
+ //////////////////////////////////////////////////////////////////////
+
+ assert(whatNext.size() == 0);
+ assert(mpiRequests.size() == 0);
+
+
+ for(int idxDescr = 0 ; idxDescr < int(neigDescriptors.size()) ; ++idxDescr){
+ NeighborDescriptor& descriptor = neigDescriptors[idxDescr];
+
+ if(descriptor.isRecv == false){
+ whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
+ mpiRequests.emplace_back();
+ AssertMpi(MPI_Isend(const_cast<partsize_t*>(&descriptor.nbParticlesToExchange),
+ 1, particles_utils::GetMpiType(partsize_t()),
+ descriptor.destProc, TAG_NB_PARTICLES,
+ current_com, &mpiRequests.back()));
+
+ if(descriptor.nbParticlesToExchange){
+ std::cout << my_rank << "] SEND_PARTICLES" << std::endl;
+ std::cout << "descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
+ std::cout << "descriptor.destProc " << descriptor.destProc << std::endl;
+ std::cout << "idxDescr " << idxDescr << std::endl;
+
+ whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
+ mpiRequests.emplace_back();
+ assert(descriptor.nbParticlesToExchange*size_particle_positions < std::numeric_limits<int>::max());
+ AssertMpi(MPI_Isend(const_cast<real_number*>(&particles_positions[particles_offset_layers[my_nb_cell_levels-descriptor.nbLevelsToExchange]]),
+ int(descriptor.nbParticlesToExchange*size_particle_positions), particles_utils::GetMpiType(real_number()),
+ descriptor.destProc, TAG_POSITION_PARTICLES,
+ current_com, &mpiRequests.back()));
+
+ assert(descriptor.toRecvAndMerge == nullptr);
+ descriptor.toRecvAndMerge.reset(new real_number[descriptor.nbParticlesToExchange*size_particle_rhs]);
+ whatNext.emplace_back(std::pair<Action,int>{MERGE_PARTICLES, int(neigDescriptors.size())});
+ mpiRequests.emplace_back();
+ assert(descriptor.nbParticlesToExchange*size_particle_rhs < std::numeric_limits<int>::max());
+ AssertMpi(MPI_Irecv(descriptor.toRecvAndMerge.get(), int(descriptor.nbParticlesToExchange*size_particle_rhs),
+ particles_utils::GetMpiType(real_number()), descriptor.destProc, TAG_RESULT_PARTICLES,
+ current_com, &mpiRequests.back()));
+ }
+ }
+ else{
+ std::cout << "RECV_PARTICLES " << RECV_PARTICLES << std::endl;
+ std::cout << "idxDescr " << idxDescr << std::endl;
+ whatNext.emplace_back(std::pair<Action,int>{RECV_PARTICLES, idxDescr});
+ mpiRequests.emplace_back();
+ AssertMpi(MPI_Irecv(&descriptor.nbParticlesToExchange,
+ 1, particles_utils::GetMpiType(partsize_t()), descriptor.destProc, TAG_NB_PARTICLES,
+ current_com, &mpiRequests.back()));
+ }
+ }
+
+ const bool more_than_one_thread = (omp_get_max_threads() > 1);
+
+ TIMEZONE_OMP_INIT_PREPARALLEL(omp_get_max_threads())
+ #pragma omp parallel default(shared)
+ {
+ #pragma omp master
+ {
+ while(mpiRequests.size()){
+ assert(mpiRequests.size() == whatNext.size());
+
+ int idxDone = int(mpiRequests.size());
+ {
+ TIMEZONE("wait");
+ AssertMpi(MPI_Waitany(int(mpiRequests.size()), mpiRequests.data(), &idxDone, MPI_STATUSES_IGNORE));
+ }
+ const std::pair<Action, int> releasedAction = whatNext[idxDone];
+ std::swap(mpiRequests[idxDone], mpiRequests[mpiRequests.size()-1]);
+ std::swap(whatNext[idxDone], whatNext[mpiRequests.size()-1]);
+ mpiRequests.pop_back();
+ whatNext.pop_back();
+
+ //////////////////////////////////////////////////////////////////////
+ /// Data to exchange particles
+ //////////////////////////////////////////////////////////////////////
+ if(releasedAction.first == RECV_PARTICLES){
+ NeighborDescriptor& descriptor = neigDescriptors[releasedAction.second];
+
+ assert(descriptor.isRecv == true);
+ const int destProc = descriptor.destProc;
+ const partsize_t NbParticlesToReceive = descriptor.nbParticlesToExchange;
+ assert(NbParticlesToReceive != -1);
+ assert(descriptor.toCompute == nullptr);
+
+ std::cout << my_rank << "] RECV_PARTICLES" << std::endl;
+ std::cout << "descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
+ std::cout << "descriptor.destProc " << descriptor.destProc << std::endl;
+ std::cout << "releasedAction.second " << releasedAction.second << std::endl;
+
+ if(NbParticlesToReceive){
+ std::cout << "MPI_Irecv " << std::endl;
+ descriptor.toCompute.reset(new real_number[NbParticlesToReceive*size_particle_positions]);
+ whatNext.emplace_back(std::pair<Action,int>{COMPUTE_PARTICLES, releasedAction.second});
+ mpiRequests.emplace_back();
+ assert(NbParticlesToReceive*size_particle_positions < std::numeric_limits<int>::max());
+ AssertMpi(MPI_Irecv(descriptor.toCompute.get(), int(NbParticlesToReceive*size_particle_positions),
+ particles_utils::GetMpiType(real_number()), destProc, TAG_POSITION_PARTICLES,
+ current_com, &mpiRequests.back()));
+ }
+ }
+
+ //////////////////////////////////////////////////////////////////////
+ /// Computation
+ //////////////////////////////////////////////////////////////////////
+ if(releasedAction.first == COMPUTE_PARTICLES){
+ NeighborDescriptor& descriptor = neigDescriptors[releasedAction.second];
+ const partsize_t NbParticlesToReceive = descriptor.nbParticlesToExchange;
+
+ assert(descriptor.toCompute != nullptr);
+ descriptor.results.reset(new real_number[NbParticlesToReceive*size_particle_rhs]);
+ // TODO in_computer.template init_result_array<size_particle_rhs>(descriptor.results.get(), NbParticlesToReceive);
+
+ // Compute
+ partsize_t idxPart = 0;
+ while(idxPart != NbParticlesToReceive){
+ const long int current_cell_idx = get_cell_idx(descriptor.toCompute[idxPart*size_particle_positions + IDX_X],
+ descriptor.toCompute[idxPart*size_particle_positions + IDX_Y],
+ descriptor.toCompute[idxPart*size_particle_positions + IDX_Z]);
+ partsize_t nb_parts_in_cell = 0;
+ while(idxPart+nb_parts_in_cell != NbParticlesToReceive
+ && current_cell_idx == get_cell_idx(descriptor.toCompute[(idxPart+nb_parts_in_cell)*size_particle_positions + IDX_X],
+ descriptor.toCompute[(idxPart+nb_parts_in_cell)*size_particle_positions + IDX_Y],
+ descriptor.toCompute[(idxPart+nb_parts_in_cell)*size_particle_positions + IDX_Z])){
+ nb_parts_in_cell += 1;
+ }
+
+ const std::vector<std::pair<partsize_t,partsize_t>>* neighbors[27];
+ const int nbNeighbors = my_tree.getNeighbors(current_cell_idx, neighbors);
+
+ // with other interval
+ for(size_t idx_neighbor = 0 ; idx_neighbor < nbNeighbors ; ++idx_neighbor){
+ for(size_t idx_2 = 0 ; idx_2 < (*neighbors[idx_neighbor]).size() ; ++idx_2){
+ for(partsize_t idx_p1 = 0 ; idx_p1 < nb_parts_in_cell ; ++idx_p1){
+ for(partsize_t idx_p2 = 0 ; idx_p2 < (*neighbors[idx_neighbor])[idx_2].second ; ++idx_p2){
+ const real_number dist_r2 = compute_distance_r2(descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_X],
+ descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_Y],
+ descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_Z],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
+ if(dist_r2 < cutoff_radius*cutoff_radius){
+ // TODO in_computer.template compute_interaction<size_particle_positions,size_particle_rhs>(
+// &descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions],
+// &descriptor.results[(idxPart+idx_p1)*size_particle_rhs],
+// &particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions],
+// &particles_current_rhs[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_rhs],
+// dist_r2);
+ }
+ }
+ }
+ }
+ }
+
+ idxPart += nb_parts_in_cell;
+ }
+
+ // Send back
+ const int destProc = descriptor.destProc;
+ whatNext.emplace_back(std::pair<Action,int>{RELEASE_BUFFER_PARTICLES, releasedAction.second});
+ mpiRequests.emplace_back();
+ assert(NbParticlesToReceive*size_particle_rhs < std::numeric_limits<int>::max());
+ AssertMpi(MPI_Isend(descriptor.results.get(), int(NbParticlesToReceive*size_particle_rhs),
+ particles_utils::GetMpiType(real_number()), destProc, TAG_RESULT_PARTICLES,
+ current_com, &mpiRequests.back()));
+ }
+ //////////////////////////////////////////////////////////////////////
+ /// Computation
+ //////////////////////////////////////////////////////////////////////
+ if(releasedAction.first == RELEASE_BUFFER_PARTICLES){
+ NeighborDescriptor& descriptor = neigDescriptors[releasedAction.second];
+ assert(descriptor.toCompute != nullptr);
+ descriptor.toCompute.release();
+ }
+ //////////////////////////////////////////////////////////////////////
+ /// Merge
+ //////////////////////////////////////////////////////////////////////
+ if(releasedAction.first == MERGE_PARTICLES && more_than_one_thread == false){
+ NeighborDescriptor& descriptor = neigDescriptors[releasedAction.second];
+ assert(descriptor.isRecv);
+ assert(descriptor.toRecvAndMerge != nullptr);
+ // TODO in_computer.template reduce_particles_rhs<size_particle_rhs>(&particles_current_rhs[0], descriptor.toRecvAndMerge.get(), descriptor.nbParticlesToExchange);
+ descriptor.toRecvAndMerge.release();
+ }
+ }
+ }
+ }
+
+ assert(whatNext.size() == 0);
+ assert(mpiRequests.size() == 0);
+
+ // Compute self data
+ for(const auto& iter_cell : my_tree){
+ const std::vector<std::pair<partsize_t,partsize_t>>& intervals = iter_cell.second;
+
+ for(size_t idx_1 = 0 ; idx_1 < intervals.size() ; ++idx_1){
+ // self interval
+ for(partsize_t idx_p1 = 0 ; idx_p1 < intervals[idx_1].second ; ++idx_p1){
+ for(partsize_t idx_p2 = idx_p1+1 ; idx_p2 < intervals[idx_1].second ; ++idx_p2){
+ const real_number dist_r2 = compute_distance_r2(particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z],
+ particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_X],
+ particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_Y],
+ particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_Z]);
+ if(dist_r2 < cutoff_radius*cutoff_radius){
+ in_computer.template compute_interaction<size_particle_positions,size_particle_rhs>(
+ &particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions],
+ &particles_current_rhs[(intervals[idx_1].first+idx_p1)*size_particle_rhs],
+ &particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions],
+ &particles_current_rhs[(intervals[idx_1].first+idx_p2)*size_particle_rhs],
+ dist_r2);
+ }
+ }
+ }
+
+ // with other interval
+ for(size_t idx_2 = idx_1+1 ; idx_2 < intervals.size() ; ++idx_2){
+ for(partsize_t idx_p1 = 0 ; idx_p1 < intervals[idx_1].second ; ++idx_p1){
+ for(partsize_t idx_p2 = 0 ; idx_p2 < intervals[idx_2].second ; ++idx_p2){
+ const real_number dist_r2 = compute_distance_r2(particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z],
+ particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
+ particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
+ particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
+ if(dist_r2 < cutoff_radius*cutoff_radius){
+ in_computer.template compute_interaction<size_particle_positions,size_particle_rhs>(
+ &particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions],
+ &particles_current_rhs[(intervals[idx_1].first+idx_p1)*size_particle_rhs],
+ &particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions],
+ &particles_current_rhs[(intervals[idx_2].first+idx_p2)*size_particle_rhs],
+ dist_r2);
+ }
+ }
+ }
+ }
+ }
+
+
+ const long int currenct_cell_idx = iter_cell.first;
+ const std::vector<std::pair<partsize_t,partsize_t>>* neighbors[27];
+ const int nbNeighbors = my_tree.getNeighbors(currenct_cell_idx, neighbors);
+
+ for(size_t idx_1 = 0 ; idx_1 < intervals.size() ; ++idx_1){
+ // with other interval
+ for(size_t idx_neighbor = 0 ; idx_neighbor < nbNeighbors ; ++idx_neighbor){
+ for(size_t idx_2 = 0 ; idx_2 < (*neighbors[idx_neighbor]).size() ; ++idx_2){
+ for(partsize_t idx_p1 = 0 ; idx_p1 < intervals[idx_1].second ; ++idx_p1){
+ for(partsize_t idx_p2 = 0 ; idx_p2 < (*neighbors[idx_neighbor])[idx_2].second ; ++idx_p2){
+ const real_number dist_r2 = compute_distance_r2(particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
+ if(dist_r2 < cutoff_radius*cutoff_radius){
+ in_computer.template compute_interaction<size_particle_positions,size_particle_rhs>(
+ &particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions],
+ &particles_current_rhs[(intervals[idx_1].first+idx_p1)*size_particle_rhs],
+ &particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions],
+ &particles_current_rhs[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_rhs],
+ dist_r2);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+};
+
+#endif
diff --git a/bfps/cpp/particles/p2p_tree.hpp b/bfps/cpp/particles/p2p_tree.hpp
new file mode 100644
index 00000000..b11cd826
--- /dev/null
+++ b/bfps/cpp/particles/p2p_tree.hpp
@@ -0,0 +1,109 @@
+#ifndef P2P_TREE_HPP
+#define P2P_TREE_HPP
+
+#include <unordered_map>
+#include <vector>
+
+template <class CellClass>
+class p2p_tree{
+ std::unordered_map<long int, CellClass> data;
+ CellClass emptyCell;
+ std::array<long int,3> nb_cell_levels;
+
+ long int get_cell_coord_x_from_index(const long int index) const{
+ return index % nb_cell_levels[IDX_X];
+ }
+
+ long int get_cell_coord_y_from_index(const long int index) const{
+ return (index - get_cell_coord_z_from_index(index)*(nb_cell_levels[IDX_X]*nb_cell_levels[IDX_Y]))
+ / nb_cell_levels[IDX_X];
+ }
+
+ long int get_cell_coord_z_from_index(const long int index) const{
+ return index / (nb_cell_levels[IDX_X]*nb_cell_levels[IDX_Y]);
+ }
+
+ long int get_cell_idx(const long int idx_x, const long int idx_y,
+ const long int idx_z) const {
+ return (((idx_z*nb_cell_levels[IDX_Y])+idx_y)*nb_cell_levels[IDX_X])+idx_x;
+ }
+
+public:
+ explicit p2p_tree(std::array<long int,3> in_nb_cell_levels)
+ : nb_cell_levels(in_nb_cell_levels){
+ }
+
+ CellClass& getCell(const long int idx){
+ return data[idx];
+ }
+
+
+ const CellClass& getCell(const long int idx) const {
+ const auto& iter = data.find(idx);
+ if(iter != data.end()){
+ return iter->second;
+ }
+ return emptyCell;
+ }
+
+ int getNeighbors(const long int idx, const CellClass* output[27]) const{
+ int nbNeighbors = 0;
+
+ const long int idx_x = get_cell_coord_x_from_index(idx);
+ const long int idx_y = get_cell_coord_y_from_index(idx);
+ const long int idx_z = get_cell_coord_z_from_index(idx);
+
+ for(long int neigh_x = - 1 ; neigh_x <= 1 ; ++neigh_x){
+ long int neigh_x_pbc = neigh_x+idx_x;
+ if(neigh_x_pbc < 0){
+ neigh_x_pbc += nb_cell_levels[IDX_X];
+ }
+ else if(nb_cell_levels[IDX_X] <= neigh_x_pbc){
+ neigh_x_pbc -= nb_cell_levels[IDX_X];
+ }
+
+ for(long int neigh_y = - 1 ; neigh_y <= 1 ; ++neigh_y){
+ long int neigh_y_pbc = neigh_y+idx_y;
+ if(neigh_y_pbc < 0){
+ neigh_y_pbc += nb_cell_levels[IDX_Y];
+ }
+ else if(nb_cell_levels[IDX_Y] <= neigh_y_pbc){
+ neigh_y_pbc -= nb_cell_levels[IDX_Y];
+ }
+
+ for(long int neigh_z = - 1 ; neigh_z <= 1 ; ++neigh_z){
+ long int neigh_z_pbc = neigh_z+idx_z;
+ if(neigh_z_pbc < 0){
+ neigh_z_pbc += nb_cell_levels[IDX_Z];
+ }
+ else if(nb_cell_levels[IDX_Z] <= neigh_z_pbc){
+ neigh_z_pbc -= nb_cell_levels[IDX_Z];
+ }
+
+ // Not the current cell
+ if(neigh_x_pbc != idx_x || neigh_y_pbc != idx_y || neigh_z_pbc != idx_z){
+ const long int idx_neigh = get_cell_idx(neigh_x_pbc,
+ neigh_y_pbc,
+ neigh_z_pbc);
+ const auto& iter = data.find(idx_neigh);
+ if(iter != data.end()){
+ output[nbNeighbors] = &(iter->second);
+ }
+ }
+ }
+ }
+ }
+
+ return nbNeighbors;
+ }
+
+ typename std::unordered_map<long int, CellClass>::iterator begin(){
+ return data.begin();
+ }
+
+ typename std::unordered_map<long int, CellClass>::iterator end(){
+ return data.end();
+ }
+};
+
+#endif
--
GitLab
From 255f493cd9bdd88f5e730c9729e719808af6834d Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Wed, 20 Sep 2017 15:50:06 +0200
Subject: [PATCH 014/342] Update method to find neighbors in grid
---
bfps/cpp/particles/p2p_distr_mpi.hpp | 105 ++++++++++++++++++++++-----
bfps/cpp/particles/p2p_tree.hpp | 14 ++--
2 files changed, 96 insertions(+), 23 deletions(-)
diff --git a/bfps/cpp/particles/p2p_distr_mpi.hpp b/bfps/cpp/particles/p2p_distr_mpi.hpp
index 7f610be1..a32c3375 100644
--- a/bfps/cpp/particles/p2p_distr_mpi.hpp
+++ b/bfps/cpp/particles/p2p_distr_mpi.hpp
@@ -246,6 +246,12 @@ public:
particles_positions[(idxPart)*size_particle_positions + IDX_Z]);
assert(my_down_z_cell_level <= get_cell_coord_z_from_index(particles_coord[idxPart]));
assert(get_cell_coord_z_from_index(particles_coord[idxPart]) <= my_top_z_cell_level);
+ if(inout_index_particles[idxPart] == 3 || inout_index_particles[idxPart] == 52){// TODO
+ printf("Coord index %ld (tree index %ld)\n", inout_index_particles[idxPart],particles_coord[idxPart]);
+ printf(">> Box index %ld - %ld - %ld\n", get_cell_coord_x_from_index(particles_coord[idxPart]),
+ get_cell_coord_y_from_index(particles_coord[idxPart]),
+ get_cell_coord_z_from_index(particles_coord[idxPart]));
+ }
}
}
@@ -540,7 +546,8 @@ public:
}
const std::vector<std::pair<partsize_t,partsize_t>>* neighbors[27];
- const int nbNeighbors = my_tree.getNeighbors(current_cell_idx, neighbors);
+ long int neighbors_indexes[27];
+ const int nbNeighbors = my_tree.getNeighbors(current_cell_idx, neighbors, neighbors_indexes, true);
// with other interval
for(size_t idx_neighbor = 0 ; idx_neighbor < nbNeighbors ; ++idx_neighbor){
@@ -625,6 +632,24 @@ public:
&particles_current_rhs[(intervals[idx_1].first+idx_p2)*size_particle_rhs],
dist_r2);
}
+ if((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 3
+ && inout_index_particles[(intervals[idx_1].first+idx_p2)] == 52)
+ || (inout_index_particles[(intervals[idx_1].first+idx_p1)] == 52
+ && inout_index_particles[(intervals[idx_1].first+idx_p2)] == 3)){// TODO
+ printf("test interaction between :\n");
+ printf("index %ld (%ld) pos %e %e %e\n",
+ (intervals[idx_1].first+idx_p1), inout_index_particles[(intervals[idx_1].first+idx_p1)],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z]);
+ printf("index %ld (%ld) pos %e %e %e\n",
+ (intervals[idx_1].first+idx_p2),
+ inout_index_particles[(intervals[idx_1].first+idx_p2)],
+ particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_X],
+ particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_Y],
+ particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_Z]);
+ printf("Radius = %e (%e)\n", sqrt(dist_r2), dist_r2);
+ }
}
}
@@ -646,6 +671,25 @@ public:
&particles_current_rhs[(intervals[idx_2].first+idx_p2)*size_particle_rhs],
dist_r2);
}
+
+ if((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 3
+ && inout_index_particles[(intervals[idx_2].first+idx_p2)] == 52)
+ || (inout_index_particles[(intervals[idx_1].first+idx_p1)] == 52
+ && inout_index_particles[(intervals[idx_2].first+idx_p2)] == 3)){// TODO
+ printf("interaction between :\n");
+ printf("index %ld (%ld) pos %e %e %e\n",
+ (intervals[idx_1].first+idx_p1), inout_index_particles[(intervals[idx_1].first+idx_p1)],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z]);
+ printf("index %ld (%ld) pos %e %e %e\n",
+ (intervals[idx_2].first+idx_p2),
+ inout_index_particles[(intervals[idx_2].first+idx_p2)],
+ particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
+ particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
+ particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
+ printf("Radius = %e (%e)\n", sqrt(dist_r2), dist_r2);
+ }
}
}
}
@@ -654,27 +698,52 @@ public:
const long int currenct_cell_idx = iter_cell.first;
const std::vector<std::pair<partsize_t,partsize_t>>* neighbors[27];
- const int nbNeighbors = my_tree.getNeighbors(currenct_cell_idx, neighbors);
+ long int neighbors_indexes[27];
+ const int nbNeighbors = my_tree.getNeighbors(currenct_cell_idx, neighbors, neighbors_indexes, false);
+
+ if(iter_cell.first == 3669){ // TODO
+ printf("Box %ld has %d neighbors\n", iter_cell.first, nbNeighbors);
+ }
for(size_t idx_1 = 0 ; idx_1 < intervals.size() ; ++idx_1){
// with other interval
for(size_t idx_neighbor = 0 ; idx_neighbor < nbNeighbors ; ++idx_neighbor){
- for(size_t idx_2 = 0 ; idx_2 < (*neighbors[idx_neighbor]).size() ; ++idx_2){
- for(partsize_t idx_p1 = 0 ; idx_p1 < intervals[idx_1].second ; ++idx_p1){
- for(partsize_t idx_p2 = 0 ; idx_p2 < (*neighbors[idx_neighbor])[idx_2].second ; ++idx_p2){
- const real_number dist_r2 = compute_distance_r2(particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
- particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
- particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z],
- particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
- particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
- particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
- if(dist_r2 < cutoff_radius*cutoff_radius){
- in_computer.template compute_interaction<size_particle_positions,size_particle_rhs>(
- &particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions],
- &particles_current_rhs[(intervals[idx_1].first+idx_p1)*size_particle_rhs],
- &particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions],
- &particles_current_rhs[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_rhs],
- dist_r2);
+ if(currenct_cell_idx < neighbors_indexes[idx_neighbor]){
+ for(size_t idx_2 = 0 ; idx_2 < (*neighbors[idx_neighbor]).size() ; ++idx_2){
+ for(partsize_t idx_p1 = 0 ; idx_p1 < intervals[idx_1].second ; ++idx_p1){
+ for(partsize_t idx_p2 = 0 ; idx_p2 < (*neighbors[idx_neighbor])[idx_2].second ; ++idx_p2){
+ const real_number dist_r2 = compute_distance_r2(particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
+ if(dist_r2 < cutoff_radius*cutoff_radius){
+ in_computer.template compute_interaction<size_particle_positions,size_particle_rhs>(
+ &particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions],
+ &particles_current_rhs[(intervals[idx_1].first+idx_p1)*size_particle_rhs],
+ &particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions],
+ &particles_current_rhs[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_rhs],
+ dist_r2);
+ }
+ if((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 3
+ && inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)] == 52)
+ || (inout_index_particles[(intervals[idx_1].first+idx_p1)] == 52
+ && inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)] == 3)){// TODO
+ printf("interaction between :\n");
+ printf("index %ld (%ld) pos %e %e %e\n",
+ (intervals[idx_1].first+idx_p1), inout_index_particles[(intervals[idx_1].first+idx_p1)],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z]);
+ printf("index %ld (%ld) pos %e %e %e\n",
+ ((*neighbors[idx_neighbor])[idx_2].first+idx_p2),
+ inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
+ printf("Radius = %e (%e)\n", sqrt(dist_r2), dist_r2);
+ }
}
}
}
diff --git a/bfps/cpp/particles/p2p_tree.hpp b/bfps/cpp/particles/p2p_tree.hpp
index b11cd826..4715dfad 100644
--- a/bfps/cpp/particles/p2p_tree.hpp
+++ b/bfps/cpp/particles/p2p_tree.hpp
@@ -46,14 +46,16 @@ public:
return emptyCell;
}
- int getNeighbors(const long int idx, const CellClass* output[27]) const{
+ int getNeighbors(const long int idx, const CellClass* output[27], long int output_indexes[27], const bool include_target) const{
int nbNeighbors = 0;
+ std::fill_n(output, 27, nullptr);
+
const long int idx_x = get_cell_coord_x_from_index(idx);
const long int idx_y = get_cell_coord_y_from_index(idx);
const long int idx_z = get_cell_coord_z_from_index(idx);
- for(long int neigh_x = - 1 ; neigh_x <= 1 ; ++neigh_x){
+ for(long int neigh_x = -1 ; neigh_x <= 1 ; ++neigh_x){
long int neigh_x_pbc = neigh_x+idx_x;
if(neigh_x_pbc < 0){
neigh_x_pbc += nb_cell_levels[IDX_X];
@@ -62,7 +64,7 @@ public:
neigh_x_pbc -= nb_cell_levels[IDX_X];
}
- for(long int neigh_y = - 1 ; neigh_y <= 1 ; ++neigh_y){
+ for(long int neigh_y = -1 ; neigh_y <= 1 ; ++neigh_y){
long int neigh_y_pbc = neigh_y+idx_y;
if(neigh_y_pbc < 0){
neigh_y_pbc += nb_cell_levels[IDX_Y];
@@ -71,7 +73,7 @@ public:
neigh_y_pbc -= nb_cell_levels[IDX_Y];
}
- for(long int neigh_z = - 1 ; neigh_z <= 1 ; ++neigh_z){
+ for(long int neigh_z = -1 ; neigh_z <= 1 ; ++neigh_z){
long int neigh_z_pbc = neigh_z+idx_z;
if(neigh_z_pbc < 0){
neigh_z_pbc += nb_cell_levels[IDX_Z];
@@ -81,13 +83,15 @@ public:
}
// Not the current cell
- if(neigh_x_pbc != idx_x || neigh_y_pbc != idx_y || neigh_z_pbc != idx_z){
+ if(include_target || neigh_x_pbc != idx_x || neigh_y_pbc != idx_y || neigh_z_pbc != idx_z){
const long int idx_neigh = get_cell_idx(neigh_x_pbc,
neigh_y_pbc,
neigh_z_pbc);
const auto& iter = data.find(idx_neigh);
if(iter != data.end()){
output[nbNeighbors] = &(iter->second);
+ output_indexes[nbNeighbors] = idx_neigh;
+ nbNeighbors += 1;
}
}
}
--
GitLab
From e2e52fa062475eeb23073e79fc8beec43bda5971 Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Wed, 20 Sep 2017 17:25:01 +0200
Subject: [PATCH 015/342] Debug
---
bfps/cpp/particles/p2p_distr_mpi.hpp | 240 +++++++++++++++------------
bfps/cpp/particles/p2p_tree.hpp | 1 -
2 files changed, 132 insertions(+), 109 deletions(-)
diff --git a/bfps/cpp/particles/p2p_distr_mpi.hpp b/bfps/cpp/particles/p2p_distr_mpi.hpp
index a32c3375..45f8bba3 100644
--- a/bfps/cpp/particles/p2p_distr_mpi.hpp
+++ b/bfps/cpp/particles/p2p_distr_mpi.hpp
@@ -25,6 +25,50 @@
- update particles property
*/
+
+
+template <class partsize_t, class real_number, int size_particle_positions, int size_particle_rhs>
+struct ParticleView{
+ partsize_t p_index;
+ real_number* ptr_particles_positions;
+ real_number* ptr_particles_current_rhs;
+ partsize_t* ptr_global_idx;
+ long int* ptr_cell_idx;
+
+ ParticleView()
+ : p_index(-1), ptr_particles_positions(nullptr),
+ ptr_particles_current_rhs(nullptr), ptr_global_idx(nullptr),
+ ptr_cell_idx(nullptr){}
+};
+
+template <class partsize_t, class real_number, int size_particle_positions, int size_particle_rhs>
+void swap(ParticleView<partsize_t, real_number, size_particle_positions,size_particle_rhs>& p1,
+ ParticleView<partsize_t, real_number, size_particle_positions,size_particle_rhs>& p2){
+ if(p1.p_index != -1 && p2.p_index != -1){
+ for(int idx_pos = 0 ; idx_pos < size_particle_positions ; ++idx_pos){
+ std::swap(p1.ptr_particles_positions[p1.p_index*size_particle_positions+idx_pos],
+ p2.ptr_particles_positions[p2.p_index*size_particle_positions+idx_pos]);
+ }
+ for(int idx_rhs = 0 ; idx_rhs < size_particle_rhs ; ++idx_rhs){
+ std::swap(p1.ptr_particles_current_rhs[p1.p_index*size_particle_rhs+idx_rhs],
+ p2.ptr_particles_current_rhs[p2.p_index*size_particle_rhs+idx_rhs]);
+ }
+ std::swap(p1.ptr_cell_idx[p1.p_index],p2.ptr_cell_idx[p2.p_index]);
+ std::swap(p1.ptr_global_idx[p1.p_index],p2.ptr_global_idx[p2.p_index]);
+ std::swap(p1.p_index,p2.p_index);
+ }
+ else if(p1.p_index != -1){
+ p2 = p1;
+ p1 = ParticleView<partsize_t, real_number, size_particle_positions,size_particle_rhs>();
+ }
+ else if(p2.p_index != -1){
+ p1 = p2;
+ p2 = ParticleView<partsize_t, real_number, size_particle_positions,size_particle_rhs>();
+ }
+}
+
+
+
template <class partsize_t, class real_number>
class p2p_distr_mpi {
protected:
@@ -183,32 +227,23 @@ public:
real_number compute_distance_r2(const real_number x1, const real_number y1, const real_number z1,
const real_number x2, const real_number y2, const real_number z2) const {
- return ((x1-x2)*(x1-x2)) + ((y1-y2)*(y1-y2)) + ((z1-z2)*(z1-z2));
- }
-
+ real_number diff_x = std::abs(x1-x2);
+ while(diff_x > spatial_box_width[IDX_X]/2){
+ diff_x = std::abs(diff_x - spatial_box_width[IDX_X]);
+ }
- template <int size_particle_positions, int size_particle_rhs>
- struct ParticleView{
- partsize_t p_index;
- real_number* ptr_particles_positions;
- real_number* ptr_particles_current_rhs;
- partsize_t* ptr_global_idx;
- long int* ptr_cell_idx;
+ real_number diff_y = std::abs(y1-y2);
+ while(diff_y > spatial_box_width[IDX_Y]/2){
+ diff_y = std::abs(diff_y - spatial_box_width[IDX_Y]);
+ }
- void swap(ParticleView& p1, ParticleView& p2){
- for(int idx_pos = 0 ; idx_pos < size_particle_positions ; ++idx_pos){
- std::swap(p1.ptr_particles_positions[p1.p_index*size_particle_positions+idx_pos],
- p2.ptr_particles_positions[p2.p_index*size_particle_positions+idx_pos]);
- }
- for(int idx_rhs = 0 ; idx_rhs < size_particle_rhs ; ++idx_rhs){
- std::swap(p1.ptr_particles_current_rhs[p1.p_index*size_particle_rhs+idx_rhs],
- p2.ptr_particles_current_rhs[p2.p_index*size_particle_rhs+idx_rhs]);
- }
- std::swap(p1.ptr_cell_idx[p1.p_index],p2.ptr_cell_idx[p2.p_index]);
- std::swap(p1.ptr_global_idx[p1.p_index],p2.ptr_global_idx[p2.p_index]);
- std::swap(p1.p_index,p2.p_index);
+ real_number diff_z = std::abs(z1-z2);
+ while(diff_z > spatial_box_width[IDX_Z]/2){
+ diff_z = std::abs(diff_z - spatial_box_width[IDX_Z]);
}
- };
+
+ return (diff_x*diff_x) + (diff_y*diff_y) + (diff_z*diff_z);
+ }
template <class computer_class, int size_particle_positions, int size_particle_rhs>
void compute_distr(computer_class& in_computer,
@@ -246,16 +281,18 @@ public:
particles_positions[(idxPart)*size_particle_positions + IDX_Z]);
assert(my_down_z_cell_level <= get_cell_coord_z_from_index(particles_coord[idxPart]));
assert(get_cell_coord_z_from_index(particles_coord[idxPart]) <= my_top_z_cell_level);
- if(inout_index_particles[idxPart] == 3 || inout_index_particles[idxPart] == 52){// TODO
- printf("Coord index %ld (tree index %ld)\n", inout_index_particles[idxPart],particles_coord[idxPart]);
+ if(inout_index_particles[idxPart] == 58576 || inout_index_particles[idxPart] == 0){// TODO
+ printf("Coord index %ld - %ld (tree index %ld)\n", idxPart, inout_index_particles[idxPart],particles_coord[idxPart]);
printf(">> Box index %ld - %ld - %ld\n", get_cell_coord_x_from_index(particles_coord[idxPart]),
get_cell_coord_y_from_index(particles_coord[idxPart]),
get_cell_coord_z_from_index(particles_coord[idxPart]));
+ printf("idxPartition %d\n", idxPartition);
}
}
}
- std::vector<ParticleView<size_particle_positions,size_particle_rhs>> part_to_sort;
+ using ParticleView_t = ParticleView<partsize_t, real_number, size_particle_positions,size_particle_rhs>;
+ std::vector<ParticleView_t> part_to_sort;
// Sort each partition in cells
for(int idxPartition = 0 ; idxPartition < current_partition_size ; ++idxPartition){
@@ -273,8 +310,10 @@ public:
assert(part_to_sort.size() == (current_offset_particles_for_partition[idxPartition+1]-current_offset_particles_for_partition[idxPartition]));
std::sort(part_to_sort.begin(), part_to_sort.end(),
- [](const ParticleView<size_particle_positions,size_particle_rhs>& p1,
- const ParticleView<size_particle_positions,size_particle_rhs>& p2){
+ [](const ParticleView_t& p1,
+ const ParticleView_t& p2){
+ assert(p1.p_index != -1 && p1.ptr_cell_idx);
+ assert(p2.p_index != -1 && p2.ptr_cell_idx);
return p1.ptr_cell_idx[p1.p_index] < p2.ptr_cell_idx[p2.p_index];
});
}
@@ -297,6 +336,13 @@ public:
current_cell_idx = particles_coord[idx_part];
current_nb_particles_in_cell = 1;
current_cell_offset = idx_part;
+ if(inout_index_particles[idx_part] == 58576 || inout_index_particles[idx_part] == 0){// TODO
+ printf("Coord index %ld - %ld (tree index %ld)\n", idx_part, inout_index_particles[idx_part],particles_coord[idx_part]);
+ printf(">> Box index %ld - %ld - %ld\n", get_cell_coord_x_from_index(particles_coord[idx_part]),
+ get_cell_coord_y_from_index(particles_coord[idx_part]),
+ get_cell_coord_z_from_index(particles_coord[idx_part]));
+ printf("current_cell_offset %ld current_nb_particles_in_cell %ld\n", current_cell_offset, current_nb_particles_in_cell);
+ }
}
}
if(current_nb_particles_in_cell){
@@ -310,13 +356,17 @@ public:
fflush(stdout); // TODO
// Offset per cell layers
+ long int previous_index = 0;
std::unique_ptr<partsize_t[]> particles_offset_layers(new partsize_t[my_nb_cell_levels+1]());
for(int idxPartition = 0 ; idxPartition < current_partition_size ; ++idxPartition){
for(partsize_t idx_part = current_offset_particles_for_partition[idxPartition] ;
idx_part != current_offset_particles_for_partition[idxPartition+1]; ++idx_part){
- assert(my_down_z_cell_level <= get_cell_coord_z_from_index(particles_coord[idx_part]));
- assert(get_cell_coord_z_from_index(particles_coord[idx_part]) <= my_top_z_cell_level);
- particles_offset_layers[get_cell_coord_z_from_index(particles_coord[idx_part])+1-my_down_z_cell_level] += 1;
+ const long int part_box_z_index = get_cell_coord_z_from_index(particles_coord[idx_part]);
+ assert(my_down_z_cell_level <= part_box_z_index);
+ assert(part_box_z_index <= my_top_z_cell_level);
+ particles_offset_layers[part_box_z_index+1-my_down_z_cell_level] += 1;
+ assert(previous_index <= part_box_z_index);
+ previous_index = part_box_z_index;
}
}
for(size_t idx_layer = 0 ; idx_layer < my_nb_cell_levels ; ++idx_layer){
@@ -363,7 +413,6 @@ public:
std::cout << "descriptor.destProc " << descriptor.destProc << std::endl;
std::cout << "descriptor.nbLevelsToExchange " << descriptor.nbLevelsToExchange << std::endl;
std::cout << "descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
- std::cout << "descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
std::cout << "descriptor.isRecv " << descriptor.isRecv << std::endl;
std::cout << "neigDescriptors.size() " << neigDescriptors.size() << std::endl;
std::cout.flush();// TODO
@@ -440,18 +489,19 @@ public:
std::cout << "descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
std::cout << "descriptor.destProc " << descriptor.destProc << std::endl;
std::cout << "idxDescr " << idxDescr << std::endl;
+ std::cout << "send from part " << particles_offset_layers[my_nb_cell_levels-descriptor.nbLevelsToExchange] << std::endl;
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
assert(descriptor.nbParticlesToExchange*size_particle_positions < std::numeric_limits<int>::max());
- AssertMpi(MPI_Isend(const_cast<real_number*>(&particles_positions[particles_offset_layers[my_nb_cell_levels-descriptor.nbLevelsToExchange]]),
+ AssertMpi(MPI_Isend(const_cast<real_number*>(&particles_positions[particles_offset_layers[my_nb_cell_levels-descriptor.nbLevelsToExchange]*size_particle_positions]),
int(descriptor.nbParticlesToExchange*size_particle_positions), particles_utils::GetMpiType(real_number()),
descriptor.destProc, TAG_POSITION_PARTICLES,
current_com, &mpiRequests.back()));
assert(descriptor.toRecvAndMerge == nullptr);
descriptor.toRecvAndMerge.reset(new real_number[descriptor.nbParticlesToExchange*size_particle_rhs]);
- whatNext.emplace_back(std::pair<Action,int>{MERGE_PARTICLES, int(neigDescriptors.size())});
+ whatNext.emplace_back(std::pair<Action,int>{MERGE_PARTICLES, idxDescr});
mpiRequests.emplace_back();
assert(descriptor.nbParticlesToExchange*size_particle_rhs < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(descriptor.toRecvAndMerge.get(), int(descriptor.nbParticlesToExchange*size_particle_rhs),
@@ -470,8 +520,6 @@ public:
}
}
- const bool more_than_one_thread = (omp_get_max_threads() > 1);
-
TIMEZONE_OMP_INIT_PREPARALLEL(omp_get_max_threads())
#pragma omp parallel default(shared)
{
@@ -496,8 +544,7 @@ public:
//////////////////////////////////////////////////////////////////////
if(releasedAction.first == RECV_PARTICLES){
NeighborDescriptor& descriptor = neigDescriptors[releasedAction.second];
-
- assert(descriptor.isRecv == true);
+ assert(descriptor.isRecv);
const int destProc = descriptor.destProc;
const partsize_t NbParticlesToReceive = descriptor.nbParticlesToExchange;
assert(NbParticlesToReceive != -1);
@@ -525,11 +572,12 @@ public:
//////////////////////////////////////////////////////////////////////
if(releasedAction.first == COMPUTE_PARTICLES){
NeighborDescriptor& descriptor = neigDescriptors[releasedAction.second];
+ assert(descriptor.isRecv);
const partsize_t NbParticlesToReceive = descriptor.nbParticlesToExchange;
assert(descriptor.toCompute != nullptr);
descriptor.results.reset(new real_number[NbParticlesToReceive*size_particle_rhs]);
- // TODO in_computer.template init_result_array<size_particle_rhs>(descriptor.results.get(), NbParticlesToReceive);
+ in_computer.template init_result_array<size_particle_rhs>(descriptor.results.get(), NbParticlesToReceive);
// Compute
partsize_t idxPart = 0;
@@ -537,7 +585,7 @@ public:
const long int current_cell_idx = get_cell_idx(descriptor.toCompute[idxPart*size_particle_positions + IDX_X],
descriptor.toCompute[idxPart*size_particle_positions + IDX_Y],
descriptor.toCompute[idxPart*size_particle_positions + IDX_Z]);
- partsize_t nb_parts_in_cell = 0;
+ partsize_t nb_parts_in_cell = 1;
while(idxPart+nb_parts_in_cell != NbParticlesToReceive
&& current_cell_idx == get_cell_idx(descriptor.toCompute[(idxPart+nb_parts_in_cell)*size_particle_positions + IDX_X],
descriptor.toCompute[(idxPart+nb_parts_in_cell)*size_particle_positions + IDX_Y],
@@ -549,6 +597,23 @@ public:
long int neighbors_indexes[27];
const int nbNeighbors = my_tree.getNeighbors(current_cell_idx, neighbors, neighbors_indexes, true);
+ for(int idx_test = 0 ; idx_test < nb_parts_in_cell ; ++idx_test){ // TODO
+ if(int(descriptor.toCompute[(idxPart+idx_test)*size_particle_positions + IDX_X]*1000) == int(1.685800e-01*1000)
+ && int(descriptor.toCompute[(idxPart+idx_test)*size_particle_positions + IDX_Y]*1000) == int(7.524981e-01*1000)
+ && int(descriptor.toCompute[(idxPart+idx_test)*size_particle_positions + IDX_Z]*1000) == int(9.999596e-01*1000)){
+ printf("Found a pos %ld\n", idxPart+idx_test);
+ printf("pos %e %e %e\n",
+ descriptor.toCompute[(idxPart+idx_test)*size_particle_positions + IDX_X],
+ descriptor.toCompute[(idxPart+idx_test)*size_particle_positions + IDX_Y],
+ descriptor.toCompute[(idxPart+idx_test)*size_particle_positions + IDX_Z]);
+ }
+ }
+ printf("Remote part from %ld for %ld at idx %ld\n", idxPart, nb_parts_in_cell, current_cell_idx); // TODO
+ printf("pos of first %e %e %e\n", descriptor.toCompute[idxPart*size_particle_positions + IDX_X],
+ descriptor.toCompute[idxPart*size_particle_positions + IDX_Y],
+ descriptor.toCompute[idxPart*size_particle_positions + IDX_Z]); // TODO
+ printf("nbNeighbors %d\n", nbNeighbors); // TODO
+
// with other interval
for(size_t idx_neighbor = 0 ; idx_neighbor < nbNeighbors ; ++idx_neighbor){
for(size_t idx_2 = 0 ; idx_2 < (*neighbors[idx_neighbor]).size() ; ++idx_2){
@@ -561,12 +626,28 @@ public:
particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
if(dist_r2 < cutoff_radius*cutoff_radius){
- // TODO in_computer.template compute_interaction<size_particle_positions,size_particle_rhs>(
-// &descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions],
-// &descriptor.results[(idxPart+idx_p1)*size_particle_rhs],
-// &particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions],
-// &particles_current_rhs[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_rhs],
-// dist_r2);
+ in_computer.template compute_interaction<size_particle_positions,size_particle_rhs>(
+ &descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions],
+ &descriptor.results[(idxPart+idx_p1)*size_particle_rhs],
+ &particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions],
+ &particles_current_rhs[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_rhs],
+ dist_r2);
+ }
+
+ if(inout_index_particles[(*neighbors[idx_neighbor])[idx_2].first+idx_p2] == 132){// TODO
+ printf("test interaction between :\n");
+ printf("index %ld (%ld) pos %e %e %e\n",
+ (idxPart+idx_p1), -1,
+ descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_X],
+ descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_Y],
+ descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_Z]);
+ printf("index %ld (%ld) pos %e %e %e\n",
+ ((*neighbors[idx_neighbor])[idx_2].first+idx_p2),
+ inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
+ printf("Radius = %e (%e)\n", sqrt(dist_r2), dist_r2);
}
}
}
@@ -591,16 +672,18 @@ public:
if(releasedAction.first == RELEASE_BUFFER_PARTICLES){
NeighborDescriptor& descriptor = neigDescriptors[releasedAction.second];
assert(descriptor.toCompute != nullptr);
+ assert(descriptor.isRecv);
descriptor.toCompute.release();
}
//////////////////////////////////////////////////////////////////////
/// Merge
//////////////////////////////////////////////////////////////////////
- if(releasedAction.first == MERGE_PARTICLES && more_than_one_thread == false){
+ if(releasedAction.first == MERGE_PARTICLES){
NeighborDescriptor& descriptor = neigDescriptors[releasedAction.second];
- assert(descriptor.isRecv);
+ assert(descriptor.isRecv == false);
assert(descriptor.toRecvAndMerge != nullptr);
- // TODO in_computer.template reduce_particles_rhs<size_particle_rhs>(&particles_current_rhs[0], descriptor.toRecvAndMerge.get(), descriptor.nbParticlesToExchange);
+ in_computer.template reduce_particles_rhs<size_particle_rhs>(&particles_current_rhs[particles_offset_layers[my_nb_cell_levels-descriptor.nbLevelsToExchange]*size_particle_rhs],
+ descriptor.toRecvAndMerge.get(), descriptor.nbParticlesToExchange);
descriptor.toRecvAndMerge.release();
}
}
@@ -632,24 +715,6 @@ public:
&particles_current_rhs[(intervals[idx_1].first+idx_p2)*size_particle_rhs],
dist_r2);
}
- if((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 3
- && inout_index_particles[(intervals[idx_1].first+idx_p2)] == 52)
- || (inout_index_particles[(intervals[idx_1].first+idx_p1)] == 52
- && inout_index_particles[(intervals[idx_1].first+idx_p2)] == 3)){// TODO
- printf("test interaction between :\n");
- printf("index %ld (%ld) pos %e %e %e\n",
- (intervals[idx_1].first+idx_p1), inout_index_particles[(intervals[idx_1].first+idx_p1)],
- particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
- particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
- particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z]);
- printf("index %ld (%ld) pos %e %e %e\n",
- (intervals[idx_1].first+idx_p2),
- inout_index_particles[(intervals[idx_1].first+idx_p2)],
- particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_X],
- particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_Y],
- particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_Z]);
- printf("Radius = %e (%e)\n", sqrt(dist_r2), dist_r2);
- }
}
}
@@ -671,25 +736,6 @@ public:
&particles_current_rhs[(intervals[idx_2].first+idx_p2)*size_particle_rhs],
dist_r2);
}
-
- if((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 3
- && inout_index_particles[(intervals[idx_2].first+idx_p2)] == 52)
- || (inout_index_particles[(intervals[idx_1].first+idx_p1)] == 52
- && inout_index_particles[(intervals[idx_2].first+idx_p2)] == 3)){// TODO
- printf("interaction between :\n");
- printf("index %ld (%ld) pos %e %e %e\n",
- (intervals[idx_1].first+idx_p1), inout_index_particles[(intervals[idx_1].first+idx_p1)],
- particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
- particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
- particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z]);
- printf("index %ld (%ld) pos %e %e %e\n",
- (intervals[idx_2].first+idx_p2),
- inout_index_particles[(intervals[idx_2].first+idx_p2)],
- particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
- particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
- particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
- printf("Radius = %e (%e)\n", sqrt(dist_r2), dist_r2);
- }
}
}
}
@@ -701,10 +747,6 @@ public:
long int neighbors_indexes[27];
const int nbNeighbors = my_tree.getNeighbors(currenct_cell_idx, neighbors, neighbors_indexes, false);
- if(iter_cell.first == 3669){ // TODO
- printf("Box %ld has %d neighbors\n", iter_cell.first, nbNeighbors);
- }
-
for(size_t idx_1 = 0 ; idx_1 < intervals.size() ; ++idx_1){
// with other interval
for(size_t idx_neighbor = 0 ; idx_neighbor < nbNeighbors ; ++idx_neighbor){
@@ -726,24 +768,6 @@ public:
&particles_current_rhs[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_rhs],
dist_r2);
}
- if((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 3
- && inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)] == 52)
- || (inout_index_particles[(intervals[idx_1].first+idx_p1)] == 52
- && inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)] == 3)){// TODO
- printf("interaction between :\n");
- printf("index %ld (%ld) pos %e %e %e\n",
- (intervals[idx_1].first+idx_p1), inout_index_particles[(intervals[idx_1].first+idx_p1)],
- particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
- particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
- particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z]);
- printf("index %ld (%ld) pos %e %e %e\n",
- ((*neighbors[idx_neighbor])[idx_2].first+idx_p2),
- inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)],
- particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
- particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
- particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
- printf("Radius = %e (%e)\n", sqrt(dist_r2), dist_r2);
- }
}
}
}
diff --git a/bfps/cpp/particles/p2p_tree.hpp b/bfps/cpp/particles/p2p_tree.hpp
index 4715dfad..9f146644 100644
--- a/bfps/cpp/particles/p2p_tree.hpp
+++ b/bfps/cpp/particles/p2p_tree.hpp
@@ -82,7 +82,6 @@ public:
neigh_z_pbc -= nb_cell_levels[IDX_Z];
}
- // Not the current cell
if(include_target || neigh_x_pbc != idx_x || neigh_y_pbc != idx_y || neigh_z_pbc != idx_z){
const long int idx_neigh = get_cell_idx(neigh_x_pbc,
neigh_y_pbc,
--
GitLab
From 69934557735f0e06488641caa4fea78f3c20631f Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Thu, 21 Sep 2017 14:11:36 +0200
Subject: [PATCH 016/342] Make it work (when there are more than one cell)
---
bfps/cpp/particles/p2p_distr_mpi.hpp | 439 +++++++++++++++++----------
1 file changed, 272 insertions(+), 167 deletions(-)
diff --git a/bfps/cpp/particles/p2p_distr_mpi.hpp b/bfps/cpp/particles/p2p_distr_mpi.hpp
index 45f8bba3..1709dc72 100644
--- a/bfps/cpp/particles/p2p_distr_mpi.hpp
+++ b/bfps/cpp/particles/p2p_distr_mpi.hpp
@@ -15,60 +15,6 @@
#include "particles_utils.hpp"
#include "p2p_tree.hpp"
-/*
-- method to reorder each particle section following the cutoff grid (will permite index too)
-- exchange particles (with upper only) and receive particle (from lower only)
-- 1 message at a time! so need the offset of each cell of the cutoff grid
-- iterate on what has been received with my own particles, fill both rhs
-- send back the rhs
-- merge rhs
-- update particles property
- */
-
-
-
-template <class partsize_t, class real_number, int size_particle_positions, int size_particle_rhs>
-struct ParticleView{
- partsize_t p_index;
- real_number* ptr_particles_positions;
- real_number* ptr_particles_current_rhs;
- partsize_t* ptr_global_idx;
- long int* ptr_cell_idx;
-
- ParticleView()
- : p_index(-1), ptr_particles_positions(nullptr),
- ptr_particles_current_rhs(nullptr), ptr_global_idx(nullptr),
- ptr_cell_idx(nullptr){}
-};
-
-template <class partsize_t, class real_number, int size_particle_positions, int size_particle_rhs>
-void swap(ParticleView<partsize_t, real_number, size_particle_positions,size_particle_rhs>& p1,
- ParticleView<partsize_t, real_number, size_particle_positions,size_particle_rhs>& p2){
- if(p1.p_index != -1 && p2.p_index != -1){
- for(int idx_pos = 0 ; idx_pos < size_particle_positions ; ++idx_pos){
- std::swap(p1.ptr_particles_positions[p1.p_index*size_particle_positions+idx_pos],
- p2.ptr_particles_positions[p2.p_index*size_particle_positions+idx_pos]);
- }
- for(int idx_rhs = 0 ; idx_rhs < size_particle_rhs ; ++idx_rhs){
- std::swap(p1.ptr_particles_current_rhs[p1.p_index*size_particle_rhs+idx_rhs],
- p2.ptr_particles_current_rhs[p2.p_index*size_particle_rhs+idx_rhs]);
- }
- std::swap(p1.ptr_cell_idx[p1.p_index],p2.ptr_cell_idx[p2.p_index]);
- std::swap(p1.ptr_global_idx[p1.p_index],p2.ptr_global_idx[p2.p_index]);
- std::swap(p1.p_index,p2.p_index);
- }
- else if(p1.p_index != -1){
- p2 = p1;
- p1 = ParticleView<partsize_t, real_number, size_particle_positions,size_particle_rhs>();
- }
- else if(p2.p_index != -1){
- p1 = p2;
- p2 = ParticleView<partsize_t, real_number, size_particle_positions,size_particle_rhs>();
- }
-}
-
-
-
template <class partsize_t, class real_number>
class p2p_distr_mpi {
protected:
@@ -126,9 +72,46 @@ protected:
std::array<real_number,3> spatial_box_width;
std::array<real_number,3> spatial_box_offset;
+ const real_number cutoff_radius_compute;
const real_number cutoff_radius;
std::array<long int,3> nb_cell_levels;
+ template <class DataType, int sizeElement>
+ static void permute_copy(const partsize_t offsetIdx, const partsize_t nbElements,
+ const std::pair<long int,partsize_t> permutation[],
+ DataType data[], std::vector<unsigned char>* buffer){
+ buffer->resize(nbElements*sizeof(DataType)*sizeElement);
+ DataType* dataBuffer = reinterpret_cast<DataType*>(buffer->data());
+
+ // Permute
+ for(partsize_t idxPart = 0 ; idxPart < nbElements ; ++idxPart){
+ const partsize_t srcData = permutation[idxPart].second;
+ const partsize_t destData = idxPart;
+ for(int idxVal = 0 ; idxVal < sizeElement ; ++idxVal){
+ dataBuffer[destData*sizeElement + idxVal]
+ = data[srcData*sizeElement + idxVal];
+ }
+ }
+
+ // Copy back
+ for(partsize_t idxPart = 0 ; idxPart < nbElements ; ++idxPart){
+ const partsize_t srcData = idxPart;
+ const partsize_t destData = idxPart+offsetIdx;
+ for(int idxVal = 0 ; idxVal < sizeElement ; ++idxVal){
+ data[destData*sizeElement + idxVal]
+ = dataBuffer[srcData*sizeElement + idxVal];
+ }
+ }
+ }
+
+ static real_number getGridCutoff(const real_number in_cutoff_radius, const std::array<real_number,3>& in_spatial_box_width){
+ int idx_factor = 1;
+ while(in_cutoff_radius <= in_spatial_box_width[IDX_Z]/real_number(idx_factor+1)){
+ idx_factor += 1;
+ }
+ return in_spatial_box_width[IDX_Z]/real_number(idx_factor);
+ }
+
public:
////////////////////////////////////////////////////////////////////////////
@@ -144,7 +127,8 @@ public:
current_partition_size(current_partition_interval.second-current_partition_interval.first),
field_grid_dim(in_field_grid_dim),
spatial_box_width(in_spatial_box_width), spatial_box_offset(in_spatial_box_offset),
- cutoff_radius(in_cutoff_radius){
+ cutoff_radius_compute(in_cutoff_radius),
+ cutoff_radius(getGridCutoff(in_cutoff_radius, in_spatial_box_width)){
AssertMpi(MPI_Comm_rank(current_com, &my_rank));
AssertMpi(MPI_Comm_size(current_com, &nb_processes));
@@ -281,18 +265,20 @@ public:
particles_positions[(idxPart)*size_particle_positions + IDX_Z]);
assert(my_down_z_cell_level <= get_cell_coord_z_from_index(particles_coord[idxPart]));
assert(get_cell_coord_z_from_index(particles_coord[idxPart]) <= my_top_z_cell_level);
- if(inout_index_particles[idxPart] == 58576 || inout_index_particles[idxPart] == 0){// TODO
- printf("Coord index %ld - %ld (tree index %ld)\n", idxPart, inout_index_particles[idxPart],particles_coord[idxPart]);
- printf(">> Box index %ld - %ld - %ld\n", get_cell_coord_x_from_index(particles_coord[idxPart]),
- get_cell_coord_y_from_index(particles_coord[idxPart]),
- get_cell_coord_z_from_index(particles_coord[idxPart]));
- printf("idxPartition %d\n", idxPartition);
- }
+// if(inout_index_particles[idxPart] == 547){// TODO
+// printf("Coord index %ld - %ld (tree index %ld)\n", idxPart, inout_index_particles[idxPart],particles_coord[idxPart]);
+// printf(">> Box index %ld - %ld - %ld\n", get_cell_coord_x_from_index(particles_coord[idxPart]),
+// get_cell_coord_y_from_index(particles_coord[idxPart]),
+// get_cell_coord_z_from_index(particles_coord[idxPart]));
+// printf(">> idxPartition %d\n", idxPartition);
+// printf(">> position %e %e %e\n", particles_positions[(idxPart)*size_particle_positions + IDX_X],
+// particles_positions[(idxPart)*size_particle_positions + IDX_Y],
+// particles_positions[(idxPart)*size_particle_positions + IDX_Z]);
+// }
}
}
- using ParticleView_t = ParticleView<partsize_t, real_number, size_particle_positions,size_particle_rhs>;
- std::vector<ParticleView_t> part_to_sort;
+ std::vector<std::pair<long int,partsize_t>> part_to_sort;
// Sort each partition in cells
for(int idxPartition = 0 ; idxPartition < current_partition_size ; ++idxPartition){
@@ -300,22 +286,36 @@ public:
for(partsize_t idxPart = current_offset_particles_for_partition[idxPartition] ; idxPart < current_offset_particles_for_partition[idxPartition+1] ; ++idxPart ){
part_to_sort.emplace_back();
- part_to_sort.back().p_index = idxPart;
- part_to_sort.back().ptr_particles_positions = particles_positions;
- part_to_sort.back().ptr_particles_current_rhs = particles_current_rhs;
- part_to_sort.back().ptr_global_idx = inout_index_particles;
- part_to_sort.back().ptr_cell_idx = particles_coord.get();
+ part_to_sort.back().first = particles_coord[idxPart];
+ part_to_sort.back().second = idxPart;
}
- assert(part_to_sort.size() == (current_offset_particles_for_partition[idxPartition+1]-current_offset_particles_for_partition[idxPartition]));
+ assert(part_to_sort.size() == (current_my_nb_particles_per_partition[idxPartition]));
std::sort(part_to_sort.begin(), part_to_sort.end(),
- [](const ParticleView_t& p1,
- const ParticleView_t& p2){
- assert(p1.p_index != -1 && p1.ptr_cell_idx);
- assert(p2.p_index != -1 && p2.ptr_cell_idx);
- return p1.ptr_cell_idx[p1.p_index] < p2.ptr_cell_idx[p2.p_index];
+ [](const std::pair<long int,partsize_t>& p1,
+ const std::pair<long int,partsize_t>& p2){
+ return p1.first < p2.first;
});
+
+// for(partsize_t idxPart = 1 ; idxPart < (long int)part_to_sort.size() ; ++idxPart){// TODO
+// assert(part_to_sort[idxPart-1].first <= part_to_sort[idxPart].first);
+// }
+
+ // Permute array using buffer
+ std::vector<unsigned char> buffer;
+ permute_copy<real_number, size_particle_positions>(current_offset_particles_for_partition[idxPartition],
+ current_my_nb_particles_per_partition[idxPartition],
+ part_to_sort.data(), particles_positions, &buffer);
+ permute_copy<real_number, size_particle_rhs>(current_offset_particles_for_partition[idxPartition],
+ current_my_nb_particles_per_partition[idxPartition],
+ part_to_sort.data(), particles_current_rhs, &buffer);
+ permute_copy<partsize_t, 1>(current_offset_particles_for_partition[idxPartition],
+ current_my_nb_particles_per_partition[idxPartition],
+ part_to_sort.data(), inout_index_particles, &buffer);
+ permute_copy<long int, 1>(current_offset_particles_for_partition[idxPartition],
+ current_my_nb_particles_per_partition[idxPartition],
+ part_to_sort.data(), particles_coord.get(), &buffer);
}
}
@@ -336,13 +336,31 @@ public:
current_cell_idx = particles_coord[idx_part];
current_nb_particles_in_cell = 1;
current_cell_offset = idx_part;
- if(inout_index_particles[idx_part] == 58576 || inout_index_particles[idx_part] == 0){// TODO
- printf("Coord index %ld - %ld (tree index %ld)\n", idx_part, inout_index_particles[idx_part],particles_coord[idx_part]);
- printf(">> Box index %ld - %ld - %ld\n", get_cell_coord_x_from_index(particles_coord[idx_part]),
- get_cell_coord_y_from_index(particles_coord[idx_part]),
- get_cell_coord_z_from_index(particles_coord[idx_part]));
- printf("current_cell_offset %ld current_nb_particles_in_cell %ld\n", current_cell_offset, current_nb_particles_in_cell);
- }
+// if(inout_index_particles[idx_part] == 547){// TODO
+// printf("idxPartition %d\n", idxPartition);
+// printf(">> Coord index %ld - %ld (tree index %ld)\n", idx_part, inout_index_particles[idx_part],particles_coord[idx_part]);
+// printf(">> Box index %ld - %ld - %ld\n", get_cell_coord_x_from_index(particles_coord[idx_part]),
+// get_cell_coord_y_from_index(particles_coord[idx_part]),
+// get_cell_coord_z_from_index(particles_coord[idx_part]));
+// printf(">> current_cell_offset %ld current_nb_particles_in_cell %ld\n", current_cell_offset, current_nb_particles_in_cell);
+// printf(">> Position %e %e %e\n", particles_positions[idx_part*size_particle_positions + IDX_X],
+// particles_positions[idx_part*size_particle_positions + IDX_Y],
+// particles_positions[idx_part*size_particle_positions + IDX_Z]);
+// }
+// if(inout_index_particles[idx_part] == 356){// TODO
+// printf("idxPartition %d\n", idxPartition);
+// printf(">> Coord index %ld - %ld (tree index %ld)\n", idx_part, inout_index_particles[idx_part],particles_coord[idx_part]);
+// printf(">> Box index %ld - %ld - %ld\n", get_cell_coord_x_from_index(particles_coord[idx_part]),
+// get_cell_coord_y_from_index(particles_coord[idx_part]),
+// get_cell_coord_z_from_index(particles_coord[idx_part]));
+// printf(">> current_cell_offset %ld current_nb_particles_in_cell %ld\n", current_cell_offset, current_nb_particles_in_cell);
+// printf(">> Position %e %e %e\n", particles_positions[idx_part*size_particle_positions + IDX_X],
+// particles_positions[idx_part*size_particle_positions + IDX_Y],
+// particles_positions[idx_part*size_particle_positions + IDX_Z]);
+// }
+ }
+ else{
+ current_nb_particles_in_cell += 1;
}
}
if(current_nb_particles_in_cell){
@@ -351,9 +369,9 @@ public:
}
}
- printf("[%d] go from cutoff level %ld to %ld\n",
- my_rank, my_down_z_cell_level, my_top_z_cell_level); // TODO remove
- fflush(stdout); // TODO
+// printf("[%d] go from cutoff level %ld to %ld\n",
+// my_rank, my_down_z_cell_level, my_top_z_cell_level); // TODO remove
+// fflush(stdout); // TODO
// Offset per cell layers
long int previous_index = 0;
@@ -369,10 +387,10 @@ public:
previous_index = part_box_z_index;
}
}
- for(size_t idx_layer = 0 ; idx_layer < my_nb_cell_levels ; ++idx_layer){
- printf("[%d] nb particles in cutoff level %llu are %ld\n",
- my_rank, idx_layer, particles_offset_layers[idx_layer+1]); // TODO remove
- fflush(stdout); // TODO
+ for(long int idx_layer = 0 ; idx_layer < my_nb_cell_levels ; ++idx_layer){
+// printf("[%d] nb particles in cutoff level %ld are %ld\n",
+// my_rank, idx_layer, particles_offset_layers[idx_layer+1]); // TODO remove
+// fflush(stdout); // TODO
particles_offset_layers[idx_layer+1] += particles_offset_layers[idx_layer];
}
@@ -383,9 +401,9 @@ public:
// Find process with at least one neighbor
{
- std::cout << my_rank << " my_top_z_cell_level " << my_top_z_cell_level << std::endl;
- std::cout << my_rank << " my_down_z_cell_level " << my_down_z_cell_level << std::endl;
- std::cout.flush();// TODO
+// std::cout << my_rank << " my_top_z_cell_level " << my_top_z_cell_level << std::endl;
+// std::cout << my_rank << " my_down_z_cell_level " << my_down_z_cell_level << std::endl;
+// std::cout.flush();// TODO
int dest_proc = (my_rank+1)%nb_processes_involved;
while(dest_proc != my_rank
@@ -398,10 +416,10 @@ public:
nb_levels_to_send += 1;
}
- std::cout << my_rank << " dest_proc " << dest_proc << std::endl;
- std::cout << my_rank << " first_cell_level_proc(dest_proc) " << first_cell_level_proc(dest_proc) << std::endl;
- std::cout << my_rank << " last_cell_level_proc(dest_proc) " << last_cell_level_proc(dest_proc) << std::endl;
- std::cout.flush();// TODO
+// std::cout << my_rank << " dest_proc " << dest_proc << std::endl;
+// std::cout << my_rank << " first_cell_level_proc(dest_proc) " << first_cell_level_proc(dest_proc) << std::endl;
+// std::cout << my_rank << " last_cell_level_proc(dest_proc) " << last_cell_level_proc(dest_proc) << std::endl;
+// std::cout.flush();// TODO
NeighborDescriptor descriptor;
descriptor.destProc = dest_proc;
@@ -409,21 +427,21 @@ public:
descriptor.nbParticlesToExchange = particles_offset_layers[my_nb_cell_levels] - particles_offset_layers[my_nb_cell_levels-nb_levels_to_send];
descriptor.isRecv = false;
- std::cout << my_rank << "SEND" << std::endl;
- std::cout << "descriptor.destProc " << descriptor.destProc << std::endl;
- std::cout << "descriptor.nbLevelsToExchange " << descriptor.nbLevelsToExchange << std::endl;
- std::cout << "descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
- std::cout << "descriptor.isRecv " << descriptor.isRecv << std::endl;
- std::cout << "neigDescriptors.size() " << neigDescriptors.size() << std::endl;
- std::cout.flush();// TODO
+// std::cout << my_rank << "SEND" << std::endl;
+// std::cout << "descriptor.destProc " << descriptor.destProc << std::endl;
+// std::cout << "descriptor.nbLevelsToExchange " << descriptor.nbLevelsToExchange << std::endl;
+// std::cout << "descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
+// std::cout << "descriptor.isRecv " << descriptor.isRecv << std::endl;
+// std::cout << "neigDescriptors.size() " << neigDescriptors.size() << std::endl;
+// std::cout.flush();// TODO
neigDescriptors.emplace_back(std::move(descriptor));
dest_proc = (dest_proc+1)%nb_processes_involved;
}
- std::cout << my_rank << " NO dest_proc " << dest_proc << std::endl;
- std::cout << my_rank << " NO first_cell_level_proc(dest_proc) " << first_cell_level_proc(dest_proc) << std::endl;
- std::cout.flush();// TODO
+// std::cout << my_rank << " NO dest_proc " << dest_proc << std::endl;
+// std::cout << my_rank << " NO first_cell_level_proc(dest_proc) " << first_cell_level_proc(dest_proc) << std::endl;
+// std::cout.flush();// TODO
int src_proc = (my_rank-1+nb_processes_involved)%nb_processes_involved;
while(src_proc != my_rank
@@ -436,9 +454,9 @@ public:
nb_levels_to_recv += 1;
}
- std::cout << my_rank << " src_proc " << src_proc << std::endl;
- std::cout << my_rank << " first_cell_level_proc(src_proc) " << first_cell_level_proc(src_proc) << std::endl;
- std::cout.flush();// TODO
+// std::cout << my_rank << " src_proc " << src_proc << std::endl;
+// std::cout << my_rank << " first_cell_level_proc(src_proc) " << first_cell_level_proc(src_proc) << std::endl;
+// std::cout.flush();// TODO
NeighborDescriptor descriptor;
descriptor.destProc = src_proc;
@@ -448,20 +466,20 @@ public:
neigDescriptors.emplace_back(std::move(descriptor));
- std::cout << my_rank << "] RECV" << std::endl;
- std::cout << "descriptor.destProc " << descriptor.destProc << std::endl;
- std::cout << "descriptor.nbLevelsToExchange " << descriptor.nbLevelsToExchange << std::endl;
- std::cout << "descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
- std::cout << "descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
- std::cout << "descriptor.isRecv " << descriptor.isRecv << std::endl;
- std::cout << "neigDescriptors.size() " << neigDescriptors.size() << std::endl;
- std::cout.flush();// TODO
+// std::cout << my_rank << "] RECV" << std::endl;
+// std::cout << "descriptor.destProc " << descriptor.destProc << std::endl;
+// std::cout << "descriptor.nbLevelsToExchange " << descriptor.nbLevelsToExchange << std::endl;
+// std::cout << "descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
+// std::cout << "descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
+// std::cout << "descriptor.isRecv " << descriptor.isRecv << std::endl;
+// std::cout << "neigDescriptors.size() " << neigDescriptors.size() << std::endl;
+// std::cout.flush();// TODO
src_proc = (src_proc-1+nb_processes_involved)%nb_processes_involved;
}
- std::cout << my_rank << " NO src_proc " << src_proc << std::endl;
- std::cout << my_rank << " NO first_cell_level_proc(src_proc) " << first_cell_level_proc(src_proc) << std::endl;
- std::cout.flush();// TODO
+// std::cout << my_rank << " NO src_proc " << src_proc << std::endl;
+// std::cout << my_rank << " NO first_cell_level_proc(src_proc) " << first_cell_level_proc(src_proc) << std::endl;
+// std::cout.flush();// TODO
}
//////////////////////////////////////////////////////////////////////
@@ -485,11 +503,11 @@ public:
current_com, &mpiRequests.back()));
if(descriptor.nbParticlesToExchange){
- std::cout << my_rank << "] SEND_PARTICLES" << std::endl;
- std::cout << "descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
- std::cout << "descriptor.destProc " << descriptor.destProc << std::endl;
- std::cout << "idxDescr " << idxDescr << std::endl;
- std::cout << "send from part " << particles_offset_layers[my_nb_cell_levels-descriptor.nbLevelsToExchange] << std::endl;
+// std::cout << my_rank << "] SEND_PARTICLES" << std::endl;
+// std::cout << "descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
+// std::cout << "descriptor.destProc " << descriptor.destProc << std::endl;
+// std::cout << "idxDescr " << idxDescr << std::endl;
+// std::cout << "send from part " << particles_offset_layers[my_nb_cell_levels-descriptor.nbLevelsToExchange] << std::endl;
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
@@ -510,8 +528,8 @@ public:
}
}
else{
- std::cout << "RECV_PARTICLES " << RECV_PARTICLES << std::endl;
- std::cout << "idxDescr " << idxDescr << std::endl;
+// std::cout << "RECV_PARTICLES " << RECV_PARTICLES << std::endl;
+// std::cout << "idxDescr " << idxDescr << std::endl;
whatNext.emplace_back(std::pair<Action,int>{RECV_PARTICLES, idxDescr});
mpiRequests.emplace_back();
AssertMpi(MPI_Irecv(&descriptor.nbParticlesToExchange,
@@ -550,13 +568,13 @@ public:
assert(NbParticlesToReceive != -1);
assert(descriptor.toCompute == nullptr);
- std::cout << my_rank << "] RECV_PARTICLES" << std::endl;
- std::cout << "descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
- std::cout << "descriptor.destProc " << descriptor.destProc << std::endl;
- std::cout << "releasedAction.second " << releasedAction.second << std::endl;
+// std::cout << my_rank << "] RECV_PARTICLES" << std::endl;
+// std::cout << "descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
+// std::cout << "descriptor.destProc " << descriptor.destProc << std::endl;
+// std::cout << "releasedAction.second " << releasedAction.second << std::endl;
if(NbParticlesToReceive){
- std::cout << "MPI_Irecv " << std::endl;
+// std::cout << "MPI_Irecv " << std::endl;
descriptor.toCompute.reset(new real_number[NbParticlesToReceive*size_particle_positions]);
whatNext.emplace_back(std::pair<Action,int>{COMPUTE_PARTICLES, releasedAction.second});
mpiRequests.emplace_back();
@@ -597,22 +615,18 @@ public:
long int neighbors_indexes[27];
const int nbNeighbors = my_tree.getNeighbors(current_cell_idx, neighbors, neighbors_indexes, true);
- for(int idx_test = 0 ; idx_test < nb_parts_in_cell ; ++idx_test){ // TODO
- if(int(descriptor.toCompute[(idxPart+idx_test)*size_particle_positions + IDX_X]*1000) == int(1.685800e-01*1000)
- && int(descriptor.toCompute[(idxPart+idx_test)*size_particle_positions + IDX_Y]*1000) == int(7.524981e-01*1000)
- && int(descriptor.toCompute[(idxPart+idx_test)*size_particle_positions + IDX_Z]*1000) == int(9.999596e-01*1000)){
- printf("Found a pos %ld\n", idxPart+idx_test);
- printf("pos %e %e %e\n",
- descriptor.toCompute[(idxPart+idx_test)*size_particle_positions + IDX_X],
- descriptor.toCompute[(idxPart+idx_test)*size_particle_positions + IDX_Y],
- descriptor.toCompute[(idxPart+idx_test)*size_particle_positions + IDX_Z]);
- }
- }
- printf("Remote part from %ld for %ld at idx %ld\n", idxPart, nb_parts_in_cell, current_cell_idx); // TODO
- printf("pos of first %e %e %e\n", descriptor.toCompute[idxPart*size_particle_positions + IDX_X],
- descriptor.toCompute[idxPart*size_particle_positions + IDX_Y],
- descriptor.toCompute[idxPart*size_particle_positions + IDX_Z]); // TODO
- printf("nbNeighbors %d\n", nbNeighbors); // TODO
+// for(int idx_test = 0 ; idx_test < nb_parts_in_cell ; ++idx_test){ // TODO
+// real_number totest[3] = {8.570442e-01, 7.173084e-02, 8.279754e-03};
+// if(int(descriptor.toCompute[(idxPart+idx_test)*size_particle_positions + IDX_X]*1000) == int(totest[0]*1000)
+// && int(descriptor.toCompute[(idxPart+idx_test)*size_particle_positions + IDX_Y]*1000) == int(totest[1]*1000)
+// && int(descriptor.toCompute[(idxPart+idx_test)*size_particle_positions + IDX_Z]*1000) == int(totest[2]*1000)){
+// printf("Found a pos %ld\n", idxPart+idx_test);
+// printf("pos %e %e %e\n",
+// descriptor.toCompute[(idxPart+idx_test)*size_particle_positions + IDX_X],
+// descriptor.toCompute[(idxPart+idx_test)*size_particle_positions + IDX_Y],
+// descriptor.toCompute[(idxPart+idx_test)*size_particle_positions + IDX_Z]);
+// }
+// }
// with other interval
for(size_t idx_neighbor = 0 ; idx_neighbor < nbNeighbors ; ++idx_neighbor){
@@ -625,7 +639,7 @@ public:
particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
- if(dist_r2 < cutoff_radius*cutoff_radius){
+ if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
in_computer.template compute_interaction<size_particle_positions,size_particle_rhs>(
&descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions],
&descriptor.results[(idxPart+idx_p1)*size_particle_rhs],
@@ -634,21 +648,21 @@ public:
dist_r2);
}
- if(inout_index_particles[(*neighbors[idx_neighbor])[idx_2].first+idx_p2] == 132){// TODO
- printf("test interaction between :\n");
- printf("index %ld (%ld) pos %e %e %e\n",
- (idxPart+idx_p1), -1,
- descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_X],
- descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_Y],
- descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_Z]);
- printf("index %ld (%ld) pos %e %e %e\n",
- ((*neighbors[idx_neighbor])[idx_2].first+idx_p2),
- inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)],
- particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
- particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
- particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
- printf("Radius = %e (%e)\n", sqrt(dist_r2), dist_r2);
- }
+// if(inout_index_particles[(*neighbors[idx_neighbor])[idx_2].first+idx_p2] == 356){// TODO
+// printf("test interaction between :\n");
+// printf("index %ld (%ld) pos %e %e %e\n",
+// (idxPart+idx_p1), -1L,
+// descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_X],
+// descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_Y],
+// descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_Z]);
+// printf("index %ld (%ld) pos %e %e %e\n",
+// ((*neighbors[idx_neighbor])[idx_2].first+idx_p2),
+// inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)],
+// particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
+// particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
+// particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
+// printf("Radius = %e (%e)\n", sqrt(dist_r2), dist_r2);
+// }
}
}
}
@@ -700,6 +714,28 @@ public:
for(size_t idx_1 = 0 ; idx_1 < intervals.size() ; ++idx_1){
// self interval
for(partsize_t idx_p1 = 0 ; idx_p1 < intervals[idx_1].second ; ++idx_p1){
+// if(((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 356))){// TODO
+// printf("box %ld:\n", iter_cell.first);
+// printf("intervals.size() %lu:\n", intervals.size());
+// printf("intervals[idx_1].second %ld:\n", intervals[idx_1].second);
+// printf("index %ld (%ld) pos %e %e %e\n",
+// (intervals[idx_1].first+idx_p1), inout_index_particles[(intervals[idx_1].first+idx_p1)],
+// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
+// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
+// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z]);
+// }
+// if(((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 547))){// TODO
+// printf("box %ld:\n", iter_cell.first);
+// printf("intervals.size() %lu:\n", intervals.size());
+// printf("intervals[idx_1].second %ld:\n", intervals[idx_1].second);
+// printf("index %ld (%ld) pos %e %e %e\n",
+// (intervals[idx_1].first+idx_p1), inout_index_particles[(intervals[idx_1].first+idx_p1)],
+// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
+// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
+// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z]);
+// }
+
+
for(partsize_t idx_p2 = idx_p1+1 ; idx_p2 < intervals[idx_1].second ; ++idx_p2){
const real_number dist_r2 = compute_distance_r2(particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
@@ -707,7 +743,7 @@ public:
particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_X],
particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_Y],
particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_Z]);
- if(dist_r2 < cutoff_radius*cutoff_radius){
+ if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
in_computer.template compute_interaction<size_particle_positions,size_particle_rhs>(
&particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions],
&particles_current_rhs[(intervals[idx_1].first+idx_p1)*size_particle_rhs],
@@ -715,6 +751,27 @@ public:
&particles_current_rhs[(intervals[idx_1].first+idx_p2)*size_particle_rhs],
dist_r2);
}
+
+// if(((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 356)
+// || inout_index_particles[(intervals[idx_1].first+idx_p2)] == 356)/*
+// && ((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 1832)
+// || inout_index_particles[(intervals[idx_1].first+idx_p2)] == 1832)
+// && ((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 547)
+// || inout_index_particles[(intervals[idx_1].first+idx_p2)] == 547)*/){// TODO
+// printf("print between :\n");
+// printf("index %ld (%ld) pos %e %e %e\n",
+// (intervals[idx_1].first+idx_p1), inout_index_particles[(intervals[idx_1].first+idx_p1)],
+// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
+// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
+// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z]);
+// printf("index %ld (%ld) pos %e %e %e\n",
+// (intervals[idx_1].first+idx_p2),
+// inout_index_particles[(intervals[idx_1].first+idx_p2)],
+// particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_X],
+// particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_Y],
+// particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_Z]);
+// printf("Radius = %e (%e)\n", sqrt(dist_r2), dist_r2);
+// }
}
}
@@ -728,7 +785,7 @@ public:
particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
- if(dist_r2 < cutoff_radius*cutoff_radius){
+ if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
in_computer.template compute_interaction<size_particle_positions,size_particle_rhs>(
&particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions],
&particles_current_rhs[(intervals[idx_1].first+idx_p1)*size_particle_rhs],
@@ -736,6 +793,27 @@ public:
&particles_current_rhs[(intervals[idx_2].first+idx_p2)*size_particle_rhs],
dist_r2);
}
+
+// if(((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 356)
+// || inout_index_particles[(intervals[idx_2].first+idx_p2)] == 356)/*
+// && ((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 547)
+// || inout_index_particles[(intervals[idx_2].first+idx_p2)] == 547)
+// && ((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 1832)
+// || inout_index_particles[(intervals[idx_2].first+idx_p2)] == 1832)*/){// TODO
+// printf("print between :\n");
+// printf("index %ld (%ld) pos %e %e %e\n",
+// (intervals[idx_1].first+idx_p1), inout_index_particles[(intervals[idx_1].first+idx_p1)],
+// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
+// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
+// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z]);
+// printf("index %ld (%ld) pos %e %e %e\n",
+// (intervals[idx_2].first+idx_p2),
+// inout_index_particles[(intervals[idx_2].first+idx_p2)],
+// particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
+// particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
+// particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
+// printf("Radius = %e (%e)\n", sqrt(dist_r2), dist_r2);
+// }
}
}
}
@@ -747,6 +825,12 @@ public:
long int neighbors_indexes[27];
const int nbNeighbors = my_tree.getNeighbors(currenct_cell_idx, neighbors, neighbors_indexes, false);
+// if(((currenct_cell_idx == 785))){// TODO
+// printf("box %ld:\n", iter_cell.first);
+// printf("intervals.size() %lu:\n", intervals.size());
+// printf("nbNeighbors %d\n",nbNeighbors);
+// }
+
for(size_t idx_1 = 0 ; idx_1 < intervals.size() ; ++idx_1){
// with other interval
for(size_t idx_neighbor = 0 ; idx_neighbor < nbNeighbors ; ++idx_neighbor){
@@ -760,7 +844,7 @@ public:
particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
- if(dist_r2 < cutoff_radius*cutoff_radius){
+ if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
in_computer.template compute_interaction<size_particle_positions,size_particle_rhs>(
&particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions],
&particles_current_rhs[(intervals[idx_1].first+idx_p1)*size_particle_rhs],
@@ -768,6 +852,27 @@ public:
&particles_current_rhs[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_rhs],
dist_r2);
}
+
+// if(((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 356)
+// || inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)] == 356)/*
+// && (inout_index_particles[(intervals[idx_1].first+idx_p1)] == 547)
+// || inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)] == 547
+// && (inout_index_particles[(intervals[idx_1].first+idx_p1)] == 1832)
+// || inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)] == 1832*/){// TODO
+// printf("print between :\n");
+// printf("index %ld (%ld) pos %e %e %e\n",
+// (intervals[idx_1].first+idx_p1), inout_index_particles[(intervals[idx_1].first+idx_p1)],
+// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
+// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
+// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z]);
+// printf("index %ld (%ld) pos %e %e %e\n",
+// ((*neighbors[idx_neighbor])[idx_2].first+idx_p2),
+// inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)],
+// particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
+// particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
+// particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
+// printf("Radius = %e (%e)\n", sqrt(dist_r2), dist_r2);
+// }
}
}
}
--
GitLab
From 70f6d0a3e61d9695e9a6692d970dc382d2e54aa3 Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Thu, 21 Sep 2017 16:15:20 +0200
Subject: [PATCH 017/342] Works well now
---
bfps/cpp/particles/p2p_computer.hpp | 5 +-
bfps/cpp/particles/p2p_distr_mpi.hpp | 84 +++++++++++++++-------------
bfps/cpp/particles/p2p_tree.hpp | 18 +++++-
3 files changed, 65 insertions(+), 42 deletions(-)
diff --git a/bfps/cpp/particles/p2p_computer.hpp b/bfps/cpp/particles/p2p_computer.hpp
index efe0e5e3..eb77729f 100644
--- a/bfps/cpp/particles/p2p_computer.hpp
+++ b/bfps/cpp/particles/p2p_computer.hpp
@@ -8,7 +8,7 @@ class p2p_computer{
public:
template <int size_particle_rhs>
void init_result_array(real_number rhs[], const partsize_t nbParticles) const{
- memset(rhs, 0, sizeof(real_number)*nbParticles);
+ memset(rhs, 0, sizeof(real_number)*nbParticles*size_particle_rhs);
}
template <int size_particle_rhs>
@@ -23,7 +23,8 @@ public:
template <int size_particle_positions, int size_particle_rhs>
void compute_interaction(const real_number pos_part1[], real_number rhs_part1[],
const real_number pos_part2[], real_number rhs_part2[],
- const real_number dist_pow2) const{
+ const real_number dist_pow2,
+ const real_number xshift_coef, const real_number yshift_coef, const real_number zshift_coef) const{
rhs_part1[0] += 1;
rhs_part2[0] += 1;
}
diff --git a/bfps/cpp/particles/p2p_distr_mpi.hpp b/bfps/cpp/particles/p2p_distr_mpi.hpp
index 1709dc72..aacd5a2f 100644
--- a/bfps/cpp/particles/p2p_distr_mpi.hpp
+++ b/bfps/cpp/particles/p2p_distr_mpi.hpp
@@ -210,21 +210,16 @@ public:
}
real_number compute_distance_r2(const real_number x1, const real_number y1, const real_number z1,
- const real_number x2, const real_number y2, const real_number z2) const {
- real_number diff_x = std::abs(x1-x2);
- while(diff_x > spatial_box_width[IDX_X]/2){
- diff_x = std::abs(diff_x - spatial_box_width[IDX_X]);
- }
+ const real_number x2, const real_number y2, const real_number z2,
+ const real_number xshift_coef, const real_number yshift_coef, const real_number zshift_coef) const {
+ real_number diff_x = std::abs(x1-x2+xshift_coef*spatial_box_width[IDX_X]);
+ assert(diff_x <= 2*cutoff_radius);
- real_number diff_y = std::abs(y1-y2);
- while(diff_y > spatial_box_width[IDX_Y]/2){
- diff_y = std::abs(diff_y - spatial_box_width[IDX_Y]);
- }
+ real_number diff_y = std::abs(y1-y2+yshift_coef*spatial_box_width[IDX_Y]);
+ assert(diff_y <= 2*cutoff_radius);
- real_number diff_z = std::abs(z1-z2);
- while(diff_z > spatial_box_width[IDX_Z]/2){
- diff_z = std::abs(diff_z - spatial_box_width[IDX_Z]);
- }
+ real_number diff_z = std::abs(z1-z2+zshift_coef*spatial_box_width[IDX_Z]);
+ assert(diff_z <= 2*cutoff_radius);
return (diff_x*diff_x) + (diff_y*diff_y) + (diff_z*diff_z);
}
@@ -401,8 +396,8 @@ public:
// Find process with at least one neighbor
{
-// std::cout << my_rank << " my_top_z_cell_level " << my_top_z_cell_level << std::endl;
-// std::cout << my_rank << " my_down_z_cell_level " << my_down_z_cell_level << std::endl;
+// std::cout << my_rank << ">> my_top_z_cell_level " << my_top_z_cell_level << std::endl;
+// std::cout << my_rank << ">> my_down_z_cell_level " << my_down_z_cell_level << std::endl;
// std::cout.flush();// TODO
int dest_proc = (my_rank+1)%nb_processes_involved;
@@ -417,8 +412,8 @@ public:
}
// std::cout << my_rank << " dest_proc " << dest_proc << std::endl;
-// std::cout << my_rank << " first_cell_level_proc(dest_proc) " << first_cell_level_proc(dest_proc) << std::endl;
-// std::cout << my_rank << " last_cell_level_proc(dest_proc) " << last_cell_level_proc(dest_proc) << std::endl;
+// std::cout << my_rank << ">> first_cell_level_proc(dest_proc) " << first_cell_level_proc(dest_proc) << std::endl;
+// std::cout << my_rank << ">> last_cell_level_proc(dest_proc) " << last_cell_level_proc(dest_proc) << std::endl;
// std::cout.flush();// TODO
NeighborDescriptor descriptor;
@@ -427,12 +422,12 @@ public:
descriptor.nbParticlesToExchange = particles_offset_layers[my_nb_cell_levels] - particles_offset_layers[my_nb_cell_levels-nb_levels_to_send];
descriptor.isRecv = false;
-// std::cout << my_rank << "SEND" << std::endl;
-// std::cout << "descriptor.destProc " << descriptor.destProc << std::endl;
-// std::cout << "descriptor.nbLevelsToExchange " << descriptor.nbLevelsToExchange << std::endl;
-// std::cout << "descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
-// std::cout << "descriptor.isRecv " << descriptor.isRecv << std::endl;
-// std::cout << "neigDescriptors.size() " << neigDescriptors.size() << std::endl;
+// std::cout << my_rank << " SEND" << std::endl;
+// std::cout << ">> descriptor.destProc " << descriptor.destProc << std::endl;
+// std::cout << ">> descriptor.nbLevelsToExchange " << descriptor.nbLevelsToExchange << std::endl;
+// std::cout << ">> descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
+// std::cout << ">> descriptor.isRecv " << descriptor.isRecv << std::endl;
+// std::cout << ">> neigDescriptors.size() " << neigDescriptors.size() << std::endl;
// std::cout.flush();// TODO
neigDescriptors.emplace_back(std::move(descriptor));
@@ -467,12 +462,12 @@ public:
neigDescriptors.emplace_back(std::move(descriptor));
// std::cout << my_rank << "] RECV" << std::endl;
-// std::cout << "descriptor.destProc " << descriptor.destProc << std::endl;
-// std::cout << "descriptor.nbLevelsToExchange " << descriptor.nbLevelsToExchange << std::endl;
-// std::cout << "descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
-// std::cout << "descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
-// std::cout << "descriptor.isRecv " << descriptor.isRecv << std::endl;
-// std::cout << "neigDescriptors.size() " << neigDescriptors.size() << std::endl;
+// std::cout << ">> descriptor.destProc " << descriptor.destProc << std::endl;
+// std::cout << ">> descriptor.nbLevelsToExchange " << descriptor.nbLevelsToExchange << std::endl;
+// std::cout << ">> descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
+// std::cout << ">> descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
+// std::cout << ">> descriptor.isRecv " << descriptor.isRecv << std::endl;
+// std::cout << ">> neigDescriptors.size() " << neigDescriptors.size() << std::endl;
// std::cout.flush();// TODO
src_proc = (src_proc-1+nb_processes_involved)%nb_processes_involved;
@@ -544,6 +539,7 @@ public:
#pragma omp master
{
while(mpiRequests.size()){
+ TIMEZONE("wait-loop");
assert(mpiRequests.size() == whatNext.size());
int idxDone = int(mpiRequests.size());
@@ -561,6 +557,7 @@ public:
/// Data to exchange particles
//////////////////////////////////////////////////////////////////////
if(releasedAction.first == RECV_PARTICLES){
+ TIMEZONE("post-recv-particles");
NeighborDescriptor& descriptor = neigDescriptors[releasedAction.second];
assert(descriptor.isRecv);
const int destProc = descriptor.destProc;
@@ -589,6 +586,7 @@ public:
/// Computation
//////////////////////////////////////////////////////////////////////
if(releasedAction.first == COMPUTE_PARTICLES){
+ TIMEZONE("compute-particles");
NeighborDescriptor& descriptor = neigDescriptors[releasedAction.second];
assert(descriptor.isRecv);
const partsize_t NbParticlesToReceive = descriptor.nbParticlesToExchange;
@@ -613,7 +611,8 @@ public:
const std::vector<std::pair<partsize_t,partsize_t>>* neighbors[27];
long int neighbors_indexes[27];
- const int nbNeighbors = my_tree.getNeighbors(current_cell_idx, neighbors, neighbors_indexes, true);
+ std::array<real_number,3> shift[27];
+ const int nbNeighbors = my_tree.getNeighbors(current_cell_idx, neighbors, neighbors_indexes, shift, true);
// for(int idx_test = 0 ; idx_test < nb_parts_in_cell ; ++idx_test){ // TODO
// real_number totest[3] = {8.570442e-01, 7.173084e-02, 8.279754e-03};
@@ -638,14 +637,15 @@ public:
descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_Z],
particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
- particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z],
+ shift[idx_neighbor][IDX_X], shift[idx_neighbor][IDX_Y], shift[idx_neighbor][IDX_Z]);
if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
in_computer.template compute_interaction<size_particle_positions,size_particle_rhs>(
&descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions],
&descriptor.results[(idxPart+idx_p1)*size_particle_rhs],
&particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions],
&particles_current_rhs[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_rhs],
- dist_r2);
+ dist_r2, shift[idx_neighbor][IDX_X], shift[idx_neighbor][IDX_Y], shift[idx_neighbor][IDX_Z]);
}
// if(inout_index_particles[(*neighbors[idx_neighbor])[idx_2].first+idx_p2] == 356){// TODO
@@ -693,6 +693,7 @@ public:
/// Merge
//////////////////////////////////////////////////////////////////////
if(releasedAction.first == MERGE_PARTICLES){
+ TIMEZONE("merge");
NeighborDescriptor& descriptor = neigDescriptors[releasedAction.second];
assert(descriptor.isRecv == false);
assert(descriptor.toRecvAndMerge != nullptr);
@@ -709,6 +710,7 @@ public:
// Compute self data
for(const auto& iter_cell : my_tree){
+ TIMEZONE("proceed-leaf");
const std::vector<std::pair<partsize_t,partsize_t>>& intervals = iter_cell.second;
for(size_t idx_1 = 0 ; idx_1 < intervals.size() ; ++idx_1){
@@ -742,14 +744,15 @@ public:
particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z],
particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_X],
particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_Y],
- particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_Z]);
+ particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_Z],
+ 0, 0, 0);
if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
in_computer.template compute_interaction<size_particle_positions,size_particle_rhs>(
&particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions],
&particles_current_rhs[(intervals[idx_1].first+idx_p1)*size_particle_rhs],
&particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions],
&particles_current_rhs[(intervals[idx_1].first+idx_p2)*size_particle_rhs],
- dist_r2);
+ dist_r2, 0, 0, 0);
}
// if(((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 356)
@@ -784,14 +787,15 @@ public:
particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z],
particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
- particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
+ particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_Z],
+ 0, 0, 0);
if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
in_computer.template compute_interaction<size_particle_positions,size_particle_rhs>(
&particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions],
&particles_current_rhs[(intervals[idx_1].first+idx_p1)*size_particle_rhs],
&particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions],
&particles_current_rhs[(intervals[idx_2].first+idx_p2)*size_particle_rhs],
- dist_r2);
+ dist_r2, 0, 0, 0);
}
// if(((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 356)
@@ -823,7 +827,8 @@ public:
const long int currenct_cell_idx = iter_cell.first;
const std::vector<std::pair<partsize_t,partsize_t>>* neighbors[27];
long int neighbors_indexes[27];
- const int nbNeighbors = my_tree.getNeighbors(currenct_cell_idx, neighbors, neighbors_indexes, false);
+ std::array<real_number,3> shift[27];
+ const int nbNeighbors = my_tree.getNeighbors(currenct_cell_idx, neighbors, neighbors_indexes, shift, false);
// if(((currenct_cell_idx == 785))){// TODO
// printf("box %ld:\n", iter_cell.first);
@@ -843,14 +848,15 @@ public:
particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z],
particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
- particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z],
+ shift[idx_neighbor][IDX_X], shift[idx_neighbor][IDX_Y], shift[idx_neighbor][IDX_Z]);
if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
in_computer.template compute_interaction<size_particle_positions,size_particle_rhs>(
&particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions],
&particles_current_rhs[(intervals[idx_1].first+idx_p1)*size_particle_rhs],
&particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions],
&particles_current_rhs[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_rhs],
- dist_r2);
+ dist_r2, shift[idx_neighbor][IDX_X], shift[idx_neighbor][IDX_Y], shift[idx_neighbor][IDX_Z]);
}
// if(((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 356)
diff --git a/bfps/cpp/particles/p2p_tree.hpp b/bfps/cpp/particles/p2p_tree.hpp
index 9f146644..52cb2f2d 100644
--- a/bfps/cpp/particles/p2p_tree.hpp
+++ b/bfps/cpp/particles/p2p_tree.hpp
@@ -46,7 +46,9 @@ public:
return emptyCell;
}
- int getNeighbors(const long int idx, const CellClass* output[27], long int output_indexes[27], const bool include_target) const{
+ template <class ShiftType>
+ int getNeighbors(const long int idx, const CellClass* output[27], long int output_indexes[27],
+ std::array<ShiftType,3> shift[27], const bool include_target) const{
int nbNeighbors = 0;
std::fill_n(output, 27, nullptr);
@@ -57,29 +59,38 @@ public:
for(long int neigh_x = -1 ; neigh_x <= 1 ; ++neigh_x){
long int neigh_x_pbc = neigh_x+idx_x;
+ ShiftType shift_x = 0;
if(neigh_x_pbc < 0){
neigh_x_pbc += nb_cell_levels[IDX_X];
+ shift_x = 1;
}
else if(nb_cell_levels[IDX_X] <= neigh_x_pbc){
neigh_x_pbc -= nb_cell_levels[IDX_X];
+ shift_x = -1;
}
for(long int neigh_y = -1 ; neigh_y <= 1 ; ++neigh_y){
long int neigh_y_pbc = neigh_y+idx_y;
+ ShiftType shift_y = 0;
if(neigh_y_pbc < 0){
neigh_y_pbc += nb_cell_levels[IDX_Y];
+ shift_y = 1;
}
else if(nb_cell_levels[IDX_Y] <= neigh_y_pbc){
neigh_y_pbc -= nb_cell_levels[IDX_Y];
+ shift_y = -1;
}
for(long int neigh_z = -1 ; neigh_z <= 1 ; ++neigh_z){
long int neigh_z_pbc = neigh_z+idx_z;
+ ShiftType shift_z = 0;
if(neigh_z_pbc < 0){
neigh_z_pbc += nb_cell_levels[IDX_Z];
+ shift_z = 1;
}
else if(nb_cell_levels[IDX_Z] <= neigh_z_pbc){
neigh_z_pbc -= nb_cell_levels[IDX_Z];
+ shift_z = -1;
}
if(include_target || neigh_x_pbc != idx_x || neigh_y_pbc != idx_y || neigh_z_pbc != idx_z){
@@ -90,6 +101,11 @@ public:
if(iter != data.end()){
output[nbNeighbors] = &(iter->second);
output_indexes[nbNeighbors] = idx_neigh;
+
+ shift[nbNeighbors][IDX_X] = shift_x;
+ shift[nbNeighbors][IDX_Y] = shift_y;
+ shift[nbNeighbors][IDX_Z] = shift_z;
+
nbNeighbors += 1;
}
}
--
GitLab
From dcd614822b1df4a996f12a8cea68f8302db195b2 Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Thu, 21 Sep 2017 17:34:08 +0200
Subject: [PATCH 018/342] Debug from updated tests (not included yet)
---
bfps/cpp/particles/p2p_distr_mpi.hpp | 107 +++++++++++++++------------
1 file changed, 59 insertions(+), 48 deletions(-)
diff --git a/bfps/cpp/particles/p2p_distr_mpi.hpp b/bfps/cpp/particles/p2p_distr_mpi.hpp
index aacd5a2f..c148e2d3 100644
--- a/bfps/cpp/particles/p2p_distr_mpi.hpp
+++ b/bfps/cpp/particles/p2p_distr_mpi.hpp
@@ -73,6 +73,7 @@ protected:
std::array<real_number,3> spatial_box_offset;
const real_number cutoff_radius_compute;
+ const int nb_cells_factor;
const real_number cutoff_radius;
std::array<long int,3> nb_cell_levels;
@@ -104,12 +105,12 @@ protected:
}
}
- static real_number getGridCutoff(const real_number in_cutoff_radius, const std::array<real_number,3>& in_spatial_box_width){
+ static int foundGridFactor(const real_number in_cutoff_radius, const std::array<real_number,3>& in_spatial_box_width){
int idx_factor = 1;
while(in_cutoff_radius <= in_spatial_box_width[IDX_Z]/real_number(idx_factor+1)){
idx_factor += 1;
}
- return in_spatial_box_width[IDX_Z]/real_number(idx_factor);
+ return idx_factor;
}
public:
@@ -128,7 +129,8 @@ public:
field_grid_dim(in_field_grid_dim),
spatial_box_width(in_spatial_box_width), spatial_box_offset(in_spatial_box_offset),
cutoff_radius_compute(in_cutoff_radius),
- cutoff_radius(getGridCutoff(in_cutoff_radius, in_spatial_box_width)){
+ nb_cells_factor(foundGridFactor(in_cutoff_radius, in_spatial_box_width)),
+ cutoff_radius(in_spatial_box_width[IDX_Z]/real_number(nb_cells_factor)){
AssertMpi(MPI_Comm_rank(current_com, &my_rank));
AssertMpi(MPI_Comm_size(current_com, &nb_processes));
@@ -158,21 +160,29 @@ public:
assert(int(field_grid_dim[IDX_Z]) == partition_interval_offset_per_proc[nb_processes_involved]);
- nb_cell_levels[IDX_X] = spatial_box_width[IDX_X]/cutoff_radius;
- nb_cell_levels[IDX_Y] = spatial_box_width[IDX_Y]/cutoff_radius;
- nb_cell_levels[IDX_Z] = spatial_box_width[IDX_Z]/cutoff_radius;
+ nb_cell_levels[IDX_X] = nb_cells_factor;
+ nb_cell_levels[IDX_Y] = nb_cells_factor;
+ nb_cell_levels[IDX_Z] = nb_cells_factor;
}
virtual ~p2p_distr_mpi(){}
////////////////////////////////////////////////////////////////////////////
+ int getGridFactor() const{
+ return nb_cells_factor;
+ }
+
+ real_number getGridCutoff() const{
+ return cutoff_radius;
+ }
+
long int get_cell_coord_x_from_index(const long int index) const{
return index % nb_cell_levels[IDX_X];
}
long int get_cell_coord_y_from_index(const long int index) const{
- return (index - get_cell_coord_z_from_index(index)*(nb_cell_levels[IDX_X]*nb_cell_levels[IDX_Y]))
+ return (index % (nb_cell_levels[IDX_X]*nb_cell_levels[IDX_Y]))
/ nb_cell_levels[IDX_X];
}
@@ -259,17 +269,18 @@ public:
particles_positions[(idxPart)*size_particle_positions + IDX_Y],
particles_positions[(idxPart)*size_particle_positions + IDX_Z]);
assert(my_down_z_cell_level <= get_cell_coord_z_from_index(particles_coord[idxPart]));
+ if(!(get_cell_coord_z_from_index(particles_coord[idxPart]) <= my_top_z_cell_level)){// TODO
+ printf("Coord index %ld - %ld (tree index %ld)\n", idxPart, inout_index_particles[idxPart],particles_coord[idxPart]);
+ printf(">> Box index %ld - %ld - %ld\n", get_cell_coord_x_from_index(particles_coord[idxPart]),
+ get_cell_coord_y_from_index(particles_coord[idxPart]),
+ get_cell_coord_z_from_index(particles_coord[idxPart]));
+ printf(">> idxPartition %d\n", idxPartition);
+ printf(">> my_top_z_cell_level %ld\n", my_top_z_cell_level);
+ printf(">> position %e %e %e\n", particles_positions[(idxPart)*size_particle_positions + IDX_X],
+ particles_positions[(idxPart)*size_particle_positions + IDX_Y],
+ particles_positions[(idxPart)*size_particle_positions + IDX_Z]);
+ }
assert(get_cell_coord_z_from_index(particles_coord[idxPart]) <= my_top_z_cell_level);
-// if(inout_index_particles[idxPart] == 547){// TODO
-// printf("Coord index %ld - %ld (tree index %ld)\n", idxPart, inout_index_particles[idxPart],particles_coord[idxPart]);
-// printf(">> Box index %ld - %ld - %ld\n", get_cell_coord_x_from_index(particles_coord[idxPart]),
-// get_cell_coord_y_from_index(particles_coord[idxPart]),
-// get_cell_coord_z_from_index(particles_coord[idxPart]));
-// printf(">> idxPartition %d\n", idxPartition);
-// printf(">> position %e %e %e\n", particles_positions[(idxPart)*size_particle_positions + IDX_X],
-// particles_positions[(idxPart)*size_particle_positions + IDX_Y],
-// particles_positions[(idxPart)*size_particle_positions + IDX_Z]);
-// }
}
}
@@ -396,9 +407,9 @@ public:
// Find process with at least one neighbor
{
-// std::cout << my_rank << ">> my_top_z_cell_level " << my_top_z_cell_level << std::endl;
-// std::cout << my_rank << ">> my_down_z_cell_level " << my_down_z_cell_level << std::endl;
-// std::cout.flush();// TODO
+ std::cout << my_rank << ">> my_top_z_cell_level " << my_top_z_cell_level << std::endl;
+ std::cout << my_rank << ">> my_down_z_cell_level " << my_down_z_cell_level << std::endl;
+ std::cout.flush();// TODO
int dest_proc = (my_rank+1)%nb_processes_involved;
while(dest_proc != my_rank
@@ -411,10 +422,10 @@ public:
nb_levels_to_send += 1;
}
-// std::cout << my_rank << " dest_proc " << dest_proc << std::endl;
-// std::cout << my_rank << ">> first_cell_level_proc(dest_proc) " << first_cell_level_proc(dest_proc) << std::endl;
-// std::cout << my_rank << ">> last_cell_level_proc(dest_proc) " << last_cell_level_proc(dest_proc) << std::endl;
-// std::cout.flush();// TODO
+ std::cout << my_rank << " dest_proc " << dest_proc << std::endl;
+ std::cout << my_rank << ">> first_cell_level_proc(dest_proc) " << first_cell_level_proc(dest_proc) << std::endl;
+ std::cout << my_rank << ">> last_cell_level_proc(dest_proc) " << last_cell_level_proc(dest_proc) << std::endl;
+ std::cout.flush();// TODO
NeighborDescriptor descriptor;
descriptor.destProc = dest_proc;
@@ -422,21 +433,21 @@ public:
descriptor.nbParticlesToExchange = particles_offset_layers[my_nb_cell_levels] - particles_offset_layers[my_nb_cell_levels-nb_levels_to_send];
descriptor.isRecv = false;
-// std::cout << my_rank << " SEND" << std::endl;
-// std::cout << ">> descriptor.destProc " << descriptor.destProc << std::endl;
-// std::cout << ">> descriptor.nbLevelsToExchange " << descriptor.nbLevelsToExchange << std::endl;
-// std::cout << ">> descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
-// std::cout << ">> descriptor.isRecv " << descriptor.isRecv << std::endl;
-// std::cout << ">> neigDescriptors.size() " << neigDescriptors.size() << std::endl;
-// std::cout.flush();// TODO
+ std::cout << my_rank << " SEND" << std::endl;
+ std::cout << ">> descriptor.destProc " << descriptor.destProc << std::endl;
+ std::cout << ">> descriptor.nbLevelsToExchange " << descriptor.nbLevelsToExchange << std::endl;
+ std::cout << ">> descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
+ std::cout << ">> descriptor.isRecv " << descriptor.isRecv << std::endl;
+ std::cout << ">> neigDescriptors.size() " << neigDescriptors.size() << std::endl;
+ std::cout.flush();// TODO
neigDescriptors.emplace_back(std::move(descriptor));
dest_proc = (dest_proc+1)%nb_processes_involved;
}
-// std::cout << my_rank << " NO dest_proc " << dest_proc << std::endl;
-// std::cout << my_rank << " NO first_cell_level_proc(dest_proc) " << first_cell_level_proc(dest_proc) << std::endl;
-// std::cout.flush();// TODO
+ std::cout << my_rank << " NO dest_proc " << dest_proc << std::endl;
+ std::cout << my_rank << " NO first_cell_level_proc(dest_proc) " << first_cell_level_proc(dest_proc) << std::endl;
+ std::cout.flush();// TODO
int src_proc = (my_rank-1+nb_processes_involved)%nb_processes_involved;
while(src_proc != my_rank
@@ -449,9 +460,9 @@ public:
nb_levels_to_recv += 1;
}
-// std::cout << my_rank << " src_proc " << src_proc << std::endl;
-// std::cout << my_rank << " first_cell_level_proc(src_proc) " << first_cell_level_proc(src_proc) << std::endl;
-// std::cout.flush();// TODO
+ std::cout << my_rank << " src_proc " << src_proc << std::endl;
+ std::cout << my_rank << " first_cell_level_proc(src_proc) " << first_cell_level_proc(src_proc) << std::endl;
+ std::cout.flush();// TODO
NeighborDescriptor descriptor;
descriptor.destProc = src_proc;
@@ -461,20 +472,20 @@ public:
neigDescriptors.emplace_back(std::move(descriptor));
-// std::cout << my_rank << "] RECV" << std::endl;
-// std::cout << ">> descriptor.destProc " << descriptor.destProc << std::endl;
-// std::cout << ">> descriptor.nbLevelsToExchange " << descriptor.nbLevelsToExchange << std::endl;
-// std::cout << ">> descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
-// std::cout << ">> descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
-// std::cout << ">> descriptor.isRecv " << descriptor.isRecv << std::endl;
-// std::cout << ">> neigDescriptors.size() " << neigDescriptors.size() << std::endl;
-// std::cout.flush();// TODO
+ std::cout << my_rank << "] RECV" << std::endl;
+ std::cout << ">> descriptor.destProc " << descriptor.destProc << std::endl;
+ std::cout << ">> descriptor.nbLevelsToExchange " << descriptor.nbLevelsToExchange << std::endl;
+ std::cout << ">> descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
+ std::cout << ">> descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
+ std::cout << ">> descriptor.isRecv " << descriptor.isRecv << std::endl;
+ std::cout << ">> neigDescriptors.size() " << neigDescriptors.size() << std::endl;
+ std::cout.flush();// TODO
src_proc = (src_proc-1+nb_processes_involved)%nb_processes_involved;
}
-// std::cout << my_rank << " NO src_proc " << src_proc << std::endl;
-// std::cout << my_rank << " NO first_cell_level_proc(src_proc) " << first_cell_level_proc(src_proc) << std::endl;
-// std::cout.flush();// TODO
+ std::cout << my_rank << " NO src_proc " << src_proc << std::endl;
+ std::cout << my_rank << " NO first_cell_level_proc(src_proc) " << first_cell_level_proc(src_proc) << std::endl;
+ std::cout.flush();// TODO
}
//////////////////////////////////////////////////////////////////////
--
GitLab
From cba0515bde712c5aa09c9b452cc15f443ec9db9a Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Fri, 22 Sep 2017 10:37:40 +0200
Subject: [PATCH 019/342] Add a data array for the particles
---
bfps/cpp/particles/p2p_computer.hpp | 6 +--
bfps/cpp/particles/p2p_distr_mpi.hpp | 57 +++++++++++++++++++++++++---
2 files changed, 54 insertions(+), 9 deletions(-)
diff --git a/bfps/cpp/particles/p2p_computer.hpp b/bfps/cpp/particles/p2p_computer.hpp
index eb77729f..cc2ed2dc 100644
--- a/bfps/cpp/particles/p2p_computer.hpp
+++ b/bfps/cpp/particles/p2p_computer.hpp
@@ -20,9 +20,9 @@ public:
}
}
- template <int size_particle_positions, int size_particle_rhs>
- void compute_interaction(const real_number pos_part1[], real_number rhs_part1[],
- const real_number pos_part2[], real_number rhs_part2[],
+ template <int size_particle_positions, int size_particle_data, int size_particle_rhs>
+ void compute_interaction(const real_number pos_part1[], const real_number data_part1[], real_number rhs_part1[],
+ const real_number pos_part2[], const real_number data_part2[], real_number rhs_part2[],
const real_number dist_pow2,
const real_number xshift_coef, const real_number yshift_coef, const real_number zshift_coef) const{
rhs_part1[0] += 1;
diff --git a/bfps/cpp/particles/p2p_distr_mpi.hpp b/bfps/cpp/particles/p2p_distr_mpi.hpp
index c148e2d3..6ed8158a 100644
--- a/bfps/cpp/particles/p2p_distr_mpi.hpp
+++ b/bfps/cpp/particles/p2p_distr_mpi.hpp
@@ -31,9 +31,11 @@ protected:
int destProc;
int nbLevelsToExchange;
bool isRecv;
+ bool positionsReceived;
std::unique_ptr<real_number[]> toRecvAndMerge;
std::unique_ptr<real_number[]> toCompute;
+ std::unique_ptr<real_number[]> toData;
std::unique_ptr<real_number[]> results;
};
@@ -41,6 +43,7 @@ protected:
NOTHING_TODO,
RECV_PARTICLES,
COMPUTE_PARTICLES,
+ CHECK_PARTICLES,
RELEASE_BUFFER_PARTICLES,
MERGE_PARTICLES,
@@ -234,10 +237,11 @@ public:
return (diff_x*diff_x) + (diff_y*diff_y) + (diff_z*diff_z);
}
- template <class computer_class, int size_particle_positions, int size_particle_rhs>
+ template <class computer_class, int size_particle_positions, int size_particle_data, int size_particle_rhs>
void compute_distr(computer_class& in_computer,
const partsize_t current_my_nb_particles_per_partition[],
real_number particles_positions[],
+ real_number particles_data[],
real_number particles_current_rhs[],
partsize_t inout_index_particles[]){
TIMEZONE("compute_distr");
@@ -313,6 +317,9 @@ public:
permute_copy<real_number, size_particle_positions>(current_offset_particles_for_partition[idxPartition],
current_my_nb_particles_per_partition[idxPartition],
part_to_sort.data(), particles_positions, &buffer);
+ permute_copy<real_number, size_particle_data>(current_offset_particles_for_partition[idxPartition],
+ current_my_nb_particles_per_partition[idxPartition],
+ part_to_sort.data(), particles_data, &buffer);
permute_copy<real_number, size_particle_rhs>(current_offset_particles_for_partition[idxPartition],
current_my_nb_particles_per_partition[idxPartition],
part_to_sort.data(), particles_current_rhs, &buffer);
@@ -432,6 +439,7 @@ public:
descriptor.nbLevelsToExchange = nb_levels_to_send;
descriptor.nbParticlesToExchange = particles_offset_layers[my_nb_cell_levels] - particles_offset_layers[my_nb_cell_levels-nb_levels_to_send];
descriptor.isRecv = false;
+ descriptor.positionsReceived = false;
std::cout << my_rank << " SEND" << std::endl;
std::cout << ">> descriptor.destProc " << descriptor.destProc << std::endl;
@@ -469,6 +477,7 @@ public:
descriptor.nbLevelsToExchange = nb_levels_to_recv;
descriptor.nbParticlesToExchange = -1;
descriptor.isRecv = true;
+ descriptor.positionsReceived = false;
neigDescriptors.emplace_back(std::move(descriptor));
@@ -523,6 +532,14 @@ public:
descriptor.destProc, TAG_POSITION_PARTICLES,
current_com, &mpiRequests.back()));
+ whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
+ mpiRequests.emplace_back();
+ assert(descriptor.nbParticlesToExchange*size_particle_data < std::numeric_limits<int>::max());
+ AssertMpi(MPI_Isend(const_cast<real_number*>(&particles_data[particles_offset_layers[my_nb_cell_levels-descriptor.nbLevelsToExchange]*size_particle_data]),
+ int(descriptor.nbParticlesToExchange*size_particle_data), particles_utils::GetMpiType(real_number()),
+ descriptor.destProc, TAG_POSITION_PARTICLES,
+ current_com, &mpiRequests.back()));
+
assert(descriptor.toRecvAndMerge == nullptr);
descriptor.toRecvAndMerge.reset(new real_number[descriptor.nbParticlesToExchange*size_particle_rhs]);
whatNext.emplace_back(std::pair<Action,int>{MERGE_PARTICLES, idxDescr});
@@ -584,12 +601,21 @@ public:
if(NbParticlesToReceive){
// std::cout << "MPI_Irecv " << std::endl;
descriptor.toCompute.reset(new real_number[NbParticlesToReceive*size_particle_positions]);
- whatNext.emplace_back(std::pair<Action,int>{COMPUTE_PARTICLES, releasedAction.second});
+ whatNext.emplace_back(std::pair<Action,int>{CHECK_PARTICLES, releasedAction.second});
mpiRequests.emplace_back();
assert(NbParticlesToReceive*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(descriptor.toCompute.get(), int(NbParticlesToReceive*size_particle_positions),
particles_utils::GetMpiType(real_number()), destProc, TAG_POSITION_PARTICLES,
current_com, &mpiRequests.back()));
+
+
+ descriptor.toData.reset(new real_number[NbParticlesToReceive*size_particle_data]);
+ whatNext.emplace_back(std::pair<Action,int>{COMPUTE_PARTICLES, releasedAction.second});
+ mpiRequests.emplace_back();
+ assert(NbParticlesToReceive*size_particle_data < std::numeric_limits<int>::max());
+ AssertMpi(MPI_Irecv(descriptor.toData.get(), int(NbParticlesToReceive*size_particle_data),
+ particles_utils::GetMpiType(real_number()), destProc, TAG_POSITION_PARTICLES,
+ current_com, &mpiRequests.back()));
}
}
@@ -603,6 +629,8 @@ public:
const partsize_t NbParticlesToReceive = descriptor.nbParticlesToExchange;
assert(descriptor.toCompute != nullptr);
+ assert(descriptor.toData != nullptr);
+ assert(descriptor.positionsReceived == true);
descriptor.results.reset(new real_number[NbParticlesToReceive*size_particle_rhs]);
in_computer.template init_result_array<size_particle_rhs>(descriptor.results.get(), NbParticlesToReceive);
@@ -651,10 +679,12 @@ public:
particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z],
shift[idx_neighbor][IDX_X], shift[idx_neighbor][IDX_Y], shift[idx_neighbor][IDX_Z]);
if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
- in_computer.template compute_interaction<size_particle_positions,size_particle_rhs>(
+ in_computer.template compute_interaction<size_particle_positions,size_particle_data, size_particle_rhs>(
&descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions],
+ &descriptor.toData[(idxPart+idx_p1)*size_particle_data],
&descriptor.results[(idxPart+idx_p1)*size_particle_rhs],
&particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions],
+ &particles_data[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_data],
&particles_current_rhs[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_rhs],
dist_r2, shift[idx_neighbor][IDX_X], shift[idx_neighbor][IDX_Y], shift[idx_neighbor][IDX_Z]);
}
@@ -694,6 +724,15 @@ public:
//////////////////////////////////////////////////////////////////////
/// Computation
//////////////////////////////////////////////////////////////////////
+ if(releasedAction.first == CHECK_PARTICLES){
+ NeighborDescriptor& descriptor = neigDescriptors[releasedAction.second];
+ assert(descriptor.toCompute != nullptr);
+ assert(descriptor.isRecv);
+ descriptor.positionsReceived = true;
+ }
+ //////////////////////////////////////////////////////////////////////
+ /// Computation
+ //////////////////////////////////////////////////////////////////////
if(releasedAction.first == RELEASE_BUFFER_PARTICLES){
NeighborDescriptor& descriptor = neigDescriptors[releasedAction.second];
assert(descriptor.toCompute != nullptr);
@@ -758,10 +797,12 @@ public:
particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_Z],
0, 0, 0);
if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
- in_computer.template compute_interaction<size_particle_positions,size_particle_rhs>(
+ in_computer.template compute_interaction<size_particle_positions,size_particle_data,size_particle_rhs>(
&particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions],
+ &particles_data[(intervals[idx_1].first+idx_p1)*size_particle_data],
&particles_current_rhs[(intervals[idx_1].first+idx_p1)*size_particle_rhs],
&particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions],
+ &particles_data[(intervals[idx_1].first+idx_p2)*size_particle_data],
&particles_current_rhs[(intervals[idx_1].first+idx_p2)*size_particle_rhs],
dist_r2, 0, 0, 0);
}
@@ -801,10 +842,12 @@ public:
particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_Z],
0, 0, 0);
if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
- in_computer.template compute_interaction<size_particle_positions,size_particle_rhs>(
+ in_computer.template compute_interaction<size_particle_positions,size_particle_data,size_particle_rhs>(
&particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions],
+ &particles_data[(intervals[idx_1].first+idx_p1)*size_particle_data],
&particles_current_rhs[(intervals[idx_1].first+idx_p1)*size_particle_rhs],
&particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions],
+ &particles_data[(intervals[idx_2].first+idx_p2)*size_particle_data],
&particles_current_rhs[(intervals[idx_2].first+idx_p2)*size_particle_rhs],
dist_r2, 0, 0, 0);
}
@@ -862,10 +905,12 @@ public:
particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z],
shift[idx_neighbor][IDX_X], shift[idx_neighbor][IDX_Y], shift[idx_neighbor][IDX_Z]);
if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
- in_computer.template compute_interaction<size_particle_positions,size_particle_rhs>(
+ in_computer.template compute_interaction<size_particle_positions,size_particle_data,size_particle_rhs>(
&particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions],
+ &particles_data[(intervals[idx_1].first+idx_p1)*size_particle_data],
&particles_current_rhs[(intervals[idx_1].first+idx_p1)*size_particle_rhs],
&particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions],
+ &particles_data[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_data],
&particles_current_rhs[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_rhs],
dist_r2, shift[idx_neighbor][IDX_X], shift[idx_neighbor][IDX_Y], shift[idx_neighbor][IDX_Z]);
}
--
GitLab
From e38f12673707a6f0ef3b489c312a3ece0c067d71 Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Fri, 22 Sep 2017 12:44:55 +0200
Subject: [PATCH 020/342] Add PBC rounding to the positions and add lock free
array to protect the cells from race conditions
---
bfps/cpp/particles/lock_free_bool_array.hpp | 35 ++
bfps/cpp/particles/p2p_distr_mpi.hpp | 431 +++++++++++---------
bfps/cpp/particles/p2p_tree.hpp | 2 +-
3 files changed, 268 insertions(+), 200 deletions(-)
create mode 100644 bfps/cpp/particles/lock_free_bool_array.hpp
diff --git a/bfps/cpp/particles/lock_free_bool_array.hpp b/bfps/cpp/particles/lock_free_bool_array.hpp
new file mode 100644
index 00000000..1ae9968e
--- /dev/null
+++ b/bfps/cpp/particles/lock_free_bool_array.hpp
@@ -0,0 +1,35 @@
+#ifndef LOCK_FREE_BOOL_ARRAY_HPP
+#define LOCK_FREE_BOOL_ARRAY_HPP
+
+#include <vector>
+#include <memory>
+
+class lock_free_bool_array{
+ std::vector<std::unique_ptr<long int>> keys;
+
+public:
+ explicit lock_free_bool_array(const long int inNbKeys = 512){
+ keys.resize(inNbKeys);
+ for(std::unique_ptr<long int>& k : keys){
+ k.reset(new long int(0));
+ }
+ }
+
+ void lock(const int inKey){
+ volatile long int* k = keys[inKey%keys.size()].get();
+ long int res = 1;
+ int cpt = 0;
+ while(res == 1){
+ res = __sync_val_compare_and_swap(k, 0, res);
+ cpt++;
+ }
+ }
+
+ void unlock(const int inKey){
+ volatile long int* k = keys[inKey%keys.size()].get();
+ assert(k && *k);
+ (*k) = 0;
+ }
+};
+
+#endif
diff --git a/bfps/cpp/particles/p2p_distr_mpi.hpp b/bfps/cpp/particles/p2p_distr_mpi.hpp
index 6ed8158a..40be1a28 100644
--- a/bfps/cpp/particles/p2p_distr_mpi.hpp
+++ b/bfps/cpp/particles/p2p_distr_mpi.hpp
@@ -14,6 +14,7 @@
#include "scope_timer.hpp"
#include "particles_utils.hpp"
#include "p2p_tree.hpp"
+#include "lock_free_bool_array.hpp"
template <class partsize_t, class real_number>
class p2p_distr_mpi {
@@ -204,11 +205,21 @@ public:
- std::numeric_limits<real_number>::epsilon())/cutoff_radius);
}
+ real_number apply_pbc(real_number pos, IDXS_3D dim) const{
+ while( pos < spatial_box_offset[dim] ){
+ pos += spatial_box_width[dim];
+ }
+ while( spatial_box_width[dim]+spatial_box_offset[dim] <= pos){
+ pos -= spatial_box_width[dim];
+ }
+ return pos;
+ }
+
std::array<long int,3> get_cell_coordinate(const real_number pos_x, const real_number pos_y,
const real_number pos_z) const {
- const real_number diff_x = pos_x - spatial_box_offset[IDX_X];
- const real_number diff_y = pos_y - spatial_box_offset[IDX_Y];
- const real_number diff_z = pos_z - spatial_box_offset[IDX_Z];
+ const real_number diff_x = apply_pbc(pos_x,IDX_X) - spatial_box_offset[IDX_X];
+ const real_number diff_y = apply_pbc(pos_y,IDX_Y) - spatial_box_offset[IDX_Y];
+ const real_number diff_z = apply_pbc(pos_z,IDX_Z) - spatial_box_offset[IDX_Z];
std::array<long int,3> coord;
coord[IDX_X] = static_cast<long int>(diff_x/cutoff_radius);
coord[IDX_Y] = static_cast<long int>(diff_y/cutoff_radius);
@@ -225,13 +236,13 @@ public:
real_number compute_distance_r2(const real_number x1, const real_number y1, const real_number z1,
const real_number x2, const real_number y2, const real_number z2,
const real_number xshift_coef, const real_number yshift_coef, const real_number zshift_coef) const {
- real_number diff_x = std::abs(x1-x2+xshift_coef*spatial_box_width[IDX_X]);
+ real_number diff_x = std::abs(apply_pbc(x1,IDX_X)-apply_pbc(x2,IDX_X)+xshift_coef*spatial_box_width[IDX_X]);
assert(diff_x <= 2*cutoff_radius);
- real_number diff_y = std::abs(y1-y2+yshift_coef*spatial_box_width[IDX_Y]);
+ real_number diff_y = std::abs(apply_pbc(y1,IDX_X)-apply_pbc(y2,IDX_X)+yshift_coef*spatial_box_width[IDX_Y]);
assert(diff_y <= 2*cutoff_radius);
- real_number diff_z = std::abs(z1-z2+zshift_coef*spatial_box_width[IDX_Z]);
+ real_number diff_z = std::abs(apply_pbc(z1,IDX_X)-apply_pbc(z2,IDX_X)+zshift_coef*spatial_box_width[IDX_Z]);
assert(diff_z <= 2*cutoff_radius);
return (diff_x*diff_x) + (diff_y*diff_y) + (diff_z*diff_z);
@@ -561,6 +572,8 @@ public:
}
}
+ lock_free_bool_array cells_locker(512);
+
TIMEZONE_OMP_INIT_PREPARALLEL(omp_get_max_threads())
#pragma omp parallel default(shared)
{
@@ -648,10 +661,12 @@ public:
nb_parts_in_cell += 1;
}
- const std::vector<std::pair<partsize_t,partsize_t>>* neighbors[27];
- long int neighbors_indexes[27];
- std::array<real_number,3> shift[27];
- const int nbNeighbors = my_tree.getNeighbors(current_cell_idx, neighbors, neighbors_indexes, shift, true);
+ #pragma omp task default(shared) firstprivate(idxPart, nb_parts_in_cell, current_cell_idx)
+ {
+ const std::vector<std::pair<partsize_t,partsize_t>>* neighbors[27];
+ long int neighbors_indexes[27];
+ std::array<real_number,3> shift[27];
+ const int nbNeighbors = my_tree.getNeighbors(current_cell_idx, neighbors, neighbors_indexes, shift, true);
// for(int idx_test = 0 ; idx_test < nb_parts_in_cell ; ++idx_test){ // TODO
// real_number totest[3] = {8.570442e-01, 7.173084e-02, 8.279754e-03};
@@ -666,52 +681,59 @@ public:
// }
// }
- // with other interval
- for(size_t idx_neighbor = 0 ; idx_neighbor < nbNeighbors ; ++idx_neighbor){
- for(size_t idx_2 = 0 ; idx_2 < (*neighbors[idx_neighbor]).size() ; ++idx_2){
- for(partsize_t idx_p1 = 0 ; idx_p1 < nb_parts_in_cell ; ++idx_p1){
- for(partsize_t idx_p2 = 0 ; idx_p2 < (*neighbors[idx_neighbor])[idx_2].second ; ++idx_p2){
- const real_number dist_r2 = compute_distance_r2(descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_X],
- descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_Y],
- descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_Z],
- particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
- particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
- particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z],
- shift[idx_neighbor][IDX_X], shift[idx_neighbor][IDX_Y], shift[idx_neighbor][IDX_Z]);
- if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
- in_computer.template compute_interaction<size_particle_positions,size_particle_data, size_particle_rhs>(
- &descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions],
- &descriptor.toData[(idxPart+idx_p1)*size_particle_data],
- &descriptor.results[(idxPart+idx_p1)*size_particle_rhs],
- &particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions],
- &particles_data[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_data],
- &particles_current_rhs[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_rhs],
- dist_r2, shift[idx_neighbor][IDX_X], shift[idx_neighbor][IDX_Y], shift[idx_neighbor][IDX_Z]);
+ // with other interval
+ for(size_t idx_neighbor = 0 ; idx_neighbor < nbNeighbors ; ++idx_neighbor){
+ cells_locker.lock(neighbors_indexes[idx_neighbor]);
+
+ for(size_t idx_2 = 0 ; idx_2 < (*neighbors[idx_neighbor]).size() ; ++idx_2){
+ for(partsize_t idx_p1 = 0 ; idx_p1 < nb_parts_in_cell ; ++idx_p1){
+ for(partsize_t idx_p2 = 0 ; idx_p2 < (*neighbors[idx_neighbor])[idx_2].second ; ++idx_p2){
+ const real_number dist_r2 = compute_distance_r2(descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_X],
+ descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_Y],
+ descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_Z],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z],
+ shift[idx_neighbor][IDX_X], shift[idx_neighbor][IDX_Y], shift[idx_neighbor][IDX_Z]);
+ if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
+ in_computer.template compute_interaction<size_particle_positions,size_particle_data, size_particle_rhs>(
+ &descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions],
+ &descriptor.toData[(idxPart+idx_p1)*size_particle_data],
+ &descriptor.results[(idxPart+idx_p1)*size_particle_rhs],
+ &particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions],
+ &particles_data[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_data],
+ &particles_current_rhs[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_rhs],
+ dist_r2, shift[idx_neighbor][IDX_X], shift[idx_neighbor][IDX_Y], shift[idx_neighbor][IDX_Z]);
+ }
+
+ // if(inout_index_particles[(*neighbors[idx_neighbor])[idx_2].first+idx_p2] == 356){// TODO
+ // printf("test interaction between :\n");
+ // printf("index %ld (%ld) pos %e %e %e\n",
+ // (idxPart+idx_p1), -1L,
+ // descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_X],
+ // descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_Y],
+ // descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_Z]);
+ // printf("index %ld (%ld) pos %e %e %e\n",
+ // ((*neighbors[idx_neighbor])[idx_2].first+idx_p2),
+ // inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)],
+ // particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
+ // particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
+ // particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
+ // printf("Radius = %e (%e)\n", sqrt(dist_r2), dist_r2);
+ // }
}
-
-// if(inout_index_particles[(*neighbors[idx_neighbor])[idx_2].first+idx_p2] == 356){// TODO
-// printf("test interaction between :\n");
-// printf("index %ld (%ld) pos %e %e %e\n",
-// (idxPart+idx_p1), -1L,
-// descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_X],
-// descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_Y],
-// descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_Z]);
-// printf("index %ld (%ld) pos %e %e %e\n",
-// ((*neighbors[idx_neighbor])[idx_2].first+idx_p2),
-// inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)],
-// particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
-// particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
-// particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
-// printf("Radius = %e (%e)\n", sqrt(dist_r2), dist_r2);
-// }
}
}
+
+ cells_locker.unlock(neighbors_indexes[idx_neighbor]);
}
}
idxPart += nb_parts_in_cell;
}
+ #pragma omp taskwait
+
// Send back
const int destProc = descriptor.destProc;
whatNext.emplace_back(std::pair<Action,int>{RELEASE_BUFFER_PARTICLES, releasedAction.second});
@@ -761,185 +783,196 @@ public:
// Compute self data
for(const auto& iter_cell : my_tree){
TIMEZONE("proceed-leaf");
- const std::vector<std::pair<partsize_t,partsize_t>>& intervals = iter_cell.second;
-
- for(size_t idx_1 = 0 ; idx_1 < intervals.size() ; ++idx_1){
- // self interval
- for(partsize_t idx_p1 = 0 ; idx_p1 < intervals[idx_1].second ; ++idx_p1){
-// if(((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 356))){// TODO
-// printf("box %ld:\n", iter_cell.first);
-// printf("intervals.size() %lu:\n", intervals.size());
-// printf("intervals[idx_1].second %ld:\n", intervals[idx_1].second);
-// printf("index %ld (%ld) pos %e %e %e\n",
-// (intervals[idx_1].first+idx_p1), inout_index_particles[(intervals[idx_1].first+idx_p1)],
-// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
-// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
-// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z]);
-// }
-// if(((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 547))){// TODO
-// printf("box %ld:\n", iter_cell.first);
-// printf("intervals.size() %lu:\n", intervals.size());
-// printf("intervals[idx_1].second %ld:\n", intervals[idx_1].second);
-// printf("index %ld (%ld) pos %e %e %e\n",
-// (intervals[idx_1].first+idx_p1), inout_index_particles[(intervals[idx_1].first+idx_p1)],
-// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
-// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
-// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z]);
-// }
-
+ const long int currenct_cell_idx = iter_cell.first;
+ const std::vector<std::pair<partsize_t,partsize_t>>* intervals_ptr = &iter_cell.second;
- for(partsize_t idx_p2 = idx_p1+1 ; idx_p2 < intervals[idx_1].second ; ++idx_p2){
- const real_number dist_r2 = compute_distance_r2(particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
- particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
- particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z],
- particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_X],
- particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_Y],
- particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_Z],
- 0, 0, 0);
- if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
- in_computer.template compute_interaction<size_particle_positions,size_particle_data,size_particle_rhs>(
- &particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions],
- &particles_data[(intervals[idx_1].first+idx_p1)*size_particle_data],
- &particles_current_rhs[(intervals[idx_1].first+idx_p1)*size_particle_rhs],
- &particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions],
- &particles_data[(intervals[idx_1].first+idx_p2)*size_particle_data],
- &particles_current_rhs[(intervals[idx_1].first+idx_p2)*size_particle_rhs],
- dist_r2, 0, 0, 0);
- }
+#pragma omp task default(shared) firstprivate(currenct_cell_idx, intervals_ptr)
+ {
+ const std::vector<std::pair<partsize_t,partsize_t>>& intervals = (*intervals_ptr);
-// if(((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 356)
-// || inout_index_particles[(intervals[idx_1].first+idx_p2)] == 356)/*
-// && ((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 1832)
-// || inout_index_particles[(intervals[idx_1].first+idx_p2)] == 1832)
-// && ((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 547)
-// || inout_index_particles[(intervals[idx_1].first+idx_p2)] == 547)*/){// TODO
-// printf("print between :\n");
-// printf("index %ld (%ld) pos %e %e %e\n",
-// (intervals[idx_1].first+idx_p1), inout_index_particles[(intervals[idx_1].first+idx_p1)],
-// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
-// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
-// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z]);
-// printf("index %ld (%ld) pos %e %e %e\n",
-// (intervals[idx_1].first+idx_p2),
-// inout_index_particles[(intervals[idx_1].first+idx_p2)],
-// particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_X],
-// particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_Y],
-// particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_Z]);
-// printf("Radius = %e (%e)\n", sqrt(dist_r2), dist_r2);
-// }
- }
- }
+ cells_locker.lock(currenct_cell_idx);
- // with other interval
- for(size_t idx_2 = idx_1+1 ; idx_2 < intervals.size() ; ++idx_2){
+ for(size_t idx_1 = 0 ; idx_1 < intervals.size() ; ++idx_1){
+ // self interval
for(partsize_t idx_p1 = 0 ; idx_p1 < intervals[idx_1].second ; ++idx_p1){
- for(partsize_t idx_p2 = 0 ; idx_p2 < intervals[idx_2].second ; ++idx_p2){
+ // if(((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 356))){// TODO
+ // printf("box %ld:\n", iter_cell.first);
+ // printf("intervals.size() %lu:\n", intervals.size());
+ // printf("intervals[idx_1].second %ld:\n", intervals[idx_1].second);
+ // printf("index %ld (%ld) pos %e %e %e\n",
+ // (intervals[idx_1].first+idx_p1), inout_index_particles[(intervals[idx_1].first+idx_p1)],
+ // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
+ // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
+ // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z]);
+ // }
+ // if(((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 547))){// TODO
+ // printf("box %ld:\n", iter_cell.first);
+ // printf("intervals.size() %lu:\n", intervals.size());
+ // printf("intervals[idx_1].second %ld:\n", intervals[idx_1].second);
+ // printf("index %ld (%ld) pos %e %e %e\n",
+ // (intervals[idx_1].first+idx_p1), inout_index_particles[(intervals[idx_1].first+idx_p1)],
+ // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
+ // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
+ // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z]);
+ // }
+
+
+ for(partsize_t idx_p2 = idx_p1+1 ; idx_p2 < intervals[idx_1].second ; ++idx_p2){
const real_number dist_r2 = compute_distance_r2(particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z],
- particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
- particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
- particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_Z],
+ particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_X],
+ particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_Y],
+ particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_Z],
0, 0, 0);
if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
in_computer.template compute_interaction<size_particle_positions,size_particle_data,size_particle_rhs>(
&particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions],
&particles_data[(intervals[idx_1].first+idx_p1)*size_particle_data],
&particles_current_rhs[(intervals[idx_1].first+idx_p1)*size_particle_rhs],
- &particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions],
- &particles_data[(intervals[idx_2].first+idx_p2)*size_particle_data],
- &particles_current_rhs[(intervals[idx_2].first+idx_p2)*size_particle_rhs],
+ &particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions],
+ &particles_data[(intervals[idx_1].first+idx_p2)*size_particle_data],
+ &particles_current_rhs[(intervals[idx_1].first+idx_p2)*size_particle_rhs],
dist_r2, 0, 0, 0);
}
-// if(((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 356)
-// || inout_index_particles[(intervals[idx_2].first+idx_p2)] == 356)/*
-// && ((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 547)
-// || inout_index_particles[(intervals[idx_2].first+idx_p2)] == 547)
-// && ((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 1832)
-// || inout_index_particles[(intervals[idx_2].first+idx_p2)] == 1832)*/){// TODO
-// printf("print between :\n");
-// printf("index %ld (%ld) pos %e %e %e\n",
-// (intervals[idx_1].first+idx_p1), inout_index_particles[(intervals[idx_1].first+idx_p1)],
-// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
-// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
-// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z]);
-// printf("index %ld (%ld) pos %e %e %e\n",
-// (intervals[idx_2].first+idx_p2),
-// inout_index_particles[(intervals[idx_2].first+idx_p2)],
-// particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
-// particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
-// particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
-// printf("Radius = %e (%e)\n", sqrt(dist_r2), dist_r2);
-// }
+ // if(((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 356)
+ // || inout_index_particles[(intervals[idx_1].first+idx_p2)] == 356)/*
+ // && ((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 1832)
+ // || inout_index_particles[(intervals[idx_1].first+idx_p2)] == 1832)
+ // && ((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 547)
+ // || inout_index_particles[(intervals[idx_1].first+idx_p2)] == 547)*/){// TODO
+ // printf("print between :\n");
+ // printf("index %ld (%ld) pos %e %e %e\n",
+ // (intervals[idx_1].first+idx_p1), inout_index_particles[(intervals[idx_1].first+idx_p1)],
+ // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
+ // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
+ // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z]);
+ // printf("index %ld (%ld) pos %e %e %e\n",
+ // (intervals[idx_1].first+idx_p2),
+ // inout_index_particles[(intervals[idx_1].first+idx_p2)],
+ // particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_X],
+ // particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_Y],
+ // particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_Z]);
+ // printf("Radius = %e (%e)\n", sqrt(dist_r2), dist_r2);
+ // }
+ }
+ }
+
+ // with other interval
+ for(size_t idx_2 = idx_1+1 ; idx_2 < intervals.size() ; ++idx_2){
+ for(partsize_t idx_p1 = 0 ; idx_p1 < intervals[idx_1].second ; ++idx_p1){
+ for(partsize_t idx_p2 = 0 ; idx_p2 < intervals[idx_2].second ; ++idx_p2){
+ const real_number dist_r2 = compute_distance_r2(particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z],
+ particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
+ particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
+ particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_Z],
+ 0, 0, 0);
+ if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
+ in_computer.template compute_interaction<size_particle_positions,size_particle_data,size_particle_rhs>(
+ &particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions],
+ &particles_data[(intervals[idx_1].first+idx_p1)*size_particle_data],
+ &particles_current_rhs[(intervals[idx_1].first+idx_p1)*size_particle_rhs],
+ &particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions],
+ &particles_data[(intervals[idx_2].first+idx_p2)*size_particle_data],
+ &particles_current_rhs[(intervals[idx_2].first+idx_p2)*size_particle_rhs],
+ dist_r2, 0, 0, 0);
+ }
+
+ // if(((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 356)
+ // || inout_index_particles[(intervals[idx_2].first+idx_p2)] == 356)/*
+ // && ((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 547)
+ // || inout_index_particles[(intervals[idx_2].first+idx_p2)] == 547)
+ // && ((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 1832)
+ // || inout_index_particles[(intervals[idx_2].first+idx_p2)] == 1832)*/){// TODO
+ // printf("print between :\n");
+ // printf("index %ld (%ld) pos %e %e %e\n",
+ // (intervals[idx_1].first+idx_p1), inout_index_particles[(intervals[idx_1].first+idx_p1)],
+ // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
+ // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
+ // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z]);
+ // printf("index %ld (%ld) pos %e %e %e\n",
+ // (intervals[idx_2].first+idx_p2),
+ // inout_index_particles[(intervals[idx_2].first+idx_p2)],
+ // particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
+ // particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
+ // particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
+ // printf("Radius = %e (%e)\n", sqrt(dist_r2), dist_r2);
+ // }
+ }
}
}
}
- }
+ const std::vector<std::pair<partsize_t,partsize_t>>* neighbors[27];
+ long int neighbors_indexes[27];
+ std::array<real_number,3> shift[27];
+ const int nbNeighbors = my_tree.getNeighbors(currenct_cell_idx, neighbors, neighbors_indexes, shift, false);
+
+ // if(((currenct_cell_idx == 785))){// TODO
+ // printf("box %ld:\n", iter_cell.first);
+ // printf("intervals.size() %lu:\n", intervals.size());
+ // printf("nbNeighbors %d\n",nbNeighbors);
+ // }
+
+ for(size_t idx_1 = 0 ; idx_1 < intervals.size() ; ++idx_1){
+ // with other interval
+ for(size_t idx_neighbor = 0 ; idx_neighbor < nbNeighbors ; ++idx_neighbor){
+ if(currenct_cell_idx < neighbors_indexes[idx_neighbor]){
+ cells_locker.lock(neighbors_indexes[idx_neighbor]);
+
+ for(size_t idx_2 = 0 ; idx_2 < (*neighbors[idx_neighbor]).size() ; ++idx_2){
+ for(partsize_t idx_p1 = 0 ; idx_p1 < intervals[idx_1].second ; ++idx_p1){
+ for(partsize_t idx_p2 = 0 ; idx_p2 < (*neighbors[idx_neighbor])[idx_2].second ; ++idx_p2){
+ const real_number dist_r2 = compute_distance_r2(particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z],
+ shift[idx_neighbor][IDX_X], shift[idx_neighbor][IDX_Y], shift[idx_neighbor][IDX_Z]);
+ if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
+ in_computer.template compute_interaction<size_particle_positions,size_particle_data,size_particle_rhs>(
+ &particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions],
+ &particles_data[(intervals[idx_1].first+idx_p1)*size_particle_data],
+ &particles_current_rhs[(intervals[idx_1].first+idx_p1)*size_particle_rhs],
+ &particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions],
+ &particles_data[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_data],
+ &particles_current_rhs[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_rhs],
+ dist_r2, shift[idx_neighbor][IDX_X], shift[idx_neighbor][IDX_Y], shift[idx_neighbor][IDX_Z]);
+ }
- const long int currenct_cell_idx = iter_cell.first;
- const std::vector<std::pair<partsize_t,partsize_t>>* neighbors[27];
- long int neighbors_indexes[27];
- std::array<real_number,3> shift[27];
- const int nbNeighbors = my_tree.getNeighbors(currenct_cell_idx, neighbors, neighbors_indexes, shift, false);
-
-// if(((currenct_cell_idx == 785))){// TODO
-// printf("box %ld:\n", iter_cell.first);
-// printf("intervals.size() %lu:\n", intervals.size());
-// printf("nbNeighbors %d\n",nbNeighbors);
-// }
-
- for(size_t idx_1 = 0 ; idx_1 < intervals.size() ; ++idx_1){
- // with other interval
- for(size_t idx_neighbor = 0 ; idx_neighbor < nbNeighbors ; ++idx_neighbor){
- if(currenct_cell_idx < neighbors_indexes[idx_neighbor]){
- for(size_t idx_2 = 0 ; idx_2 < (*neighbors[idx_neighbor]).size() ; ++idx_2){
- for(partsize_t idx_p1 = 0 ; idx_p1 < intervals[idx_1].second ; ++idx_p1){
- for(partsize_t idx_p2 = 0 ; idx_p2 < (*neighbors[idx_neighbor])[idx_2].second ; ++idx_p2){
- const real_number dist_r2 = compute_distance_r2(particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
- particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
- particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z],
- particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
- particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
- particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z],
- shift[idx_neighbor][IDX_X], shift[idx_neighbor][IDX_Y], shift[idx_neighbor][IDX_Z]);
- if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
- in_computer.template compute_interaction<size_particle_positions,size_particle_data,size_particle_rhs>(
- &particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions],
- &particles_data[(intervals[idx_1].first+idx_p1)*size_particle_data],
- &particles_current_rhs[(intervals[idx_1].first+idx_p1)*size_particle_rhs],
- &particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions],
- &particles_data[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_data],
- &particles_current_rhs[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_rhs],
- dist_r2, shift[idx_neighbor][IDX_X], shift[idx_neighbor][IDX_Y], shift[idx_neighbor][IDX_Z]);
+ // if(((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 356)
+ // || inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)] == 356)/*
+ // && (inout_index_particles[(intervals[idx_1].first+idx_p1)] == 547)
+ // || inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)] == 547
+ // && (inout_index_particles[(intervals[idx_1].first+idx_p1)] == 1832)
+ // || inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)] == 1832*/){// TODO
+ // printf("print between :\n");
+ // printf("index %ld (%ld) pos %e %e %e\n",
+ // (intervals[idx_1].first+idx_p1), inout_index_particles[(intervals[idx_1].first+idx_p1)],
+ // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
+ // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
+ // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z]);
+ // printf("index %ld (%ld) pos %e %e %e\n",
+ // ((*neighbors[idx_neighbor])[idx_2].first+idx_p2),
+ // inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)],
+ // particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
+ // particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
+ // particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
+ // printf("Radius = %e (%e)\n", sqrt(dist_r2), dist_r2);
+ // }
}
-
-// if(((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 356)
-// || inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)] == 356)/*
-// && (inout_index_particles[(intervals[idx_1].first+idx_p1)] == 547)
-// || inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)] == 547
-// && (inout_index_particles[(intervals[idx_1].first+idx_p1)] == 1832)
-// || inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)] == 1832*/){// TODO
-// printf("print between :\n");
-// printf("index %ld (%ld) pos %e %e %e\n",
-// (intervals[idx_1].first+idx_p1), inout_index_particles[(intervals[idx_1].first+idx_p1)],
-// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
-// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
-// particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z]);
-// printf("index %ld (%ld) pos %e %e %e\n",
-// ((*neighbors[idx_neighbor])[idx_2].first+idx_p2),
-// inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)],
-// particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
-// particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
-// particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
-// printf("Radius = %e (%e)\n", sqrt(dist_r2), dist_r2);
-// }
}
}
+ cells_locker.unlock(neighbors_indexes[idx_neighbor]);
}
}
}
+
+ cells_locker.unlock(currenct_cell_idx);
}
}
}
diff --git a/bfps/cpp/particles/p2p_tree.hpp b/bfps/cpp/particles/p2p_tree.hpp
index 52cb2f2d..3d92c4e5 100644
--- a/bfps/cpp/particles/p2p_tree.hpp
+++ b/bfps/cpp/particles/p2p_tree.hpp
@@ -15,7 +15,7 @@ class p2p_tree{
}
long int get_cell_coord_y_from_index(const long int index) const{
- return (index - get_cell_coord_z_from_index(index)*(nb_cell_levels[IDX_X]*nb_cell_levels[IDX_Y]))
+ return (index % (nb_cell_levels[IDX_X]*nb_cell_levels[IDX_Y]))
/ nb_cell_levels[IDX_X];
}
--
GitLab
From 0de739dd2b978f18aeea01f1d9cd0b6853d35a2b Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Fri, 22 Sep 2017 13:03:10 +0200
Subject: [PATCH 021/342] Remove useless statements
---
bfps/cpp/particles/lock_free_bool_array.hpp | 2 --
1 file changed, 2 deletions(-)
diff --git a/bfps/cpp/particles/lock_free_bool_array.hpp b/bfps/cpp/particles/lock_free_bool_array.hpp
index 1ae9968e..d0b2aa2b 100644
--- a/bfps/cpp/particles/lock_free_bool_array.hpp
+++ b/bfps/cpp/particles/lock_free_bool_array.hpp
@@ -18,10 +18,8 @@ public:
void lock(const int inKey){
volatile long int* k = keys[inKey%keys.size()].get();
long int res = 1;
- int cpt = 0;
while(res == 1){
res = __sync_val_compare_and_swap(k, 0, res);
- cpt++;
}
}
--
GitLab
From b8eaa38d71bef40ba10e698b22c11cb00a42ce35 Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Fri, 22 Sep 2017 14:01:59 +0200
Subject: [PATCH 022/342] Remove tests and comments
---
bfps/cpp/particles/p2p_computer.hpp | 4 +-
bfps/cpp/particles/p2p_distr_mpi.hpp | 214 ---------------------------
2 files changed, 2 insertions(+), 216 deletions(-)
diff --git a/bfps/cpp/particles/p2p_computer.hpp b/bfps/cpp/particles/p2p_computer.hpp
index cc2ed2dc..cb384950 100644
--- a/bfps/cpp/particles/p2p_computer.hpp
+++ b/bfps/cpp/particles/p2p_computer.hpp
@@ -25,8 +25,8 @@ public:
const real_number pos_part2[], const real_number data_part2[], real_number rhs_part2[],
const real_number dist_pow2,
const real_number xshift_coef, const real_number yshift_coef, const real_number zshift_coef) const{
- rhs_part1[0] += 1;
- rhs_part2[0] += 1;
+ // TODO put the kernel here
+ static_assert(size_particle_positions == 3, "This kernel works only with 3 values for one position");
}
};
diff --git a/bfps/cpp/particles/p2p_distr_mpi.hpp b/bfps/cpp/particles/p2p_distr_mpi.hpp
index 40be1a28..12b349dc 100644
--- a/bfps/cpp/particles/p2p_distr_mpi.hpp
+++ b/bfps/cpp/particles/p2p_distr_mpi.hpp
@@ -284,17 +284,6 @@ public:
particles_positions[(idxPart)*size_particle_positions + IDX_Y],
particles_positions[(idxPart)*size_particle_positions + IDX_Z]);
assert(my_down_z_cell_level <= get_cell_coord_z_from_index(particles_coord[idxPart]));
- if(!(get_cell_coord_z_from_index(particles_coord[idxPart]) <= my_top_z_cell_level)){// TODO
- printf("Coord index %ld - %ld (tree index %ld)\n", idxPart, inout_index_particles[idxPart],particles_coord[idxPart]);
- printf(">> Box index %ld - %ld - %ld\n", get_cell_coord_x_from_index(particles_coord[idxPart]),
- get_cell_coord_y_from_index(particles_coord[idxPart]),
- get_cell_coord_z_from_index(particles_coord[idxPart]));
- printf(">> idxPartition %d\n", idxPartition);
- printf(">> my_top_z_cell_level %ld\n", my_top_z_cell_level);
- printf(">> position %e %e %e\n", particles_positions[(idxPart)*size_particle_positions + IDX_X],
- particles_positions[(idxPart)*size_particle_positions + IDX_Y],
- particles_positions[(idxPart)*size_particle_positions + IDX_Z]);
- }
assert(get_cell_coord_z_from_index(particles_coord[idxPart]) <= my_top_z_cell_level);
}
}
@@ -319,10 +308,6 @@ public:
return p1.first < p2.first;
});
-// for(partsize_t idxPart = 1 ; idxPart < (long int)part_to_sort.size() ; ++idxPart){// TODO
-// assert(part_to_sort[idxPart-1].first <= part_to_sort[idxPart].first);
-// }
-
// Permute array using buffer
std::vector<unsigned char> buffer;
permute_copy<real_number, size_particle_positions>(current_offset_particles_for_partition[idxPartition],
@@ -360,28 +345,6 @@ public:
current_cell_idx = particles_coord[idx_part];
current_nb_particles_in_cell = 1;
current_cell_offset = idx_part;
-// if(inout_index_particles[idx_part] == 547){// TODO
-// printf("idxPartition %d\n", idxPartition);
-// printf(">> Coord index %ld - %ld (tree index %ld)\n", idx_part, inout_index_particles[idx_part],particles_coord[idx_part]);
-// printf(">> Box index %ld - %ld - %ld\n", get_cell_coord_x_from_index(particles_coord[idx_part]),
-// get_cell_coord_y_from_index(particles_coord[idx_part]),
-// get_cell_coord_z_from_index(particles_coord[idx_part]));
-// printf(">> current_cell_offset %ld current_nb_particles_in_cell %ld\n", current_cell_offset, current_nb_particles_in_cell);
-// printf(">> Position %e %e %e\n", particles_positions[idx_part*size_particle_positions + IDX_X],
-// particles_positions[idx_part*size_particle_positions + IDX_Y],
-// particles_positions[idx_part*size_particle_positions + IDX_Z]);
-// }
-// if(inout_index_particles[idx_part] == 356){// TODO
-// printf("idxPartition %d\n", idxPartition);
-// printf(">> Coord index %ld - %ld (tree index %ld)\n", idx_part, inout_index_particles[idx_part],particles_coord[idx_part]);
-// printf(">> Box index %ld - %ld - %ld\n", get_cell_coord_x_from_index(particles_coord[idx_part]),
-// get_cell_coord_y_from_index(particles_coord[idx_part]),
-// get_cell_coord_z_from_index(particles_coord[idx_part]));
-// printf(">> current_cell_offset %ld current_nb_particles_in_cell %ld\n", current_cell_offset, current_nb_particles_in_cell);
-// printf(">> Position %e %e %e\n", particles_positions[idx_part*size_particle_positions + IDX_X],
-// particles_positions[idx_part*size_particle_positions + IDX_Y],
-// particles_positions[idx_part*size_particle_positions + IDX_Z]);
-// }
}
else{
current_nb_particles_in_cell += 1;
@@ -393,10 +356,6 @@ public:
}
}
-// printf("[%d] go from cutoff level %ld to %ld\n",
-// my_rank, my_down_z_cell_level, my_top_z_cell_level); // TODO remove
-// fflush(stdout); // TODO
-
// Offset per cell layers
long int previous_index = 0;
std::unique_ptr<partsize_t[]> particles_offset_layers(new partsize_t[my_nb_cell_levels+1]());
@@ -412,9 +371,6 @@ public:
}
}
for(long int idx_layer = 0 ; idx_layer < my_nb_cell_levels ; ++idx_layer){
-// printf("[%d] nb particles in cutoff level %ld are %ld\n",
-// my_rank, idx_layer, particles_offset_layers[idx_layer+1]); // TODO remove
-// fflush(stdout); // TODO
particles_offset_layers[idx_layer+1] += particles_offset_layers[idx_layer];
}
@@ -425,10 +381,6 @@ public:
// Find process with at least one neighbor
{
- std::cout << my_rank << ">> my_top_z_cell_level " << my_top_z_cell_level << std::endl;
- std::cout << my_rank << ">> my_down_z_cell_level " << my_down_z_cell_level << std::endl;
- std::cout.flush();// TODO
-
int dest_proc = (my_rank+1)%nb_processes_involved;
while(dest_proc != my_rank
&& (my_top_z_cell_level == first_cell_level_proc(dest_proc)
@@ -440,11 +392,6 @@ public:
nb_levels_to_send += 1;
}
- std::cout << my_rank << " dest_proc " << dest_proc << std::endl;
- std::cout << my_rank << ">> first_cell_level_proc(dest_proc) " << first_cell_level_proc(dest_proc) << std::endl;
- std::cout << my_rank << ">> last_cell_level_proc(dest_proc) " << last_cell_level_proc(dest_proc) << std::endl;
- std::cout.flush();// TODO
-
NeighborDescriptor descriptor;
descriptor.destProc = dest_proc;
descriptor.nbLevelsToExchange = nb_levels_to_send;
@@ -452,21 +399,10 @@ public:
descriptor.isRecv = false;
descriptor.positionsReceived = false;
- std::cout << my_rank << " SEND" << std::endl;
- std::cout << ">> descriptor.destProc " << descriptor.destProc << std::endl;
- std::cout << ">> descriptor.nbLevelsToExchange " << descriptor.nbLevelsToExchange << std::endl;
- std::cout << ">> descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
- std::cout << ">> descriptor.isRecv " << descriptor.isRecv << std::endl;
- std::cout << ">> neigDescriptors.size() " << neigDescriptors.size() << std::endl;
- std::cout.flush();// TODO
-
neigDescriptors.emplace_back(std::move(descriptor));
dest_proc = (dest_proc+1)%nb_processes_involved;
}
- std::cout << my_rank << " NO dest_proc " << dest_proc << std::endl;
- std::cout << my_rank << " NO first_cell_level_proc(dest_proc) " << first_cell_level_proc(dest_proc) << std::endl;
- std::cout.flush();// TODO
int src_proc = (my_rank-1+nb_processes_involved)%nb_processes_involved;
while(src_proc != my_rank
@@ -479,10 +415,6 @@ public:
nb_levels_to_recv += 1;
}
- std::cout << my_rank << " src_proc " << src_proc << std::endl;
- std::cout << my_rank << " first_cell_level_proc(src_proc) " << first_cell_level_proc(src_proc) << std::endl;
- std::cout.flush();// TODO
-
NeighborDescriptor descriptor;
descriptor.destProc = src_proc;
descriptor.nbLevelsToExchange = nb_levels_to_recv;
@@ -492,20 +424,8 @@ public:
neigDescriptors.emplace_back(std::move(descriptor));
- std::cout << my_rank << "] RECV" << std::endl;
- std::cout << ">> descriptor.destProc " << descriptor.destProc << std::endl;
- std::cout << ">> descriptor.nbLevelsToExchange " << descriptor.nbLevelsToExchange << std::endl;
- std::cout << ">> descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
- std::cout << ">> descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
- std::cout << ">> descriptor.isRecv " << descriptor.isRecv << std::endl;
- std::cout << ">> neigDescriptors.size() " << neigDescriptors.size() << std::endl;
- std::cout.flush();// TODO
-
src_proc = (src_proc-1+nb_processes_involved)%nb_processes_involved;
}
- std::cout << my_rank << " NO src_proc " << src_proc << std::endl;
- std::cout << my_rank << " NO first_cell_level_proc(src_proc) " << first_cell_level_proc(src_proc) << std::endl;
- std::cout.flush();// TODO
}
//////////////////////////////////////////////////////////////////////
@@ -529,12 +449,6 @@ public:
current_com, &mpiRequests.back()));
if(descriptor.nbParticlesToExchange){
-// std::cout << my_rank << "] SEND_PARTICLES" << std::endl;
-// std::cout << "descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
-// std::cout << "descriptor.destProc " << descriptor.destProc << std::endl;
-// std::cout << "idxDescr " << idxDescr << std::endl;
-// std::cout << "send from part " << particles_offset_layers[my_nb_cell_levels-descriptor.nbLevelsToExchange] << std::endl;
-
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
assert(descriptor.nbParticlesToExchange*size_particle_positions < std::numeric_limits<int>::max());
@@ -562,8 +476,6 @@ public:
}
}
else{
-// std::cout << "RECV_PARTICLES " << RECV_PARTICLES << std::endl;
-// std::cout << "idxDescr " << idxDescr << std::endl;
whatNext.emplace_back(std::pair<Action,int>{RECV_PARTICLES, idxDescr});
mpiRequests.emplace_back();
AssertMpi(MPI_Irecv(&descriptor.nbParticlesToExchange,
@@ -606,13 +518,7 @@ public:
assert(NbParticlesToReceive != -1);
assert(descriptor.toCompute == nullptr);
-// std::cout << my_rank << "] RECV_PARTICLES" << std::endl;
-// std::cout << "descriptor.nbParticlesToExchange " << descriptor.nbParticlesToExchange << std::endl;
-// std::cout << "descriptor.destProc " << descriptor.destProc << std::endl;
-// std::cout << "releasedAction.second " << releasedAction.second << std::endl;
-
if(NbParticlesToReceive){
-// std::cout << "MPI_Irecv " << std::endl;
descriptor.toCompute.reset(new real_number[NbParticlesToReceive*size_particle_positions]);
whatNext.emplace_back(std::pair<Action,int>{CHECK_PARTICLES, releasedAction.second});
mpiRequests.emplace_back();
@@ -668,19 +574,6 @@ public:
std::array<real_number,3> shift[27];
const int nbNeighbors = my_tree.getNeighbors(current_cell_idx, neighbors, neighbors_indexes, shift, true);
-// for(int idx_test = 0 ; idx_test < nb_parts_in_cell ; ++idx_test){ // TODO
-// real_number totest[3] = {8.570442e-01, 7.173084e-02, 8.279754e-03};
-// if(int(descriptor.toCompute[(idxPart+idx_test)*size_particle_positions + IDX_X]*1000) == int(totest[0]*1000)
-// && int(descriptor.toCompute[(idxPart+idx_test)*size_particle_positions + IDX_Y]*1000) == int(totest[1]*1000)
-// && int(descriptor.toCompute[(idxPart+idx_test)*size_particle_positions + IDX_Z]*1000) == int(totest[2]*1000)){
-// printf("Found a pos %ld\n", idxPart+idx_test);
-// printf("pos %e %e %e\n",
-// descriptor.toCompute[(idxPart+idx_test)*size_particle_positions + IDX_X],
-// descriptor.toCompute[(idxPart+idx_test)*size_particle_positions + IDX_Y],
-// descriptor.toCompute[(idxPart+idx_test)*size_particle_positions + IDX_Z]);
-// }
-// }
-
// with other interval
for(size_t idx_neighbor = 0 ; idx_neighbor < nbNeighbors ; ++idx_neighbor){
cells_locker.lock(neighbors_indexes[idx_neighbor]);
@@ -705,22 +598,6 @@ public:
&particles_current_rhs[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_rhs],
dist_r2, shift[idx_neighbor][IDX_X], shift[idx_neighbor][IDX_Y], shift[idx_neighbor][IDX_Z]);
}
-
- // if(inout_index_particles[(*neighbors[idx_neighbor])[idx_2].first+idx_p2] == 356){// TODO
- // printf("test interaction between :\n");
- // printf("index %ld (%ld) pos %e %e %e\n",
- // (idxPart+idx_p1), -1L,
- // descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_X],
- // descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_Y],
- // descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_Z]);
- // printf("index %ld (%ld) pos %e %e %e\n",
- // ((*neighbors[idx_neighbor])[idx_2].first+idx_p2),
- // inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)],
- // particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
- // particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
- // particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
- // printf("Radius = %e (%e)\n", sqrt(dist_r2), dist_r2);
- // }
}
}
}
@@ -795,28 +672,6 @@ public:
for(size_t idx_1 = 0 ; idx_1 < intervals.size() ; ++idx_1){
// self interval
for(partsize_t idx_p1 = 0 ; idx_p1 < intervals[idx_1].second ; ++idx_p1){
- // if(((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 356))){// TODO
- // printf("box %ld:\n", iter_cell.first);
- // printf("intervals.size() %lu:\n", intervals.size());
- // printf("intervals[idx_1].second %ld:\n", intervals[idx_1].second);
- // printf("index %ld (%ld) pos %e %e %e\n",
- // (intervals[idx_1].first+idx_p1), inout_index_particles[(intervals[idx_1].first+idx_p1)],
- // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
- // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
- // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z]);
- // }
- // if(((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 547))){// TODO
- // printf("box %ld:\n", iter_cell.first);
- // printf("intervals.size() %lu:\n", intervals.size());
- // printf("intervals[idx_1].second %ld:\n", intervals[idx_1].second);
- // printf("index %ld (%ld) pos %e %e %e\n",
- // (intervals[idx_1].first+idx_p1), inout_index_particles[(intervals[idx_1].first+idx_p1)],
- // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
- // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
- // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z]);
- // }
-
-
for(partsize_t idx_p2 = idx_p1+1 ; idx_p2 < intervals[idx_1].second ; ++idx_p2){
const real_number dist_r2 = compute_distance_r2(particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
@@ -835,27 +690,6 @@ public:
&particles_current_rhs[(intervals[idx_1].first+idx_p2)*size_particle_rhs],
dist_r2, 0, 0, 0);
}
-
- // if(((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 356)
- // || inout_index_particles[(intervals[idx_1].first+idx_p2)] == 356)/*
- // && ((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 1832)
- // || inout_index_particles[(intervals[idx_1].first+idx_p2)] == 1832)
- // && ((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 547)
- // || inout_index_particles[(intervals[idx_1].first+idx_p2)] == 547)*/){// TODO
- // printf("print between :\n");
- // printf("index %ld (%ld) pos %e %e %e\n",
- // (intervals[idx_1].first+idx_p1), inout_index_particles[(intervals[idx_1].first+idx_p1)],
- // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
- // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
- // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z]);
- // printf("index %ld (%ld) pos %e %e %e\n",
- // (intervals[idx_1].first+idx_p2),
- // inout_index_particles[(intervals[idx_1].first+idx_p2)],
- // particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_X],
- // particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_Y],
- // particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_Z]);
- // printf("Radius = %e (%e)\n", sqrt(dist_r2), dist_r2);
- // }
}
}
@@ -880,27 +714,6 @@ public:
&particles_current_rhs[(intervals[idx_2].first+idx_p2)*size_particle_rhs],
dist_r2, 0, 0, 0);
}
-
- // if(((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 356)
- // || inout_index_particles[(intervals[idx_2].first+idx_p2)] == 356)/*
- // && ((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 547)
- // || inout_index_particles[(intervals[idx_2].first+idx_p2)] == 547)
- // && ((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 1832)
- // || inout_index_particles[(intervals[idx_2].first+idx_p2)] == 1832)*/){// TODO
- // printf("print between :\n");
- // printf("index %ld (%ld) pos %e %e %e\n",
- // (intervals[idx_1].first+idx_p1), inout_index_particles[(intervals[idx_1].first+idx_p1)],
- // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
- // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
- // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z]);
- // printf("index %ld (%ld) pos %e %e %e\n",
- // (intervals[idx_2].first+idx_p2),
- // inout_index_particles[(intervals[idx_2].first+idx_p2)],
- // particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
- // particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
- // particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
- // printf("Radius = %e (%e)\n", sqrt(dist_r2), dist_r2);
- // }
}
}
}
@@ -911,12 +724,6 @@ public:
std::array<real_number,3> shift[27];
const int nbNeighbors = my_tree.getNeighbors(currenct_cell_idx, neighbors, neighbors_indexes, shift, false);
- // if(((currenct_cell_idx == 785))){// TODO
- // printf("box %ld:\n", iter_cell.first);
- // printf("intervals.size() %lu:\n", intervals.size());
- // printf("nbNeighbors %d\n",nbNeighbors);
- // }
-
for(size_t idx_1 = 0 ; idx_1 < intervals.size() ; ++idx_1){
// with other interval
for(size_t idx_neighbor = 0 ; idx_neighbor < nbNeighbors ; ++idx_neighbor){
@@ -943,27 +750,6 @@ public:
&particles_current_rhs[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_rhs],
dist_r2, shift[idx_neighbor][IDX_X], shift[idx_neighbor][IDX_Y], shift[idx_neighbor][IDX_Z]);
}
-
- // if(((inout_index_particles[(intervals[idx_1].first+idx_p1)] == 356)
- // || inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)] == 356)/*
- // && (inout_index_particles[(intervals[idx_1].first+idx_p1)] == 547)
- // || inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)] == 547
- // && (inout_index_particles[(intervals[idx_1].first+idx_p1)] == 1832)
- // || inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)] == 1832*/){// TODO
- // printf("print between :\n");
- // printf("index %ld (%ld) pos %e %e %e\n",
- // (intervals[idx_1].first+idx_p1), inout_index_particles[(intervals[idx_1].first+idx_p1)],
- // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
- // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
- // particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z]);
- // printf("index %ld (%ld) pos %e %e %e\n",
- // ((*neighbors[idx_neighbor])[idx_2].first+idx_p2),
- // inout_index_particles[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)],
- // particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
- // particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
- // particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z]);
- // printf("Radius = %e (%e)\n", sqrt(dist_r2), dist_r2);
- // }
}
}
}
--
GitLab
From 99d5fe727a8825da6f17693196e2e4f3e9866d77 Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Fri, 22 Sep 2017 15:53:10 +0200
Subject: [PATCH 023/342] Update BFPS interface to include the new P2P classes
---
bfps/cpp/full_code/NSVEparticlesP2P.cpp | 113 ++++++++++++++++++
bfps/cpp/full_code/NSVEparticlesP2P.hpp | 82 +++++++++++++
.../particles/abstract_particles_system.hpp | 2 +
bfps/cpp/particles/lock_free_bool_array.hpp | 4 +-
bfps/cpp/particles/p2p_computer.hpp | 6 +
bfps/cpp/particles/p2p_computer_empty.hpp | 31 +++++
bfps/cpp/particles/p2p_distr_mpi.hpp | 6 +-
bfps/cpp/particles/particles_distr_mpi.hpp | 112 ++++++++++++++---
bfps/cpp/particles/particles_system.hpp | 37 +++++-
.../particles/particles_system_builder.hpp | 30 ++++-
setup.py | 3 +-
11 files changed, 393 insertions(+), 33 deletions(-)
create mode 100644 bfps/cpp/full_code/NSVEparticlesP2P.cpp
create mode 100644 bfps/cpp/full_code/NSVEparticlesP2P.hpp
create mode 100644 bfps/cpp/particles/p2p_computer_empty.hpp
diff --git a/bfps/cpp/full_code/NSVEparticlesP2P.cpp b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
new file mode 100644
index 00000000..08326119
--- /dev/null
+++ b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
@@ -0,0 +1,113 @@
+#include <string>
+#include <cmath>
+#include "NSVEparticlesP2P.hpp"
+#include "scope_timer.hpp"
+#include "particles/particles_sampling.hpp"
+#include "particles/p2p_computer.hpp"
+
+template <typename rnumber>
+int NSVEparticlesP2P<rnumber>::initialize(void)
+{
+ this->NSVE<rnumber>::initialize();
+
+ this->ps = particles_system_builder(
+ this->fs->cvelocity, // (field object)
+ this->fs->kk, // (kspace object, contains dkx, dky, dkz)
+ tracers0_integration_steps, // to check coherency between parameters and hdf input file (nb rhs)
+ (long long int)nparticles, // to check coherency between parameters and hdf input file
+ this->fs->get_current_fname(), // particles input filename
+ std::string("/tracers0/state/") + std::to_string(this->fs->iteration), // dataset name for initial input
+ std::string("/tracers0/rhs/") + std::to_string(this->fs->iteration), // dataset name for initial input
+ tracers0_neighbours, // parameter (interpolation no neighbours)
+ tracers0_smoothness, // parameter
+ this->comm,
+ this->fs->iteration+1);
+ // TODO P2P write particle data too
+ this->particles_output_writer_mpi = new particles_output_hdf5<
+ long long int, double, 3, 3>(
+ MPI_COMM_WORLD,
+ "tracers0",
+ nparticles,
+ tracers0_integration_steps);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVEparticlesP2P<rnumber>::step(void)
+{
+ this->fs->compute_velocity(this->fs->cvorticity);
+ this->fs->cvelocity->ift();
+ this->ps->completeLoop(this->dt);
+ this->NSVE<rnumber>::step();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVEparticlesP2P<rnumber>::write_checkpoint(void)
+{
+ this->NSVE<rnumber>::write_checkpoint();
+ this->particles_output_writer_mpi->open_file(this->fs->get_current_fname());
+ // TODO P2P write particle data too
+ this->particles_output_writer_mpi->save(
+ this->ps->getParticlesPositions(),
+ this->ps->getParticlesRhs(),
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->fs->iteration);
+ this->particles_output_writer_mpi->close_file();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVEparticlesP2P<rnumber>::finalize(void)
+{
+ this->ps.release();
+ delete this->particles_output_writer_mpi;
+ this->NSVE<rnumber>::finalize();
+ return EXIT_SUCCESS;
+}
+
+/** \brief Compute fluid stats and sample fields at particle locations.
+ */
+
+template <typename rnumber>
+int NSVEparticlesP2P<rnumber>::do_stats()
+{
+ /// fluid stats go here
+ this->NSVE<rnumber>::do_stats();
+
+
+ if (!(this->iteration % this->niter_part == 0))
+ return EXIT_SUCCESS;
+
+ /// sample position
+ sample_particles_system_position(
+ this->ps,
+ (this->simname + "_particles.h5"), // filename
+ "tracers0", // hdf5 parent group
+ "position" // dataset basename TODO
+ );
+
+ /// sample velocity
+ sample_from_particles_system(*this->tmp_vec_field, // field to save
+ this->ps,
+ (this->simname + "_particles.h5"), // filename
+ "tracers0", // hdf5 parent group
+ "velocity" // dataset basename TODO
+ );
+
+ /// compute acceleration and sample it
+ this->fs->compute_Lagrangian_acceleration(this->tmp_vec_field);
+ this->tmp_vec_field->ift();
+ sample_from_particles_system(*this->tmp_vec_field,
+ this->ps,
+ (this->simname + "_particles.h5"),
+ "tracers0",
+ "acceleration");
+
+ return EXIT_SUCCESS;
+}
+
+template class NSVEparticlesP2P<float>;
+template class NSVEparticlesP2P<double>;
+
diff --git a/bfps/cpp/full_code/NSVEparticlesP2P.hpp b/bfps/cpp/full_code/NSVEparticlesP2P.hpp
new file mode 100644
index 00000000..9b015659
--- /dev/null
+++ b/bfps/cpp/full_code/NSVEparticlesP2P.hpp
@@ -0,0 +1,82 @@
+/**********************************************************************
+* *
+* Copyright 2017 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef NSVEPARTICLESP2P_HPP
+#define NSVEPARTICLESP2P_HPP
+
+
+
+#include <cstdlib>
+#include "base.hpp"
+#include "vorticity_equation.hpp"
+#include "full_code/NSVE.hpp"
+#include "particles/particles_system_builder.hpp"
+#include "particles/particles_output_hdf5.hpp"
+
+/** \brief Navier-Stokes solver that includes simple Lagrangian tracers.
+ *
+ * Child of Navier Stokes vorticity equation solver, this class calls all the
+ * methods from `NSVE`, and in addition integrates simple Lagrangian tracers
+ * in the resulting velocity field.
+ */
+
+template <typename rnumber>
+class NSVEparticlesP2P: public NSVE<rnumber>
+{
+ public:
+
+ /* parameters that are read in read_parameters */
+ int niter_part;
+ int nparticles;
+ int tracers0_integration_steps;
+ int tracers0_neighbours;
+ int tracers0_smoothness;
+
+ /* other stuff */
+ std::unique_ptr<abstract_particles_system<long long int, double>> ps;
+ // TODO P2P use a reader with particle data
+ particles_output_hdf5<long long int, double,3,3> *particles_output_writer_mpi;
+
+
+ NSVEparticlesP2P(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ NSVE<rnumber>(
+ COMMUNICATOR,
+ simulation_name){}
+ ~NSVEparticlesP2P(){}
+
+ int initialize(void);
+ int step(void);
+ int finalize(void);
+
+ int read_parameters(void);
+ int write_checkpoint(void);
+ int do_stats(void);
+};
+
+#endif//NSVEPARTICLESP2P_HPP
+
diff --git a/bfps/cpp/particles/abstract_particles_system.hpp b/bfps/cpp/particles/abstract_particles_system.hpp
index 1c8592f3..26ce7198 100644
--- a/bfps/cpp/particles/abstract_particles_system.hpp
+++ b/bfps/cpp/particles/abstract_particles_system.hpp
@@ -14,6 +14,8 @@ class abstract_particles_system {
public:
virtual void compute() = 0;
+ virtual void compute_p2p() = 0;
+
virtual void move(const real_number dt) = 0;
virtual void redistribute() = 0;
diff --git a/bfps/cpp/particles/lock_free_bool_array.hpp b/bfps/cpp/particles/lock_free_bool_array.hpp
index d0b2aa2b..928c1753 100644
--- a/bfps/cpp/particles/lock_free_bool_array.hpp
+++ b/bfps/cpp/particles/lock_free_bool_array.hpp
@@ -15,7 +15,7 @@ public:
}
}
- void lock(const int inKey){
+ void lock(const long int inKey){
volatile long int* k = keys[inKey%keys.size()].get();
long int res = 1;
while(res == 1){
@@ -23,7 +23,7 @@ public:
}
}
- void unlock(const int inKey){
+ void unlock(const long int inKey){
volatile long int* k = keys[inKey%keys.size()].get();
assert(k && *k);
(*k) = 0;
diff --git a/bfps/cpp/particles/p2p_computer.hpp b/bfps/cpp/particles/p2p_computer.hpp
index cb384950..de024ff7 100644
--- a/bfps/cpp/particles/p2p_computer.hpp
+++ b/bfps/cpp/particles/p2p_computer.hpp
@@ -6,6 +6,8 @@
template <class real_number, class partsize_t>
class p2p_computer{
public:
+ constexpr static int size_data = 3;
+
template <int size_particle_rhs>
void init_result_array(real_number rhs[], const partsize_t nbParticles) const{
memset(rhs, 0, sizeof(real_number)*nbParticles*size_particle_rhs);
@@ -28,6 +30,10 @@ public:
// TODO put the kernel here
static_assert(size_particle_positions == 3, "This kernel works only with 3 values for one position");
}
+
+ constexpr static bool isEmpty() {
+ return false;
+ }
};
#endif
diff --git a/bfps/cpp/particles/p2p_computer_empty.hpp b/bfps/cpp/particles/p2p_computer_empty.hpp
new file mode 100644
index 00000000..7076061e
--- /dev/null
+++ b/bfps/cpp/particles/p2p_computer_empty.hpp
@@ -0,0 +1,31 @@
+#ifndef P2P_COMPUTER_EMPTY_HPP
+#define P2P_COMPUTER_EMPTY_HPP
+
+#include <cstring>
+
+template <class real_number, class partsize_t>
+class p2p_computer_empty{
+public:
+ constexpr int static size_data = 0;
+
+ template <int size_particle_rhs>
+ void init_result_array(real_number /*rhs*/[], const partsize_t /*nbParticles*/) const{
+ }
+
+ template <int size_particle_rhs>
+ void reduce_particles_rhs(real_number /*rhs_dst*/[], const real_number /*rhs_src*/[], const partsize_t /*nbParticles*/) const{
+ }
+
+ template <int size_particle_positions, int size_particle_data, int size_particle_rhs>
+ void compute_interaction(const real_number /*pos_part1*/[], const real_number /*data_part1*/[], real_number /*rhs_part1*/[],
+ const real_number /*pos_part2*/[], const real_number /*data_part2*/[], real_number /*rhs_part2*/[],
+ const real_number /*dist_pow2*/,
+ const real_number /*xshift_coef*/, const real_number /*yshift_coef*/, const real_number /*zshift_coef*/) const{
+ }
+
+ constexpr static bool isEmpty() {
+ return true;
+ }
+};
+
+#endif
diff --git a/bfps/cpp/particles/p2p_distr_mpi.hpp b/bfps/cpp/particles/p2p_distr_mpi.hpp
index 12b349dc..ccc236d6 100644
--- a/bfps/cpp/particles/p2p_distr_mpi.hpp
+++ b/bfps/cpp/particles/p2p_distr_mpi.hpp
@@ -300,7 +300,7 @@ public:
part_to_sort.back().second = idxPart;
}
- assert(part_to_sort.size() == (current_my_nb_particles_per_partition[idxPartition]));
+ assert(partsize_t(part_to_sort.size()) == (current_my_nb_particles_per_partition[idxPartition]));
std::sort(part_to_sort.begin(), part_to_sort.end(),
[](const std::pair<long int,partsize_t>& p1,
@@ -575,7 +575,7 @@ public:
const int nbNeighbors = my_tree.getNeighbors(current_cell_idx, neighbors, neighbors_indexes, shift, true);
// with other interval
- for(size_t idx_neighbor = 0 ; idx_neighbor < nbNeighbors ; ++idx_neighbor){
+ for(int idx_neighbor = 0 ; idx_neighbor < nbNeighbors ; ++idx_neighbor){
cells_locker.lock(neighbors_indexes[idx_neighbor]);
for(size_t idx_2 = 0 ; idx_2 < (*neighbors[idx_neighbor]).size() ; ++idx_2){
@@ -726,7 +726,7 @@ public:
for(size_t idx_1 = 0 ; idx_1 < intervals.size() ; ++idx_1){
// with other interval
- for(size_t idx_neighbor = 0 ; idx_neighbor < nbNeighbors ; ++idx_neighbor){
+ for(int idx_neighbor = 0 ; idx_neighbor < nbNeighbors ; ++idx_neighbor){
if(currenct_cell_idx < neighbors_indexes[idx_neighbor]){
cells_locker.lock(neighbors_indexes[idx_neighbor]);
diff --git a/bfps/cpp/particles/particles_distr_mpi.hpp b/bfps/cpp/particles/particles_distr_mpi.hpp
index 48559518..f0c09fd9 100644
--- a/bfps/cpp/particles/particles_distr_mpi.hpp
+++ b/bfps/cpp/particles/particles_distr_mpi.hpp
@@ -35,6 +35,9 @@ protected:
TAG_LOW_UP_MOVED_PARTICLES_INDEXES,
TAG_UP_LOW_MOVED_PARTICLES_INDEXES,
+ TAG_LOW_UP_MOVED_PARTICLES_DATA,
+ TAG_UP_LOW_MOVED_PARTICLES_DATA,
+
TAG_LOW_UP_MOVED_PARTICLES_RHS,
TAG_LOW_UP_MOVED_PARTICLES_RHS_MAX = TAG_LOW_UP_MOVED_PARTICLES_RHS+MaxNbRhs,
@@ -503,13 +506,14 @@ public:
////////////////////////////////////////////////////////////////////////////
- template <class computer_class, int size_particle_positions, int size_particle_rhs, int size_particle_index>
+ template <class computer_class, int size_particle_positions, int size_particle_data, int size_particle_rhs, int size_particle_index>
void redistribute(computer_class& in_computer,
partsize_t current_my_nb_particles_per_partition[],
partsize_t* nb_particles,
std::unique_ptr<real_number[]>* inout_positions_particles,
std::unique_ptr<real_number[]> inout_rhs_particles[], const int in_nb_rhs,
- std::unique_ptr<partsize_t[]>* inout_index_particles){
+ std::unique_ptr<partsize_t[]>* inout_index_particles,
+ std::unique_ptr<real_number[]>* inout_data_particles){
TIMEZONE("redistribute");
// Some latest processes might not be involved
@@ -537,7 +541,13 @@ public:
},
[&](const partsize_t idx1, const partsize_t idx2){
for(int idx_val = 0 ; idx_val < size_particle_index ; ++idx_val){
- std::swap((*inout_index_particles)[idx1], (*inout_index_particles)[idx2]);
+ std::swap((*inout_index_particles)[size_particle_index*idx1+idx_val],
+ (*inout_index_particles)[size_particle_index*idx2+idx_val]);
+ }
+
+ for(int idx_val = 0 ; idx_val < size_particle_data ; ++idx_val){
+ std::swap((*inout_data_particles)[size_particle_data*idx1+idx_val],
+ (*inout_data_particles)[size_particle_data*idx2+idx_val]);
}
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
@@ -562,7 +572,13 @@ public:
},
[&](const partsize_t idx1, const partsize_t idx2){
for(int idx_val = 0 ; idx_val < size_particle_index ; ++idx_val){
- std::swap((*inout_index_particles)[idx1], (*inout_index_particles)[idx2]);
+ std::swap((*inout_index_particles)[size_particle_index*idx1+idx_val],
+ (*inout_index_particles)[size_particle_index*idx2+idx_val]);
+ }
+
+ for(int idx_val = 0 ; idx_val < size_particle_data ; ++idx_val){
+ std::swap((*inout_data_particles)[size_particle_data*idx1+idx_val],
+ (*inout_data_particles)[size_particle_data*idx2+idx_val]);
}
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
@@ -581,6 +597,8 @@ public:
std::unique_ptr<real_number[]> newParticlesUp;
std::unique_ptr<partsize_t[]> newParticlesLowIndexes;
std::unique_ptr<partsize_t[]> newParticlesUpIndexes;
+ std::unique_ptr<real_number[]> newParticlesLowData;
+ std::unique_ptr<real_number[]> newParticlesUpData;
std::vector<std::unique_ptr<real_number[]>> newParticlesLowRhs(in_nb_rhs);
std::vector<std::unique_ptr<real_number[]>> newParticlesUpRhs(in_nb_rhs);
@@ -607,13 +625,24 @@ public:
assert(nbOutLower*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(&(*inout_positions_particles)[0], int(nbOutLower*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
+
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
- assert(nbOutLower < std::numeric_limits<int>::max());
- AssertMpi(MPI_Isend(&(*inout_index_particles)[0], int(nbOutLower), particles_utils::GetMpiType(partsize_t()),
+ assert(nbOutLower*size_particle_index < std::numeric_limits<int>::max());
+ AssertMpi(MPI_Isend(&(*inout_index_particles)[0], int(nbOutLower*size_particle_index), particles_utils::GetMpiType(partsize_t()),
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES_INDEXES,
MPI_COMM_WORLD, &mpiRequests.back()));
+ if(size_particle_data){
+ whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
+ mpiRequests.emplace_back();
+ assert(nbOutLower*size_particle_data < std::numeric_limits<int>::max());
+ AssertMpi(MPI_Isend(&(*inout_data_particles)[0], int(nbOutLower*size_particle_data),
+ particles_utils::GetMpiType(partsize_t()),
+ (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES_DATA,
+ MPI_COMM_WORLD, &mpiRequests.back()));
+ }
+
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
@@ -643,13 +672,23 @@ public:
AssertMpi(MPI_Isend(&(*inout_positions_particles)[(myTotalNbParticles-nbOutUpper)*size_particle_positions],
int(nbOutUpper*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
+
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
- assert(nbOutUpper < std::numeric_limits<int>::max());
- AssertMpi(MPI_Isend(&(*inout_index_particles)[(myTotalNbParticles-nbOutUpper)], int(nbOutUpper),
+ assert(nbOutUpper*size_particle_index < std::numeric_limits<int>::max());
+ AssertMpi(MPI_Isend(&(*inout_index_particles)[(myTotalNbParticles-nbOutUpper)*size_particle_index], int(nbOutUpper*size_particle_index),
particles_utils::GetMpiType(partsize_t()), (my_rank+1)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES_INDEXES,
MPI_COMM_WORLD, &mpiRequests.back()));
+ if(size_particle_data){
+ whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
+ mpiRequests.emplace_back();
+ assert(nbOutUpper*size_particle_data < std::numeric_limits<int>::max());
+ AssertMpi(MPI_Isend(&(*inout_data_particles)[(myTotalNbParticles-nbOutUpper)*size_particle_data],
+ int(nbOutUpper*size_particle_data),
+ particles_utils::GetMpiType(partsize_t()), (my_rank+1)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES_DATA,
+ MPI_COMM_WORLD, &mpiRequests.back()));
+ }
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
@@ -684,14 +723,26 @@ public:
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
- newParticlesLowIndexes.reset(new partsize_t[nbNewFromLow]);
+ newParticlesLowIndexes.reset(new partsize_t[nbNewFromLow*size_particle_index]);
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
- assert(nbNewFromLow < std::numeric_limits<int>::max());
- AssertMpi(MPI_Irecv(&newParticlesLowIndexes[0], int(nbNewFromLow), particles_utils::GetMpiType(partsize_t()),
+ assert(nbNewFromLow*size_particle_index < std::numeric_limits<int>::max());
+ AssertMpi(MPI_Irecv(&newParticlesLowIndexes[0], int(nbNewFromLow*size_particle_index),
+ particles_utils::GetMpiType(partsize_t()),
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES_INDEXES,
MPI_COMM_WORLD, &mpiRequests.back()));
+ if(size_particle_data){
+ newParticlesLowData.reset(new real_number[nbNewFromLow*size_particle_data]);
+ whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
+ mpiRequests.emplace_back();
+ assert(nbNewFromLow*size_particle_data < std::numeric_limits<int>::max());
+ AssertMpi(MPI_Irecv(&newParticlesLowData[0], int(nbNewFromLow*size_particle_data),
+ particles_utils::GetMpiType(real_number()),
+ (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES_DATA,
+ MPI_COMM_WORLD, &mpiRequests.back()));
+ }
+
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
newParticlesLowRhs[idx_rhs].reset(new real_number[nbNewFromLow*size_particle_rhs]);
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
@@ -713,14 +764,26 @@ public:
AssertMpi(MPI_Irecv(&newParticlesUp[0], int(nbNewFromUp*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank+1)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
- newParticlesUpIndexes.reset(new partsize_t[nbNewFromUp]);
+ newParticlesUpIndexes.reset(new partsize_t[nbNewFromUp*size_particle_index]);
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
- assert(nbNewFromUp < std::numeric_limits<int>::max());
- AssertMpi(MPI_Irecv(&newParticlesUpIndexes[0], int(nbNewFromUp), particles_utils::GetMpiType(partsize_t()),
+ assert(nbNewFromUp*size_particle_index < std::numeric_limits<int>::max());
+ AssertMpi(MPI_Irecv(&newParticlesUpIndexes[0], int(nbNewFromUp*size_particle_index),
+ particles_utils::GetMpiType(partsize_t()),
(my_rank+1)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES_INDEXES,
MPI_COMM_WORLD, &mpiRequests.back()));
+ if(size_particle_data){
+ newParticlesUpData.reset(new real_number[nbNewFromUp*size_particle_data]);
+ whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
+ mpiRequests.emplace_back();
+ assert(nbNewFromUp*size_particle_data < std::numeric_limits<int>::max());
+ AssertMpi(MPI_Irecv(&newParticlesUpData[0], int(nbNewFromUp*size_particle_data),
+ particles_utils::GetMpiType(real_number()),
+ (my_rank+1)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES_DATA,
+ MPI_COMM_WORLD, &mpiRequests.back()));
+ }
+
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
newParticlesUpRhs[idx_rhs].reset(new real_number[nbNewFromUp*size_particle_rhs]);
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
@@ -750,7 +813,8 @@ public:
const partsize_t myTotalNewNbParticles = nbOldParticlesInside + nbNewFromLow + nbNewFromUp;
std::unique_ptr<real_number[]> newArray(new real_number[myTotalNewNbParticles*size_particle_positions]);
- std::unique_ptr<partsize_t[]> newArrayIndexes(new partsize_t[myTotalNewNbParticles]);
+ std::unique_ptr<partsize_t[]> newArrayIndexes(new partsize_t[myTotalNewNbParticles*size_particle_index]);
+ std::unique_ptr<real_number[]> newArrayData(new real_number[myTotalNewNbParticles*size_particle_data]);
std::vector<std::unique_ptr<real_number[]>> newArrayRhs(in_nb_rhs);
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
newArrayRhs[idx_rhs].reset(new real_number[myTotalNewNbParticles*size_particle_rhs]);
@@ -760,7 +824,8 @@ public:
if(nbNewFromLow){
const particles_utils::fixed_copy fcp(0, 0, nbNewFromLow);
fcp.copy(newArray, newParticlesLow, size_particle_positions);
- fcp.copy(newArrayIndexes, newParticlesLowIndexes);
+ fcp.copy(newArrayIndexes, newParticlesLowIndexes, size_particle_index);
+ fcp.copy(newArrayData, newParticlesLowData, size_particle_data);
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
fcp.copy(newArrayRhs[idx_rhs], newParticlesLowRhs[idx_rhs], size_particle_rhs);
}
@@ -770,7 +835,8 @@ public:
{
const particles_utils::fixed_copy fcp(nbNewFromLow, nbOutLower, nbOldParticlesInside);
fcp.copy(newArray, (*inout_positions_particles), size_particle_positions);
- fcp.copy(newArrayIndexes, (*inout_index_particles));
+ fcp.copy(newArrayIndexes, (*inout_index_particles), size_particle_index);
+ fcp.copy(newArrayData, (*inout_data_particles), size_particle_data);
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
fcp.copy(newArrayRhs[idx_rhs], inout_rhs_particles[idx_rhs], size_particle_rhs);
}
@@ -780,7 +846,8 @@ public:
if(nbNewFromUp){
const particles_utils::fixed_copy fcp(nbNewFromLow+nbOldParticlesInside, 0, nbNewFromUp);
fcp.copy(newArray, newParticlesUp, size_particle_positions);
- fcp.copy(newArrayIndexes, newParticlesUpIndexes);
+ fcp.copy(newArrayIndexes, newParticlesUpIndexes, size_particle_index);
+ fcp.copy(newArrayData, newParticlesUpData, size_particle_data);
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
fcp.copy(newArrayRhs[idx_rhs], newParticlesUpRhs[idx_rhs], size_particle_rhs);
}
@@ -788,6 +855,7 @@ public:
(*inout_positions_particles) = std::move(newArray);
(*inout_index_particles) = std::move(newArrayIndexes);
+ (*inout_data_particles) = std::move(newArrayData);
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
inout_rhs_particles[idx_rhs] = std::move(newArrayRhs[idx_rhs]);
}
@@ -808,7 +876,13 @@ public:
},
[&](const partsize_t idx1, const partsize_t idx2){
for(int idx_val = 0 ; idx_val < size_particle_index ; ++idx_val){
- std::swap((*inout_index_particles)[idx1], (*inout_index_particles)[idx2]);
+ std::swap((*inout_index_particles)[size_particle_index*idx1 + idx_val],
+ (*inout_index_particles)[size_particle_index*idx2 + idx_val]);
+ }
+
+ for(int idx_val = 0 ; idx_val < size_particle_data ; ++idx_val){
+ std::swap((*inout_data_particles)[size_particle_data*idx1 + idx_val],
+ (*inout_data_particles)[size_particle_data*idx2 + idx_val]);
}
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
diff --git a/bfps/cpp/particles/particles_system.hpp b/bfps/cpp/particles/particles_system.hpp
index 02767a8b..12bf6c29 100644
--- a/bfps/cpp/particles/particles_system.hpp
+++ b/bfps/cpp/particles/particles_system.hpp
@@ -12,8 +12,10 @@
#include "particles_adams_bashforth.hpp"
#include "scope_timer.hpp"
+#include "p2p_distr_mpi.hpp"
+
template <class partsize_t, class real_number, class field_rnumber, class field_class, class interpolator_class, int interp_neighbours,
- int size_particle_rhs>
+ int size_particle_rhs, class p2p_computer_class, int size_particle_data>
class particles_system : public abstract_particles_system<partsize_t, real_number> {
MPI_Comm mpi_com;
@@ -47,6 +49,10 @@ class particles_system : public abstract_particles_system<partsize_t, real_numbe
int step_idx;
+ p2p_distr_mpi<partsize_t, real_number> distr_p2p;
+ p2p_computer_class computer_p2p;
+ std::unique_ptr<real_number[]> my_particles_data;
+
public:
particles_system(const std::array<size_t,3>& field_grid_dim, const std::array<real_number,3>& in_spatial_box_width,
const std::array<real_number,3>& in_spatial_box_offset,
@@ -57,7 +63,8 @@ public:
const field_class& in_field,
MPI_Comm in_mpi_com,
const partsize_t in_total_nb_particles,
- const int in_current_iteration = 1)
+ const int in_current_iteration = 1,
+ const real_number in_cutoff = 1.)
: mpi_com(in_mpi_com),
current_partition_interval({in_local_field_offset[IDX_Z], in_local_field_offset[IDX_Z] + in_local_field_dims[IDX_Z]}),
partition_interval_size(current_partition_interval.second - current_partition_interval.first),
@@ -69,7 +76,8 @@ public:
default_field(in_field),
spatial_box_width(in_spatial_box_width), spatial_partition_width(in_spatial_partition_width),
my_spatial_low_limit(in_my_spatial_low_limit), my_spatial_up_limit(in_my_spatial_up_limit),
- my_nb_particles(0), total_nb_particles(in_total_nb_particles), step_idx(in_current_iteration){
+ my_nb_particles(0), total_nb_particles(in_total_nb_particles), step_idx(in_current_iteration),
+ distr_p2p(in_mpi_com, current_partition_interval,field_grid_dim, spatial_box_width, in_spatial_box_offset, in_cutoff){
current_my_nb_particles_per_partition.reset(new partsize_t[partition_interval_size]);
current_offset_particles_for_partition.reset(new partsize_t[partition_interval_size+1]);
@@ -85,6 +93,8 @@ public:
my_particles_positions_indexes = particles_input.getMyParticlesIndexes();
my_particles_rhs = particles_input.getMyRhs();
my_nb_particles = particles_input.getLocalNbParticles();
+ // TODO P2P get it from loader
+ my_particles_data.reset(new real_number[my_nb_particles*size_particle_data]());
for(partsize_t idx_part = 0 ; idx_part < my_nb_particles ; ++idx_part){ // TODO remove me
const int partition_level = computer.pbc_field_layer(my_particles_positions[idx_part*3+IDX_Z], IDX_Z);
@@ -107,6 +117,10 @@ public:
my_particles_rhs[idx_rhs][idx2*size_particle_rhs + idx_val]);
}
}
+ for(int idx_val = 0 ; idx_val < size_particle_data ; ++idx_val){
+ std::swap(my_particles_data[idx1*size_particle_data + idx_val],
+ my_particles_data[idx2*size_particle_data + idx_val]);
+ }
});
{// TODO remove
@@ -131,6 +145,17 @@ public:
interp_neighbours);
}
+ void compute_p2p() final {
+ // TODO P2P
+ if(p2p_computer_class::isEmpty() == false){
+ TIMEZONE("particles_system::compute_p2p");
+ distr_p2p.template compute_distr<p2p_computer_class, 3, size_particle_data, size_particle_rhs>(
+ computer_p2p, current_my_nb_particles_per_partition.get(),
+ my_particles_positions.get(), my_particles_data.get(), my_particles_rhs.front().get(),
+ my_particles_positions_indexes.get());
+ }
+ }
+
template <class sample_field_class, int sample_size_particle_rhs>
void sample_compute(const sample_field_class& sample_field,
real_number sample_rhs[]) {
@@ -179,13 +204,14 @@ public:
void redistribute() final {
TIMEZONE("particles_system::redistribute");
- particles_distr.template redistribute<computer_class, 3, size_particle_rhs, 1>(
+ particles_distr.template redistribute<computer_class, 3, size_particle_data, size_particle_rhs, 1>(
computer,
current_my_nb_particles_per_partition.get(),
&my_nb_particles,
&my_particles_positions,
my_particles_rhs.data(), int(my_particles_rhs.size()),
- &my_particles_positions_indexes);
+ &my_particles_positions_indexes,
+ &my_particles_data);
}
void inc_step_idx() final {
@@ -210,6 +236,7 @@ public:
void completeLoop(const real_number dt) final {
TIMEZONE("particles_system::completeLoop");
compute();
+ compute_p2p();
move(dt);
redistribute();
inc_step_idx();
diff --git a/bfps/cpp/particles/particles_system_builder.hpp b/bfps/cpp/particles/particles_system_builder.hpp
index a3bc689d..f9ce512d 100644
--- a/bfps/cpp/particles/particles_system_builder.hpp
+++ b/bfps/cpp/particles/particles_system_builder.hpp
@@ -8,6 +8,7 @@
#include "particles_system.hpp"
#include "particles_input_hdf5.hpp"
#include "particles_generic_interp.hpp"
+#include "p2p_computer_empty.hpp"
#include "field.hpp"
#include "kspace.hpp"
@@ -109,7 +110,7 @@ inline RetType evaluate(IterType1 value1, IterType2 value2, Args... args){
///
//////////////////////////////////////////////////////////////////////////////
-template <class partsize_t, class field_rnumber, field_backend be, field_components fc, class particles_rnumber>
+template <class partsize_t, class field_rnumber, field_backend be, field_components fc, class particles_rnumber, class p2p_computer_class>
struct particles_system_build_container {
template <const int interpolation_size, const int spline_mode>
static std::unique_ptr<abstract_particles_system<partsize_t, particles_rnumber>> instanciate(
@@ -196,7 +197,8 @@ struct particles_system_build_container {
using particles_system_type = particles_system<partsize_t, particles_rnumber, field_rnumber,
field<field_rnumber, be, fc>,
particles_generic_interp<particles_rnumber, interpolation_size,spline_mode>,
- interpolation_size, ncomp(fc)>;
+ interpolation_size, ncomp(fc),
+ p2p_computer_class, p2p_computer_class::size_data>;
particles_system_type* part_sys = new particles_system_type(field_grid_dim,
spatial_box_width,
spatial_box_offset,
@@ -210,6 +212,7 @@ struct particles_system_build_container {
nparticles,
in_current_iteration);
+ // TODO P2P load particle data too
// Load particles from hdf5
particles_input_hdf5<partsize_t, particles_rnumber, 3,3> generator(mpi_comm, fname_input,
inDatanameState, inDatanameRhs, my_spatial_low_limit_z, my_spatial_up_limit_z);
@@ -251,7 +254,28 @@ inline std::unique_ptr<abstract_particles_system<partsize_t, particles_rnumber>>
return Template_double_for_if::evaluate<std::unique_ptr<abstract_particles_system<partsize_t, particles_rnumber>>,
int, 1, 11, 1, // interpolation_size
int, 0, 3, 1, // spline_mode
- particles_system_build_container<partsize_t, field_rnumber,be,fc,particles_rnumber>>(
+ particles_system_build_container<partsize_t, field_rnumber,be,fc,particles_rnumber, p2p_computer_empty<particles_rnumber,partsize_t>>>(
+ interpolation_size, // template iterator 1
+ spline_mode, // template iterator 2
+ fs_field,fs_kk, nsteps, nparticles, fname_input, inDatanameState, inDatanameRhs, mpi_comm, in_current_iteration);
+}
+
+template <class partsize_t, class field_rnumber, field_backend be, field_components fc, class p2p_computer_class, class particles_rnumber = double>
+inline std::unique_ptr<abstract_particles_system<partsize_t, particles_rnumber>> particles_system_builder_with_p2p(
+ const field<field_rnumber, be, fc>* fs_field, // (field object)
+ const kspace<be, SMOOTH>* fs_kk, // (kspace object, contains dkx, dky, dkz)
+ const int nsteps, // to check coherency between parameters and hdf input file (nb rhs)
+ const partsize_t nparticles, // to check coherency between parameters and hdf input file
+ const std::string& fname_input, // particles input filename
+ const std::string& inDatanameState, const std::string& inDatanameRhs, // input dataset names
+ const int interpolation_size,
+ const int spline_mode,
+ MPI_Comm mpi_comm,
+ const int in_current_iteration){
+ return Template_double_for_if::evaluate<std::unique_ptr<abstract_particles_system<partsize_t, particles_rnumber>>,
+ int, 1, 11, 1, // interpolation_size
+ int, 0, 3, 1, // spline_mode
+ particles_system_build_container<partsize_t, field_rnumber,be,fc,particles_rnumber,p2p_computer_class>>(
interpolation_size, // template iterator 1
spline_mode, // template iterator 2
fs_field,fs_kk, nsteps, nparticles, fname_input, inDatanameState, inDatanameRhs, mpi_comm, in_current_iteration);
diff --git a/setup.py b/setup.py
index 9bba1701..f257fd7f 100644
--- a/setup.py
+++ b/setup.py
@@ -127,7 +127,8 @@ src_file_list = ['full_code/joint_acc_vel_stats',
'spline_n10',
'Lagrange_polys',
'scope_timer',
- 'full_code/NSVEparticles']
+ 'full_code/NSVEparticles',
+ 'full_code/NSVEparticlesP2P']
particle_headers = [
'cpp/particles/particles_distr_mpi.hpp',
--
GitLab
From 134dd8103fbff8b1d28a82ab05c94f8e13336896 Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Thu, 28 Sep 2017 21:51:40 +0200
Subject: [PATCH 024/342] add preliminary field test
---
bfps/TEST.py | 6 ++
bfps/cpp/full_code/field_test.cpp | 105 ++++++++++++++++++++++++++++++
bfps/cpp/full_code/field_test.hpp | 63 ++++++++++++++++++
bfps/cpp/full_code/test.cpp | 2 +-
setup.py | 1 +
5 files changed, 176 insertions(+), 1 deletion(-)
create mode 100644 bfps/cpp/full_code/field_test.cpp
create mode 100644 bfps/cpp/full_code/field_test.hpp
diff --git a/bfps/TEST.py b/bfps/TEST.py
index 5f573403..f7e8e24e 100644
--- a/bfps/TEST.py
+++ b/bfps/TEST.py
@@ -257,6 +257,12 @@ class TEST(_code):
self.simulation_parser_arguments(parser_filter_test)
self.job_parser_arguments(parser_filter_test)
self.parameters_to_parser_arguments(parser_filter_test)
+ parser_field_test = subparsers.add_parser(
+ 'field_test',
+ help = 'plain field test')
+ self.simulation_parser_arguments(parser_field_test)
+ self.job_parser_arguments(parser_field_test)
+ self.parameters_to_parser_arguments(parser_field_test)
return None
def prepare_launch(
self,
diff --git a/bfps/cpp/full_code/field_test.cpp b/bfps/cpp/full_code/field_test.cpp
new file mode 100644
index 00000000..acee3617
--- /dev/null
+++ b/bfps/cpp/full_code/field_test.cpp
@@ -0,0 +1,105 @@
+#include <string>
+#include <cmath>
+#include <random>
+#include "field_test.hpp"
+#include "scope_timer.hpp"
+
+
+template <typename rnumber>
+int field_test<rnumber>::initialize(void)
+{
+ this->read_parameters();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int field_test<rnumber>::finalize(void)
+{
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int field_test<rnumber>::read_parameters()
+{
+ this->test::read_parameters();
+ // in case any parameters are needed, this is where they should be read
+ hid_t parameter_file;
+ hid_t dset, memtype, space;
+ parameter_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ dset = H5Dopen(parameter_file, "/parameters/filter_length", H5P_DEFAULT);
+ H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->filter_length);
+ H5Dclose(dset);
+ H5Fclose(parameter_file);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int field_test<rnumber>::do_work(void)
+{
+ // allocate
+ field<rnumber, FFTW, ONE> *scal_field = new field<rnumber, FFTW, ONE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ DEFAULT_FFTW_FLAG);
+ field<rnumber, FFTW, ONE> *scal_field_alt = new field<rnumber, FFTW, ONE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ DEFAULT_FFTW_FLAG);
+ std::default_random_engine rgen;
+ std::normal_distribution<rnumber> rdist;
+ rgen.seed(1);
+ //auto gaussian = std::bind(rgen, rdist);
+ kspace<FFTW,SMOOTH> *kk = new kspace<FFTW, SMOOTH>(
+ scal_field->clayout, this->dkx, this->dky, this->dkz);
+
+ if (this->myrank == 0)
+ {
+ hid_t stat_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDWR,
+ H5P_DEFAULT);
+ kk->store(stat_file);
+ H5Fclose(stat_file);
+ }
+
+ // fill up scal_field
+ scal_field->real_space_representation = true;
+ scal_field->RLOOP(
+ [&](ptrdiff_t rindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex){
+ scal_field->rval(rindex) = rdist(rgen);
+ });
+
+ *scal_field_alt = scal_field->get_rdata();
+ scal_field->dft();
+ scal_field->ift();
+ scal_field->normalize();
+
+ double max_error = 0;
+ scal_field->RLOOP(
+ [&](ptrdiff_t rindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex){
+ double tval = fabs(scal_field->rval(rindex) - scal_field_alt->rval(rindex));
+ if (max_error < tval)
+ max_error = tval;
+ });
+
+ DEBUG_MSG("maximum error is %g\n", max_error);
+
+ // deallocate
+ delete kk;
+ delete scal_field;
+ delete scal_field_alt;
+ return EXIT_SUCCESS;
+}
+
+template class field_test<float>;
+template class field_test<double>;
+
diff --git a/bfps/cpp/full_code/field_test.hpp b/bfps/cpp/full_code/field_test.hpp
new file mode 100644
index 00000000..5339feb8
--- /dev/null
+++ b/bfps/cpp/full_code/field_test.hpp
@@ -0,0 +1,63 @@
+/**********************************************************************
+* *
+* Copyright 2017 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef FILTER_TEST_HPP
+#define FILTER_TEST_HPP
+
+
+
+#include <cstdlib>
+#include "base.hpp"
+#include "kspace.hpp"
+#include "field.hpp"
+#include "full_code/test.hpp"
+
+/** \brief A class for testing basic field class functionality.
+ */
+
+template <typename rnumber>
+class field_test: public test
+{
+ public:
+ double filter_length;
+ // kspace, in case we want to compute spectra or smth
+
+ field_test(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ test(
+ COMMUNICATOR,
+ simulation_name){}
+ ~field_test(){}
+
+ int initialize(void);
+ int do_work(void);
+ int finalize(void);
+ int read_parameters(void);
+};
+
+#endif//FILTER_TEST_HPP
+
diff --git a/bfps/cpp/full_code/test.cpp b/bfps/cpp/full_code/test.cpp
index 4f7a402c..fd2192a0 100644
--- a/bfps/cpp/full_code/test.cpp
+++ b/bfps/cpp/full_code/test.cpp
@@ -14,7 +14,7 @@ int test::main_loop(void)
this->start_simple_timer();
this->do_work();
this->print_simple_timer(
- "do_work required " + std::to_string(this->iteration));
+ "do_work required ");
return EXIT_SUCCESS;
}
diff --git a/setup.py b/setup.py
index 9bba1701..b03bd4f4 100644
--- a/setup.py
+++ b/setup.py
@@ -91,6 +91,7 @@ print('This is bfps version ' + VERSION)
src_file_list = ['full_code/joint_acc_vel_stats',
'full_code/test',
'full_code/filter_test',
+ 'full_code/field_test',
'hdf5_tools',
'full_code/get_rfields',
'full_code/NSVE_field_stats',
--
GitLab
From ffa26d9be811bee27f3bf7b37857552d36694571 Mon Sep 17 00:00:00 2001
From: Cristian Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 29 Sep 2017 06:47:13 +0200
Subject: [PATCH 025/342] set up environment for IBM load leveler
---
bfps/_code.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/bfps/_code.py b/bfps/_code.py
index 7fc4642d..3791092a 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -380,6 +380,8 @@ class _code(_base):
# If Ibm is used should be : script_file.write('# @ job_type = parallel\n')
script_file.write('# @ job_type = MPICH\n')
+ assert(type(self.host_info['environment']) != type(None))
+ script_file.write('# @ class = {0}\n'.format(self.host_info['environment']))
script_file.write('# @ node_usage = not_shared\n')
script_file.write('# @ notification = complete\n')
@@ -414,7 +416,6 @@ class _code(_base):
script_file.write('# @ resources = ConsumableCpus({})\n'.format(nb_threads_per_process))
script_file.write('# @ network.MPI = sn_all,not_shared,us\n')
script_file.write('# @ wall_clock_limit = {0}:{1:0>2d}:00\n'.format(hours, minutes))
- assert(type(self.host_info['environment']) != type(None))
script_file.write('# @ node = {0}\n'.format(nb_nodes))
script_file.write('# @ tasks_per_node = {0}\n'.format(nb_processes_per_node))
if (first_node_tasks > 0):
--
GitLab
From a8dab3bdfb8963b47244b18175c038b381961f6c Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Fri, 29 Sep 2017 16:03:06 +0200
Subject: [PATCH 026/342] WIP Update fftw interface to split mpi many calls and
replace direct calls to fftw by calls to our interface
---
bfps/cpp/fftw_interface.hpp | 440 ++++++++++++++++++++++++++++++
bfps/cpp/field.cpp | 2 +-
bfps/cpp/field.hpp | 4 +-
bfps/cpp/field_descriptor.cpp | 2 +-
bfps/cpp/fluid_solver.cpp | 16 +-
bfps/cpp/fluid_solver.hpp | 12 +-
bfps/cpp/fluid_solver_base.cpp | 8 +-
bfps/cpp/slab_field_particles.cpp | 20 +-
8 files changed, 472 insertions(+), 32 deletions(-)
diff --git a/bfps/cpp/fftw_interface.hpp b/bfps/cpp/fftw_interface.hpp
index 495ec9fa..682850c8 100644
--- a/bfps/cpp/fftw_interface.hpp
+++ b/bfps/cpp/fftw_interface.hpp
@@ -34,6 +34,15 @@
#define DEFAULT_FFTW_FLAG FFTW_PATIENT
#endif
+// To have multiple calls to c2r/r2c
+#define SPLIT_FFTW_MANY
+#ifdef SPLIT_FFTW_MANY
+#include <vector>
+#include <memory>
+#include <algorithm>
+#include <cassert>
+#endif
+
template <class realtype>
class fftw_interface;
@@ -45,6 +54,31 @@ public:
using complex = fftwf_complex;
using plan = fftwf_plan;
using iodim = fftwf_iodim;
+#ifdef SPLIT_FFTW_MANY
+ struct many_plan_container{
+ int rnk;
+ std::vector<ptrdiff_t> n;
+ int howmany;
+ ptrdiff_t iblock;
+ ptrdiff_t oblock;
+ std::shared_ptr<real> buffer;
+ plan plan_to_use;
+
+ ptrdiff_t local_n0, local_0_start;
+ ptrdiff_t local_n1, local_1_start;
+
+ bool is_r2c;
+ void* in;
+ void* out;
+
+ ptrdiff_t nb_real_to_copy;
+ ptrdiff_t nb_complex_to_copy;
+ };
+
+ using many_plan = many_plan_container;
+#else
+ using many_plan = fftwf_plan;
+#endif
static complex* alloc_complex(const size_t in_size){
return fftwf_alloc_complex(in_size);
@@ -66,6 +100,16 @@ public:
fftwf_destroy_plan(in_plan);
}
+ template <class ... Params>
+ static ptrdiff_t mpi_local_size_many_transposed(Params ... params){
+ return fftwf_mpi_local_size_many_transposed(params...);
+ }
+
+ template <class ... Params>
+ static ptrdiff_t mpi_local_size_many(Params ... params){
+ return fftwf_mpi_local_size_many(params...);
+ }
+
template <class ... Params>
static plan mpi_plan_transpose(Params ... params){
return fftwf_mpi_plan_transpose(params...);
@@ -86,6 +130,175 @@ public:
return fftwf_plan_guru_dft(params...);
}
+#ifdef SPLIT_FFTW_MANY
+ static many_plan mpi_plan_many_dft_c2r(int rnk, const ptrdiff_t *n, ptrdiff_t howmany,
+ ptrdiff_t iblock, ptrdiff_t oblock,
+ complex *in, real *out,
+ MPI_Comm comm, unsigned flags){
+ assert(iblock == FFTW_MPI_DEFAULT_BLOCK);
+ assert(oblock == FFTW_MPI_DEFAULT_BLOCK);
+
+ many_plan c2r_plan;
+ c2r_plan.rnk = rnk;
+ c2r_plan.n.insert(c2r_plan.n.end(), n, n+rnk);
+ c2r_plan.howmany = howmany;
+ c2r_plan.iblock = iblock;
+ c2r_plan.oblock = oblock;
+ c2r_plan.is_r2c = false;
+ c2r_plan.in = in;
+ c2r_plan.out = out;
+
+ // If 1 then use default without copy
+ if(howmany == 1){
+ c2r_plan.plan_to_use = mpi_plan_dft_c2r(rnk, n,
+ (complex*)in,
+ out,
+ comm, flags);
+ return c2r_plan;
+ }
+
+ // We need to find out the size of the buffer to allocate
+ mpi_local_size_many_transposed(
+ rnk, n, howmany,
+ FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK, comm,
+ &c2r_plan.local_n0, &c2r_plan.local_0_start,
+ &c2r_plan.local_n1, &c2r_plan.local_1_start);
+
+ ptrdiff_t sizeBuffer = c2r_plan.local_n0/howmany;
+ for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
+ sizeBuffer *= n[idxrnk];
+ }
+ sizeBuffer *= n[rnk-1]+2;
+
+ c2r_plan.buffer.reset(alloc_real(sizeBuffer));
+ // Init the plan
+ c2r_plan.plan_to_use = mpi_plan_dft_c2r(rnk, n,
+ (complex*)c2r_plan.buffer.get(),
+ c2r_plan.buffer.get(),
+ comm, flags);
+
+ c2r_plan.nb_real_to_copy = c2r_plan.local_n0/howmany;
+ c2r_plan.nb_complex_to_copy = c2r_plan.local_n0/howmany;
+ for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
+ c2r_plan.nb_real_to_copy *= n[idxrnk];
+ c2r_plan.nb_complex_to_copy *= n[idxrnk];
+ }
+ c2r_plan.nb_real_to_copy *= n[rnk-1];
+ c2r_plan.nb_complex_to_copy *= n[rnk-1]/2 + 1;
+
+ return c2r_plan;
+ }
+
+ static many_plan mpi_plan_many_dft_r2c(int rnk, const ptrdiff_t *n, ptrdiff_t howmany,
+ ptrdiff_t iblock, ptrdiff_t oblock,
+ real *in, complex *out,
+ MPI_Comm comm, unsigned flags){
+ assert(iblock == FFTW_MPI_DEFAULT_BLOCK);
+ assert(oblock == FFTW_MPI_DEFAULT_BLOCK);
+
+ many_plan r2c_plan;
+ r2c_plan.rnk = rnk;
+ r2c_plan.n.insert(r2c_plan.n.end(), n, n+rnk);
+ r2c_plan.howmany = howmany;
+ r2c_plan.iblock = iblock;
+ r2c_plan.oblock = oblock;
+ r2c_plan.is_r2c = true;
+ r2c_plan.in = in;
+ r2c_plan.out = out;
+
+ // If 1 then use default without copy
+ if(howmany == 1){
+ r2c_plan.plan_to_use = mpi_plan_dft_r2c(rnk, n,
+ in,
+ (complex*)out,
+ comm, flags);
+ return r2c_plan;
+ }
+
+ // We need to find out the size of the buffer to allocate
+ mpi_local_size_many_transposed(
+ rnk, n, howmany,
+ FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK, comm,
+ &r2c_plan.local_n0, &r2c_plan.local_0_start,
+ &r2c_plan.local_n1, &r2c_plan.local_1_start);
+
+ ptrdiff_t sizeBuffer = r2c_plan.local_n0/howmany;
+ for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
+ sizeBuffer *= n[idxrnk];
+ }
+ sizeBuffer *= n[rnk-1]+2;
+
+ r2c_plan.buffer.reset(alloc_real(sizeBuffer));
+ // Init the plan
+ r2c_plan.plan_to_use = mpi_plan_dft_r2c(rnk, n,
+ r2c_plan.buffer.get(),
+ (complex*)r2c_plan.buffer.get(),
+ comm, flags);
+
+
+ r2c_plan.nb_real_to_copy = r2c_plan.local_n0/howmany;
+ r2c_plan.nb_complex_to_copy = r2c_plan.local_n0/howmany;
+ for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
+ r2c_plan.nb_real_to_copy *= n[idxrnk];
+ r2c_plan.nb_complex_to_copy *= n[idxrnk];
+ }
+ r2c_plan.nb_real_to_copy *= n[rnk-1];
+ r2c_plan.nb_complex_to_copy *= n[rnk-1]/2 + 1;
+
+ return r2c_plan;
+ }
+
+ static void execute(many_plan& in_plan){
+ if(in_plan.howmany == 1){
+ execute(in_plan.plan_to_use);
+ return;
+ }
+
+ const ptrdiff_t nb_to_copy_to_buffer = (in_plan.is_r2c?in_plan.nb_real_to_copy:in_plan.nb_complex_to_copy);
+ const ptrdiff_t nb_to_copy_from_buffer = (!in_plan.is_r2c?in_plan.nb_real_to_copy:in_plan.nb_complex_to_copy);
+
+ for(int idx_howmany = 0 ; idx_howmany < in_plan.howmany ; ++idx_howmany){
+ // Copy to buffer
+ if(in_plan.is_r2c){
+ real* dest = in_plan.buffer.get();
+ const real* src = ((const real*)in_plan.in)+idx_howmany;
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_to_buffer ; ++idx_copy){
+ dest[idx_copy] = src[idx_copy*in_plan.howmany];
+ }
+ }
+ else{
+ complex* dest = (complex*)in_plan.buffer.get();
+ const complex* src = ((const complex*)in_plan.in)+idx_howmany;
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_to_buffer ; ++idx_copy){
+ dest[idx_copy][0] = src[idx_copy*in_plan.howmany][0];
+ dest[idx_copy][1] = src[idx_copy*in_plan.howmany][1];
+ }
+ }
+
+ execute(in_plan.plan_to_use);
+ // Copy result from buffer
+ if(in_plan.is_r2c){
+ complex* dest = ((complex*)in_plan.in)+idx_howmany;
+ const complex* src = (const complex*)in_plan.buffer.get();
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_from_buffer ; ++idx_copy){
+ dest[idx_copy*in_plan.howmany][0] = src[idx_copy][0];
+ dest[idx_copy*in_plan.howmany][1] = src[idx_copy][1];
+ }
+ }
+ else{
+ real* dest = ((real*)in_plan.in)+idx_howmany;
+ const real* src = in_plan.buffer.get();
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_from_buffer ; ++idx_copy){
+ dest[idx_copy*in_plan.howmany] = src[idx_copy];
+ }
+ }
+ }
+ }
+
+ static void destroy_plan(many_plan& in_plan){
+ destroy_plan(in_plan.plan_to_use);
+ }
+#else
template <class ... Params>
static plan mpi_plan_many_dft_c2r(Params ... params){
return fftwf_mpi_plan_many_dft_c2r(params...);
@@ -95,6 +308,17 @@ public:
static plan mpi_plan_many_dft_r2c(Params ... params){
return fftwf_mpi_plan_many_dft_r2c(params...);
}
+#endif
+
+ template <class ... Params>
+ static plan mpi_plan_dft_c2r(Params ... params){
+ return fftwf_mpi_plan_dft_c2r(params...);
+ }
+
+ template <class ... Params>
+ static plan mpi_plan_dft_r2c(Params ... params){
+ return fftwf_mpi_plan_dft_r2c(params...);
+ }
template <class ... Params>
static plan mpi_plan_dft_c2r_3d(Params ... params){
@@ -110,6 +334,31 @@ public:
using complex = fftw_complex;
using plan = fftw_plan;
using iodim = fftw_iodim;
+#ifdef SPLIT_FFTW_MANY
+ struct many_plan_container{
+ int rnk;
+ std::vector<ptrdiff_t> n;
+ int howmany;
+ ptrdiff_t iblock;
+ ptrdiff_t oblock;
+ std::shared_ptr<real> buffer;
+ plan plan_to_use;
+
+ ptrdiff_t local_n0, local_0_start;
+ ptrdiff_t local_n1, local_1_start;
+
+ bool is_r2c;
+ void* in;
+ void* out;
+
+ ptrdiff_t nb_real_to_copy;
+ ptrdiff_t nb_complex_to_copy;
+ };
+
+ using many_plan = many_plan_container;
+#else
+ using many_plan = fftw_plan;
+#endif
static complex* alloc_complex(const size_t in_size){
return fftw_alloc_complex(in_size);
@@ -131,6 +380,16 @@ public:
fftw_destroy_plan(in_plan);
}
+ template <class ... Params>
+ static ptrdiff_t mpi_local_size_many_transposed(Params ... params){
+ return fftw_mpi_local_size_many_transposed(params...);
+ }
+
+ template <class ... Params>
+ static ptrdiff_t mpi_local_size_many(Params ... params){
+ return fftw_mpi_local_size_many(params...);
+ }
+
template <class ... Params>
static plan mpi_plan_transpose(Params ... params){
return fftw_mpi_plan_transpose(params...);
@@ -151,6 +410,176 @@ public:
return fftw_plan_guru_dft(params...);
}
+
+#ifdef SPLIT_FFTW_MANY
+ static many_plan mpi_plan_many_dft_c2r(int rnk, const ptrdiff_t *n, ptrdiff_t howmany,
+ ptrdiff_t iblock, ptrdiff_t oblock,
+ complex *in, real *out,
+ MPI_Comm comm, unsigned flags){
+ assert(iblock == FFTW_MPI_DEFAULT_BLOCK);
+ assert(oblock == FFTW_MPI_DEFAULT_BLOCK);
+
+ many_plan c2r_plan;
+ c2r_plan.rnk = rnk;
+ c2r_plan.n.insert(c2r_plan.n.end(), n, n+rnk);
+ c2r_plan.howmany = howmany;
+ c2r_plan.iblock = iblock;
+ c2r_plan.oblock = oblock;
+ c2r_plan.is_r2c = false;
+ c2r_plan.in = in;
+ c2r_plan.out = out;
+
+ // If 1 then use default without copy
+ if(howmany == 1){
+ c2r_plan.plan_to_use = mpi_plan_dft_c2r(rnk, n,
+ (complex*)in,
+ out,
+ comm, flags);
+ return c2r_plan;
+ }
+
+ // We need to find out the size of the buffer to allocate
+ mpi_local_size_many_transposed(
+ rnk, n, howmany,
+ FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK, comm,
+ &c2r_plan.local_n0, &c2r_plan.local_0_start,
+ &c2r_plan.local_n1, &c2r_plan.local_1_start);
+
+ ptrdiff_t sizeBuffer = c2r_plan.local_n0/howmany;
+ for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
+ sizeBuffer *= n[idxrnk];
+ }
+ sizeBuffer *= n[rnk-1]+2;
+
+ c2r_plan.buffer.reset(alloc_real(sizeBuffer));
+ // Init the plan
+ c2r_plan.plan_to_use = mpi_plan_dft_c2r(rnk, n,
+ (complex*)c2r_plan.buffer.get(),
+ c2r_plan.buffer.get(),
+ comm, flags);
+
+ c2r_plan.nb_real_to_copy = c2r_plan.local_n0/howmany;
+ c2r_plan.nb_complex_to_copy = c2r_plan.local_n0/howmany;
+ for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
+ c2r_plan.nb_real_to_copy *= n[idxrnk];
+ c2r_plan.nb_complex_to_copy *= n[idxrnk];
+ }
+ c2r_plan.nb_real_to_copy *= n[rnk-1];
+ c2r_plan.nb_complex_to_copy *= n[rnk-1]/2 + 1;
+
+ return c2r_plan;
+ }
+
+ static many_plan mpi_plan_many_dft_r2c(int rnk, const ptrdiff_t *n, ptrdiff_t howmany,
+ ptrdiff_t iblock, ptrdiff_t oblock,
+ real *in, complex *out,
+ MPI_Comm comm, unsigned flags){
+ assert(iblock == FFTW_MPI_DEFAULT_BLOCK);
+ assert(oblock == FFTW_MPI_DEFAULT_BLOCK);
+
+ many_plan r2c_plan;
+ r2c_plan.rnk = rnk;
+ r2c_plan.n.insert(r2c_plan.n.end(), n, n+rnk);
+ r2c_plan.howmany = howmany;
+ r2c_plan.iblock = iblock;
+ r2c_plan.oblock = oblock;
+ r2c_plan.is_r2c = true;
+ r2c_plan.in = in;
+ r2c_plan.out = out;
+
+ // If 1 then use default without copy
+ if(howmany == 1){
+ r2c_plan.plan_to_use = mpi_plan_dft_r2c(rnk, n,
+ in,
+ (complex*)out,
+ comm, flags);
+ return r2c_plan;
+ }
+
+ // We need to find out the size of the buffer to allocate
+ mpi_local_size_many_transposed(
+ rnk, n, howmany,
+ FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK, comm,
+ &r2c_plan.local_n0, &r2c_plan.local_0_start,
+ &r2c_plan.local_n1, &r2c_plan.local_1_start);
+
+ ptrdiff_t sizeBuffer = r2c_plan.local_n0/howmany;
+ for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
+ sizeBuffer *= n[idxrnk];
+ }
+ sizeBuffer *= n[rnk-1]+2;
+
+ r2c_plan.buffer.reset(alloc_real(sizeBuffer));
+ // Init the plan
+ r2c_plan.plan_to_use = mpi_plan_dft_r2c(rnk, n,
+ r2c_plan.buffer.get(),
+ (complex*)r2c_plan.buffer.get(),
+ comm, flags);
+
+
+ r2c_plan.nb_real_to_copy = r2c_plan.local_n0/howmany;
+ r2c_plan.nb_complex_to_copy = r2c_plan.local_n0/howmany;
+ for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
+ r2c_plan.nb_real_to_copy *= n[idxrnk];
+ r2c_plan.nb_complex_to_copy *= n[idxrnk];
+ }
+ r2c_plan.nb_real_to_copy *= n[rnk-1];
+ r2c_plan.nb_complex_to_copy *= n[rnk-1]/2 + 1;
+
+ return r2c_plan;
+ }
+
+ static void execute(many_plan& in_plan){
+ if(in_plan.howmany == 1){
+ execute(in_plan.plan_to_use);
+ return;
+ }
+
+ const ptrdiff_t nb_to_copy_to_buffer = (in_plan.is_r2c?in_plan.nb_real_to_copy:in_plan.nb_complex_to_copy);
+ const ptrdiff_t nb_to_copy_from_buffer = (!in_plan.is_r2c?in_plan.nb_real_to_copy:in_plan.nb_complex_to_copy);
+
+ for(int idx_howmany = 0 ; idx_howmany < in_plan.howmany ; ++idx_howmany){
+ // Copy to buffer
+ if(in_plan.is_r2c){
+ real* dest = in_plan.buffer.get();
+ const real* src = ((const real*)in_plan.in)+idx_howmany;
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_to_buffer ; ++idx_copy){
+ dest[idx_copy] = src[idx_copy*in_plan.howmany];
+ }
+ }
+ else{
+ complex* dest = (complex*)in_plan.buffer.get();
+ const complex* src = ((const complex*)in_plan.in)+idx_howmany;
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_to_buffer ; ++idx_copy){
+ dest[idx_copy][0] = src[idx_copy*in_plan.howmany][0];
+ dest[idx_copy][1] = src[idx_copy*in_plan.howmany][1];
+ }
+ }
+
+ execute(in_plan.plan_to_use);
+ // Copy result from buffer
+ if(in_plan.is_r2c){
+ complex* dest = ((complex*)in_plan.in)+idx_howmany;
+ const complex* src = (const complex*)in_plan.buffer.get();
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_from_buffer ; ++idx_copy){
+ dest[idx_copy*in_plan.howmany][0] = src[idx_copy][0];
+ dest[idx_copy*in_plan.howmany][1] = src[idx_copy][1];
+ }
+ }
+ else{
+ real* dest = ((real*)in_plan.in)+idx_howmany;
+ const real* src = in_plan.buffer.get();
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_from_buffer ; ++idx_copy){
+ dest[idx_copy*in_plan.howmany] = src[idx_copy];
+ }
+ }
+ }
+ }
+
+ static void destroy_plan(many_plan& in_plan){
+ destroy_plan(in_plan.plan_to_use);
+ }
+#else
template <class ... Params>
static plan mpi_plan_many_dft_c2r(Params ... params){
return fftw_mpi_plan_many_dft_c2r(params...);
@@ -160,6 +589,17 @@ public:
static plan mpi_plan_many_dft_r2c(Params ... params){
return fftw_mpi_plan_many_dft_r2c(params...);
}
+#endif
+
+ template <class ... Params>
+ static plan mpi_plan_dft_c2r(Params ... params){
+ return fftw_mpi_plan_dft_c2r(params...);
+ }
+
+ template <class ... Params>
+ static plan mpi_plan_dft_r2c(Params ... params){
+ return fftw_mpi_plan_dft_r2c(params...);
+ }
template <class ... Params>
static plan mpi_plan_dft_c2r_3d(Params ... params){
diff --git a/bfps/cpp/field.cpp b/bfps/cpp/field.cpp
index 6f2ff938..e20207c1 100644
--- a/bfps/cpp/field.cpp
+++ b/bfps/cpp/field.cpp
@@ -77,7 +77,7 @@ field<rnumber, be, fc>::field(
ptrdiff_t local_n0, local_0_start;
ptrdiff_t local_n1, local_1_start;
//tmp_local_size = fftw_mpi_local_size_many_transposed(
- fftw_mpi_local_size_many_transposed(
+ fftw_interface<rnumber>::mpi_local_size_many_transposed(
3, nfftw, ncomp(fc),
FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK, this->comm,
&local_n0, &local_0_start,
diff --git a/bfps/cpp/field.hpp b/bfps/cpp/field.hpp
index 9a5ab1be..d8038517 100644
--- a/bfps/cpp/field.hpp
+++ b/bfps/cpp/field.hpp
@@ -72,8 +72,8 @@ class field
field_layout<fc> *clayout, *rlayout, *rmemlayout;
/* FFT plans */
- typename fftw_interface<rnumber>::plan c2r_plan;
- typename fftw_interface<rnumber>::plan r2c_plan;
+ typename fftw_interface<rnumber>::many_plan c2r_plan;
+ typename fftw_interface<rnumber>::many_plan r2c_plan;
unsigned fftw_plan_rigor;
/* HDF5 data types for arrays */
diff --git a/bfps/cpp/field_descriptor.cpp b/bfps/cpp/field_descriptor.cpp
index 20c63426..cb7da995 100644
--- a/bfps/cpp/field_descriptor.cpp
+++ b/bfps/cpp/field_descriptor.cpp
@@ -62,7 +62,7 @@ field_descriptor<rnumber>::field_descriptor(
ptrdiff_t local_n0, local_0_start;
for (int i = 0; i < this->ndims; i++)
nfftw[i] = n[i];
- this->local_size = fftw_mpi_local_size_many(
+ this->local_size = fftw_interface<rnumber>::mpi_local_size_many(
this->ndims,
&nfftw.front(),
1,
diff --git a/bfps/cpp/fluid_solver.cpp b/bfps/cpp/fluid_solver.cpp
index 31918610..7ec0c978 100644
--- a/bfps/cpp/fluid_solver.cpp
+++ b/bfps/cpp/fluid_solver.cpp
@@ -86,10 +86,10 @@ fluid_solver<rnumber>::fluid_solver(
this->rv[1] = fftw_interface<rnumber>::alloc_real(this->cd->local_size*2);
this->rv[2] = this->rv[1];
- this->c2r_vorticity = new typename fftw_interface<rnumber>::plan;
- this->r2c_vorticity = new typename fftw_interface<rnumber>::plan;
- this->c2r_velocity = new typename fftw_interface<rnumber>::plan;
- this->r2c_velocity = new typename fftw_interface<rnumber>::plan;
+ this->c2r_vorticity = new typename fftw_interface<rnumber>::many_plan;
+ this->r2c_vorticity = new typename fftw_interface<rnumber>::many_plan;
+ this->c2r_velocity = new typename fftw_interface<rnumber>::many_plan;
+ this->r2c_velocity = new typename fftw_interface<rnumber>::many_plan;
ptrdiff_t sizes[] = {nz,
ny,
@@ -120,10 +120,10 @@ fluid_solver<rnumber>::fluid_solver(
this->vc2r[0] = this->c2r_vorticity;
this->vr2c[0] = this->r2c_vorticity;
- this->vc2r[1] = new typename fftw_interface<rnumber>::plan;
- this->vr2c[1] = new typename fftw_interface<rnumber>::plan;
- this->vc2r[2] = new typename fftw_interface<rnumber>::plan;
- this->vr2c[2] = new typename fftw_interface<rnumber>::plan;
+ this->vc2r[1] = new typename fftw_interface<rnumber>::many_plan;
+ this->vr2c[1] = new typename fftw_interface<rnumber>::many_plan;
+ this->vc2r[2] = new typename fftw_interface<rnumber>::many_plan;
+ this->vr2c[2] = new typename fftw_interface<rnumber>::many_plan;
*(this->vc2r[1]) = fftw_interface<rnumber>::mpi_plan_many_dft_c2r(
3, sizes, 3, FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
diff --git a/bfps/cpp/fluid_solver.hpp b/bfps/cpp/fluid_solver.hpp
index 4cc75cee..aaddbb59 100644
--- a/bfps/cpp/fluid_solver.hpp
+++ b/bfps/cpp/fluid_solver.hpp
@@ -55,12 +55,12 @@ class fluid_solver:public fluid_solver_base<rnumber>
typename fluid_solver_base<rnumber>::cnumber *cu, *cv[4];
/* plans */
- typename fftw_interface<rnumber>::plan *c2r_vorticity;
- typename fftw_interface<rnumber>::plan *r2c_vorticity;
- typename fftw_interface<rnumber>::plan *c2r_velocity;
- typename fftw_interface<rnumber>::plan *r2c_velocity;
- typename fftw_interface<rnumber>::plan *uc2r, *ur2c;
- typename fftw_interface<rnumber>::plan *vr2c[3], *vc2r[3];
+ typename fftw_interface<rnumber>::many_plan *c2r_vorticity;
+ typename fftw_interface<rnumber>::many_plan *r2c_vorticity;
+ typename fftw_interface<rnumber>::many_plan *c2r_velocity;
+ typename fftw_interface<rnumber>::many_plan *r2c_velocity;
+ typename fftw_interface<rnumber>::many_plan *uc2r, *ur2c;
+ typename fftw_interface<rnumber>::many_plan *vr2c[3], *vc2r[3];
/* physical parameters */
double nu;
diff --git a/bfps/cpp/fluid_solver_base.cpp b/bfps/cpp/fluid_solver_base.cpp
index 6e4fd333..b1d64ef5 100644
--- a/bfps/cpp/fluid_solver_base.cpp
+++ b/bfps/cpp/fluid_solver_base.cpp
@@ -52,7 +52,7 @@ void fluid_solver_base<rnumber>::clean_up_real_space(rnumber *a, int howmany)
template <class rnumber>
double fluid_solver_base<rnumber>::autocorrel(cnumber *a)
{
- double *spec = fftw_alloc_real(this->nshells*9);
+ double *spec = fftw_interface<double>::alloc_real(this->nshells*9);
double sum_local;
this->cospectrum(a, a, spec);
sum_local = 0.0;
@@ -60,7 +60,7 @@ double fluid_solver_base<rnumber>::autocorrel(cnumber *a)
{
sum_local += spec[n*9] + spec[n*9 + 4] + spec[n*9 + 8];
}
- fftw_free(spec);
+ fftw_interface<double>::free(spec);
return sum_local;
}
@@ -427,7 +427,7 @@ template <class rnumber>
void fluid_solver_base<rnumber>::write_spectrum(const char *fname, cnumber *a, const double k2exponent)
{
TIMEZONE("fluid_solver_base::write_spectrum");
- double *spec = fftw_alloc_real(this->nshells);
+ double *spec = fftw_interface<double>::alloc_real(this->nshells);
this->cospectrum(a, a, spec, k2exponent);
if (this->cd->myrank == 0)
{
@@ -439,7 +439,7 @@ void fluid_solver_base<rnumber>::write_spectrum(const char *fname, cnumber *a, c
fwrite((void*)spec, sizeof(double), this->nshells, spec_file);
fclose(spec_file);
}
- fftw_free(spec);
+ fftw_interface<double>::free(spec);
}
/*****************************************************************************/
diff --git a/bfps/cpp/slab_field_particles.cpp b/bfps/cpp/slab_field_particles.cpp
index 15fa363f..e3c84574 100644
--- a/bfps/cpp/slab_field_particles.cpp
+++ b/bfps/cpp/slab_field_particles.cpp
@@ -69,11 +69,11 @@ slab_field_particles<rnumber>::slab_field_particles(
this->buffer_width = this->interp_neighbours+1;
this->buffer_size = this->buffer_width*this->fs->rd->slice_size;
this->array_size = this->nparticles * this->ncomponents;
- this->state = fftw_alloc_real(this->array_size);
+ this->state = fftw_interface<rnumber>::alloc_real(this->array_size);
std::fill_n(this->state, this->array_size, 0.0);
for (int i=0; i < this->integration_steps; i++)
{
- this->rhs[i] = fftw_alloc_real(this->array_size);
+ this->rhs[i] = fftw_interface<rnumber>::alloc_real(this->array_size);
std::fill_n(this->rhs[i], this->array_size, 0.0);
}
this->watching = new bool[this->fs->rd->nprocs*nparticles];
@@ -131,10 +131,10 @@ slab_field_particles<rnumber>::~slab_field_particles()
{
delete[] this->computing;
delete[] this->watching;
- fftw_free(this->state);
+ fftw_interface<rnumber>::free(this->state);
for (int i=0; i < this->integration_steps; i++)
{
- fftw_free(this->rhs[i]);
+ fftw_interface<rnumber>::free(this->rhs[i]);
}
delete[] this->lbound;
delete[] this->ubound;
@@ -193,7 +193,7 @@ void slab_field_particles<rnumber>::synchronize_single_particle_state(int p, dou
template <class rnumber>
void slab_field_particles<rnumber>::synchronize()
{
- double *tstate = fftw_alloc_real(this->array_size);
+ double *tstate = fftw_interface<double>::alloc_real(this->array_size);
// first, synchronize state and jump across CPUs
std::fill_n(tstate, this->array_size, 0.0);
for (int p=0; p<this->nparticles; p++)
@@ -236,14 +236,14 @@ void slab_field_particles<rnumber>::synchronize()
this->fs->rd->comm);
}
}
- fftw_free(tstate);
+ fftw_interface<double>::free(tstate);
// assignment of particles
for (int p=0; p<this->nparticles; p++)
{
this->computing[p] = this->get_rank(this->state[p*this->ncomponents + 2]);
//DEBUG_MSG("synchronizing particles, particle %d computing is %d\n", p, this->computing[p]);
}
- double *jump = fftw_alloc_real(this->nparticles);
+ double *jump = fftw_interface<double>::alloc_real(this->nparticles);
this->jump_estimate(jump);
// now, see who needs to watch
bool *local_watching = new bool[this->fs->rd->nprocs*this->nparticles];
@@ -255,7 +255,7 @@ void slab_field_particles<rnumber>::synchronize()
local_watching[this->get_rank(this->state[this->ncomponents*p+2]-jump[p])*this->nparticles+p] = true;
local_watching[this->get_rank(this->state[this->ncomponents*p+2]+jump[p])*this->nparticles+p] = true;
}
- fftw_free(jump);
+ fftw_interface<double>::free(jump);
MPI_Allreduce(
local_watching,
this->watching,
@@ -389,7 +389,7 @@ void slab_field_particles<rnumber>::step()
template <class rnumber>
void slab_field_particles<rnumber>::Euler()
{
- double *y = fftw_alloc_real(this->array_size);
+ double *y = fftw_interface<double>::alloc_real(this->array_size);
this->get_rhs(this->state, y);
for (int p=0; p<this->nparticles; p++) if (this->fs->rd->myrank == this->computing[p])
{
@@ -399,7 +399,7 @@ void slab_field_particles<rnumber>::Euler()
// "particle %d state is %lg %lg %lg\n",
// p, this->state[p*this->ncomponents], this->state[p*this->ncomponents+1], this->state[p*this->ncomponents+2]);
}
- fftw_free(y);
+ fftw_interface<double>::free(y);
}
--
GitLab
From 00f9a47c70afd849a276e7de2c10e6d2681edec2 Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Fri, 29 Sep 2017 16:18:14 +0200
Subject: [PATCH 027/342] Initialize allocated memory and use local1 for
complex
---
bfps/cpp/fftw_interface.hpp | 13 +++++++++----
1 file changed, 9 insertions(+), 4 deletions(-)
diff --git a/bfps/cpp/fftw_interface.hpp b/bfps/cpp/fftw_interface.hpp
index 682850c8..7d56d336 100644
--- a/bfps/cpp/fftw_interface.hpp
+++ b/bfps/cpp/fftw_interface.hpp
@@ -41,6 +41,7 @@
#include <memory>
#include <algorithm>
#include <cassert>
+#include <cstring>
#endif
template <class realtype>
@@ -171,6 +172,7 @@ public:
sizeBuffer *= n[rnk-1]+2;
c2r_plan.buffer.reset(alloc_real(sizeBuffer));
+ memset(c2r_plan.buffer.get(), 0, sizeof(real)*sizeBuffer);
// Init the plan
c2r_plan.plan_to_use = mpi_plan_dft_c2r(rnk, n,
(complex*)c2r_plan.buffer.get(),
@@ -178,7 +180,7 @@ public:
comm, flags);
c2r_plan.nb_real_to_copy = c2r_plan.local_n0/howmany;
- c2r_plan.nb_complex_to_copy = c2r_plan.local_n0/howmany;
+ c2r_plan.nb_complex_to_copy = c2r_plan.local_n1/howmany;
for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
c2r_plan.nb_real_to_copy *= n[idxrnk];
c2r_plan.nb_complex_to_copy *= n[idxrnk];
@@ -229,6 +231,7 @@ public:
sizeBuffer *= n[rnk-1]+2;
r2c_plan.buffer.reset(alloc_real(sizeBuffer));
+ memset(r2c_plan.buffer.get(), 0, sizeof(real)*sizeBuffer);
// Init the plan
r2c_plan.plan_to_use = mpi_plan_dft_r2c(rnk, n,
r2c_plan.buffer.get(),
@@ -237,7 +240,7 @@ public:
r2c_plan.nb_real_to_copy = r2c_plan.local_n0/howmany;
- r2c_plan.nb_complex_to_copy = r2c_plan.local_n0/howmany;
+ r2c_plan.nb_complex_to_copy = r2c_plan.local_n1/howmany;
for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
r2c_plan.nb_real_to_copy *= n[idxrnk];
r2c_plan.nb_complex_to_copy *= n[idxrnk];
@@ -452,6 +455,7 @@ public:
sizeBuffer *= n[rnk-1]+2;
c2r_plan.buffer.reset(alloc_real(sizeBuffer));
+ memset(c2r_plan.buffer.get(), 0, sizeof(real)*sizeBuffer);
// Init the plan
c2r_plan.plan_to_use = mpi_plan_dft_c2r(rnk, n,
(complex*)c2r_plan.buffer.get(),
@@ -459,7 +463,7 @@ public:
comm, flags);
c2r_plan.nb_real_to_copy = c2r_plan.local_n0/howmany;
- c2r_plan.nb_complex_to_copy = c2r_plan.local_n0/howmany;
+ c2r_plan.nb_complex_to_copy = c2r_plan.local_n1/howmany;
for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
c2r_plan.nb_real_to_copy *= n[idxrnk];
c2r_plan.nb_complex_to_copy *= n[idxrnk];
@@ -510,6 +514,7 @@ public:
sizeBuffer *= n[rnk-1]+2;
r2c_plan.buffer.reset(alloc_real(sizeBuffer));
+ memset(r2c_plan.buffer.get(), 0, sizeof(real)*sizeBuffer);
// Init the plan
r2c_plan.plan_to_use = mpi_plan_dft_r2c(rnk, n,
r2c_plan.buffer.get(),
@@ -518,7 +523,7 @@ public:
r2c_plan.nb_real_to_copy = r2c_plan.local_n0/howmany;
- r2c_plan.nb_complex_to_copy = r2c_plan.local_n0/howmany;
+ r2c_plan.nb_complex_to_copy = r2c_plan.local_n1/howmany;
for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
r2c_plan.nb_real_to_copy *= n[idxrnk];
r2c_plan.nb_complex_to_copy *= n[idxrnk];
--
GitLab
From 5ba28dffe718f3a523a0354b9d9d0ccc18c69f9e Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Fri, 29 Sep 2017 16:36:13 +0200
Subject: [PATCH 028/342] Do not divide by local size (so I do not know where
howmany is located)
---
bfps/cpp/fftw_interface.hpp | 24 ++++++++++++------------
1 file changed, 12 insertions(+), 12 deletions(-)
diff --git a/bfps/cpp/fftw_interface.hpp b/bfps/cpp/fftw_interface.hpp
index 7d56d336..6753e658 100644
--- a/bfps/cpp/fftw_interface.hpp
+++ b/bfps/cpp/fftw_interface.hpp
@@ -165,7 +165,7 @@ public:
&c2r_plan.local_n0, &c2r_plan.local_0_start,
&c2r_plan.local_n1, &c2r_plan.local_1_start);
- ptrdiff_t sizeBuffer = c2r_plan.local_n0/howmany;
+ ptrdiff_t sizeBuffer = c2r_plan.local_n0;
for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
sizeBuffer *= n[idxrnk];
}
@@ -179,8 +179,8 @@ public:
c2r_plan.buffer.get(),
comm, flags);
- c2r_plan.nb_real_to_copy = c2r_plan.local_n0/howmany;
- c2r_plan.nb_complex_to_copy = c2r_plan.local_n1/howmany;
+ c2r_plan.nb_real_to_copy = c2r_plan.local_n0;
+ c2r_plan.nb_complex_to_copy = c2r_plan.local_n1;
for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
c2r_plan.nb_real_to_copy *= n[idxrnk];
c2r_plan.nb_complex_to_copy *= n[idxrnk];
@@ -224,7 +224,7 @@ public:
&r2c_plan.local_n0, &r2c_plan.local_0_start,
&r2c_plan.local_n1, &r2c_plan.local_1_start);
- ptrdiff_t sizeBuffer = r2c_plan.local_n0/howmany;
+ ptrdiff_t sizeBuffer = r2c_plan.local_n0;
for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
sizeBuffer *= n[idxrnk];
}
@@ -239,8 +239,8 @@ public:
comm, flags);
- r2c_plan.nb_real_to_copy = r2c_plan.local_n0/howmany;
- r2c_plan.nb_complex_to_copy = r2c_plan.local_n1/howmany;
+ r2c_plan.nb_real_to_copy = r2c_plan.local_n0;
+ r2c_plan.nb_complex_to_copy = r2c_plan.local_n1;
for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
r2c_plan.nb_real_to_copy *= n[idxrnk];
r2c_plan.nb_complex_to_copy *= n[idxrnk];
@@ -448,7 +448,7 @@ public:
&c2r_plan.local_n0, &c2r_plan.local_0_start,
&c2r_plan.local_n1, &c2r_plan.local_1_start);
- ptrdiff_t sizeBuffer = c2r_plan.local_n0/howmany;
+ ptrdiff_t sizeBuffer = c2r_plan.local_n0;
for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
sizeBuffer *= n[idxrnk];
}
@@ -462,8 +462,8 @@ public:
c2r_plan.buffer.get(),
comm, flags);
- c2r_plan.nb_real_to_copy = c2r_plan.local_n0/howmany;
- c2r_plan.nb_complex_to_copy = c2r_plan.local_n1/howmany;
+ c2r_plan.nb_real_to_copy = c2r_plan.local_n0;
+ c2r_plan.nb_complex_to_copy = c2r_plan.local_n1;
for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
c2r_plan.nb_real_to_copy *= n[idxrnk];
c2r_plan.nb_complex_to_copy *= n[idxrnk];
@@ -507,7 +507,7 @@ public:
&r2c_plan.local_n0, &r2c_plan.local_0_start,
&r2c_plan.local_n1, &r2c_plan.local_1_start);
- ptrdiff_t sizeBuffer = r2c_plan.local_n0/howmany;
+ ptrdiff_t sizeBuffer = r2c_plan.local_n0;
for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
sizeBuffer *= n[idxrnk];
}
@@ -522,8 +522,8 @@ public:
comm, flags);
- r2c_plan.nb_real_to_copy = r2c_plan.local_n0/howmany;
- r2c_plan.nb_complex_to_copy = r2c_plan.local_n1/howmany;
+ r2c_plan.nb_real_to_copy = r2c_plan.local_n0;
+ r2c_plan.nb_complex_to_copy = r2c_plan.local_n1;
for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
r2c_plan.nb_real_to_copy *= n[idxrnk];
r2c_plan.nb_complex_to_copy *= n[idxrnk];
--
GitLab
From a2c931078fe71a7f8698ab6c8eefab5258cf894c Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Fri, 29 Sep 2017 17:05:15 +0200
Subject: [PATCH 029/342] Update indexing
---
bfps/cpp/fftw_interface.hpp | 82 ++++++++++++++++++++++++++-----------
1 file changed, 58 insertions(+), 24 deletions(-)
diff --git a/bfps/cpp/fftw_interface.hpp b/bfps/cpp/fftw_interface.hpp
index 6753e658..ed8dbbf2 100644
--- a/bfps/cpp/fftw_interface.hpp
+++ b/bfps/cpp/fftw_interface.hpp
@@ -170,6 +170,7 @@ public:
sizeBuffer *= n[idxrnk];
}
sizeBuffer *= n[rnk-1]+2;
+ sizeBuffer *= rnk;
c2r_plan.buffer.reset(alloc_real(sizeBuffer));
memset(c2r_plan.buffer.get(), 0, sizeof(real)*sizeBuffer);
@@ -183,10 +184,11 @@ public:
c2r_plan.nb_complex_to_copy = c2r_plan.local_n1;
for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
c2r_plan.nb_real_to_copy *= n[idxrnk];
- c2r_plan.nb_complex_to_copy *= n[idxrnk];
+ c2r_plan.nb_complex_to_copy *= n[(idxrnk==1)?0:idxrnk];
}
c2r_plan.nb_real_to_copy *= n[rnk-1];
c2r_plan.nb_complex_to_copy *= n[rnk-1]/2 + 1;
+ assert(c2r_plan.nb_real_to_copy*rnk <= sizeBuffer);
return c2r_plan;
}
@@ -229,6 +231,7 @@ public:
sizeBuffer *= n[idxrnk];
}
sizeBuffer *= n[rnk-1]+2;
+ sizeBuffer *= rnk;
r2c_plan.buffer.reset(alloc_real(sizeBuffer));
memset(r2c_plan.buffer.get(), 0, sizeof(real)*sizeBuffer);
@@ -243,10 +246,11 @@ public:
r2c_plan.nb_complex_to_copy = r2c_plan.local_n1;
for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
r2c_plan.nb_real_to_copy *= n[idxrnk];
- r2c_plan.nb_complex_to_copy *= n[idxrnk];
+ r2c_plan.nb_complex_to_copy *= n[(idxrnk==1)?0:idxrnk];
}
r2c_plan.nb_real_to_copy *= n[rnk-1];
r2c_plan.nb_complex_to_copy *= n[rnk-1]/2 + 1;
+ assert(r2c_plan.nb_real_to_copy*rnk <= sizeBuffer);
return r2c_plan;
}
@@ -264,35 +268,49 @@ public:
// Copy to buffer
if(in_plan.is_r2c){
real* dest = in_plan.buffer.get();
- const real* src = ((const real*)in_plan.in)+idx_howmany;
+ const real* src = ((const real*)in_plan.in)+idx_howmany*in_plan.rnk;
for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_to_buffer ; ++idx_copy){
- dest[idx_copy] = src[idx_copy*in_plan.howmany];
+ for(int idx_n = 0 ; idx_n < in_plan.rnk ; ++idx_n){
+ dest[idx_copy*in_plan.rnk + idx_n] = src[idx_copy*in_plan.howmany*in_plan.rnk
+ +idx_howmany*in_plan.rnk + idx_n];
+ }
}
}
else{
complex* dest = (complex*)in_plan.buffer.get();
- const complex* src = ((const complex*)in_plan.in)+idx_howmany;
+ const complex* src = ((const complex*)in_plan.in)+idx_howmany*in_plan.rnk;
for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_to_buffer ; ++idx_copy){
- dest[idx_copy][0] = src[idx_copy*in_plan.howmany][0];
- dest[idx_copy][1] = src[idx_copy*in_plan.howmany][1];
+ for(int idx_n = 0 ; idx_n < in_plan.rnk ; ++idx_n){
+ dest[idx_copy*in_plan.rnk + idx_n][0] = src[idx_copy*in_plan.howmany*in_plan.rnk
+ +idx_howmany*in_plan.rnk + idx_n][0];
+ dest[idx_copy*in_plan.rnk + idx_n][1] = src[idx_copy*in_plan.howmany*in_plan.rnk
+ +idx_howmany*in_plan.rnk + idx_n][1];
+ }
}
}
execute(in_plan.plan_to_use);
// Copy result from buffer
if(in_plan.is_r2c){
- complex* dest = ((complex*)in_plan.in)+idx_howmany;
+ complex* dest = ((complex*)in_plan.in)+idx_howmany*in_plan.rnk;
const complex* src = (const complex*)in_plan.buffer.get();
for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_from_buffer ; ++idx_copy){
- dest[idx_copy*in_plan.howmany][0] = src[idx_copy][0];
- dest[idx_copy*in_plan.howmany][1] = src[idx_copy][1];
+ for(int idx_n = 0 ; idx_n < in_plan.rnk ; ++idx_n){
+ dest[idx_copy*in_plan.howmany*in_plan.rnk
+ +idx_howmany*in_plan.rnk + idx_n][0] = src[idx_copy*in_plan.rnk + idx_n][0];
+ dest[idx_copy*in_plan.howmany*in_plan.rnk
+ +idx_howmany*in_plan.rnk + idx_n][1] = src[idx_copy*in_plan.rnk + idx_n][1];
+ }
}
}
else{
- real* dest = ((real*)in_plan.in)+idx_howmany;
+ real* dest = ((real*)in_plan.in)+idx_howmany*in_plan.rnk;
const real* src = in_plan.buffer.get();
for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_from_buffer ; ++idx_copy){
- dest[idx_copy*in_plan.howmany] = src[idx_copy];
+ for(int idx_n = 0 ; idx_n < in_plan.rnk ; ++idx_n){
+ dest[idx_copy*in_plan.howmany*in_plan.rnk
+ +idx_howmany*in_plan.rnk + idx_n] = src[idx_copy*in_plan.rnk + idx_n];
+ }
}
}
}
@@ -453,6 +471,7 @@ public:
sizeBuffer *= n[idxrnk];
}
sizeBuffer *= n[rnk-1]+2;
+ sizeBuffer *= rnk;
c2r_plan.buffer.reset(alloc_real(sizeBuffer));
memset(c2r_plan.buffer.get(), 0, sizeof(real)*sizeBuffer);
@@ -466,7 +485,7 @@ public:
c2r_plan.nb_complex_to_copy = c2r_plan.local_n1;
for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
c2r_plan.nb_real_to_copy *= n[idxrnk];
- c2r_plan.nb_complex_to_copy *= n[idxrnk];
+ c2r_plan.nb_complex_to_copy *= n[(idxrnk==1)?0:idxrnk];
}
c2r_plan.nb_real_to_copy *= n[rnk-1];
c2r_plan.nb_complex_to_copy *= n[rnk-1]/2 + 1;
@@ -512,6 +531,7 @@ public:
sizeBuffer *= n[idxrnk];
}
sizeBuffer *= n[rnk-1]+2;
+ sizeBuffer *= rnk;
r2c_plan.buffer.reset(alloc_real(sizeBuffer));
memset(r2c_plan.buffer.get(), 0, sizeof(real)*sizeBuffer);
@@ -526,7 +546,7 @@ public:
r2c_plan.nb_complex_to_copy = r2c_plan.local_n1;
for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
r2c_plan.nb_real_to_copy *= n[idxrnk];
- r2c_plan.nb_complex_to_copy *= n[idxrnk];
+ r2c_plan.nb_complex_to_copy *= n[(idxrnk==1)?0:idxrnk];
}
r2c_plan.nb_real_to_copy *= n[rnk-1];
r2c_plan.nb_complex_to_copy *= n[rnk-1]/2 + 1;
@@ -547,35 +567,49 @@ public:
// Copy to buffer
if(in_plan.is_r2c){
real* dest = in_plan.buffer.get();
- const real* src = ((const real*)in_plan.in)+idx_howmany;
+ const real* src = ((const real*)in_plan.in)+idx_howmany*in_plan.rnk;
for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_to_buffer ; ++idx_copy){
- dest[idx_copy] = src[idx_copy*in_plan.howmany];
+ for(int idx_n = 0 ; idx_n < in_plan.rnk ; ++idx_n){
+ dest[idx_copy*in_plan.rnk + idx_n] = src[idx_copy*in_plan.howmany*in_plan.rnk
+ +idx_howmany*in_plan.rnk + idx_n];
+ }
}
}
else{
complex* dest = (complex*)in_plan.buffer.get();
- const complex* src = ((const complex*)in_plan.in)+idx_howmany;
+ const complex* src = ((const complex*)in_plan.in)+idx_howmany*in_plan.rnk;
for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_to_buffer ; ++idx_copy){
- dest[idx_copy][0] = src[idx_copy*in_plan.howmany][0];
- dest[idx_copy][1] = src[idx_copy*in_plan.howmany][1];
+ for(int idx_n = 0 ; idx_n < in_plan.rnk ; ++idx_n){
+ dest[idx_copy*in_plan.rnk + idx_n][0] = src[idx_copy*in_plan.howmany*in_plan.rnk
+ +idx_howmany*in_plan.rnk + idx_n][0];
+ dest[idx_copy*in_plan.rnk + idx_n][1] = src[idx_copy*in_plan.howmany*in_plan.rnk
+ +idx_howmany*in_plan.rnk + idx_n][1];
+ }
}
}
execute(in_plan.plan_to_use);
// Copy result from buffer
if(in_plan.is_r2c){
- complex* dest = ((complex*)in_plan.in)+idx_howmany;
+ complex* dest = ((complex*)in_plan.in)+idx_howmany*in_plan.rnk;
const complex* src = (const complex*)in_plan.buffer.get();
for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_from_buffer ; ++idx_copy){
- dest[idx_copy*in_plan.howmany][0] = src[idx_copy][0];
- dest[idx_copy*in_plan.howmany][1] = src[idx_copy][1];
+ for(int idx_n = 0 ; idx_n < in_plan.rnk ; ++idx_n){
+ dest[idx_copy*in_plan.howmany*in_plan.rnk
+ +idx_howmany*in_plan.rnk + idx_n][0] = src[idx_copy*in_plan.rnk + idx_n][0];
+ dest[idx_copy*in_plan.howmany*in_plan.rnk
+ +idx_howmany*in_plan.rnk + idx_n][1] = src[idx_copy*in_plan.rnk + idx_n][1];
+ }
}
}
else{
- real* dest = ((real*)in_plan.in)+idx_howmany;
+ real* dest = ((real*)in_plan.in)+idx_howmany*in_plan.rnk;
const real* src = in_plan.buffer.get();
for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_from_buffer ; ++idx_copy){
- dest[idx_copy*in_plan.howmany] = src[idx_copy];
+ for(int idx_n = 0 ; idx_n < in_plan.rnk ; ++idx_n){
+ dest[idx_copy*in_plan.howmany*in_plan.rnk
+ +idx_howmany*in_plan.rnk + idx_n] = src[idx_copy*in_plan.rnk + idx_n];
+ }
}
}
}
--
GitLab
From 8b0d2997ae064e3254fd8c87b091e01d94729262 Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Mon, 2 Oct 2017 13:00:58 +0200
Subject: [PATCH 030/342] Update to correct memory access (still problem in the
result that makes the particules not moving correctly)
---
bfps/cpp/fftw_interface.hpp | 102 ++++++++++++++++--------------------
1 file changed, 44 insertions(+), 58 deletions(-)
diff --git a/bfps/cpp/fftw_interface.hpp b/bfps/cpp/fftw_interface.hpp
index ed8dbbf2..91f8a002 100644
--- a/bfps/cpp/fftw_interface.hpp
+++ b/bfps/cpp/fftw_interface.hpp
@@ -74,6 +74,8 @@ public:
ptrdiff_t nb_real_to_copy;
ptrdiff_t nb_complex_to_copy;
+
+ ptrdiff_t sizeBuffer;
};
using many_plan = many_plan_container;
@@ -148,6 +150,7 @@ public:
c2r_plan.is_r2c = false;
c2r_plan.in = in;
c2r_plan.out = out;
+ c2r_plan.sizeBuffer = 0;
// If 1 then use default without copy
if(howmany == 1){
@@ -170,10 +173,10 @@ public:
sizeBuffer *= n[idxrnk];
}
sizeBuffer *= n[rnk-1]+2;
- sizeBuffer *= rnk;
c2r_plan.buffer.reset(alloc_real(sizeBuffer));
memset(c2r_plan.buffer.get(), 0, sizeof(real)*sizeBuffer);
+ c2r_plan.sizeBuffer = sizeBuffer;
// Init the plan
c2r_plan.plan_to_use = mpi_plan_dft_c2r(rnk, n,
(complex*)c2r_plan.buffer.get(),
@@ -186,9 +189,10 @@ public:
c2r_plan.nb_real_to_copy *= n[idxrnk];
c2r_plan.nb_complex_to_copy *= n[(idxrnk==1)?0:idxrnk];
}
- c2r_plan.nb_real_to_copy *= n[rnk-1];
+ c2r_plan.nb_real_to_copy *= n[rnk-1] + 2;
c2r_plan.nb_complex_to_copy *= n[rnk-1]/2 + 1;
- assert(c2r_plan.nb_real_to_copy*rnk <= sizeBuffer);
+ assert(c2r_plan.nb_real_to_copy == sizeBuffer);
+ assert(c2r_plan.nb_complex_to_copy <= sizeBuffer/2);
return c2r_plan;
}
@@ -209,6 +213,7 @@ public:
r2c_plan.is_r2c = true;
r2c_plan.in = in;
r2c_plan.out = out;
+ r2c_plan.sizeBuffer = 0;
// If 1 then use default without copy
if(howmany == 1){
@@ -231,10 +236,10 @@ public:
sizeBuffer *= n[idxrnk];
}
sizeBuffer *= n[rnk-1]+2;
- sizeBuffer *= rnk;
r2c_plan.buffer.reset(alloc_real(sizeBuffer));
memset(r2c_plan.buffer.get(), 0, sizeof(real)*sizeBuffer);
+ r2c_plan.sizeBuffer = sizeBuffer;
// Init the plan
r2c_plan.plan_to_use = mpi_plan_dft_r2c(rnk, n,
r2c_plan.buffer.get(),
@@ -248,9 +253,10 @@ public:
r2c_plan.nb_real_to_copy *= n[idxrnk];
r2c_plan.nb_complex_to_copy *= n[(idxrnk==1)?0:idxrnk];
}
- r2c_plan.nb_real_to_copy *= n[rnk-1];
+ r2c_plan.nb_real_to_copy *= n[rnk-1] + 2;
r2c_plan.nb_complex_to_copy *= n[rnk-1]/2 + 1;
- assert(r2c_plan.nb_real_to_copy*rnk <= sizeBuffer);
+ assert(r2c_plan.nb_real_to_copy == sizeBuffer);
+ assert(r2c_plan.nb_complex_to_copy <= sizeBuffer/2);
return r2c_plan;
}
@@ -268,49 +274,35 @@ public:
// Copy to buffer
if(in_plan.is_r2c){
real* dest = in_plan.buffer.get();
- const real* src = ((const real*)in_plan.in)+idx_howmany*in_plan.rnk;
+ const real* src = ((const real*)in_plan.in)+idx_howmany;
for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_to_buffer ; ++idx_copy){
- for(int idx_n = 0 ; idx_n < in_plan.rnk ; ++idx_n){
- dest[idx_copy*in_plan.rnk + idx_n] = src[idx_copy*in_plan.howmany*in_plan.rnk
- +idx_howmany*in_plan.rnk + idx_n];
- }
+ dest[idx_copy] = src[idx_copy*in_plan.howmany];
}
}
else{
complex* dest = (complex*)in_plan.buffer.get();
- const complex* src = ((const complex*)in_plan.in)+idx_howmany*in_plan.rnk;
+ const complex* src = ((const complex*)in_plan.in)+idx_howmany;
for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_to_buffer ; ++idx_copy){
- for(int idx_n = 0 ; idx_n < in_plan.rnk ; ++idx_n){
- dest[idx_copy*in_plan.rnk + idx_n][0] = src[idx_copy*in_plan.howmany*in_plan.rnk
- +idx_howmany*in_plan.rnk + idx_n][0];
- dest[idx_copy*in_plan.rnk + idx_n][1] = src[idx_copy*in_plan.howmany*in_plan.rnk
- +idx_howmany*in_plan.rnk + idx_n][1];
- }
+ dest[idx_copy][0] = src[idx_copy*in_plan.howmany][0];
+ dest[idx_copy][1] = src[idx_copy*in_plan.howmany][1];
}
}
execute(in_plan.plan_to_use);
// Copy result from buffer
if(in_plan.is_r2c){
- complex* dest = ((complex*)in_plan.in)+idx_howmany*in_plan.rnk;
+ complex* dest = ((complex*)in_plan.in)+idx_howmany;
const complex* src = (const complex*)in_plan.buffer.get();
for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_from_buffer ; ++idx_copy){
- for(int idx_n = 0 ; idx_n < in_plan.rnk ; ++idx_n){
- dest[idx_copy*in_plan.howmany*in_plan.rnk
- +idx_howmany*in_plan.rnk + idx_n][0] = src[idx_copy*in_plan.rnk + idx_n][0];
- dest[idx_copy*in_plan.howmany*in_plan.rnk
- +idx_howmany*in_plan.rnk + idx_n][1] = src[idx_copy*in_plan.rnk + idx_n][1];
- }
+ dest[idx_copy*in_plan.howmany][0] = src[idx_copy][0];
+ dest[idx_copy*in_plan.howmany][1] = src[idx_copy][1];
}
}
else{
- real* dest = ((real*)in_plan.in)+idx_howmany*in_plan.rnk;
+ real* dest = ((real*)in_plan.in)+idx_howmany;
const real* src = in_plan.buffer.get();
for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_from_buffer ; ++idx_copy){
- for(int idx_n = 0 ; idx_n < in_plan.rnk ; ++idx_n){
- dest[idx_copy*in_plan.howmany*in_plan.rnk
- +idx_howmany*in_plan.rnk + idx_n] = src[idx_copy*in_plan.rnk + idx_n];
- }
+ dest[idx_copy*in_plan.howmany] = src[idx_copy];
}
}
}
@@ -374,6 +366,8 @@ public:
ptrdiff_t nb_real_to_copy;
ptrdiff_t nb_complex_to_copy;
+
+ ptrdiff_t sizeBuffer;
};
using many_plan = many_plan_container;
@@ -449,6 +443,7 @@ public:
c2r_plan.is_r2c = false;
c2r_plan.in = in;
c2r_plan.out = out;
+ c2r_plan.sizeBuffer = 0;
// If 1 then use default without copy
if(howmany == 1){
@@ -471,10 +466,10 @@ public:
sizeBuffer *= n[idxrnk];
}
sizeBuffer *= n[rnk-1]+2;
- sizeBuffer *= rnk;
c2r_plan.buffer.reset(alloc_real(sizeBuffer));
memset(c2r_plan.buffer.get(), 0, sizeof(real)*sizeBuffer);
+ c2r_plan.sizeBuffer = sizeBuffer;
// Init the plan
c2r_plan.plan_to_use = mpi_plan_dft_c2r(rnk, n,
(complex*)c2r_plan.buffer.get(),
@@ -487,8 +482,10 @@ public:
c2r_plan.nb_real_to_copy *= n[idxrnk];
c2r_plan.nb_complex_to_copy *= n[(idxrnk==1)?0:idxrnk];
}
- c2r_plan.nb_real_to_copy *= n[rnk-1];
+ c2r_plan.nb_real_to_copy *= n[rnk-1] + 2;
c2r_plan.nb_complex_to_copy *= n[rnk-1]/2 + 1;
+ assert(c2r_plan.nb_real_to_copy == sizeBuffer);
+ assert(c2r_plan.nb_complex_to_copy <= sizeBuffer/2);
return c2r_plan;
}
@@ -509,6 +506,7 @@ public:
r2c_plan.is_r2c = true;
r2c_plan.in = in;
r2c_plan.out = out;
+ r2c_plan.sizeBuffer = 0;
// If 1 then use default without copy
if(howmany == 1){
@@ -531,10 +529,10 @@ public:
sizeBuffer *= n[idxrnk];
}
sizeBuffer *= n[rnk-1]+2;
- sizeBuffer *= rnk;
r2c_plan.buffer.reset(alloc_real(sizeBuffer));
memset(r2c_plan.buffer.get(), 0, sizeof(real)*sizeBuffer);
+ r2c_plan.sizeBuffer = sizeBuffer;
// Init the plan
r2c_plan.plan_to_use = mpi_plan_dft_r2c(rnk, n,
r2c_plan.buffer.get(),
@@ -548,8 +546,10 @@ public:
r2c_plan.nb_real_to_copy *= n[idxrnk];
r2c_plan.nb_complex_to_copy *= n[(idxrnk==1)?0:idxrnk];
}
- r2c_plan.nb_real_to_copy *= n[rnk-1];
+ r2c_plan.nb_real_to_copy *= n[rnk-1] + 2;
r2c_plan.nb_complex_to_copy *= n[rnk-1]/2 + 1;
+ assert(r2c_plan.nb_real_to_copy == sizeBuffer);
+ assert(r2c_plan.nb_complex_to_copy <= sizeBuffer/2);
return r2c_plan;
}
@@ -567,49 +567,35 @@ public:
// Copy to buffer
if(in_plan.is_r2c){
real* dest = in_plan.buffer.get();
- const real* src = ((const real*)in_plan.in)+idx_howmany*in_plan.rnk;
+ const real* src = ((const real*)in_plan.in)+idx_howmany;
for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_to_buffer ; ++idx_copy){
- for(int idx_n = 0 ; idx_n < in_plan.rnk ; ++idx_n){
- dest[idx_copy*in_plan.rnk + idx_n] = src[idx_copy*in_plan.howmany*in_plan.rnk
- +idx_howmany*in_plan.rnk + idx_n];
- }
+ dest[idx_copy] = src[idx_copy*in_plan.howmany];
}
}
else{
complex* dest = (complex*)in_plan.buffer.get();
- const complex* src = ((const complex*)in_plan.in)+idx_howmany*in_plan.rnk;
+ const complex* src = ((const complex*)in_plan.in)+idx_howmany;
for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_to_buffer ; ++idx_copy){
- for(int idx_n = 0 ; idx_n < in_plan.rnk ; ++idx_n){
- dest[idx_copy*in_plan.rnk + idx_n][0] = src[idx_copy*in_plan.howmany*in_plan.rnk
- +idx_howmany*in_plan.rnk + idx_n][0];
- dest[idx_copy*in_plan.rnk + idx_n][1] = src[idx_copy*in_plan.howmany*in_plan.rnk
- +idx_howmany*in_plan.rnk + idx_n][1];
- }
+ dest[idx_copy][0] = src[idx_copy*in_plan.howmany][0];
+ dest[idx_copy][1] = src[idx_copy*in_plan.howmany][1];
}
}
execute(in_plan.plan_to_use);
// Copy result from buffer
if(in_plan.is_r2c){
- complex* dest = ((complex*)in_plan.in)+idx_howmany*in_plan.rnk;
+ complex* dest = ((complex*)in_plan.in)+idx_howmany;
const complex* src = (const complex*)in_plan.buffer.get();
for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_from_buffer ; ++idx_copy){
- for(int idx_n = 0 ; idx_n < in_plan.rnk ; ++idx_n){
- dest[idx_copy*in_plan.howmany*in_plan.rnk
- +idx_howmany*in_plan.rnk + idx_n][0] = src[idx_copy*in_plan.rnk + idx_n][0];
- dest[idx_copy*in_plan.howmany*in_plan.rnk
- +idx_howmany*in_plan.rnk + idx_n][1] = src[idx_copy*in_plan.rnk + idx_n][1];
- }
+ dest[idx_copy*in_plan.howmany][0] = src[idx_copy][0];
+ dest[idx_copy*in_plan.howmany][1] = src[idx_copy][1];
}
}
else{
- real* dest = ((real*)in_plan.in)+idx_howmany*in_plan.rnk;
+ real* dest = ((real*)in_plan.in)+idx_howmany;
const real* src = in_plan.buffer.get();
for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_from_buffer ; ++idx_copy){
- for(int idx_n = 0 ; idx_n < in_plan.rnk ; ++idx_n){
- dest[idx_copy*in_plan.howmany*in_plan.rnk
- +idx_howmany*in_plan.rnk + idx_n] = src[idx_copy*in_plan.rnk + idx_n];
- }
+ dest[idx_copy*in_plan.howmany] = src[idx_copy];
}
}
}
--
GitLab
From 83fb9f9aa89028f8b2865584fc668659c1c5de9b Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Mon, 2 Oct 2017 15:28:52 +0200
Subject: [PATCH 031/342] Copy in buffer completly otherwise it is overwritten
---
bfps/cpp/fftw_interface.hpp | 294 ++++++++++++++++++++++++++----------
1 file changed, 217 insertions(+), 77 deletions(-)
diff --git a/bfps/cpp/fftw_interface.hpp b/bfps/cpp/fftw_interface.hpp
index 91f8a002..4e86f63e 100644
--- a/bfps/cpp/fftw_interface.hpp
+++ b/bfps/cpp/fftw_interface.hpp
@@ -72,8 +72,10 @@ public:
void* in;
void* out;
- ptrdiff_t nb_real_to_copy;
- ptrdiff_t nb_complex_to_copy;
+ ptrdiff_t nb_sections_real;
+ ptrdiff_t size_real_section;
+ ptrdiff_t nb_sections_complex;
+ ptrdiff_t size_complex_section;
ptrdiff_t sizeBuffer;
};
@@ -167,6 +169,16 @@ public:
FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK, comm,
&c2r_plan.local_n0, &c2r_plan.local_0_start,
&c2r_plan.local_n1, &c2r_plan.local_1_start);
+ if(rnk == 3){
+ ptrdiff_t local_n0, local_0_start, local_n1, local_1_start;
+ fftw_mpi_local_size_3d_transposed(n[0], n[1], n[2], comm,
+ &local_n0, &local_0_start,
+ &local_n1, &local_1_start);
+ assert(c2r_plan.local_n0 == local_n0);
+ assert(c2r_plan.local_0_start == local_0_start);
+ assert(c2r_plan.local_n1 == local_n1);
+ assert(c2r_plan.local_1_start == local_1_start);
+ }
ptrdiff_t sizeBuffer = c2r_plan.local_n0;
for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
@@ -183,16 +195,23 @@ public:
c2r_plan.buffer.get(),
comm, flags);
- c2r_plan.nb_real_to_copy = c2r_plan.local_n0;
- c2r_plan.nb_complex_to_copy = c2r_plan.local_n1;
+ c2r_plan.nb_sections_real = c2r_plan.local_n0;
+ for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
+ c2r_plan.nb_sections_real *= n[idxrnk];
+ c2r_plan.nb_sections_complex *= n[idxrnk];
+ }
+ c2r_plan.size_real_section = (n[rnk-1] + 2);
+
+ c2r_plan.nb_sections_complex = c2r_plan.local_n1;
for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
- c2r_plan.nb_real_to_copy *= n[idxrnk];
- c2r_plan.nb_complex_to_copy *= n[(idxrnk==1)?0:idxrnk];
+ if(idxrnk == 1){
+ c2r_plan.nb_sections_complex *= n[0];
+ }
+ else{
+ c2r_plan.nb_sections_complex *= n[idxrnk];
+ }
}
- c2r_plan.nb_real_to_copy *= n[rnk-1] + 2;
- c2r_plan.nb_complex_to_copy *= n[rnk-1]/2 + 1;
- assert(c2r_plan.nb_real_to_copy == sizeBuffer);
- assert(c2r_plan.nb_complex_to_copy <= sizeBuffer/2);
+ c2r_plan.size_complex_section = (n[rnk-1]/2 + 1);
return c2r_plan;
}
@@ -230,6 +249,16 @@ public:
FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK, comm,
&r2c_plan.local_n0, &r2c_plan.local_0_start,
&r2c_plan.local_n1, &r2c_plan.local_1_start);
+ if(rnk == 3){
+ ptrdiff_t local_n0, local_0_start, local_n1, local_1_start;
+ fftw_mpi_local_size_3d_transposed(n[0], n[1], n[2], comm,
+ &local_n0, &local_0_start,
+ &local_n1, &local_1_start);
+ assert(r2c_plan.local_n0 == local_n0);
+ assert(r2c_plan.local_0_start == local_0_start);
+ assert(r2c_plan.local_n1 == local_n1);
+ assert(r2c_plan.local_1_start == local_1_start);
+ }
ptrdiff_t sizeBuffer = r2c_plan.local_n0;
for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
@@ -246,17 +275,23 @@ public:
(complex*)r2c_plan.buffer.get(),
comm, flags);
+ r2c_plan.nb_sections_real = r2c_plan.local_n0;
+ for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
+ r2c_plan.nb_sections_real *= n[idxrnk];
+ r2c_plan.nb_sections_complex *= n[idxrnk];
+ }
+ r2c_plan.size_real_section = (n[rnk-1] + 2);
- r2c_plan.nb_real_to_copy = r2c_plan.local_n0;
- r2c_plan.nb_complex_to_copy = r2c_plan.local_n1;
+ r2c_plan.nb_sections_complex = r2c_plan.local_n1;
for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
- r2c_plan.nb_real_to_copy *= n[idxrnk];
- r2c_plan.nb_complex_to_copy *= n[(idxrnk==1)?0:idxrnk];
+ if(idxrnk == 1){
+ r2c_plan.nb_sections_complex *= n[0];
+ }
+ else{
+ r2c_plan.nb_sections_complex *= n[idxrnk];
+ }
}
- r2c_plan.nb_real_to_copy *= n[rnk-1] + 2;
- r2c_plan.nb_complex_to_copy *= n[rnk-1]/2 + 1;
- assert(r2c_plan.nb_real_to_copy == sizeBuffer);
- assert(r2c_plan.nb_complex_to_copy <= sizeBuffer/2);
+ r2c_plan.size_complex_section = (n[rnk-1]/2 + 1);
return r2c_plan;
}
@@ -267,42 +302,77 @@ public:
return;
}
- const ptrdiff_t nb_to_copy_to_buffer = (in_plan.is_r2c?in_plan.nb_real_to_copy:in_plan.nb_complex_to_copy);
- const ptrdiff_t nb_to_copy_from_buffer = (!in_plan.is_r2c?in_plan.nb_real_to_copy:in_plan.nb_complex_to_copy);
+ std::unique_ptr<real[]> in_copy;
+ if(in_plan.is_r2c){
+ in_copy.reset(new real[in_plan.nb_sections_real * in_plan.size_real_section * in_plan.howmany]);
+
+ for(int idx_section = 0 ; idx_section < in_plan.nb_sections_real ; ++idx_section){
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < in_plan.n[in_plan.rnk-1] ; ++idx_copy){
+ for(int idx_howmany = 0 ; idx_howmany < in_plan.howmany ; ++idx_howmany){
+ in_copy[idx_howmany + idx_copy*in_plan.howmany + idx_section*in_plan.size_real_section*in_plan.howmany] =
+ ((const real*)in_plan.in)[idx_howmany + idx_copy*in_plan.howmany + idx_section*in_plan.size_real_section*in_plan.howmany];
+ }
+ }
+ }
+ }
+ else{
+ in_copy.reset((real*)new complex[in_plan.nb_sections_complex * in_plan.size_complex_section * in_plan.howmany]);
+
+ for(int idx_section = 0 ; idx_section < in_plan.nb_sections_complex ; ++idx_section){
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < in_plan.n[in_plan.rnk-1]/2+1 ; ++idx_copy){
+ for(int idx_howmany = 0 ; idx_howmany < in_plan.howmany ; ++idx_howmany){
+ ((complex*)in_copy.get())[idx_howmany + idx_copy*in_plan.howmany + idx_section*in_plan.size_complex_section*in_plan.howmany][0] =
+ ((const complex*)in_plan.in)[idx_howmany + idx_copy*in_plan.howmany + idx_section*in_plan.size_complex_section*in_plan.howmany][0];
+ ((complex*)in_copy.get())[idx_howmany + idx_copy*in_plan.howmany + idx_section*in_plan.size_complex_section*in_plan.howmany][1] =
+ ((const complex*)in_plan.in)[idx_howmany + idx_copy*in_plan.howmany + idx_section*in_plan.size_complex_section*in_plan.howmany][1];
+ }
+ }
+ }
+ }
for(int idx_howmany = 0 ; idx_howmany < in_plan.howmany ; ++idx_howmany){
// Copy to buffer
if(in_plan.is_r2c){
- real* dest = in_plan.buffer.get();
- const real* src = ((const real*)in_plan.in)+idx_howmany;
- for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_to_buffer ; ++idx_copy){
- dest[idx_copy] = src[idx_copy*in_plan.howmany];
+ for(int idx_section = 0 ; idx_section < in_plan.nb_sections_real ; ++idx_section){
+ real* dest = in_plan.buffer.get() + idx_section*in_plan.size_real_section;
+ const real* src = in_copy.get()+idx_howmany + idx_section*in_plan.size_real_section*in_plan.howmany;
+
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < in_plan.n[in_plan.rnk-1] ; ++idx_copy){
+ dest[idx_copy] = src[idx_copy*in_plan.howmany];
+ }
}
}
else{
- complex* dest = (complex*)in_plan.buffer.get();
- const complex* src = ((const complex*)in_plan.in)+idx_howmany;
- for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_to_buffer ; ++idx_copy){
- dest[idx_copy][0] = src[idx_copy*in_plan.howmany][0];
- dest[idx_copy][1] = src[idx_copy*in_plan.howmany][1];
+ for(int idx_section = 0 ; idx_section < in_plan.nb_sections_complex ; ++idx_section){
+ complex* dest = ((complex*)in_plan.buffer.get()) + idx_section*in_plan.size_complex_section;
+ const complex* src = ((const complex*)in_copy.get()) + idx_howmany + idx_section*in_plan.size_complex_section*in_plan.howmany;
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < in_plan.n[in_plan.rnk-1]/2+1 ; ++idx_copy){
+ dest[idx_copy][0] = src[idx_copy*in_plan.howmany][0];
+ dest[idx_copy][1] = src[idx_copy*in_plan.howmany][1];
+ }
}
}
execute(in_plan.plan_to_use);
// Copy result from buffer
if(in_plan.is_r2c){
- complex* dest = ((complex*)in_plan.in)+idx_howmany;
- const complex* src = (const complex*)in_plan.buffer.get();
- for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_from_buffer ; ++idx_copy){
- dest[idx_copy*in_plan.howmany][0] = src[idx_copy][0];
- dest[idx_copy*in_plan.howmany][1] = src[idx_copy][1];
+ for(int idx_section = 0 ; idx_section < in_plan.nb_sections_complex ; ++idx_section){
+ complex* dest = ((complex*)in_plan.out) + idx_howmany + idx_section*in_plan.size_complex_section*in_plan.howmany;
+ const complex* src = ((const complex*)in_plan.buffer.get()) + idx_section*in_plan.size_complex_section;
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < in_plan.n[in_plan.rnk-1]/2+1 ; ++idx_copy){
+ dest[idx_copy*in_plan.howmany][0] = src[idx_copy][0];
+ dest[idx_copy*in_plan.howmany][1] = src[idx_copy][1];
+ }
}
}
else{
- real* dest = ((real*)in_plan.in)+idx_howmany;
- const real* src = in_plan.buffer.get();
- for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_from_buffer ; ++idx_copy){
- dest[idx_copy*in_plan.howmany] = src[idx_copy];
+ for(int idx_section = 0 ; idx_section < in_plan.nb_sections_real ; ++idx_section){
+ real* dest = ((real*)in_plan.out)+idx_howmany + idx_section*in_plan.size_real_section*in_plan.howmany;
+ const real* src = in_plan.buffer.get() + idx_section*in_plan.size_real_section;
+
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < in_plan.n[in_plan.rnk-1] ; ++idx_copy){
+ dest[idx_copy*in_plan.howmany] = src[idx_copy];
+ }
}
}
}
@@ -364,8 +434,10 @@ public:
void* in;
void* out;
- ptrdiff_t nb_real_to_copy;
- ptrdiff_t nb_complex_to_copy;
+ ptrdiff_t nb_sections_real;
+ ptrdiff_t size_real_section;
+ ptrdiff_t nb_sections_complex;
+ ptrdiff_t size_complex_section;
ptrdiff_t sizeBuffer;
};
@@ -426,7 +498,7 @@ public:
}
-#ifdef SPLIT_FFTW_MANY
+#ifdef SPLIT_FFTW_MANY
static many_plan mpi_plan_many_dft_c2r(int rnk, const ptrdiff_t *n, ptrdiff_t howmany,
ptrdiff_t iblock, ptrdiff_t oblock,
complex *in, real *out,
@@ -460,6 +532,16 @@ public:
FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK, comm,
&c2r_plan.local_n0, &c2r_plan.local_0_start,
&c2r_plan.local_n1, &c2r_plan.local_1_start);
+ if(rnk == 3){
+ ptrdiff_t local_n0, local_0_start, local_n1, local_1_start;
+ fftw_mpi_local_size_3d_transposed(n[0], n[1], n[2], comm,
+ &local_n0, &local_0_start,
+ &local_n1, &local_1_start);
+ assert(c2r_plan.local_n0 == local_n0);
+ assert(c2r_plan.local_0_start == local_0_start);
+ assert(c2r_plan.local_n1 == local_n1);
+ assert(c2r_plan.local_1_start == local_1_start);
+ }
ptrdiff_t sizeBuffer = c2r_plan.local_n0;
for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
@@ -476,16 +558,23 @@ public:
c2r_plan.buffer.get(),
comm, flags);
- c2r_plan.nb_real_to_copy = c2r_plan.local_n0;
- c2r_plan.nb_complex_to_copy = c2r_plan.local_n1;
+ c2r_plan.nb_sections_real = c2r_plan.local_n0;
for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
- c2r_plan.nb_real_to_copy *= n[idxrnk];
- c2r_plan.nb_complex_to_copy *= n[(idxrnk==1)?0:idxrnk];
+ c2r_plan.nb_sections_real *= n[idxrnk];
+ c2r_plan.nb_sections_complex *= n[idxrnk];
}
- c2r_plan.nb_real_to_copy *= n[rnk-1] + 2;
- c2r_plan.nb_complex_to_copy *= n[rnk-1]/2 + 1;
- assert(c2r_plan.nb_real_to_copy == sizeBuffer);
- assert(c2r_plan.nb_complex_to_copy <= sizeBuffer/2);
+ c2r_plan.size_real_section = (n[rnk-1] + 2);
+
+ c2r_plan.nb_sections_complex = c2r_plan.local_n1;
+ for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
+ if(idxrnk == 1){
+ c2r_plan.nb_sections_complex *= n[0];
+ }
+ else{
+ c2r_plan.nb_sections_complex *= n[idxrnk];
+ }
+ }
+ c2r_plan.size_complex_section = (n[rnk-1]/2 + 1);
return c2r_plan;
}
@@ -523,6 +612,16 @@ public:
FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK, comm,
&r2c_plan.local_n0, &r2c_plan.local_0_start,
&r2c_plan.local_n1, &r2c_plan.local_1_start);
+ if(rnk == 3){
+ ptrdiff_t local_n0, local_0_start, local_n1, local_1_start;
+ fftw_mpi_local_size_3d_transposed(n[0], n[1], n[2], comm,
+ &local_n0, &local_0_start,
+ &local_n1, &local_1_start);
+ assert(r2c_plan.local_n0 == local_n0);
+ assert(r2c_plan.local_0_start == local_0_start);
+ assert(r2c_plan.local_n1 == local_n1);
+ assert(r2c_plan.local_1_start == local_1_start);
+ }
ptrdiff_t sizeBuffer = r2c_plan.local_n0;
for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
@@ -539,17 +638,23 @@ public:
(complex*)r2c_plan.buffer.get(),
comm, flags);
+ r2c_plan.nb_sections_real = r2c_plan.local_n0;
+ for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
+ r2c_plan.nb_sections_real *= n[idxrnk];
+ r2c_plan.nb_sections_complex *= n[idxrnk];
+ }
+ r2c_plan.size_real_section = (n[rnk-1] + 2);
- r2c_plan.nb_real_to_copy = r2c_plan.local_n0;
- r2c_plan.nb_complex_to_copy = r2c_plan.local_n1;
+ r2c_plan.nb_sections_complex = r2c_plan.local_n1;
for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
- r2c_plan.nb_real_to_copy *= n[idxrnk];
- r2c_plan.nb_complex_to_copy *= n[(idxrnk==1)?0:idxrnk];
+ if(idxrnk == 1){
+ r2c_plan.nb_sections_complex *= n[0];
+ }
+ else{
+ r2c_plan.nb_sections_complex *= n[idxrnk];
+ }
}
- r2c_plan.nb_real_to_copy *= n[rnk-1] + 2;
- r2c_plan.nb_complex_to_copy *= n[rnk-1]/2 + 1;
- assert(r2c_plan.nb_real_to_copy == sizeBuffer);
- assert(r2c_plan.nb_complex_to_copy <= sizeBuffer/2);
+ r2c_plan.size_complex_section = (n[rnk-1]/2 + 1);
return r2c_plan;
}
@@ -560,42 +665,77 @@ public:
return;
}
- const ptrdiff_t nb_to_copy_to_buffer = (in_plan.is_r2c?in_plan.nb_real_to_copy:in_plan.nb_complex_to_copy);
- const ptrdiff_t nb_to_copy_from_buffer = (!in_plan.is_r2c?in_plan.nb_real_to_copy:in_plan.nb_complex_to_copy);
+ std::unique_ptr<real[]> in_copy;
+ if(in_plan.is_r2c){
+ in_copy.reset(new real[in_plan.nb_sections_real * in_plan.size_real_section * in_plan.howmany]);
+
+ for(int idx_section = 0 ; idx_section < in_plan.nb_sections_real ; ++idx_section){
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < in_plan.n[in_plan.rnk-1] ; ++idx_copy){
+ for(int idx_howmany = 0 ; idx_howmany < in_plan.howmany ; ++idx_howmany){
+ in_copy[idx_howmany + idx_copy*in_plan.howmany + idx_section*in_plan.size_real_section*in_plan.howmany] =
+ ((const real*)in_plan.in)[idx_howmany + idx_copy*in_plan.howmany + idx_section*in_plan.size_real_section*in_plan.howmany];
+ }
+ }
+ }
+ }
+ else{
+ in_copy.reset((real*)new complex[in_plan.nb_sections_complex * in_plan.size_complex_section * in_plan.howmany]);
+
+ for(int idx_section = 0 ; idx_section < in_plan.nb_sections_complex ; ++idx_section){
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < in_plan.n[in_plan.rnk-1]/2+1 ; ++idx_copy){
+ for(int idx_howmany = 0 ; idx_howmany < in_plan.howmany ; ++idx_howmany){
+ ((complex*)in_copy.get())[idx_howmany + idx_copy*in_plan.howmany + idx_section*in_plan.size_complex_section*in_plan.howmany][0] =
+ ((const complex*)in_plan.in)[idx_howmany + idx_copy*in_plan.howmany + idx_section*in_plan.size_complex_section*in_plan.howmany][0];
+ ((complex*)in_copy.get())[idx_howmany + idx_copy*in_plan.howmany + idx_section*in_plan.size_complex_section*in_plan.howmany][1] =
+ ((const complex*)in_plan.in)[idx_howmany + idx_copy*in_plan.howmany + idx_section*in_plan.size_complex_section*in_plan.howmany][1];
+ }
+ }
+ }
+ }
for(int idx_howmany = 0 ; idx_howmany < in_plan.howmany ; ++idx_howmany){
// Copy to buffer
if(in_plan.is_r2c){
- real* dest = in_plan.buffer.get();
- const real* src = ((const real*)in_plan.in)+idx_howmany;
- for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_to_buffer ; ++idx_copy){
- dest[idx_copy] = src[idx_copy*in_plan.howmany];
+ for(int idx_section = 0 ; idx_section < in_plan.nb_sections_real ; ++idx_section){
+ real* dest = in_plan.buffer.get() + idx_section*in_plan.size_real_section;
+ const real* src = in_copy.get()+idx_howmany + idx_section*in_plan.size_real_section*in_plan.howmany;
+
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < in_plan.n[in_plan.rnk-1] ; ++idx_copy){
+ dest[idx_copy] = src[idx_copy*in_plan.howmany];
+ }
}
}
else{
- complex* dest = (complex*)in_plan.buffer.get();
- const complex* src = ((const complex*)in_plan.in)+idx_howmany;
- for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_to_buffer ; ++idx_copy){
- dest[idx_copy][0] = src[idx_copy*in_plan.howmany][0];
- dest[idx_copy][1] = src[idx_copy*in_plan.howmany][1];
+ for(int idx_section = 0 ; idx_section < in_plan.nb_sections_complex ; ++idx_section){
+ complex* dest = ((complex*)in_plan.buffer.get()) + idx_section*in_plan.size_complex_section;
+ const complex* src = ((const complex*)in_copy.get()) + idx_howmany + idx_section*in_plan.size_complex_section*in_plan.howmany;
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < in_plan.n[in_plan.rnk-1]/2+1 ; ++idx_copy){
+ dest[idx_copy][0] = src[idx_copy*in_plan.howmany][0];
+ dest[idx_copy][1] = src[idx_copy*in_plan.howmany][1];
+ }
}
}
execute(in_plan.plan_to_use);
// Copy result from buffer
if(in_plan.is_r2c){
- complex* dest = ((complex*)in_plan.in)+idx_howmany;
- const complex* src = (const complex*)in_plan.buffer.get();
- for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_from_buffer ; ++idx_copy){
- dest[idx_copy*in_plan.howmany][0] = src[idx_copy][0];
- dest[idx_copy*in_plan.howmany][1] = src[idx_copy][1];
+ for(int idx_section = 0 ; idx_section < in_plan.nb_sections_complex ; ++idx_section){
+ complex* dest = ((complex*)in_plan.out) + idx_howmany + idx_section*in_plan.size_complex_section*in_plan.howmany;
+ const complex* src = ((const complex*)in_plan.buffer.get()) + idx_section*in_plan.size_complex_section;
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < in_plan.n[in_plan.rnk-1]/2+1 ; ++idx_copy){
+ dest[idx_copy*in_plan.howmany][0] = src[idx_copy][0];
+ dest[idx_copy*in_plan.howmany][1] = src[idx_copy][1];
+ }
}
}
else{
- real* dest = ((real*)in_plan.in)+idx_howmany;
- const real* src = in_plan.buffer.get();
- for(ptrdiff_t idx_copy = 0 ; idx_copy < nb_to_copy_from_buffer ; ++idx_copy){
- dest[idx_copy*in_plan.howmany] = src[idx_copy];
+ for(int idx_section = 0 ; idx_section < in_plan.nb_sections_real ; ++idx_section){
+ real* dest = ((real*)in_plan.out)+idx_howmany + idx_section*in_plan.size_real_section*in_plan.howmany;
+ const real* src = in_plan.buffer.get() + idx_section*in_plan.size_real_section;
+
+ for(ptrdiff_t idx_copy = 0 ; idx_copy < in_plan.n[in_plan.rnk-1] ; ++idx_copy){
+ dest[idx_copy*in_plan.howmany] = src[idx_copy];
+ }
}
}
}
--
GitLab
From 4ffc2d91537d55fe75527037dbb792ea6ac25acc Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Mon, 2 Oct 2017 16:22:44 +0200
Subject: [PATCH 032/342] Use a deleter to capture the free of the unique ptr
allocated with fftw - update to use only ref of the field in particle system
(avoid copy or move)
---
bfps/cpp/fftw_interface.hpp | 14 ++++++++++++--
bfps/cpp/particles/particles_distr_mpi.hpp | 2 +-
bfps/cpp/particles/particles_system.hpp | 2 +-
3 files changed, 14 insertions(+), 4 deletions(-)
diff --git a/bfps/cpp/fftw_interface.hpp b/bfps/cpp/fftw_interface.hpp
index 4e86f63e..68eed567 100644
--- a/bfps/cpp/fftw_interface.hpp
+++ b/bfps/cpp/fftw_interface.hpp
@@ -42,6 +42,16 @@
#include <algorithm>
#include <cassert>
#include <cstring>
+#include <type_traits>
+
+// To mix unique ptr with allocation from fftw
+struct fftw_free_deleter{
+ template <typename T>
+ void operator()(T *p) const {
+ fftwf_free(const_cast<typename std::remove_const<T>::type*>(p));
+ }
+};
+
#endif
template <class realtype>
@@ -62,7 +72,7 @@ public:
int howmany;
ptrdiff_t iblock;
ptrdiff_t oblock;
- std::shared_ptr<real> buffer;
+ std::unique_ptr<real[], fftw_free_deleter> buffer;
plan plan_to_use;
ptrdiff_t local_n0, local_0_start;
@@ -424,7 +434,7 @@ public:
int howmany;
ptrdiff_t iblock;
ptrdiff_t oblock;
- std::shared_ptr<real> buffer;
+ std::unique_ptr<real[], fftw_free_deleter> buffer;
plan plan_to_use;
ptrdiff_t local_n0, local_0_start;
diff --git a/bfps/cpp/particles/particles_distr_mpi.hpp b/bfps/cpp/particles/particles_distr_mpi.hpp
index 48559518..ebc24750 100644
--- a/bfps/cpp/particles/particles_distr_mpi.hpp
+++ b/bfps/cpp/particles/particles_distr_mpi.hpp
@@ -136,7 +136,7 @@ public:
template <class computer_class, class field_class, int size_particle_positions, int size_particle_rhs>
void compute_distr(computer_class& in_computer,
- field_class& in_field,
+ const field_class& in_field,
const partsize_t current_my_nb_particles_per_partition[],
const real_number particles_positions[],
real_number particles_current_rhs[],
diff --git a/bfps/cpp/particles/particles_system.hpp b/bfps/cpp/particles/particles_system.hpp
index 02767a8b..081f4b4b 100644
--- a/bfps/cpp/particles/particles_system.hpp
+++ b/bfps/cpp/particles/particles_system.hpp
@@ -29,7 +29,7 @@ class particles_system : public abstract_particles_system<partsize_t, real_numbe
using computer_class = particles_field_computer<partsize_t, real_number, interpolator_class, interp_neighbours>;
computer_class computer;
- field_class default_field;
+ const field_class& default_field;
std::unique_ptr<partsize_t[]> current_my_nb_particles_per_partition;
std::unique_ptr<partsize_t[]> current_offset_particles_for_partition;
--
GitLab
From ae3ff38a26165ef5fc718c575f4b4046632a1462 Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Mon, 2 Oct 2017 16:49:41 +0200
Subject: [PATCH 033/342] Use the NOT many local size when the fftw call is
split too, in order to have the correct size (usefull for corner cases only)
---
bfps/cpp/fftw_interface.hpp | 116 ++++++++++++++++--------------------
1 file changed, 52 insertions(+), 64 deletions(-)
diff --git a/bfps/cpp/fftw_interface.hpp b/bfps/cpp/fftw_interface.hpp
index 68eed567..00c71d27 100644
--- a/bfps/cpp/fftw_interface.hpp
+++ b/bfps/cpp/fftw_interface.hpp
@@ -115,11 +115,6 @@ public:
fftwf_destroy_plan(in_plan);
}
- template <class ... Params>
- static ptrdiff_t mpi_local_size_many_transposed(Params ... params){
- return fftwf_mpi_local_size_many_transposed(params...);
- }
-
template <class ... Params>
static ptrdiff_t mpi_local_size_many(Params ... params){
return fftwf_mpi_local_size_many(params...);
@@ -145,7 +140,23 @@ public:
return fftwf_plan_guru_dft(params...);
}
+ template <class ... Params>
+ static ptrdiff_t mpi_local_size_transposed(Params ... params){
+ return fftwf_mpi_local_size_transposed(params...);
+ }
+
#ifdef SPLIT_FFTW_MANY
+ static ptrdiff_t mpi_local_size_many_transposed(int rnk, const ptrdiff_t *n, ptrdiff_t howmany,
+ ptrdiff_t block0, ptrdiff_t block1, MPI_Comm comm,
+ ptrdiff_t *local_n0, ptrdiff_t *local_0_start,
+ ptrdiff_t *local_n1, ptrdiff_t *local_1_start){
+ assert(block0 == FFTW_MPI_DEFAULT_BLOCK);
+ assert(block1 == FFTW_MPI_DEFAULT_BLOCK);
+ return howmany*mpi_local_size_transposed(rnk, n, comm,
+ local_n0, local_0_start,
+ local_n1, local_1_start);
+ }
+
static many_plan mpi_plan_many_dft_c2r(int rnk, const ptrdiff_t *n, ptrdiff_t howmany,
ptrdiff_t iblock, ptrdiff_t oblock,
complex *in, real *out,
@@ -174,21 +185,10 @@ public:
}
// We need to find out the size of the buffer to allocate
- mpi_local_size_many_transposed(
- rnk, n, howmany,
- FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK, comm,
+ mpi_local_size_transposed(
+ rnk, n, comm,
&c2r_plan.local_n0, &c2r_plan.local_0_start,
&c2r_plan.local_n1, &c2r_plan.local_1_start);
- if(rnk == 3){
- ptrdiff_t local_n0, local_0_start, local_n1, local_1_start;
- fftw_mpi_local_size_3d_transposed(n[0], n[1], n[2], comm,
- &local_n0, &local_0_start,
- &local_n1, &local_1_start);
- assert(c2r_plan.local_n0 == local_n0);
- assert(c2r_plan.local_0_start == local_0_start);
- assert(c2r_plan.local_n1 == local_n1);
- assert(c2r_plan.local_1_start == local_1_start);
- }
ptrdiff_t sizeBuffer = c2r_plan.local_n0;
for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
@@ -254,21 +254,10 @@ public:
}
// We need to find out the size of the buffer to allocate
- mpi_local_size_many_transposed(
- rnk, n, howmany,
- FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK, comm,
+ mpi_local_size_transposed(
+ rnk, n, comm,
&r2c_plan.local_n0, &r2c_plan.local_0_start,
&r2c_plan.local_n1, &r2c_plan.local_1_start);
- if(rnk == 3){
- ptrdiff_t local_n0, local_0_start, local_n1, local_1_start;
- fftw_mpi_local_size_3d_transposed(n[0], n[1], n[2], comm,
- &local_n0, &local_0_start,
- &local_n1, &local_1_start);
- assert(r2c_plan.local_n0 == local_n0);
- assert(r2c_plan.local_0_start == local_0_start);
- assert(r2c_plan.local_n1 == local_n1);
- assert(r2c_plan.local_1_start == local_1_start);
- }
ptrdiff_t sizeBuffer = r2c_plan.local_n0;
for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
@@ -392,6 +381,12 @@ public:
destroy_plan(in_plan.plan_to_use);
}
#else
+
+ template <class ... Params>
+ static ptrdiff_t mpi_local_size_many_transposed(Params ... params){
+ return fftwf_mpi_local_size_many_transposed(params...);
+ }
+
template <class ... Params>
static plan mpi_plan_many_dft_c2r(Params ... params){
return fftwf_mpi_plan_many_dft_c2r(params...);
@@ -477,11 +472,6 @@ public:
fftw_destroy_plan(in_plan);
}
- template <class ... Params>
- static ptrdiff_t mpi_local_size_many_transposed(Params ... params){
- return fftw_mpi_local_size_many_transposed(params...);
- }
-
template <class ... Params>
static ptrdiff_t mpi_local_size_many(Params ... params){
return fftw_mpi_local_size_many(params...);
@@ -507,8 +497,23 @@ public:
return fftw_plan_guru_dft(params...);
}
+ template <class ... Params>
+ static ptrdiff_t mpi_local_size_transposed(Params ... params){
+ return fftw_mpi_local_size_transposed(params...);
+ }
+
+#ifdef SPLIT_FFTW_MANY
+ static ptrdiff_t mpi_local_size_many_transposed(int rnk, const ptrdiff_t *n, ptrdiff_t howmany,
+ ptrdiff_t block0, ptrdiff_t block1, MPI_Comm comm,
+ ptrdiff_t *local_n0, ptrdiff_t *local_0_start,
+ ptrdiff_t *local_n1, ptrdiff_t *local_1_start){
+ assert(block0 == FFTW_MPI_DEFAULT_BLOCK);
+ assert(block1 == FFTW_MPI_DEFAULT_BLOCK);
+ return howmany*mpi_local_size_transposed(rnk, n, comm,
+ local_n0, local_0_start,
+ local_n1, local_1_start);
+ }
-#ifdef SPLIT_FFTW_MANY
static many_plan mpi_plan_many_dft_c2r(int rnk, const ptrdiff_t *n, ptrdiff_t howmany,
ptrdiff_t iblock, ptrdiff_t oblock,
complex *in, real *out,
@@ -537,21 +542,10 @@ public:
}
// We need to find out the size of the buffer to allocate
- mpi_local_size_many_transposed(
- rnk, n, howmany,
- FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK, comm,
+ mpi_local_size_transposed(
+ rnk, n, comm,
&c2r_plan.local_n0, &c2r_plan.local_0_start,
&c2r_plan.local_n1, &c2r_plan.local_1_start);
- if(rnk == 3){
- ptrdiff_t local_n0, local_0_start, local_n1, local_1_start;
- fftw_mpi_local_size_3d_transposed(n[0], n[1], n[2], comm,
- &local_n0, &local_0_start,
- &local_n1, &local_1_start);
- assert(c2r_plan.local_n0 == local_n0);
- assert(c2r_plan.local_0_start == local_0_start);
- assert(c2r_plan.local_n1 == local_n1);
- assert(c2r_plan.local_1_start == local_1_start);
- }
ptrdiff_t sizeBuffer = c2r_plan.local_n0;
for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
@@ -617,21 +611,10 @@ public:
}
// We need to find out the size of the buffer to allocate
- mpi_local_size_many_transposed(
- rnk, n, howmany,
- FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK, comm,
+ mpi_local_size_transposed(
+ rnk, n, comm,
&r2c_plan.local_n0, &r2c_plan.local_0_start,
&r2c_plan.local_n1, &r2c_plan.local_1_start);
- if(rnk == 3){
- ptrdiff_t local_n0, local_0_start, local_n1, local_1_start;
- fftw_mpi_local_size_3d_transposed(n[0], n[1], n[2], comm,
- &local_n0, &local_0_start,
- &local_n1, &local_1_start);
- assert(r2c_plan.local_n0 == local_n0);
- assert(r2c_plan.local_0_start == local_0_start);
- assert(r2c_plan.local_n1 == local_n1);
- assert(r2c_plan.local_1_start == local_1_start);
- }
ptrdiff_t sizeBuffer = r2c_plan.local_n0;
for(int idxrnk = 1 ; idxrnk < rnk-1 ; ++idxrnk){
@@ -754,7 +737,12 @@ public:
static void destroy_plan(many_plan& in_plan){
destroy_plan(in_plan.plan_to_use);
}
-#else
+#else
+ template <class ... Params>
+ static ptrdiff_t mpi_local_size_many_transposed(Params ... params){
+ return fftw_mpi_local_size_many_transposed(params...);
+ }
+
template <class ... Params>
static plan mpi_plan_many_dft_c2r(Params ... params){
return fftw_mpi_plan_many_dft_c2r(params...);
--
GitLab
From 5f78b2716635f981137db331426db10581681d6e Mon Sep 17 00:00:00 2001
From: Cristian Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 6 Oct 2017 09:02:35 +0200
Subject: [PATCH 034/342] fix ibm load leveller class usage
---
bfps/_code.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/bfps/_code.py b/bfps/_code.py
index 3791092a..6c098cd5 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -477,6 +477,8 @@ class _code(_base):
script_file.write('# @ output = ' + os.path.join(self.work_dir, out_file) + '\n')
# If Ibm is used should be : script_file.write('# @ job_type = parallel\n')
script_file.write('# @ job_type = MPICH\n')
+ assert(type(self.host_info['environment']) != type(None))
+ script_file.write('# @ class = {0}\n'.format(self.host_info['environment']))
script_file.write('# @ node_usage = not_shared\n')
script_file.write('#\n')
--
GitLab
From 920f623e4248ce1e878f285863a88a6a50cafac8 Mon Sep 17 00:00:00 2001
From: Cristian Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 6 Oct 2017 15:45:07 +0200
Subject: [PATCH 035/342] handle custom library inclusion
for linking against some particular libraries, it's necessary to add
"extra_libraries" such as '-Wl,--start-group'.
Therefore when the compile command is now constructed, "extra_libraries"
are checked to see whether they start with '-' or '/' (for static
libraries given as full file names).
---
bfps/_code.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/bfps/_code.py b/bfps/_code.py
index 6c098cd5..9064f972 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -214,7 +214,10 @@ class _code(_base):
command_strings.append('-Wl,-rpath=' + bfps.lib_dir)
for libname in libraries:
- command_strings += ['-l' + libname]
+ if libname[0] not in ['-', '/']:
+ command_strings += ['-l' + libname]
+ else:
+ command_strings += [libname]
command_strings += ['-fopenmp']
--
GitLab
From 1bab0435a0add6f3fac604913fa737a3fcf15144 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 10 Oct 2017 15:52:43 +0200
Subject: [PATCH 036/342] add preliminary particle orientation sync
---
bfps/cpp/particles/p2p_computer.hpp | 49 ++++++++++++++++++++++++++++-
1 file changed, 48 insertions(+), 1 deletion(-)
diff --git a/bfps/cpp/particles/p2p_computer.hpp b/bfps/cpp/particles/p2p_computer.hpp
index de024ff7..6611a088 100644
--- a/bfps/cpp/particles/p2p_computer.hpp
+++ b/bfps/cpp/particles/p2p_computer.hpp
@@ -3,6 +3,22 @@
#include <cstring>
+/** \brief A simple distance weighting function.
+ *
+ * This function returns 1 if a distance is smaller than a cut-off length,
+ * i.e. particle 1 interacts with particle 2 if particle 2 is inside a
+ * sphere of radius `cutoff' centered on particle 1.
+ */
+double dumb_distance_weight(
+ double distance,
+ double cutoff)
+{
+ // this function should only be called for interacting particles,
+ // and particles interact if they are closer than cutoff.
+ assert(distance < cutoff);
+ return 1.0;
+}
+
template <class real_number, class partsize_t>
class p2p_computer{
public:
@@ -27,10 +43,41 @@ public:
const real_number pos_part2[], const real_number data_part2[], real_number rhs_part2[],
const real_number dist_pow2,
const real_number xshift_coef, const real_number yshift_coef, const real_number zshift_coef) const{
- // TODO put the kernel here
static_assert(size_particle_positions == 3, "This kernel works only with 3 values for one position");
+ // TODO:
+ // Should I put 0 in the rhs corresponding to the orientations?
+ // Or should I just add the interaction term?
+ // In other words: is this method called after the vorticity and strain terms have been computed?
+ // The following two lines set the rhs to 0:
+ //std::fill_n(rhs_part1+3, 3, 0);
+ //std::fill_n(rhs_part2+3, 3, 0);
+ real_number distance = sqrt(dist_pow2);
+ double max_distance = 1.0;
+ double tau = 1.0;
+ if (distance >= max_distance)
+ return;
+ // TODO: a reasonable way of choosing between different distance_weight functions should be thought of.
+ // We need to ask Michael about how flexible this distance_weight needs to be.
+ double ww = dumb_distance_weight(distance, max_distance);
+ ///
+ /// term in equation is:
+ ///
+ /// \f[
+ /// (4 / \tau) \sum_j W_\ell ( | x^i - x^j | ) (p^i \cdot p^j)p^j
+ /// \f]
+ ///
+ double dot_product = (data_part1[0]*data_part2[0] +
+ data_part1[1]*data_part2[1] +
+ data_part1[2]*data_part2[2])
+ rhs_part1[3] += data_part2[0] * 4 * ww * dot_product;
+ rhs_part1[4] += data_part2[1] * 4 * ww * dot_product;
+ rhs_part1[5] += data_part2[2] * 4 * ww * dot_product;
+ rhs_part2[3] += data_part1[0] * 4 * ww * dot_product;
+ rhs_part2[4] += data_part1[1] * 4 * ww * dot_product;
+ rhs_part2[5] += data_part1[2] * 4 * ww * dot_product;
}
+
constexpr static bool isEmpty() {
return false;
}
--
GitLab
From 359ef9ab029b5ccc379c0fa7a8e018823aa04f23 Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Tue, 10 Oct 2017 16:49:18 +0200
Subject: [PATCH 037/342] update P2P computation
---
bfps/cpp/particles/p2p_computer.hpp | 23 +++++++----------------
1 file changed, 7 insertions(+), 16 deletions(-)
diff --git a/bfps/cpp/particles/p2p_computer.hpp b/bfps/cpp/particles/p2p_computer.hpp
index 6611a088..bd6a4001 100644
--- a/bfps/cpp/particles/p2p_computer.hpp
+++ b/bfps/cpp/particles/p2p_computer.hpp
@@ -43,22 +43,13 @@ public:
const real_number pos_part2[], const real_number data_part2[], real_number rhs_part2[],
const real_number dist_pow2,
const real_number xshift_coef, const real_number yshift_coef, const real_number zshift_coef) const{
- static_assert(size_particle_positions == 3, "This kernel works only with 3 values for one position");
- // TODO:
- // Should I put 0 in the rhs corresponding to the orientations?
- // Or should I just add the interaction term?
- // In other words: is this method called after the vorticity and strain terms have been computed?
- // The following two lines set the rhs to 0:
- //std::fill_n(rhs_part1+3, 3, 0);
- //std::fill_n(rhs_part2+3, 3, 0);
- real_number distance = sqrt(dist_pow2);
- double max_distance = 1.0;
- double tau = 1.0;
- if (distance >= max_distance)
- return;
+ static_assert(size_particle_positions == 3, "This kernel works only with 3 values for one particle's position");
+ static_assert(size_particle_rhs >= 6, "This kernel works only with more than 6 values per particle's rhs");
+ static_assert(size_particle_data == 3, "This kernel works only with 3 values per particle's' data");
+
// TODO: a reasonable way of choosing between different distance_weight functions should be thought of.
// We need to ask Michael about how flexible this distance_weight needs to be.
- double ww = dumb_distance_weight(distance, max_distance);
+ const double ww = dumb_distance_weight(distance, max_distance);
///
/// term in equation is:
///
@@ -66,9 +57,9 @@ public:
/// (4 / \tau) \sum_j W_\ell ( | x^i - x^j | ) (p^i \cdot p^j)p^j
/// \f]
///
- double dot_product = (data_part1[0]*data_part2[0] +
+ const double dot_product = (data_part1[0]*data_part2[0] +
data_part1[1]*data_part2[1] +
- data_part1[2]*data_part2[2])
+ data_part1[2]*data_part2[2]);
rhs_part1[3] += data_part2[0] * 4 * ww * dot_product;
rhs_part1[4] += data_part2[1] * 4 * ww * dot_product;
rhs_part1[5] += data_part2[2] * 4 * ww * dot_product;
--
GitLab
From 7d3bb09554c1981c7851d64f2cc06c502743fd1b Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Thu, 12 Oct 2017 17:07:27 +0200
Subject: [PATCH 038/342] Update the kernels and make it possible to update
parameters at runtime
---
bfps/cpp/field.hpp | 4 +
bfps/cpp/full_code/NSVEparticlesP2P.cpp | 25 ++++--
bfps/cpp/full_code/NSVEparticlesP2P.hpp | 10 ++-
.../particles/abstract_particles_system.hpp | 16 ++++
bfps/cpp/particles/p2p_computer.hpp | 84 ++++++++++---------
bfps/cpp/particles/p2p_computer_empty.hpp | 14 ++--
bfps/cpp/particles/p2p_distr_mpi.hpp | 49 ++---------
.../particles/particles_adams_bashforth.hpp | 29 +++++--
bfps/cpp/particles/particles_distr_mpi.hpp | 72 +---------------
.../particles/particles_field_computer.hpp | 6 +-
.../particles/particles_inner_computer.hpp | 65 ++++++++++++++
.../particles_inner_computer_empty.hpp | 24 ++++++
bfps/cpp/particles/particles_system.hpp | 79 +++++++++++------
.../particles/particles_system_builder.hpp | 43 +++++++---
14 files changed, 313 insertions(+), 207 deletions(-)
create mode 100644 bfps/cpp/particles/particles_inner_computer.hpp
create mode 100644 bfps/cpp/particles/particles_inner_computer_empty.hpp
diff --git a/bfps/cpp/field.hpp b/bfps/cpp/field.hpp
index 9a5ab1be..2d93a0c9 100644
--- a/bfps/cpp/field.hpp
+++ b/bfps/cpp/field.hpp
@@ -58,6 +58,10 @@ class field
private:
rnumber *__restrict__ data; /**< data array */
public:
+ constexpr int nb_components() {
+ return ncomp(fc);
+ }
+
hsize_t npoints; /**< total number of grid points. Useful for normalization. */
bool real_space_representation; /**< `true` if field is in real space representation. */
diff --git a/bfps/cpp/full_code/NSVEparticlesP2P.cpp b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
index 08326119..9574b226 100644
--- a/bfps/cpp/full_code/NSVEparticlesP2P.cpp
+++ b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
@@ -4,13 +4,20 @@
#include "scope_timer.hpp"
#include "particles/particles_sampling.hpp"
#include "particles/p2p_computer.hpp"
+#include "particles/particles_inner_computer.hpp"
template <typename rnumber>
int NSVEparticlesP2P<rnumber>::initialize(void)
{
this->NSVE<rnumber>::initialize();
- this->ps = particles_system_builder(
+ p2p_computer<rnumber, long long int> current_p2p_computer;
+ current_p2p_computer.setEnable(enable_p2p);
+
+ particles_inner_computer<rnumber, long long int> current_particles_inner_computer(inner_v0);
+ current_particles_inner_computer.setEnable(enable_inner);
+
+ this->ps = particles_system_builder_with_p2p(
this->fs->cvelocity, // (field object)
this->fs->kk, // (kspace object, contains dkx, dky, dkz)
tracers0_integration_steps, // to check coherency between parameters and hdf input file (nb rhs)
@@ -21,10 +28,13 @@ int NSVEparticlesP2P<rnumber>::initialize(void)
tracers0_neighbours, // parameter (interpolation no neighbours)
tracers0_smoothness, // parameter
this->comm,
- this->fs->iteration+1);
- // TODO P2P write particle data too
+ this->fs->iteration+1,
+ std::move(current_p2p_computer),
+ std::move(current_particles_inner_computer),
+ cutoff);
+
this->particles_output_writer_mpi = new particles_output_hdf5<
- long long int, double, 3, 3>(
+ long long int, double, 6, 6>(
MPI_COMM_WORLD,
"tracers0",
nparticles,
@@ -37,7 +47,12 @@ int NSVEparticlesP2P<rnumber>::step(void)
{
this->fs->compute_velocity(this->fs->cvorticity);
this->fs->cvelocity->ift();
- this->ps->completeLoop(this->dt);
+ if(enable_vorticity_omega){
+ this->ps->completeLoopWithVorticity(this->dt, *this->fs->cvorticity);
+ }
+ else{
+ this->ps->completeLoop(this->dt);
+ }
this->NSVE<rnumber>::step();
return EXIT_SUCCESS;
}
diff --git a/bfps/cpp/full_code/NSVEparticlesP2P.hpp b/bfps/cpp/full_code/NSVEparticlesP2P.hpp
index 9b015659..73427bca 100644
--- a/bfps/cpp/full_code/NSVEparticlesP2P.hpp
+++ b/bfps/cpp/full_code/NSVEparticlesP2P.hpp
@@ -55,10 +55,15 @@ class NSVEparticlesP2P: public NSVE<rnumber>
int tracers0_neighbours;
int tracers0_smoothness;
+ double cutoff;
+ bool enable_p2p;
+ bool enable_inner;
+ bool enable_vorticity_omega;
+
/* other stuff */
std::unique_ptr<abstract_particles_system<long long int, double>> ps;
// TODO P2P use a reader with particle data
- particles_output_hdf5<long long int, double,3,3> *particles_output_writer_mpi;
+ particles_output_hdf5<long long int, double,6,6> *particles_output_writer_mpi;
NSVEparticlesP2P(
@@ -66,7 +71,8 @@ class NSVEparticlesP2P: public NSVE<rnumber>
const std::string &simulation_name):
NSVE<rnumber>(
COMMUNICATOR,
- simulation_name){}
+ simulation_name),
+ cutoff(std::numeric_limits<double>::max()){}
~NSVEparticlesP2P(){}
int initialize(void);
diff --git a/bfps/cpp/particles/abstract_particles_system.hpp b/bfps/cpp/particles/abstract_particles_system.hpp
index 26ce7198..b2d09566 100644
--- a/bfps/cpp/particles/abstract_particles_system.hpp
+++ b/bfps/cpp/particles/abstract_particles_system.hpp
@@ -16,6 +16,10 @@ public:
virtual void compute_p2p() = 0;
+ virtual void compute_particles_inner() = 0;
+
+ virtual void compute_particles_inner(const real_number particle_extra_rhs[]) = 0;
+
virtual void move(const real_number dt) = 0;
virtual void redistribute() = 0;
@@ -26,6 +30,9 @@ public:
virtual void completeLoop(const real_number dt) = 0;
+ virtual void completeLoopWithVorticity(const real_number dt,
+ const real_number particle_extra_rhs[]) = 0;
+
virtual const real_number* getParticlesPositions() const = 0;
virtual const std::unique_ptr<real_number[]>* getParticlesRhs() const = 0;
@@ -54,6 +61,15 @@ public:
virtual void sample_compute_field(const field<double, FFTW, THREExTHREE>& sample_field,
real_number sample_rhs[]) = 0;
//- Not generic to enable sampling end
+
+ template <typename rnumber, field_backend be, field_components fc>
+ void completeLoopWithVorticity(const real_number dt,
+ const field<rnumber, be, fc>& in_field) {
+ static_assert(fc == THREE, "only THREE is supported for now");
+ std::unique_ptr<real_number[]> extra_rhs(new real_number[getLocalNbParticles()*3]());
+ sample_compute_field(in_field, extra_rhs.get());
+ completeLoopWithVorticity(dt, extra_rhs.get());
+ }
};
#endif
diff --git a/bfps/cpp/particles/p2p_computer.hpp b/bfps/cpp/particles/p2p_computer.hpp
index bd6a4001..46328c3a 100644
--- a/bfps/cpp/particles/p2p_computer.hpp
+++ b/bfps/cpp/particles/p2p_computer.hpp
@@ -2,27 +2,31 @@
#define P2P_COMPUTER_HPP
#include <cstring>
-
-/** \brief A simple distance weighting function.
- *
- * This function returns 1 if a distance is smaller than a cut-off length,
- * i.e. particle 1 interacts with particle 2 if particle 2 is inside a
- * sphere of radius `cutoff' centered on particle 1.
- */
-double dumb_distance_weight(
- double distance,
- double cutoff)
-{
- // this function should only be called for interacting particles,
- // and particles interact if they are closer than cutoff.
- assert(distance < cutoff);
- return 1.0;
-}
+#include <cassert>
template <class real_number, class partsize_t>
class p2p_computer{
+
+ bool isActive;
+
+ /** \brief A simple distance weighting function.
+ *
+ * This function returns 1 if a distance is smaller than a cut-off length,
+ * i.e. particle 1 interacts with particle 2 if particle 2 is inside a
+ * sphere of radius `cutoff' centered on particle 1.
+ */
+ static double dumb_distance_weight(
+ const double dist_pow2,
+ const double cutoff){
+ // this function should only be called for interacting particles,
+ // and particles interact if they are closer than cutoff.
+ assert(dist_pow2 < cutoff*cutoff);
+ return 1.0;
+ }
+
+
public:
- constexpr static int size_data = 3;
+ p2p_computer() : isActive(true){}
template <int size_particle_rhs>
void init_result_array(real_number rhs[], const partsize_t nbParticles) const{
@@ -31,25 +35,26 @@ public:
template <int size_particle_rhs>
void reduce_particles_rhs(real_number rhs_dst[], const real_number rhs_src[], const partsize_t nbParticles) const{
+ static_assert(size_particle_rhs == 6, "This kernel works only with 6 values per particle's rhs");
for(int idx_part = 0 ; idx_part < nbParticles ; ++idx_part){
- for(int idx_rhs = 0 ; idx_rhs < size_particle_rhs ; ++idx_rhs){
+ // We merge only the values modified by the current kernel (3-5)
+ for(int idx_rhs = 3 ; idx_rhs < size_particle_rhs ; ++idx_rhs){
rhs_dst[idx_part*size_particle_rhs+idx_rhs] += rhs_src[idx_part*size_particle_rhs+idx_rhs];
}
}
}
- template <int size_particle_positions, int size_particle_data, int size_particle_rhs>
- void compute_interaction(const real_number pos_part1[], const real_number data_part1[], real_number rhs_part1[],
- const real_number pos_part2[], const real_number data_part2[], real_number rhs_part2[],
- const real_number dist_pow2,
- const real_number xshift_coef, const real_number yshift_coef, const real_number zshift_coef) const{
- static_assert(size_particle_positions == 3, "This kernel works only with 3 values for one particle's position");
- static_assert(size_particle_rhs >= 6, "This kernel works only with more than 6 values per particle's rhs");
- static_assert(size_particle_data == 3, "This kernel works only with 3 values per particle's' data");
+ template <int size_particle_positions, int size_particle_rhs>
+ void compute_interaction(const real_number pos_part1[], real_number rhs_part1[],
+ const real_number pos_part2[], real_number rhs_part2[],
+ const real_number dist_pow2, const real_number cutoff,
+ const real_number /*xshift_coef*/, const real_number /*yshift_coef*/, const real_number /*zshift_coef*/) const{
+ static_assert(size_particle_positions == 6, "This kernel works only with 6 values for one particle's position+orientation");
+ static_assert(size_particle_rhs == 6, "This kernel works only with 6 values per particle's rhs");
// TODO: a reasonable way of choosing between different distance_weight functions should be thought of.
// We need to ask Michael about how flexible this distance_weight needs to be.
- const double ww = dumb_distance_weight(distance, max_distance);
+ const double ww = dumb_distance_weight(dist_pow2, cutoff);
///
/// term in equation is:
///
@@ -57,20 +62,23 @@ public:
/// (4 / \tau) \sum_j W_\ell ( | x^i - x^j | ) (p^i \cdot p^j)p^j
/// \f]
///
- const double dot_product = (data_part1[0]*data_part2[0] +
- data_part1[1]*data_part2[1] +
- data_part1[2]*data_part2[2]);
- rhs_part1[3] += data_part2[0] * 4 * ww * dot_product;
- rhs_part1[4] += data_part2[1] * 4 * ww * dot_product;
- rhs_part1[5] += data_part2[2] * 4 * ww * dot_product;
- rhs_part2[3] += data_part1[0] * 4 * ww * dot_product;
- rhs_part2[4] += data_part1[1] * 4 * ww * dot_product;
- rhs_part2[5] += data_part1[2] * 4 * ww * dot_product;
+ const double dot_product = (pos_part1[3+IDX_X]*pos_part2[3+IDX_X] +
+ pos_part1[3+IDX_Y]*pos_part2[3+IDX_Y] +
+ pos_part1[3+IDX_Z]*pos_part2[3+IDX_Z]);
+ rhs_part1[3+IDX_X] += pos_part2[3+IDX_X] * 4 * ww * dot_product;
+ rhs_part1[3+IDX_Y] += pos_part2[3+IDX_Y] * 4 * ww * dot_product;
+ rhs_part1[3+IDX_Z] += pos_part2[3+IDX_Z] * 4 * ww * dot_product;
+ rhs_part2[3+IDX_X] += pos_part1[3+IDX_X] * 4 * ww * dot_product;
+ rhs_part2[3+IDX_Y] += pos_part1[3+IDX_Y] * 4 * ww * dot_product;
+ rhs_part2[3+IDX_Z] += pos_part1[3+IDX_Z] * 4 * ww * dot_product;
}
+ bool isEnable() const {
+ return isActive;
+ }
- constexpr static bool isEmpty() {
- return false;
+ void setEnable(const bool inIsActive) {
+ isActive = inIsActive;
}
};
diff --git a/bfps/cpp/particles/p2p_computer_empty.hpp b/bfps/cpp/particles/p2p_computer_empty.hpp
index 7076061e..5e442b86 100644
--- a/bfps/cpp/particles/p2p_computer_empty.hpp
+++ b/bfps/cpp/particles/p2p_computer_empty.hpp
@@ -6,8 +6,6 @@
template <class real_number, class partsize_t>
class p2p_computer_empty{
public:
- constexpr int static size_data = 0;
-
template <int size_particle_rhs>
void init_result_array(real_number /*rhs*/[], const partsize_t /*nbParticles*/) const{
}
@@ -16,15 +14,15 @@ public:
void reduce_particles_rhs(real_number /*rhs_dst*/[], const real_number /*rhs_src*/[], const partsize_t /*nbParticles*/) const{
}
- template <int size_particle_positions, int size_particle_data, int size_particle_rhs>
- void compute_interaction(const real_number /*pos_part1*/[], const real_number /*data_part1*/[], real_number /*rhs_part1*/[],
- const real_number /*pos_part2*/[], const real_number /*data_part2*/[], real_number /*rhs_part2*/[],
- const real_number /*dist_pow2*/,
+ template <int size_particle_positions, int size_particle_rhs>
+ void compute_interaction(const real_number /*pos_part1*/[], real_number /*rhs_part1*/[],
+ const real_number /*pos_part2*/[], real_number /*rhs_part2*/[],
+ const real_number /*dist_pow2*/, const real_number /*cutoff*/,
const real_number /*xshift_coef*/, const real_number /*yshift_coef*/, const real_number /*zshift_coef*/) const{
}
- constexpr static bool isEmpty() {
- return true;
+ constexpr static bool isEnable() {
+ return false;
}
};
diff --git a/bfps/cpp/particles/p2p_distr_mpi.hpp b/bfps/cpp/particles/p2p_distr_mpi.hpp
index ccc236d6..b009a57e 100644
--- a/bfps/cpp/particles/p2p_distr_mpi.hpp
+++ b/bfps/cpp/particles/p2p_distr_mpi.hpp
@@ -36,7 +36,6 @@ protected:
std::unique_ptr<real_number[]> toRecvAndMerge;
std::unique_ptr<real_number[]> toCompute;
- std::unique_ptr<real_number[]> toData;
std::unique_ptr<real_number[]> results;
};
@@ -248,11 +247,10 @@ public:
return (diff_x*diff_x) + (diff_y*diff_y) + (diff_z*diff_z);
}
- template <class computer_class, int size_particle_positions, int size_particle_data, int size_particle_rhs>
+ template <class computer_class, int size_particle_positions, int size_particle_rhs>
void compute_distr(computer_class& in_computer,
const partsize_t current_my_nb_particles_per_partition[],
real_number particles_positions[],
- real_number particles_data[],
real_number particles_current_rhs[],
partsize_t inout_index_particles[]){
TIMEZONE("compute_distr");
@@ -313,9 +311,6 @@ public:
permute_copy<real_number, size_particle_positions>(current_offset_particles_for_partition[idxPartition],
current_my_nb_particles_per_partition[idxPartition],
part_to_sort.data(), particles_positions, &buffer);
- permute_copy<real_number, size_particle_data>(current_offset_particles_for_partition[idxPartition],
- current_my_nb_particles_per_partition[idxPartition],
- part_to_sort.data(), particles_data, &buffer);
permute_copy<real_number, size_particle_rhs>(current_offset_particles_for_partition[idxPartition],
current_my_nb_particles_per_partition[idxPartition],
part_to_sort.data(), particles_current_rhs, &buffer);
@@ -457,14 +452,6 @@ public:
descriptor.destProc, TAG_POSITION_PARTICLES,
current_com, &mpiRequests.back()));
- whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
- mpiRequests.emplace_back();
- assert(descriptor.nbParticlesToExchange*size_particle_data < std::numeric_limits<int>::max());
- AssertMpi(MPI_Isend(const_cast<real_number*>(&particles_data[particles_offset_layers[my_nb_cell_levels-descriptor.nbLevelsToExchange]*size_particle_data]),
- int(descriptor.nbParticlesToExchange*size_particle_data), particles_utils::GetMpiType(real_number()),
- descriptor.destProc, TAG_POSITION_PARTICLES,
- current_com, &mpiRequests.back()));
-
assert(descriptor.toRecvAndMerge == nullptr);
descriptor.toRecvAndMerge.reset(new real_number[descriptor.nbParticlesToExchange*size_particle_rhs]);
whatNext.emplace_back(std::pair<Action,int>{MERGE_PARTICLES, idxDescr});
@@ -526,15 +513,6 @@ public:
AssertMpi(MPI_Irecv(descriptor.toCompute.get(), int(NbParticlesToReceive*size_particle_positions),
particles_utils::GetMpiType(real_number()), destProc, TAG_POSITION_PARTICLES,
current_com, &mpiRequests.back()));
-
-
- descriptor.toData.reset(new real_number[NbParticlesToReceive*size_particle_data]);
- whatNext.emplace_back(std::pair<Action,int>{COMPUTE_PARTICLES, releasedAction.second});
- mpiRequests.emplace_back();
- assert(NbParticlesToReceive*size_particle_data < std::numeric_limits<int>::max());
- AssertMpi(MPI_Irecv(descriptor.toData.get(), int(NbParticlesToReceive*size_particle_data),
- particles_utils::GetMpiType(real_number()), destProc, TAG_POSITION_PARTICLES,
- current_com, &mpiRequests.back()));
}
}
@@ -548,7 +526,6 @@ public:
const partsize_t NbParticlesToReceive = descriptor.nbParticlesToExchange;
assert(descriptor.toCompute != nullptr);
- assert(descriptor.toData != nullptr);
assert(descriptor.positionsReceived == true);
descriptor.results.reset(new real_number[NbParticlesToReceive*size_particle_rhs]);
in_computer.template init_result_array<size_particle_rhs>(descriptor.results.get(), NbParticlesToReceive);
@@ -589,14 +566,12 @@ public:
particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z],
shift[idx_neighbor][IDX_X], shift[idx_neighbor][IDX_Y], shift[idx_neighbor][IDX_Z]);
if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
- in_computer.template compute_interaction<size_particle_positions,size_particle_data, size_particle_rhs>(
+ in_computer.template compute_interaction<size_particle_positions, size_particle_rhs>(
&descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions],
- &descriptor.toData[(idxPart+idx_p1)*size_particle_data],
&descriptor.results[(idxPart+idx_p1)*size_particle_rhs],
&particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions],
- &particles_data[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_data],
&particles_current_rhs[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_rhs],
- dist_r2, shift[idx_neighbor][IDX_X], shift[idx_neighbor][IDX_Y], shift[idx_neighbor][IDX_Z]);
+ dist_r2, cutoff_radius_compute, shift[idx_neighbor][IDX_X], shift[idx_neighbor][IDX_Y], shift[idx_neighbor][IDX_Z]);
}
}
}
@@ -681,14 +656,12 @@ public:
particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_Z],
0, 0, 0);
if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
- in_computer.template compute_interaction<size_particle_positions,size_particle_data,size_particle_rhs>(
+ in_computer.template compute_interaction<size_particle_positions,size_particle_rhs>(
&particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions],
- &particles_data[(intervals[idx_1].first+idx_p1)*size_particle_data],
&particles_current_rhs[(intervals[idx_1].first+idx_p1)*size_particle_rhs],
&particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions],
- &particles_data[(intervals[idx_1].first+idx_p2)*size_particle_data],
&particles_current_rhs[(intervals[idx_1].first+idx_p2)*size_particle_rhs],
- dist_r2, 0, 0, 0);
+ dist_r2, cutoff_radius_compute, 0, 0, 0);
}
}
}
@@ -705,14 +678,12 @@ public:
particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_Z],
0, 0, 0);
if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
- in_computer.template compute_interaction<size_particle_positions,size_particle_data,size_particle_rhs>(
+ in_computer.template compute_interaction<size_particle_positions,size_particle_rhs>(
&particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions],
- &particles_data[(intervals[idx_1].first+idx_p1)*size_particle_data],
&particles_current_rhs[(intervals[idx_1].first+idx_p1)*size_particle_rhs],
&particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions],
- &particles_data[(intervals[idx_2].first+idx_p2)*size_particle_data],
&particles_current_rhs[(intervals[idx_2].first+idx_p2)*size_particle_rhs],
- dist_r2, 0, 0, 0);
+ dist_r2, cutoff_radius_compute, 0, 0, 0);
}
}
}
@@ -741,14 +712,12 @@ public:
particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z],
shift[idx_neighbor][IDX_X], shift[idx_neighbor][IDX_Y], shift[idx_neighbor][IDX_Z]);
if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
- in_computer.template compute_interaction<size_particle_positions,size_particle_data,size_particle_rhs>(
+ in_computer.template compute_interaction<size_particle_positions,size_particle_rhs>(
&particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions],
- &particles_data[(intervals[idx_1].first+idx_p1)*size_particle_data],
&particles_current_rhs[(intervals[idx_1].first+idx_p1)*size_particle_rhs],
&particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions],
- &particles_data[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_data],
&particles_current_rhs[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_rhs],
- dist_r2, shift[idx_neighbor][IDX_X], shift[idx_neighbor][IDX_Y], shift[idx_neighbor][IDX_Z]);
+ dist_r2, cutoff_radius_compute, shift[idx_neighbor][IDX_X], shift[idx_neighbor][IDX_Y], shift[idx_neighbor][IDX_Z]);
}
}
}
diff --git a/bfps/cpp/particles/particles_adams_bashforth.hpp b/bfps/cpp/particles/particles_adams_bashforth.hpp
index 2fb61462..20c1ea50 100644
--- a/bfps/cpp/particles/particles_adams_bashforth.hpp
+++ b/bfps/cpp/particles/particles_adams_bashforth.hpp
@@ -7,11 +7,30 @@
#include "scope_timer.hpp"
#include "particles_utils.hpp"
-template <class partsize_t, class real_number, int size_particle_positions = 3, int size_particle_rhs = 3>
-class particles_adams_bashforth {
- static_assert(size_particle_positions == size_particle_rhs,
- "Not having the same dimension for positions and rhs looks like a bug,"
- "otherwise comment this assertion.");
+template <class partsize_t, class real_number, int size_particle_positions, int size_particle_rhs>
+class particles_adams_bashforth;
+
+
+template <class partsize_t, class real_number>
+class particles_adams_bashforth<partsize_t,real_number,6,6>{
+ static const int size_particle_positions = 6;
+ static const int size_particle_rhs = 6;
+public:
+ static const int Max_steps = 6;
+
+ void move_particles(real_number*__restrict__ particles_positions,
+ const partsize_t nb_particles,
+ const std::unique_ptr<real_number[]> particles_rhs[],
+ const int nb_rhs, const real_number dt) const{
+ // TODO
+ }
+};
+
+
+template <class partsize_t, class real_number>
+class particles_adams_bashforth<partsize_t,real_number,3,3>{
+ static const int size_particle_positions = 3;
+ static const int size_particle_rhs = 3;
public:
static const int Max_steps = 6;
diff --git a/bfps/cpp/particles/particles_distr_mpi.hpp b/bfps/cpp/particles/particles_distr_mpi.hpp
index f0c09fd9..79ff2e8b 100644
--- a/bfps/cpp/particles/particles_distr_mpi.hpp
+++ b/bfps/cpp/particles/particles_distr_mpi.hpp
@@ -35,9 +35,6 @@ protected:
TAG_LOW_UP_MOVED_PARTICLES_INDEXES,
TAG_UP_LOW_MOVED_PARTICLES_INDEXES,
- TAG_LOW_UP_MOVED_PARTICLES_DATA,
- TAG_UP_LOW_MOVED_PARTICLES_DATA,
-
TAG_LOW_UP_MOVED_PARTICLES_RHS,
TAG_LOW_UP_MOVED_PARTICLES_RHS_MAX = TAG_LOW_UP_MOVED_PARTICLES_RHS+MaxNbRhs,
@@ -506,14 +503,13 @@ public:
////////////////////////////////////////////////////////////////////////////
- template <class computer_class, int size_particle_positions, int size_particle_data, int size_particle_rhs, int size_particle_index>
+ template <class computer_class, int size_particle_positions, int size_particle_rhs, int size_particle_index>
void redistribute(computer_class& in_computer,
partsize_t current_my_nb_particles_per_partition[],
partsize_t* nb_particles,
std::unique_ptr<real_number[]>* inout_positions_particles,
std::unique_ptr<real_number[]> inout_rhs_particles[], const int in_nb_rhs,
- std::unique_ptr<partsize_t[]>* inout_index_particles,
- std::unique_ptr<real_number[]>* inout_data_particles){
+ std::unique_ptr<partsize_t[]>* inout_index_particles){
TIMEZONE("redistribute");
// Some latest processes might not be involved
@@ -545,11 +541,6 @@ public:
(*inout_index_particles)[size_particle_index*idx2+idx_val]);
}
- for(int idx_val = 0 ; idx_val < size_particle_data ; ++idx_val){
- std::swap((*inout_data_particles)[size_particle_data*idx1+idx_val],
- (*inout_data_particles)[size_particle_data*idx2+idx_val]);
- }
-
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
for(int idx_val = 0 ; idx_val < size_particle_rhs ; ++idx_val){
std::swap(inout_rhs_particles[idx_rhs][idx1*size_particle_rhs + idx_val],
@@ -576,11 +567,6 @@ public:
(*inout_index_particles)[size_particle_index*idx2+idx_val]);
}
- for(int idx_val = 0 ; idx_val < size_particle_data ; ++idx_val){
- std::swap((*inout_data_particles)[size_particle_data*idx1+idx_val],
- (*inout_data_particles)[size_particle_data*idx2+idx_val]);
- }
-
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
for(int idx_val = 0 ; idx_val < size_particle_rhs ; ++idx_val){
std::swap(inout_rhs_particles[idx_rhs][idx1*size_particle_rhs + idx_val],
@@ -597,8 +583,6 @@ public:
std::unique_ptr<real_number[]> newParticlesUp;
std::unique_ptr<partsize_t[]> newParticlesLowIndexes;
std::unique_ptr<partsize_t[]> newParticlesUpIndexes;
- std::unique_ptr<real_number[]> newParticlesLowData;
- std::unique_ptr<real_number[]> newParticlesUpData;
std::vector<std::unique_ptr<real_number[]>> newParticlesLowRhs(in_nb_rhs);
std::vector<std::unique_ptr<real_number[]>> newParticlesUpRhs(in_nb_rhs);
@@ -633,16 +617,6 @@ public:
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES_INDEXES,
MPI_COMM_WORLD, &mpiRequests.back()));
- if(size_particle_data){
- whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
- mpiRequests.emplace_back();
- assert(nbOutLower*size_particle_data < std::numeric_limits<int>::max());
- AssertMpi(MPI_Isend(&(*inout_data_particles)[0], int(nbOutLower*size_particle_data),
- particles_utils::GetMpiType(partsize_t()),
- (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES_DATA,
- MPI_COMM_WORLD, &mpiRequests.back()));
- }
-
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
@@ -680,16 +654,6 @@ public:
particles_utils::GetMpiType(partsize_t()), (my_rank+1)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES_INDEXES,
MPI_COMM_WORLD, &mpiRequests.back()));
- if(size_particle_data){
- whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
- mpiRequests.emplace_back();
- assert(nbOutUpper*size_particle_data < std::numeric_limits<int>::max());
- AssertMpi(MPI_Isend(&(*inout_data_particles)[(myTotalNbParticles-nbOutUpper)*size_particle_data],
- int(nbOutUpper*size_particle_data),
- particles_utils::GetMpiType(partsize_t()), (my_rank+1)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES_DATA,
- MPI_COMM_WORLD, &mpiRequests.back()));
- }
-
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
@@ -732,17 +696,6 @@ public:
(my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES_INDEXES,
MPI_COMM_WORLD, &mpiRequests.back()));
- if(size_particle_data){
- newParticlesLowData.reset(new real_number[nbNewFromLow*size_particle_data]);
- whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
- mpiRequests.emplace_back();
- assert(nbNewFromLow*size_particle_data < std::numeric_limits<int>::max());
- AssertMpi(MPI_Irecv(&newParticlesLowData[0], int(nbNewFromLow*size_particle_data),
- particles_utils::GetMpiType(real_number()),
- (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_UP_LOW_MOVED_PARTICLES_DATA,
- MPI_COMM_WORLD, &mpiRequests.back()));
- }
-
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
newParticlesLowRhs[idx_rhs].reset(new real_number[nbNewFromLow*size_particle_rhs]);
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
@@ -773,17 +726,6 @@ public:
(my_rank+1)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES_INDEXES,
MPI_COMM_WORLD, &mpiRequests.back()));
- if(size_particle_data){
- newParticlesUpData.reset(new real_number[nbNewFromUp*size_particle_data]);
- whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
- mpiRequests.emplace_back();
- assert(nbNewFromUp*size_particle_data < std::numeric_limits<int>::max());
- AssertMpi(MPI_Irecv(&newParticlesUpData[0], int(nbNewFromUp*size_particle_data),
- particles_utils::GetMpiType(real_number()),
- (my_rank+1)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES_DATA,
- MPI_COMM_WORLD, &mpiRequests.back()));
- }
-
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
newParticlesUpRhs[idx_rhs].reset(new real_number[nbNewFromUp*size_particle_rhs]);
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
@@ -814,7 +756,6 @@ public:
std::unique_ptr<real_number[]> newArray(new real_number[myTotalNewNbParticles*size_particle_positions]);
std::unique_ptr<partsize_t[]> newArrayIndexes(new partsize_t[myTotalNewNbParticles*size_particle_index]);
- std::unique_ptr<real_number[]> newArrayData(new real_number[myTotalNewNbParticles*size_particle_data]);
std::vector<std::unique_ptr<real_number[]>> newArrayRhs(in_nb_rhs);
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
newArrayRhs[idx_rhs].reset(new real_number[myTotalNewNbParticles*size_particle_rhs]);
@@ -825,7 +766,6 @@ public:
const particles_utils::fixed_copy fcp(0, 0, nbNewFromLow);
fcp.copy(newArray, newParticlesLow, size_particle_positions);
fcp.copy(newArrayIndexes, newParticlesLowIndexes, size_particle_index);
- fcp.copy(newArrayData, newParticlesLowData, size_particle_data);
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
fcp.copy(newArrayRhs[idx_rhs], newParticlesLowRhs[idx_rhs], size_particle_rhs);
}
@@ -836,7 +776,6 @@ public:
const particles_utils::fixed_copy fcp(nbNewFromLow, nbOutLower, nbOldParticlesInside);
fcp.copy(newArray, (*inout_positions_particles), size_particle_positions);
fcp.copy(newArrayIndexes, (*inout_index_particles), size_particle_index);
- fcp.copy(newArrayData, (*inout_data_particles), size_particle_data);
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
fcp.copy(newArrayRhs[idx_rhs], inout_rhs_particles[idx_rhs], size_particle_rhs);
}
@@ -847,7 +786,6 @@ public:
const particles_utils::fixed_copy fcp(nbNewFromLow+nbOldParticlesInside, 0, nbNewFromUp);
fcp.copy(newArray, newParticlesUp, size_particle_positions);
fcp.copy(newArrayIndexes, newParticlesUpIndexes, size_particle_index);
- fcp.copy(newArrayData, newParticlesUpData, size_particle_data);
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
fcp.copy(newArrayRhs[idx_rhs], newParticlesUpRhs[idx_rhs], size_particle_rhs);
}
@@ -855,7 +793,6 @@ public:
(*inout_positions_particles) = std::move(newArray);
(*inout_index_particles) = std::move(newArrayIndexes);
- (*inout_data_particles) = std::move(newArrayData);
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
inout_rhs_particles[idx_rhs] = std::move(newArrayRhs[idx_rhs]);
}
@@ -880,11 +817,6 @@ public:
(*inout_index_particles)[size_particle_index*idx2 + idx_val]);
}
- for(int idx_val = 0 ; idx_val < size_particle_data ; ++idx_val){
- std::swap((*inout_data_particles)[size_particle_data*idx1 + idx_val],
- (*inout_data_particles)[size_particle_data*idx2 + idx_val]);
- }
-
for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
for(int idx_val = 0 ; idx_val < size_particle_rhs ; ++idx_val){
std::swap(inout_rhs_particles[idx_rhs][idx1*size_particle_rhs + idx_val],
diff --git a/bfps/cpp/particles/particles_field_computer.hpp b/bfps/cpp/particles/particles_field_computer.hpp
index f68f2fc0..70da4757 100644
--- a/bfps/cpp/particles/particles_field_computer.hpp
+++ b/bfps/cpp/particles/particles_field_computer.hpp
@@ -67,8 +67,9 @@ public:
const real_number particles_positions[],
real_number particles_current_rhs[],
const partsize_t nb_particles) const {
+ static_assert(field_class::nb_components() <= size_particle_rhs, "Cannot store all the component in the given array");
TIMEZONE("particles_field_computer::apply_computation");
- //DEBUG_MSG("just entered particles_field_computer::apply_computation\n");
+
for(partsize_t idxPart = 0 ; idxPart < nb_particles ; ++idxPart){
const real_number reltv_x = get_norm_pos_in_cell(particles_positions[idxPart*3+IDX_X], IDX_X);
const real_number reltv_y = get_norm_pos_in_cell(particles_positions[idxPart*3+IDX_Y], IDX_Y);
@@ -146,7 +147,8 @@ public:
const ptrdiff_t tindex = field.get_rindex_from_global(idx_x_pbc, idx_y_pbc, idx_z_pbc);
// getValue does not necessary return real_number
- for(int idx_rhs_val = 0 ; idx_rhs_val < size_particle_rhs ; ++idx_rhs_val){
+ // size_particle_rhs is just for the leading dimension of the array
+ for(int idx_rhs_val = 0 ; idx_rhs_val < field_class::nb_components() ; ++idx_rhs_val){
particles_current_rhs[idxPart*size_particle_rhs+idx_rhs_val] += real_number(field.rval(tindex,idx_rhs_val))*coef;
}
}
diff --git a/bfps/cpp/particles/particles_inner_computer.hpp b/bfps/cpp/particles/particles_inner_computer.hpp
new file mode 100644
index 00000000..4d0a6678
--- /dev/null
+++ b/bfps/cpp/particles/particles_inner_computer.hpp
@@ -0,0 +1,65 @@
+#ifndef PARTICLES_INNER_COMPUTER_HPP
+#define PARTICLES_INNER_COMPUTER_HPP
+
+#include <cstring>
+#include <cassert>
+
+template <class real_number, class partsize_t>
+class particles_inner_computer{
+ bool isActive;
+ const real_number v0;
+
+public:
+ explicit particles_inner_computer(const real_number inV0) : isActive(true), v0(inV0){
+ }
+
+ template <int size_particle_positions, int size_particle_rhs>
+ void compute_interaction(const partsize_t nb_particles, real_number pos_part[], real_number rhs_part[]) const{
+ static_assert(size_particle_positions == 6, "This kernel works only with 6 values for one particle's position");
+ static_assert(size_particle_rhs == 6, "This kernel works only with 6 values per particle's rhs");
+
+ #pragma omp parallel for
+ for(partsize_t idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
+ // Add attr × V0 to the field interpolation
+ rhs_part[idx_part*size_particle_rhs + IDX_X] += pos_part[idx_part*size_particle_positions + 3+IDX_X]*v0;
+ rhs_part[idx_part*size_particle_rhs + IDX_Y] += pos_part[idx_part*size_particle_positions + 3+IDX_Y]*v0;
+ rhs_part[idx_part*size_particle_rhs + IDX_Z] += pos_part[idx_part*size_particle_positions + 3+IDX_Z]*v0;
+
+ real_number alpha[3]= {0}; // TODO compute aplha
+
+ rhs_part[idx_part*size_particle_rhs + 3+IDX_X] += alpha[IDX_X];
+ rhs_part[idx_part*size_particle_rhs + 3+IDX_Y] += alpha[IDX_Y];
+ rhs_part[idx_part*size_particle_rhs + 3+IDX_Z] += alpha[IDX_Z];
+ }
+ }
+
+ template <int size_particle_positions, int size_particle_rhs, int size_particle_rhs_extra>
+ void compute_interaction_with_extra(const partsize_t nb_particles, real_number pos_part[], real_number rhs_part[],
+ const real_number rhs_part_extra[]) const{
+ static_assert(size_particle_rhs_extra == 3, "This kernel works only with 3 values for one particle's rhs extra");
+
+ compute_interaction<size_particle_positions, size_particle_rhs>(nb_particles, pos_part, rhs_part);
+
+ #pragma omp parallel for
+ for(partsize_t idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
+ // Cross product vorticity/orientation
+ rhs_part[idx_part*size_particle_rhs + 3+IDX_X] += rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Y]*pos_part[idx_part*size_particle_positions + 3+IDX_Z]
+ - rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Z]*pos_part[idx_part*size_particle_positions + 3+IDX_Y];
+ rhs_part[idx_part*size_particle_rhs + 3+IDX_Y] += rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Z]*pos_part[idx_part*size_particle_positions + 3+IDX_X]
+ - rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_X]*pos_part[idx_part*size_particle_positions + 3+IDX_Z];
+ rhs_part[idx_part*size_particle_rhs + 3+IDX_Z] += rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_X]*pos_part[idx_part*size_particle_positions + 3+IDX_Y]
+ - rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Y]*pos_part[idx_part*size_particle_positions + 3+IDX_X];
+ }
+ }
+
+
+ bool isEnable() const {
+ return isActive;
+ }
+
+ void setEnable(const bool inIsActive) {
+ isActive = inIsActive;
+ }
+};
+
+#endif
diff --git a/bfps/cpp/particles/particles_inner_computer_empty.hpp b/bfps/cpp/particles/particles_inner_computer_empty.hpp
new file mode 100644
index 00000000..1bd3b1ec
--- /dev/null
+++ b/bfps/cpp/particles/particles_inner_computer_empty.hpp
@@ -0,0 +1,24 @@
+#ifndef PARTICLES_INNER_COMPUTER_EMPTY_HPP
+#define PARTICLES_INNER_COMPUTER_EMPTY_HPP
+
+#include <cstring>
+#include <cassert>
+
+template <class real_number, class partsize_t>
+class particles_inner_computer_empty{
+public:
+ template <int size_particle_positions, int size_particle_rhs>
+ void compute_interaction(const partsize_t /*nb_particles*/, real_number /*pos_part*/[], real_number /*rhs_part*/[]) const{
+ }
+
+ template <int size_particle_positions, int size_particle_rhs, int size_particle_rhs_extra>
+ void compute_interaction_with_extra(const partsize_t /*nb_particles*/, real_number /*pos_part*/[], real_number /*rhs_part*/[],
+ const real_number /*rhs_part_extra*/[]) const{
+ }
+
+ constexpr static bool isEnable() {
+ return false;
+ }
+};
+
+#endif
diff --git a/bfps/cpp/particles/particles_system.hpp b/bfps/cpp/particles/particles_system.hpp
index 12bf6c29..2b56cae8 100644
--- a/bfps/cpp/particles/particles_system.hpp
+++ b/bfps/cpp/particles/particles_system.hpp
@@ -15,7 +15,7 @@
#include "p2p_distr_mpi.hpp"
template <class partsize_t, class real_number, class field_rnumber, class field_class, class interpolator_class, int interp_neighbours,
- int size_particle_rhs, class p2p_computer_class, int size_particle_data>
+ int size_particle_positions, int size_particle_rhs, class p2p_computer_class, class particles_inner_computer_class>
class particles_system : public abstract_particles_system<partsize_t, real_number> {
MPI_Comm mpi_com;
@@ -26,7 +26,7 @@ class particles_system : public abstract_particles_system<partsize_t, real_numbe
particles_distr_mpi<partsize_t, real_number> particles_distr;
- particles_adams_bashforth<partsize_t, real_number, 3, size_particle_rhs> positions_updater;
+ particles_adams_bashforth<partsize_t, real_number, size_particle_positions, size_particle_rhs> positions_updater;
using computer_class = particles_field_computer<partsize_t, real_number, interpolator_class, interp_neighbours>;
computer_class computer;
@@ -51,7 +51,7 @@ class particles_system : public abstract_particles_system<partsize_t, real_numbe
p2p_distr_mpi<partsize_t, real_number> distr_p2p;
p2p_computer_class computer_p2p;
- std::unique_ptr<real_number[]> my_particles_data;
+ particles_inner_computer_class computer_particules_inner;
public:
particles_system(const std::array<size_t,3>& field_grid_dim, const std::array<real_number,3>& in_spatial_box_width,
@@ -63,8 +63,10 @@ public:
const field_class& in_field,
MPI_Comm in_mpi_com,
const partsize_t in_total_nb_particles,
- const int in_current_iteration = 1,
- const real_number in_cutoff = 1.)
+ const real_number in_cutoff,
+ p2p_computer_class in_computer_p2p,
+ particles_inner_computer_class in_computer_particules_inner,
+ const int in_current_iteration = 1)
: mpi_com(in_mpi_com),
current_partition_interval({in_local_field_offset[IDX_Z], in_local_field_offset[IDX_Z] + in_local_field_dims[IDX_Z]}),
partition_interval_size(current_partition_interval.second - current_partition_interval.first),
@@ -77,7 +79,8 @@ public:
spatial_box_width(in_spatial_box_width), spatial_partition_width(in_spatial_partition_width),
my_spatial_low_limit(in_my_spatial_low_limit), my_spatial_up_limit(in_my_spatial_up_limit),
my_nb_particles(0), total_nb_particles(in_total_nb_particles), step_idx(in_current_iteration),
- distr_p2p(in_mpi_com, current_partition_interval,field_grid_dim, spatial_box_width, in_spatial_box_offset, in_cutoff){
+ distr_p2p(in_mpi_com, current_partition_interval,field_grid_dim, spatial_box_width, in_spatial_box_offset, in_cutoff),
+ computer_p2p(std::move(in_computer_p2p)), computer_particules_inner(std::move(in_computer_particules_inner)){
current_my_nb_particles_per_partition.reset(new partsize_t[partition_interval_size]);
current_offset_particles_for_partition.reset(new partsize_t[partition_interval_size+1]);
@@ -93,16 +96,14 @@ public:
my_particles_positions_indexes = particles_input.getMyParticlesIndexes();
my_particles_rhs = particles_input.getMyRhs();
my_nb_particles = particles_input.getLocalNbParticles();
- // TODO P2P get it from loader
- my_particles_data.reset(new real_number[my_nb_particles*size_particle_data]());
for(partsize_t idx_part = 0 ; idx_part < my_nb_particles ; ++idx_part){ // TODO remove me
- const int partition_level = computer.pbc_field_layer(my_particles_positions[idx_part*3+IDX_Z], IDX_Z);
+ const int partition_level = computer.pbc_field_layer(my_particles_positions[idx_part*size_particle_positions+IDX_Z], IDX_Z);
assert(partition_level >= current_partition_interval.first);
assert(partition_level < current_partition_interval.second);
}
- particles_utils::partition_extra_z<partsize_t, 3>(&my_particles_positions[0], my_nb_particles, partition_interval_size,
+ particles_utils::partition_extra_z<partsize_t, size_particle_positions>(&my_particles_positions[0], my_nb_particles, partition_interval_size,
current_my_nb_particles_per_partition.get(), current_offset_particles_for_partition.get(),
[&](const real_number& z_pos){
const int partition_level = computer.pbc_field_layer(z_pos, IDX_Z);
@@ -117,10 +118,6 @@ public:
my_particles_rhs[idx_rhs][idx2*size_particle_rhs + idx_val]);
}
}
- for(int idx_val = 0 ; idx_val < size_particle_data ; ++idx_val){
- std::swap(my_particles_data[idx1*size_particle_data + idx_val],
- my_particles_data[idx2*size_particle_data + idx_val]);
- }
});
{// TODO remove
@@ -128,16 +125,15 @@ public:
assert(current_my_nb_particles_per_partition[idxPartition] ==
current_offset_particles_for_partition[idxPartition+1] - current_offset_particles_for_partition[idxPartition]);
for(partsize_t idx = current_offset_particles_for_partition[idxPartition] ; idx < current_offset_particles_for_partition[idxPartition+1] ; ++idx){
- assert(computer.pbc_field_layer(my_particles_positions[idx*3+IDX_Z], IDX_Z)-current_partition_interval.first == idxPartition);
+ assert(computer.pbc_field_layer(my_particles_positions[idx*size_particle_positions+IDX_Z], IDX_Z)-current_partition_interval.first == idxPartition);
}
}
}
}
-
void compute() final {
TIMEZONE("particles_system::compute");
- particles_distr.template compute_distr<computer_class, field_class, 3, size_particle_rhs>(
+ particles_distr.template compute_distr<computer_class, field_class, size_particle_positions, size_particle_rhs>(
computer, default_field,
current_my_nb_particles_per_partition.get(),
my_particles_positions.get(),
@@ -147,20 +143,37 @@ public:
void compute_p2p() final {
// TODO P2P
- if(p2p_computer_class::isEmpty() == false){
+ if(computer_p2p.isEnable() == true){
TIMEZONE("particles_system::compute_p2p");
- distr_p2p.template compute_distr<p2p_computer_class, 3, size_particle_data, size_particle_rhs>(
+ distr_p2p.template compute_distr<p2p_computer_class, size_particle_positions, size_particle_rhs>(
computer_p2p, current_my_nb_particles_per_partition.get(),
- my_particles_positions.get(), my_particles_data.get(), my_particles_rhs.front().get(),
+ my_particles_positions.get(), my_particles_rhs.front().get(),
my_particles_positions_indexes.get());
}
}
+ void compute_particles_inner() final {
+ if(computer_particules_inner.isEnable() == true){
+ TIMEZONE("particles_system::compute_particles_inner");
+ computer_particules_inner.template compute_interaction<size_particle_positions, size_particle_rhs>(
+ my_nb_particles, my_particles_positions.get(), my_particles_rhs.front().get());
+ }
+ }
+
+ void compute_particles_inner(const real_number particle_extra_rhs[]) final {
+ if(computer_particules_inner.isEnable() == true){
+ TIMEZONE("particles_system::compute_particles_inner");
+ computer_particules_inner.template compute_interaction_with_extra<size_particle_positions, size_particle_rhs, 3>(
+ my_nb_particles, my_particles_positions.get(), my_particles_rhs.front().get(),
+ particle_extra_rhs);
+ }
+ }
+
template <class sample_field_class, int sample_size_particle_rhs>
void sample_compute(const sample_field_class& sample_field,
real_number sample_rhs[]) {
TIMEZONE("particles_system::compute");
- particles_distr.template compute_distr<computer_class, sample_field_class, 3, sample_size_particle_rhs>(
+ particles_distr.template compute_distr<computer_class, sample_field_class, size_particle_positions, sample_size_particle_rhs>(
computer, sample_field,
current_my_nb_particles_per_partition.get(),
my_particles_positions.get(),
@@ -204,14 +217,13 @@ public:
void redistribute() final {
TIMEZONE("particles_system::redistribute");
- particles_distr.template redistribute<computer_class, 3, size_particle_data, size_particle_rhs, 1>(
+ particles_distr.template redistribute<computer_class, size_particle_positions, size_particle_rhs, 1>(
computer,
current_my_nb_particles_per_partition.get(),
&my_nb_particles,
&my_particles_positions,
my_particles_rhs.data(), int(my_particles_rhs.size()),
- &my_particles_positions_indexes,
- &my_particles_data);
+ &my_particles_positions_indexes);
}
void inc_step_idx() final {
@@ -237,6 +249,19 @@ public:
TIMEZONE("particles_system::completeLoop");
compute();
compute_p2p();
+ compute_particles_inner();
+ move(dt);
+ redistribute();
+ inc_step_idx();
+ shift_rhs_vectors();
+ }
+
+ void completeLoopWithVorticity(const real_number dt,
+ const real_number particle_extra_rhs[]) final {
+ TIMEZONE("particles_system::completeLoop");
+ compute();
+ compute_p2p();
+ compute_particles_inner(particle_extra_rhs);
move(dt);
redistribute();
inc_step_idx();
@@ -269,9 +294,9 @@ public:
void checkNan() const { // TODO remove
for(partsize_t idx_part = 0 ; idx_part < my_nb_particles ; ++idx_part){ // TODO remove me
- assert(std::isnan(my_particles_positions[idx_part*3+IDX_X]) == false);
- assert(std::isnan(my_particles_positions[idx_part*3+IDX_Y]) == false);
- assert(std::isnan(my_particles_positions[idx_part*3+IDX_Z]) == false);
+ assert(std::isnan(my_particles_positions[idx_part*size_particle_positions+IDX_X]) == false);
+ assert(std::isnan(my_particles_positions[idx_part*size_particle_positions+IDX_Y]) == false);
+ assert(std::isnan(my_particles_positions[idx_part*size_particle_positions+IDX_Z]) == false);
for(int idx_rhs = 0 ; idx_rhs < my_particles_rhs.size() ; ++idx_rhs){
for(int idx_rhs_val = 0 ; idx_rhs_val < size_particle_rhs ; ++idx_rhs_val){
diff --git a/bfps/cpp/particles/particles_system_builder.hpp b/bfps/cpp/particles/particles_system_builder.hpp
index f9ce512d..ba078000 100644
--- a/bfps/cpp/particles/particles_system_builder.hpp
+++ b/bfps/cpp/particles/particles_system_builder.hpp
@@ -9,6 +9,7 @@
#include "particles_input_hdf5.hpp"
#include "particles_generic_interp.hpp"
#include "p2p_computer_empty.hpp"
+#include "particles_inner_computer_empty.hpp"
#include "field.hpp"
#include "kspace.hpp"
@@ -110,7 +111,8 @@ inline RetType evaluate(IterType1 value1, IterType2 value2, Args... args){
///
//////////////////////////////////////////////////////////////////////////////
-template <class partsize_t, class field_rnumber, field_backend be, field_components fc, class particles_rnumber, class p2p_computer_class>
+template <class partsize_t, class field_rnumber, field_backend be, field_components fc, class particles_rnumber, class p2p_computer_class,
+ class particles_inner_computer_class, int size_particle_positions, int size_particle_rhs>
struct particles_system_build_container {
template <const int interpolation_size, const int spline_mode>
static std::unique_ptr<abstract_particles_system<partsize_t, particles_rnumber>> instanciate(
@@ -121,7 +123,10 @@ struct particles_system_build_container {
const std::string& fname_input, // particles input filename
const std::string& inDatanameState, const std::string& inDatanameRhs, // input dataset names
MPI_Comm mpi_comm,
- const int in_current_iteration){
+ const int in_current_iteration,
+ p2p_computer_class p2p_computer,
+ particles_inner_computer_class inner_computer,
+ const particles_rnumber cutoff = std::numeric_limits<particles_rnumber>::max()){
// The size of the field grid (global size) all_size seems
std::array<size_t,3> field_grid_dim;
@@ -197,8 +202,10 @@ struct particles_system_build_container {
using particles_system_type = particles_system<partsize_t, particles_rnumber, field_rnumber,
field<field_rnumber, be, fc>,
particles_generic_interp<particles_rnumber, interpolation_size,spline_mode>,
- interpolation_size, ncomp(fc),
- p2p_computer_class, p2p_computer_class::size_data>;
+ interpolation_size,
+ size_particle_positions, size_particle_rhs,
+ p2p_computer_class,
+ particles_inner_computer_class>;
particles_system_type* part_sys = new particles_system_type(field_grid_dim,
spatial_box_width,
spatial_box_offset,
@@ -210,6 +217,9 @@ struct particles_system_build_container {
(*fs_field),
mpi_comm,
nparticles,
+ cutoff,
+ p2p_computer,
+ inner_computer,
in_current_iteration);
// TODO P2P load particle data too
@@ -254,13 +264,19 @@ inline std::unique_ptr<abstract_particles_system<partsize_t, particles_rnumber>>
return Template_double_for_if::evaluate<std::unique_ptr<abstract_particles_system<partsize_t, particles_rnumber>>,
int, 1, 11, 1, // interpolation_size
int, 0, 3, 1, // spline_mode
- particles_system_build_container<partsize_t, field_rnumber,be,fc,particles_rnumber, p2p_computer_empty<particles_rnumber,partsize_t>>>(
+ particles_system_build_container<partsize_t, field_rnumber,be,fc,particles_rnumber,
+ p2p_computer_empty<particles_rnumber,partsize_t>,
+ particles_inner_computer_empty<particles_rnumber,partsize_t>,
+ 3,3>>(
interpolation_size, // template iterator 1
spline_mode, // template iterator 2
- fs_field,fs_kk, nsteps, nparticles, fname_input, inDatanameState, inDatanameRhs, mpi_comm, in_current_iteration);
+ fs_field,fs_kk, nsteps, nparticles, fname_input, inDatanameState, inDatanameRhs, mpi_comm, in_current_iteration,
+ p2p_computer_empty<particles_rnumber,partsize_t>(), particles_inner_computer_empty<particles_rnumber,partsize_t>());
}
-template <class partsize_t, class field_rnumber, field_backend be, field_components fc, class p2p_computer_class, class particles_rnumber = double>
+template <class partsize_t, class field_rnumber, field_backend be, field_components fc,
+ class p2p_computer_class, class particles_inner_computer_class,
+ class particles_rnumber = double>
inline std::unique_ptr<abstract_particles_system<partsize_t, particles_rnumber>> particles_system_builder_with_p2p(
const field<field_rnumber, be, fc>* fs_field, // (field object)
const kspace<be, SMOOTH>* fs_kk, // (kspace object, contains dkx, dky, dkz)
@@ -271,14 +287,21 @@ inline std::unique_ptr<abstract_particles_system<partsize_t, particles_rnumber>>
const int interpolation_size,
const int spline_mode,
MPI_Comm mpi_comm,
- const int in_current_iteration){
+ const int in_current_iteration,
+ p2p_computer_class p2p_computer,
+ particles_inner_computer_class inner_computer,
+ const particles_rnumber cutoff){
return Template_double_for_if::evaluate<std::unique_ptr<abstract_particles_system<partsize_t, particles_rnumber>>,
int, 1, 11, 1, // interpolation_size
int, 0, 3, 1, // spline_mode
- particles_system_build_container<partsize_t, field_rnumber,be,fc,particles_rnumber,p2p_computer_class>>(
+ particles_system_build_container<partsize_t, field_rnumber,be,fc,particles_rnumber,
+ p2p_computer_class,
+ particles_inner_computer_class,
+ 6,6>>(
interpolation_size, // template iterator 1
spline_mode, // template iterator 2
- fs_field,fs_kk, nsteps, nparticles, fname_input, inDatanameState, inDatanameRhs, mpi_comm, in_current_iteration);
+ fs_field,fs_kk, nsteps, nparticles, fname_input, inDatanameState, inDatanameRhs, mpi_comm, in_current_iteration,
+ std::move(p2p_computer), std::move(inner_computer), cutoff);
}
--
GitLab
From 4587eece04fa93d9b28fdc386b311b60d36b5194 Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Fri, 13 Oct 2017 11:39:25 +0200
Subject: [PATCH 039/342] Make it compile, but I would have prefer a class
attribute instead of method to decompose template of field
---
bfps/cpp/field.hpp | 4 ---
bfps/cpp/full_code/NSVEparticlesP2P.cpp | 4 +--
bfps/cpp/full_code/NSVEparticlesP2P.hpp | 1 +
bfps/cpp/particles/particles_distr_mpi.hpp | 8 ++---
.../particles/particles_field_computer.hpp | 29 +++++++++++++------
5 files changed, 27 insertions(+), 19 deletions(-)
diff --git a/bfps/cpp/field.hpp b/bfps/cpp/field.hpp
index 2d93a0c9..9a5ab1be 100644
--- a/bfps/cpp/field.hpp
+++ b/bfps/cpp/field.hpp
@@ -58,10 +58,6 @@ class field
private:
rnumber *__restrict__ data; /**< data array */
public:
- constexpr int nb_components() {
- return ncomp(fc);
- }
-
hsize_t npoints; /**< total number of grid points. Useful for normalization. */
bool real_space_representation; /**< `true` if field is in real space representation. */
diff --git a/bfps/cpp/full_code/NSVEparticlesP2P.cpp b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
index 9574b226..3f648308 100644
--- a/bfps/cpp/full_code/NSVEparticlesP2P.cpp
+++ b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
@@ -11,10 +11,10 @@ int NSVEparticlesP2P<rnumber>::initialize(void)
{
this->NSVE<rnumber>::initialize();
- p2p_computer<rnumber, long long int> current_p2p_computer;
+ p2p_computer<double, long long int> current_p2p_computer;
current_p2p_computer.setEnable(enable_p2p);
- particles_inner_computer<rnumber, long long int> current_particles_inner_computer(inner_v0);
+ particles_inner_computer<double, long long int> current_particles_inner_computer(inner_v0);
current_particles_inner_computer.setEnable(enable_inner);
this->ps = particles_system_builder_with_p2p(
diff --git a/bfps/cpp/full_code/NSVEparticlesP2P.hpp b/bfps/cpp/full_code/NSVEparticlesP2P.hpp
index 73427bca..6bfaa3a8 100644
--- a/bfps/cpp/full_code/NSVEparticlesP2P.hpp
+++ b/bfps/cpp/full_code/NSVEparticlesP2P.hpp
@@ -56,6 +56,7 @@ class NSVEparticlesP2P: public NSVE<rnumber>
int tracers0_smoothness;
double cutoff;
+ double inner_v0;
bool enable_p2p;
bool enable_inner;
bool enable_vorticity_omega;
diff --git a/bfps/cpp/particles/particles_distr_mpi.hpp b/bfps/cpp/particles/particles_distr_mpi.hpp
index 79ff2e8b..979c8d1e 100644
--- a/bfps/cpp/particles/particles_distr_mpi.hpp
+++ b/bfps/cpp/particles/particles_distr_mpi.hpp
@@ -383,7 +383,7 @@ public:
in_computer.template init_result_array<size_particle_rhs>(descriptor.results.get(), NbParticlesToReceive);
if(more_than_one_thread == false){
- in_computer.template apply_computation<field_class, size_particle_rhs>(in_field, descriptor.toCompute.get(), descriptor.results.get(), NbParticlesToReceive);
+ in_computer.template apply_computation<field_class, size_particle_positions, size_particle_rhs>(in_field, descriptor.toCompute.get(), descriptor.results.get(), NbParticlesToReceive);
}
else{
TIMEZONE_OMP_INIT_PRETASK(timeZoneTaskKey)
@@ -396,7 +396,7 @@ public:
TIMEZONE_OMP_PRAGMA_TASK_KEY(timeZoneTaskKey)
{
TIMEZONE_OMP_TASK("in_computer.apply_computation", timeZoneTaskKey);
- in_computer.template apply_computation<field_class, size_particle_rhs>(in_field, &ptr_descriptor->toCompute[idxPart*size_particle_positions],
+ in_computer.template apply_computation<field_class, size_particle_positions, size_particle_rhs>(in_field, &ptr_descriptor->toCompute[idxPart*size_particle_positions],
&ptr_descriptor->results[idxPart*size_particle_rhs], sizeToDo);
}
}
@@ -456,7 +456,7 @@ public:
#pragma omp task default(shared) firstprivate(idxPart, sizeToDo) priority(0) TIMEZONE_OMP_PRAGMA_TASK_KEY(timeZoneTaskKey)
{
TIMEZONE_OMP_TASK("in_computer.apply_computation", timeZoneTaskKey);
- in_computer.template apply_computation<field_class, size_particle_rhs>(in_field, &particles_positions[idxPart*size_particle_positions],
+ in_computer.template apply_computation<field_class, size_particle_positions, size_particle_rhs>(in_field, &particles_positions[idxPart*size_particle_positions],
&particles_current_rhs[idxPart*size_particle_rhs],
sizeToDo);
}
@@ -492,7 +492,7 @@ public:
TIMEZONE("compute-my_compute");
// Compute my particles
if(myTotalNbParticles){
- in_computer.template apply_computation<field_class, size_particle_rhs>(in_field, particles_positions, particles_current_rhs, myTotalNbParticles);
+ in_computer.template apply_computation<field_class, size_particle_positions, size_particle_rhs>(in_field, particles_positions, particles_current_rhs, myTotalNbParticles);
}
}
diff --git a/bfps/cpp/particles/particles_field_computer.hpp b/bfps/cpp/particles/particles_field_computer.hpp
index 70da4757..330763c8 100644
--- a/bfps/cpp/particles/particles_field_computer.hpp
+++ b/bfps/cpp/particles/particles_field_computer.hpp
@@ -12,6 +12,16 @@ template <class partsize_t,
class interpolator_class,
int interp_neighbours>
class particles_field_computer {
+ // TODO but not critical, add in field:
+ // static const int nb_components = ncomp(fc);
+ // and use it as field_class::nb_components
+ // but failed up to now....
+ template <typename rnumber,
+ field_backend be,
+ field_components fc>
+ static constexpr int nbcomp(const field<rnumber, be, fc>& /*field*/){
+ return ncomp(fc);
+ }
const std::array<int,3> field_grid_dim;
const std::pair<int,int> current_partition_interval;
@@ -62,18 +72,19 @@ public:
return pos_in_cell;
}
- template <class field_class, int size_particle_rhs>
+ template <class field_class, int size_particle_positions, int size_particle_rhs>
void apply_computation(const field_class& field,
const real_number particles_positions[],
real_number particles_current_rhs[],
const partsize_t nb_particles) const {
- static_assert(field_class::nb_components() <= size_particle_rhs, "Cannot store all the component in the given array");
+ constexpr int nb_components_in_field = nbcomp(field);
+ static_assert(nb_components_in_field <= size_particle_rhs, "Cannot store all the component in the given array");
TIMEZONE("particles_field_computer::apply_computation");
for(partsize_t idxPart = 0 ; idxPart < nb_particles ; ++idxPart){
- const real_number reltv_x = get_norm_pos_in_cell(particles_positions[idxPart*3+IDX_X], IDX_X);
- const real_number reltv_y = get_norm_pos_in_cell(particles_positions[idxPart*3+IDX_Y], IDX_Y);
- const real_number reltv_z = get_norm_pos_in_cell(particles_positions[idxPart*3+IDX_Z], IDX_Z);
+ const real_number reltv_x = get_norm_pos_in_cell(particles_positions[idxPart*size_particle_positions+IDX_X], IDX_X);
+ const real_number reltv_y = get_norm_pos_in_cell(particles_positions[idxPart*size_particle_positions+IDX_Y], IDX_Y);
+ const real_number reltv_z = get_norm_pos_in_cell(particles_positions[idxPart*size_particle_positions+IDX_Z], IDX_Z);
typename interpolator_class::real_number
bx[interp_neighbours*2+2],
@@ -83,9 +94,9 @@ public:
interpolator.compute_beta(deriv[IDX_Y], reltv_y, by);
interpolator.compute_beta(deriv[IDX_Z], reltv_z, bz);
- const int partGridIdx_x = pbc_field_layer(particles_positions[idxPart*3+IDX_X], IDX_X);
- const int partGridIdx_y = pbc_field_layer(particles_positions[idxPart*3+IDX_Y], IDX_Y);
- const int partGridIdx_z = pbc_field_layer(particles_positions[idxPart*3+IDX_Z], IDX_Z);
+ const int partGridIdx_x = pbc_field_layer(particles_positions[idxPart*size_particle_positions+IDX_X], IDX_X);
+ const int partGridIdx_y = pbc_field_layer(particles_positions[idxPart*size_particle_positions+IDX_Y], IDX_Y);
+ const int partGridIdx_z = pbc_field_layer(particles_positions[idxPart*size_particle_positions+IDX_Z], IDX_Z);
assert(0 <= partGridIdx_x && partGridIdx_x < int(field_grid_dim[IDX_X]));
assert(0 <= partGridIdx_y && partGridIdx_y < int(field_grid_dim[IDX_Y]));
@@ -148,7 +159,7 @@ public:
// getValue does not necessary return real_number
// size_particle_rhs is just for the leading dimension of the array
- for(int idx_rhs_val = 0 ; idx_rhs_val < field_class::nb_components() ; ++idx_rhs_val){
+ for(int idx_rhs_val = 0 ; idx_rhs_val < nb_components_in_field ; ++idx_rhs_val){
particles_current_rhs[idxPart*size_particle_rhs+idx_rhs_val] += real_number(field.rval(tindex,idx_rhs_val))*coef;
}
}
--
GitLab
From b3e45e2003ec1441428932f7edcd066831d90a2c Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 18 Oct 2017 16:34:12 +0200
Subject: [PATCH 040/342] change particle I/O file interaction
the "simname_particles.h5" file is now only opened once, and then
samples are written the same as before.
some sanity checks of the velocity and acceleration data should still be
performed.
---
bfps/cpp/full_code/NSVEparticles.cpp | 78 +++++++++++----
bfps/cpp/full_code/NSVEparticles.hpp | 2 +
bfps/cpp/full_code/field_test.cpp | 2 +-
bfps/cpp/full_code/filter_test.cpp | 2 +-
bfps/cpp/full_code/test.cpp | 3 +-
.../particles/abstract_particles_output.hpp | 4 +
.../particles_output_sampling_hdf5.hpp | 95 +++++++++++++++----
setup.py | 6 +-
8 files changed, 151 insertions(+), 41 deletions(-)
diff --git a/bfps/cpp/full_code/NSVEparticles.cpp b/bfps/cpp/full_code/NSVEparticles.cpp
index 2384e3f1..7536145e 100644
--- a/bfps/cpp/full_code/NSVEparticles.cpp
+++ b/bfps/cpp/full_code/NSVEparticles.cpp
@@ -2,7 +2,6 @@
#include <cmath>
#include "NSVEparticles.hpp"
#include "scope_timer.hpp"
-#include "particles/particles_sampling.hpp"
template <typename rnumber>
int NSVEparticles<rnumber>::initialize(void)
@@ -27,6 +26,13 @@ int NSVEparticles<rnumber>::initialize(void)
"tracers0",
nparticles,
tracers0_integration_steps);
+ this->particles_sample_writer_mpi = new particles_output_sampling_hdf5<
+ long long int, double, 3, 3>(
+ MPI_COMM_WORLD,
+ this->ps->getGlobalNbParticles(),
+ (this->simname + "_particles.h5"),
+ "tracers0",
+ "position/0");
return EXIT_SUCCESS;
}
@@ -60,6 +66,7 @@ int NSVEparticles<rnumber>::finalize(void)
{
this->ps.release();
delete this->particles_output_writer_mpi;
+ delete this->particles_sample_writer_mpi;
this->NSVE<rnumber>::finalize();
return EXIT_SUCCESS;
}
@@ -77,30 +84,65 @@ int NSVEparticles<rnumber>::do_stats()
if (!(this->iteration % this->niter_part == 0))
return EXIT_SUCCESS;
+ // allocate temporary data array
+ std::unique_ptr<double[]> pdata(new double[3*this->ps->getLocalNbParticles()]);
+
+ // copy position data
+ std::copy(this->ps->getParticlesPositions(),
+ this->ps->getParticlesPositions()+this->ps->getLocalNbParticles(),
+ pdata.get());
+
/// sample position
- sample_particles_system_position(
- this->ps,
- (this->simname + "_particles.h5"), // filename
- "tracers0", // hdf5 parent group
- "position" // dataset basename TODO
- );
+ this->particles_sample_writer_mpi->save_dataset(
+ "tracers0",
+ "position",
+ this->ps->getParticlesPositions(),
+ &pdata,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx());
+
+ ////sample_particles_system_position(
+ //// this->ps,
+ //// (this->simname + "_particles.h5"), // filename
+ //// "tracers0", // hdf5 parent group
+ //// "position" // dataset basename TODO
+ //// );
/// sample velocity
- sample_from_particles_system(*this->tmp_vec_field, // field to save
- this->ps,
- (this->simname + "_particles.h5"), // filename
- "tracers0", // hdf5 parent group
- "velocity" // dataset basename TODO
- );
+ this->ps->sample_compute_field(*this->tmp_vec_field, pdata.get());
+ this->particles_sample_writer_mpi->save_dataset(
+ "tracers0",
+ "velocity",
+ this->ps->getParticlesPositions(),
+ &pdata,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx());
+ //sample_from_particles_system(*this->tmp_vec_field, // field to save
+ // this->ps,
+ // (this->simname + "_particles.h5"), // filename
+ // "tracers0", // hdf5 parent group
+ // "velocity" // dataset basename TODO
+ // );
/// compute acceleration and sample it
this->fs->compute_Lagrangian_acceleration(this->tmp_vec_field);
this->tmp_vec_field->ift();
- sample_from_particles_system(*this->tmp_vec_field,
- this->ps,
- (this->simname + "_particles.h5"),
- "tracers0",
- "acceleration");
+ this->ps->sample_compute_field(*this->tmp_vec_field, pdata.get());
+ this->particles_sample_writer_mpi->save_dataset(
+ "tracers0",
+ "acceleration",
+ this->ps->getParticlesPositions(),
+ &pdata,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx());
+ //sample_from_particles_system(*this->tmp_vec_field,
+ // this->ps,
+ // (this->simname + "_particles.h5"),
+ // "tracers0",
+ // "acceleration");
return EXIT_SUCCESS;
}
diff --git a/bfps/cpp/full_code/NSVEparticles.hpp b/bfps/cpp/full_code/NSVEparticles.hpp
index ccafe6ee..03d45aaa 100644
--- a/bfps/cpp/full_code/NSVEparticles.hpp
+++ b/bfps/cpp/full_code/NSVEparticles.hpp
@@ -35,6 +35,7 @@
#include "full_code/NSVE.hpp"
#include "particles/particles_system_builder.hpp"
#include "particles/particles_output_hdf5.hpp"
+#include "particles/particles_sampling.hpp"
/** \brief Navier-Stokes solver that includes simple Lagrangian tracers.
*
@@ -58,6 +59,7 @@ class NSVEparticles: public NSVE<rnumber>
/* other stuff */
std::unique_ptr<abstract_particles_system<long long int, double>> ps;
particles_output_hdf5<long long int, double,3,3> *particles_output_writer_mpi;
+ particles_output_sampling_hdf5<long long int, double, 3, 3> *particles_sample_writer_mpi;
NSVEparticles(
diff --git a/bfps/cpp/full_code/field_test.cpp b/bfps/cpp/full_code/field_test.cpp
index acee3617..b07f9b39 100644
--- a/bfps/cpp/full_code/field_test.cpp
+++ b/bfps/cpp/full_code/field_test.cpp
@@ -24,7 +24,7 @@ int field_test<rnumber>::read_parameters()
this->test::read_parameters();
// in case any parameters are needed, this is where they should be read
hid_t parameter_file;
- hid_t dset, memtype, space;
+ hid_t dset;
parameter_file = H5Fopen(
(this->simname + std::string(".h5")).c_str(),
H5F_ACC_RDONLY,
diff --git a/bfps/cpp/full_code/filter_test.cpp b/bfps/cpp/full_code/filter_test.cpp
index aeedfbe7..80c4f83d 100644
--- a/bfps/cpp/full_code/filter_test.cpp
+++ b/bfps/cpp/full_code/filter_test.cpp
@@ -40,7 +40,7 @@ int filter_test<rnumber>::read_parameters()
{
this->test::read_parameters();
hid_t parameter_file;
- hid_t dset, memtype, space;
+ hid_t dset;
parameter_file = H5Fopen(
(this->simname + std::string(".h5")).c_str(),
H5F_ACC_RDONLY,
diff --git a/bfps/cpp/full_code/test.cpp b/bfps/cpp/full_code/test.cpp
index fd2192a0..9c2e4e67 100644
--- a/bfps/cpp/full_code/test.cpp
+++ b/bfps/cpp/full_code/test.cpp
@@ -22,9 +22,8 @@ int test::main_loop(void)
int test::read_parameters()
{
hid_t parameter_file;
- hid_t dset, memtype, space;
+ hid_t dset;
char fname[256];
- char *string_data;
sprintf(fname, "%s.h5", this->simname.c_str());
parameter_file = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT);
dset = H5Dopen(parameter_file, "/parameters/dealias_type", H5P_DEFAULT);
diff --git a/bfps/cpp/particles/abstract_particles_output.hpp b/bfps/cpp/particles/abstract_particles_output.hpp
index a6eccaea..5285c90f 100644
--- a/bfps/cpp/particles/abstract_particles_output.hpp
+++ b/bfps/cpp/particles/abstract_particles_output.hpp
@@ -41,6 +41,10 @@ class abstract_particles_output {
partsize_t particles_chunk_current_offset;
protected:
+ MPI_Comm& getCom(){
+ return mpi_com;
+ }
+
MPI_Comm& getComWriter(){
return mpi_com_writer;
}
diff --git a/bfps/cpp/particles/particles_output_sampling_hdf5.hpp b/bfps/cpp/particles/particles_output_sampling_hdf5.hpp
index 238c9acf..64faffdd 100644
--- a/bfps/cpp/particles/particles_output_sampling_hdf5.hpp
+++ b/bfps/cpp/particles/particles_output_sampling_hdf5.hpp
@@ -9,10 +9,11 @@ template <class partsize_t,
class real_number,
int size_particle_positions,
int size_particle_rhs>
-class particles_output_sampling_hdf5 : public abstract_particles_output<partsize_t,
- real_number,
- size_particle_positions,
- size_particle_rhs>{
+class particles_output_sampling_hdf5 : public abstract_particles_output<
+ partsize_t,
+ real_number,
+ size_particle_positions,
+ size_particle_rhs>{
using Parent = abstract_particles_output<partsize_t,
real_number,
size_particle_positions,
@@ -20,7 +21,7 @@ class particles_output_sampling_hdf5 : public abstract_particles_output<partsize
hid_t file_id, pgroup_id;
- const std::string dataset_name;
+ std::string dataset_name;
const bool use_collective_io;
public:
@@ -34,7 +35,6 @@ public:
int dataset_exists = -1;
if(my_rank == 0){
- // Parallel HDF5 write
hid_t file_id = H5Fopen(
in_filename.c_str(),
H5F_ACC_RDWR | H5F_ACC_DEBUG,
@@ -54,16 +54,18 @@ public:
return dataset_exists;
}
- particles_output_sampling_hdf5(MPI_Comm in_mpi_com,
- const partsize_t inTotalNbParticles,
- const std::string& in_filename,
- const std::string& in_groupname,
- const std::string& in_dataset_name,
- const bool in_use_collective_io = false)
+ particles_output_sampling_hdf5(
+ MPI_Comm in_mpi_com,
+ const partsize_t inTotalNbParticles,
+ const std::string& in_filename,
+ const std::string& in_groupname,
+ const std::string& in_dataset_name,
+ const bool in_use_collective_io = false)
: Parent(in_mpi_com, inTotalNbParticles, 1),
dataset_name(in_dataset_name),
use_collective_io(in_use_collective_io){
if(Parent::isInvolved()){
+ // prepare parallel MPI access property list
hid_t plist_id_par = H5Pcreate(H5P_FILE_ACCESS);
assert(plist_id_par >= 0);
int retTest = H5Pset_fapl_mpio(
@@ -72,7 +74,7 @@ public:
MPI_INFO_NULL);
assert(retTest >= 0);
- // Parallel HDF5 write
+ // open file for parallel HDF5 access
file_id = H5Fopen(
in_filename.c_str(),
H5F_ACC_RDWR | H5F_ACC_DEBUG,
@@ -81,6 +83,7 @@ public:
retTest = H5Pclose(plist_id_par);
assert(retTest >= 0);
+ // open group
pgroup_id = H5Gopen(
file_id,
in_groupname.c_str(),
@@ -91,13 +94,65 @@ public:
~particles_output_sampling_hdf5(){
if(Parent::isInvolved()){
+ // close group
int retTest = H5Gclose(pgroup_id);
assert(retTest >= 0);
+ // close file
retTest = H5Fclose(file_id);
assert(retTest >= 0);
}
}
+ int switch_to_group(
+ const std::string &in_groupname)
+ {
+ if(Parent::isInvolved()){
+ // close old group
+ int retTest = H5Gclose(pgroup_id);
+ assert(retTest >= 0);
+
+ // open new group
+ pgroup_id = H5Gopen(
+ file_id,
+ in_groupname.c_str(),
+ H5P_DEFAULT);
+ assert(pgroup_id >= 0);
+ }
+ return EXIT_SUCCESS;
+ }
+
+ int save_dataset(
+ const std::string& in_groupname,
+ const std::string& in_dataset_name,
+ const real_number input_particles_positions[],
+ const std::unique_ptr<real_number[]> input_particles_rhs[],
+ const partsize_t index_particles[],
+ const partsize_t nb_particles,
+ const int idx_time_step)
+ {
+ // update group
+ int retTest = this->switch_to_group(
+ in_groupname);
+ assert(retTest == EXIT_SUCCESS);
+ // update dataset name
+ dataset_name = in_dataset_name + "/" + std::to_string(idx_time_step);
+ int dataset_exists;
+ if (this->getMyRank() == 0)
+ dataset_exists = H5Lexists(
+ pgroup_id,
+ dataset_name.c_str(),
+ H5P_DEFAULT);
+ AssertMpi(MPI_Bcast(&dataset_exists, 1, MPI_INT, 0, this->getCom()));
+ if (dataset_exists == 0)
+ this->save(
+ input_particles_positions,
+ input_particles_rhs,
+ index_particles,
+ nb_particles,
+ idx_time_step);
+ return EXIT_SUCCESS;
+ }
+
void write(
const int /*idx_time_step*/,
const real_number* /*particles_positions*/,
@@ -108,18 +163,26 @@ public:
TIMEZONE("particles_output_hdf5::write");
- assert(particles_idx_offset < Parent::getTotalNbParticles() || (particles_idx_offset == Parent::getTotalNbParticles() && nb_particles == 0));
+ assert(particles_idx_offset < Parent::getTotalNbParticles() ||
+ (particles_idx_offset == Parent::getTotalNbParticles() &&
+ nb_particles == 0));
assert(particles_idx_offset+nb_particles <= Parent::getTotalNbParticles());
static_assert(std::is_same<real_number, double>::value ||
std::is_same<real_number, float>::value,
"real_number must be double or float");
- const hid_t type_id = (sizeof(real_number) == 8 ? H5T_NATIVE_DOUBLE : H5T_NATIVE_FLOAT);
+ const hid_t type_id = (sizeof(real_number) == 8 ?
+ H5T_NATIVE_DOUBLE :
+ H5T_NATIVE_FLOAT);
hid_t plist_id = H5Pcreate(H5P_DATASET_XFER);
assert(plist_id >= 0);
{
- int rethdf = H5Pset_dxpl_mpio(plist_id, use_collective_io ? H5FD_MPIO_COLLECTIVE : H5FD_MPIO_INDEPENDENT);
+ int rethdf = H5Pset_dxpl_mpio(
+ plist_id,
+ (use_collective_io ?
+ H5FD_MPIO_COLLECTIVE :
+ H5FD_MPIO_INDEPENDENT));
assert(rethdf >= 0);
}
{
diff --git a/setup.py b/setup.py
index b03bd4f4..1fae430b 100644
--- a/setup.py
+++ b/setup.py
@@ -88,7 +88,8 @@ print('This is bfps version ' + VERSION)
### lists of files and MANIFEST.in
-src_file_list = ['full_code/joint_acc_vel_stats',
+src_file_list = ['full_code/NSVEparticles',
+ 'full_code/joint_acc_vel_stats',
'full_code/test',
'full_code/filter_test',
'full_code/field_test',
@@ -127,8 +128,7 @@ src_file_list = ['full_code/joint_acc_vel_stats',
'spline_n9',
'spline_n10',
'Lagrange_polys',
- 'scope_timer',
- 'full_code/NSVEparticles']
+ 'scope_timer']
particle_headers = [
'cpp/particles/particles_distr_mpi.hpp',
--
GitLab
From 60fc09f7f3241e64eff4679b119f44bbc82a5413 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 18 Oct 2017 17:11:49 +0200
Subject: [PATCH 041/342] fix position sampling
---
bfps/cpp/full_code/NSVEparticles.cpp | 33 ++++++++--------------------
1 file changed, 9 insertions(+), 24 deletions(-)
diff --git a/bfps/cpp/full_code/NSVEparticles.cpp b/bfps/cpp/full_code/NSVEparticles.cpp
index 7536145e..0b1adfdf 100644
--- a/bfps/cpp/full_code/NSVEparticles.cpp
+++ b/bfps/cpp/full_code/NSVEparticles.cpp
@@ -88,11 +88,11 @@ int NSVEparticles<rnumber>::do_stats()
std::unique_ptr<double[]> pdata(new double[3*this->ps->getLocalNbParticles()]);
// copy position data
- std::copy(this->ps->getParticlesPositions(),
- this->ps->getParticlesPositions()+this->ps->getLocalNbParticles(),
- pdata.get());
/// sample position
+ std::copy(this->ps->getParticlesPositions(),
+ this->ps->getParticlesPositions()+3*this->ps->getLocalNbParticles(),
+ pdata.get());
this->particles_sample_writer_mpi->save_dataset(
"tracers0",
"position",
@@ -100,14 +100,7 @@ int NSVEparticles<rnumber>::do_stats()
&pdata,
this->ps->getParticlesIndexes(),
this->ps->getLocalNbParticles(),
- this->ps->get_step_idx());
-
- ////sample_particles_system_position(
- //// this->ps,
- //// (this->simname + "_particles.h5"), // filename
- //// "tracers0", // hdf5 parent group
- //// "position" // dataset basename TODO
- //// );
+ this->ps->get_step_idx()-1);
/// sample velocity
this->ps->sample_compute_field(*this->tmp_vec_field, pdata.get());
@@ -118,13 +111,7 @@ int NSVEparticles<rnumber>::do_stats()
&pdata,
this->ps->getParticlesIndexes(),
this->ps->getLocalNbParticles(),
- this->ps->get_step_idx());
- //sample_from_particles_system(*this->tmp_vec_field, // field to save
- // this->ps,
- // (this->simname + "_particles.h5"), // filename
- // "tracers0", // hdf5 parent group
- // "velocity" // dataset basename TODO
- // );
+ this->ps->get_step_idx()-1);
/// compute acceleration and sample it
this->fs->compute_Lagrangian_acceleration(this->tmp_vec_field);
@@ -137,12 +124,10 @@ int NSVEparticles<rnumber>::do_stats()
&pdata,
this->ps->getParticlesIndexes(),
this->ps->getLocalNbParticles(),
- this->ps->get_step_idx());
- //sample_from_particles_system(*this->tmp_vec_field,
- // this->ps,
- // (this->simname + "_particles.h5"),
- // "tracers0",
- // "acceleration");
+ this->ps->get_step_idx()-1);
+
+ // deallocate temporary data array
+ pdata.release();
return EXIT_SUCCESS;
}
--
GitLab
From 754e27d315d31672c9fe74a8c827aa2041c9e718 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 19 Oct 2017 10:38:32 +0200
Subject: [PATCH 042/342] add computation of alpha
---
bfps/cpp/particles/particles_inner_computer.hpp | 14 +++++++++++++-
1 file changed, 13 insertions(+), 1 deletion(-)
diff --git a/bfps/cpp/particles/particles_inner_computer.hpp b/bfps/cpp/particles/particles_inner_computer.hpp
index 4d0a6678..0498bd95 100644
--- a/bfps/cpp/particles/particles_inner_computer.hpp
+++ b/bfps/cpp/particles/particles_inner_computer.hpp
@@ -25,8 +25,20 @@ public:
rhs_part[idx_part*size_particle_rhs + IDX_Y] += pos_part[idx_part*size_particle_positions + 3+IDX_Y]*v0;
rhs_part[idx_part*size_particle_rhs + IDX_Z] += pos_part[idx_part*size_particle_positions + 3+IDX_Z]*v0;
- real_number alpha[3]= {0}; // TODO compute aplha
+ real_number alpha[3] = {0, 0, 0};
+ // I call "rotation" to be the right hand side of the orientation part of the ODE
+ // project rotation on orientation:
+ real_number projection = (
+ pos_part[idx_part*size_particle_positions + 3+IDX_X]*rhs_part[idx_part*size_particle_rhs + 3+IDX_X] +
+ pos_part[idx_part*size_particle_positions + 3+IDX_Y]*rhs_part[idx_part*size_particle_rhs + 3+IDX_Y] +
+ pos_part[idx_part*size_particle_positions + 3+IDX_Z]*rhs_part[idx_part*size_particle_rhs + 3+IDX_Z]);
+ // alpha is the vector that makes rotation perpendicular to orientation.
+ // note that the following three lines assume the current orientation is a unit vector.
+ alpha[IDX_X] = -pos_part[idx_part*size_particle_positions + 3+IDX_X]*projection;
+ alpha[IDX_Y] = -pos_part[idx_part*size_particle_positions + 3+IDX_Z]*projection;
+ alpha[IDX_Z] = -pos_part[idx_part*size_particle_positions + 3+IDX_Y]*projection;
+ // now add alpha term to orientation ODE right-hand side.
rhs_part[idx_part*size_particle_rhs + 3+IDX_X] += alpha[IDX_X];
rhs_part[idx_part*size_particle_rhs + 3+IDX_Y] += alpha[IDX_Y];
rhs_part[idx_part*size_particle_rhs + 3+IDX_Z] += alpha[IDX_Z];
--
GitLab
From f669814b8fba1369cd171ac25aa6bedef7b2efca Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 19 Oct 2017 10:49:15 +0200
Subject: [PATCH 043/342] realign some equations
a lot easier to read now
---
bfps/cpp/particles/particles_inner_computer.hpp | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/bfps/cpp/particles/particles_inner_computer.hpp b/bfps/cpp/particles/particles_inner_computer.hpp
index 0498bd95..b20abfda 100644
--- a/bfps/cpp/particles/particles_inner_computer.hpp
+++ b/bfps/cpp/particles/particles_inner_computer.hpp
@@ -55,12 +55,12 @@ public:
#pragma omp parallel for
for(partsize_t idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
// Cross product vorticity/orientation
- rhs_part[idx_part*size_particle_rhs + 3+IDX_X] += rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Y]*pos_part[idx_part*size_particle_positions + 3+IDX_Z]
- - rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Z]*pos_part[idx_part*size_particle_positions + 3+IDX_Y];
- rhs_part[idx_part*size_particle_rhs + 3+IDX_Y] += rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Z]*pos_part[idx_part*size_particle_positions + 3+IDX_X]
- - rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_X]*pos_part[idx_part*size_particle_positions + 3+IDX_Z];
- rhs_part[idx_part*size_particle_rhs + 3+IDX_Z] += rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_X]*pos_part[idx_part*size_particle_positions + 3+IDX_Y]
- - rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Y]*pos_part[idx_part*size_particle_positions + 3+IDX_X];
+ rhs_part[idx_part*size_particle_rhs + 3+IDX_X] += (rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Y]*pos_part[idx_part*size_particle_positions + 3+IDX_Z] -
+ rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Z]*pos_part[idx_part*size_particle_positions + 3+IDX_Y]);
+ rhs_part[idx_part*size_particle_rhs + 3+IDX_Y] += (rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Z]*pos_part[idx_part*size_particle_positions + 3+IDX_X] -
+ rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_X]*pos_part[idx_part*size_particle_positions + 3+IDX_Z]);
+ rhs_part[idx_part*size_particle_rhs + 3+IDX_Z] += (rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_X]*pos_part[idx_part*size_particle_positions + 3+IDX_Y] -
+ rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Y]*pos_part[idx_part*size_particle_positions + 3+IDX_X]);
}
}
@@ -75,3 +75,4 @@ public:
};
#endif
+
--
GitLab
From 6f214d33adc6ea5fb9ddbd8140eecb4aba062886 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 19 Oct 2017 11:09:24 +0200
Subject: [PATCH 044/342] update particle header file list
---
setup.py | 19 +++++++++++++------
1 file changed, 13 insertions(+), 6 deletions(-)
diff --git a/setup.py b/setup.py
index f257fd7f..3d8d05ca 100644
--- a/setup.py
+++ b/setup.py
@@ -131,23 +131,30 @@ src_file_list = ['full_code/joint_acc_vel_stats',
'full_code/NSVEparticlesP2P']
particle_headers = [
- 'cpp/particles/particles_distr_mpi.hpp',
'cpp/particles/abstract_particles_input.hpp',
'cpp/particles/abstract_particles_output.hpp',
'cpp/particles/abstract_particles_system.hpp',
'cpp/particles/alltoall_exchanger.hpp',
+ 'cpp/particles/env_utils.hpp',
+ 'cpp/particles/lock_free_bool_array.hpp',
+ 'cpp/particles/p2p_computer_empty.hpp',
+ 'cpp/particles/p2p_computer.hpp',
+ 'cpp/particles/p2p_distr_mpi.hpp',
+ 'cpp/particles/p2p_tree.hpp',
'cpp/particles/particles_adams_bashforth.hpp',
+ 'cpp/particles/particles_distr_mpi.hpp',
'cpp/particles/particles_field_computer.hpp',
- 'cpp/particles/particles_input_hdf5.hpp',
'cpp/particles/particles_generic_interp.hpp',
+ 'cpp/particles/particles_inner_computer_empty.hpp',
+ 'cpp/particles/particles_inner_computer.hpp',
+ 'cpp/particles/particles_input_hdf5.hpp',
'cpp/particles/particles_output_hdf5.hpp',
'cpp/particles/particles_output_mpiio.hpp',
- 'cpp/particles/particles_system_builder.hpp',
- 'cpp/particles/particles_system.hpp',
- 'cpp/particles/particles_utils.hpp',
'cpp/particles/particles_output_sampling_hdf5.hpp',
'cpp/particles/particles_sampling.hpp',
- 'cpp/particles/env_utils.hpp']
+ 'cpp/particles/particles_system_builder.hpp',
+ 'cpp/particles/particles_system.hpp',
+ 'cpp/particles/particles_utils.hpp']
full_code_headers = ['cpp/full_code/main_code.hpp',
'cpp/full_code/codes_with_no_output.hpp',
--
GitLab
From 8b2867ce2de1a11b38e1439808adcf4bf33ae6e0 Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Thu, 19 Oct 2017 14:20:51 +0200
Subject: [PATCH 045/342] generate initial conditions forparticles with
orientation
---
bfps/DNS.py | 46 +++++++++++++++++++++++++++++++++++-----------
1 file changed, 35 insertions(+), 11 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 8cbe8d9c..be973e17 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -127,7 +127,7 @@ class DNS(_code):
template_class = '{0}<{1}>::'.format(self.dns_type, rnumber),
template_prefix = 'template '.format(rnumber),
just_declaration = True) + '\n\n')
- if self.dns_type in ['NSVEparticles', 'NSVE_no_output', 'NSVEparticles_no_output']:
+ if self.dns_type in ['NSVEparticles', 'NSVE_no_output', 'NSVEparticles_no_output', 'NSVEparticlesP2P']:
outfile.write('template <typename rnumber> int NSVE<rnumber>::read_parameters(){return EXIT_SUCCESS;}\n')
outfile.write('template int NSVE<float>::read_parameters();\n')
outfile.write('template int NSVE<double>::read_parameters();\n\n')
@@ -377,7 +377,7 @@ class DNS(_code):
assert (self.parameters['niter_todo'] % self.parameters['niter_stat'] == 0)
assert (self.parameters['niter_todo'] % self.parameters['niter_out'] == 0)
assert (self.parameters['niter_out'] % self.parameters['niter_stat'] == 0)
- if self.dns_type in ['NSVEparticles_no_output', 'NSVEparticles']:
+ if self.dns_type in ['NSVEparticles_no_output', 'NSVEparticlesP2P', 'NSVEparticles']:
assert (self.parameters['niter_todo'] % self.parameters['niter_part'] == 0)
assert (self.parameters['niter_out'] % self.parameters['niter_part'] == 0)
_code.write_par(self, iter0 = iter0)
@@ -432,6 +432,9 @@ class DNS(_code):
number_of_particles = 1
for val in pbase_shape[1:]:
number_of_particles *= val
+ ncomponents = 3
+ if self.dns_type in ['NSVEparticlesP2P']:
+ ncomponents = 6
with h5py.File(self.get_checkpoint_0_fname(), 'a') as ofile:
s = 0
ofile.create_group('tracers{0}'.format(s))
@@ -442,13 +445,13 @@ class DNS(_code):
shape = (
(self.parameters['tracers{0}_integration_steps'.format(s)],) +
pbase_shape +
- (3,)),
+ (ncomponents,)),
dtype = np.float)
ofile['tracers{0}/state'.format(s)].create_dataset(
'0',
shape = (
pbase_shape +
- (3,)),
+ (ncomponents,)),
dtype = np.float)
return None
def job_parser_arguments(
@@ -621,6 +624,17 @@ class DNS(_code):
self.parameters_to_parser_arguments(
parser_NSVEp2,
self.NSVEp_extra_parameters)
+
+ parser_NSVEp2p = subparsers.add_parser(
+ 'NSVEparticlesP2P',
+ help = 'plain Navier-Stokes vorticity formulation, with basic fluid tracers')
+ self.simulation_parser_arguments(parser_NSVEp2p)
+ self.job_parser_arguments(parser_NSVEp2p)
+ self.particle_parser_arguments(parser_NSVEp2p)
+ self.parameters_to_parser_arguments(parser_NSVEp2p)
+ self.parameters_to_parser_arguments(
+ parser_NSVEp2p,
+ self.NSVEp_extra_parameters)
return None
def prepare_launch(
self,
@@ -656,7 +670,7 @@ class DNS(_code):
self.dns_type = opt.DNS_class
self.name = self.dns_type + '-' + self.fluid_precision + '-v' + bfps.__version__
# merge parameters if needed
- if self.dns_type in ['NSVEparticles', 'NSVEparticles_no_output']:
+ if self.dns_type in ['NSVEparticles', 'NSVEparticlesP2P', 'NSVEparticles_no_output']:
for k in self.NSVEp_extra_parameters.keys():
self.parameters[k] = self.NSVEp_extra_parameters[k]
if type(extra_parameters) != type(None):
@@ -690,7 +704,7 @@ class DNS(_code):
# hardcoded FFTW complex representation size
field_size = 3*(opt.nx+2)*opt.ny*opt.nz*self.fluid_dtype.itemsize
checkpoint_size = field_size
- if self.dns_type in ['NSVEparticles', 'NSVEparticles_no_output']:
+ if self.dns_type in ['NSVEparticles', 'NSVEparticlesP2P', 'NSVEparticles_no_output']:
rhs_size = self.parameters['tracers0_integration_steps']
if type(opt.tracers0_integration_steps) != type(None):
rhs_size = opt.tracers0_integration_steps
@@ -726,14 +740,24 @@ class DNS(_code):
nn = self.parameters['nparticles']
cc = int(0)
batch_size = int(1e6)
+ def get_random_phases(npoints):
+ return np.random.random(
+ (npoints, 3))*2*np.pi
+ def get_random_versors(npoints):
+ bla = np.random.normal(
+ size = (npoints, 3))
+ bla /= np.sum(bla**2, axis = 1)[:, None]
+ return bla
while nn > 0:
if nn > batch_size:
- dset[cc*batch_size:(cc+1)*batch_size] = np.random.random(
- (batch_size, 3))*2*np.pi
+ dset[cc*batch_size:(cc+1)*batch_size, :3] = get_random_phases(batch_size)
+ if dset.shape[1] == 6:
+ dset[cc*batch_size:(cc+1)*batch_size, 3:] = get_random_versors(batch_size)
nn -= batch_size
else:
- dset[cc*batch_size:cc*batch_size+nn] = np.random.random(
- (nn, 3))*2*np.pi
+ dset[cc*batch_size:cc*batch_size+nn, :3] = get_random_phases(nn)
+ if dset.shape[1] == 6:
+ dset[cc*batch_size:cc*batch_size+nn, 3:] = get_random_versors(nn)
nn = 0
cc += 1
return None
@@ -943,7 +967,7 @@ class DNS(_code):
# particle_initial_condition[..., 2] += onedarray[None, :, None, None]
self.write_par(
particle_ic = None)
- if self.dns_type in ['NSVEparticles', 'NSVEparticles_no_output']:
+ if self.dns_type in ['NSVEparticles', 'NSVEparticlesP2P', 'NSVEparticles_no_output']:
self.generate_particle_data(opt = opt)
self.run(
nb_processes = opt.nb_processes,
--
GitLab
From cd6bc8730c88ea76ee467707e47133df5d3e82f8 Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Thu, 19 Oct 2017 15:34:10 +0200
Subject: [PATCH 046/342] update initial condition reader template
---
bfps/cpp/particles/particles_system_builder.hpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bfps/cpp/particles/particles_system_builder.hpp b/bfps/cpp/particles/particles_system_builder.hpp
index ba078000..3e72ceaa 100644
--- a/bfps/cpp/particles/particles_system_builder.hpp
+++ b/bfps/cpp/particles/particles_system_builder.hpp
@@ -224,7 +224,7 @@ struct particles_system_build_container {
// TODO P2P load particle data too
// Load particles from hdf5
- particles_input_hdf5<partsize_t, particles_rnumber, 3,3> generator(mpi_comm, fname_input,
+ particles_input_hdf5<partsize_t, particles_rnumber, size_particle_positions, size_particle_rhs> generator(mpi_comm, fname_input,
inDatanameState, inDatanameRhs, my_spatial_low_limit_z, my_spatial_up_limit_z);
// Ensure parameters match the input file
--
GitLab
From 56b77c039dfea9bca6c8523d22e46a6b222de79e Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Thu, 19 Oct 2017 15:57:01 +0200
Subject: [PATCH 047/342] Remove a times 3 if there are more than 3 values for
each particles in the position array
---
bfps/cpp/particles/particles_distr_mpi.hpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bfps/cpp/particles/particles_distr_mpi.hpp b/bfps/cpp/particles/particles_distr_mpi.hpp
index 979c8d1e..32e86066 100644
--- a/bfps/cpp/particles/particles_distr_mpi.hpp
+++ b/bfps/cpp/particles/particles_distr_mpi.hpp
@@ -830,7 +830,7 @@ public:
assert(current_my_nb_particles_per_partition[idxPartition] ==
current_offset_particles_for_partition[idxPartition+1] - current_offset_particles_for_partition[idxPartition]);
for(partsize_t idx = current_offset_particles_for_partition[idxPartition] ; idx < current_offset_particles_for_partition[idxPartition+1] ; ++idx){
- assert(in_computer.pbc_field_layer((*inout_positions_particles)[idx*3+IDX_Z], IDX_Z)-current_partition_interval.first == idxPartition);
+ assert(in_computer.pbc_field_layer((*inout_positions_particles)[idx*size_particle_positions+IDX_Z], IDX_Z)-current_partition_interval.first == idxPartition);
}
}
}
--
GitLab
From c5f7042b6783060d6da77e3e30cca8471e87750c Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 20 Oct 2017 14:16:16 +0200
Subject: [PATCH 048/342] move compilation of particles to last
---
setup.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/setup.py b/setup.py
index 1fae430b..b03bd4f4 100644
--- a/setup.py
+++ b/setup.py
@@ -88,8 +88,7 @@ print('This is bfps version ' + VERSION)
### lists of files and MANIFEST.in
-src_file_list = ['full_code/NSVEparticles',
- 'full_code/joint_acc_vel_stats',
+src_file_list = ['full_code/joint_acc_vel_stats',
'full_code/test',
'full_code/filter_test',
'full_code/field_test',
@@ -128,7 +127,8 @@ src_file_list = ['full_code/NSVEparticles',
'spline_n9',
'spline_n10',
'Lagrange_polys',
- 'scope_timer']
+ 'scope_timer',
+ 'full_code/NSVEparticles']
particle_headers = [
'cpp/particles/particles_distr_mpi.hpp',
--
GitLab
From 3af022f395298c8f6e60c27e7a1b52ee9cc33f6c Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Fri, 20 Oct 2017 15:47:03 +0200
Subject: [PATCH 049/342] Debug -- add default value for cutoff and options --
use different tag values to be idle if missmatch
---
bfps/cpp/full_code/NSVEparticlesP2P.hpp | 2 +-
bfps/cpp/particles/p2p_distr_mpi.hpp | 32 ++++++-------------------
2 files changed, 8 insertions(+), 26 deletions(-)
diff --git a/bfps/cpp/full_code/NSVEparticlesP2P.hpp b/bfps/cpp/full_code/NSVEparticlesP2P.hpp
index 6bfaa3a8..62ae8165 100644
--- a/bfps/cpp/full_code/NSVEparticlesP2P.hpp
+++ b/bfps/cpp/full_code/NSVEparticlesP2P.hpp
@@ -73,7 +73,7 @@ class NSVEparticlesP2P: public NSVE<rnumber>
NSVE<rnumber>(
COMMUNICATOR,
simulation_name),
- cutoff(std::numeric_limits<double>::max()){}
+ cutoff(10), inner_v0(1), enable_p2p(true), enable_inner(true), enable_vorticity_omega(true){}
~NSVEparticlesP2P(){}
int initialize(void);
diff --git a/bfps/cpp/particles/p2p_distr_mpi.hpp b/bfps/cpp/particles/p2p_distr_mpi.hpp
index b009a57e..d607666a 100644
--- a/bfps/cpp/particles/p2p_distr_mpi.hpp
+++ b/bfps/cpp/particles/p2p_distr_mpi.hpp
@@ -32,7 +32,6 @@ protected:
int destProc;
int nbLevelsToExchange;
bool isRecv;
- bool positionsReceived;
std::unique_ptr<real_number[]> toRecvAndMerge;
std::unique_ptr<real_number[]> toCompute;
@@ -40,17 +39,11 @@ protected:
};
enum Action{
- NOTHING_TODO,
+ NOTHING_TODO = 512,
RECV_PARTICLES,
COMPUTE_PARTICLES,
- CHECK_PARTICLES,
RELEASE_BUFFER_PARTICLES,
- MERGE_PARTICLES,
-
- RECV_MOVE_NB_LOW,
- RECV_MOVE_NB_UP,
- RECV_MOVE_LOW,
- RECV_MOVE_UP
+ MERGE_PARTICLES
};
MPI_Comm current_com;
@@ -392,7 +385,6 @@ public:
descriptor.nbLevelsToExchange = nb_levels_to_send;
descriptor.nbParticlesToExchange = particles_offset_layers[my_nb_cell_levels] - particles_offset_layers[my_nb_cell_levels-nb_levels_to_send];
descriptor.isRecv = false;
- descriptor.positionsReceived = false;
neigDescriptors.emplace_back(std::move(descriptor));
@@ -415,7 +407,6 @@ public:
descriptor.nbLevelsToExchange = nb_levels_to_recv;
descriptor.nbParticlesToExchange = -1;
descriptor.isRecv = true;
- descriptor.positionsReceived = false;
neigDescriptors.emplace_back(std::move(descriptor));
@@ -507,7 +498,7 @@ public:
if(NbParticlesToReceive){
descriptor.toCompute.reset(new real_number[NbParticlesToReceive*size_particle_positions]);
- whatNext.emplace_back(std::pair<Action,int>{CHECK_PARTICLES, releasedAction.second});
+ whatNext.emplace_back(std::pair<Action,int>{COMPUTE_PARTICLES, releasedAction.second});
mpiRequests.emplace_back();
assert(NbParticlesToReceive*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Irecv(descriptor.toCompute.get(), int(NbParticlesToReceive*size_particle_positions),
@@ -526,7 +517,6 @@ public:
const partsize_t NbParticlesToReceive = descriptor.nbParticlesToExchange;
assert(descriptor.toCompute != nullptr);
- assert(descriptor.positionsReceived == true);
descriptor.results.reset(new real_number[NbParticlesToReceive*size_particle_rhs]);
in_computer.template init_result_array<size_particle_rhs>(descriptor.results.get(), NbParticlesToReceive);
@@ -594,24 +584,16 @@ public:
AssertMpi(MPI_Isend(descriptor.results.get(), int(NbParticlesToReceive*size_particle_rhs),
particles_utils::GetMpiType(real_number()), destProc, TAG_RESULT_PARTICLES,
current_com, &mpiRequests.back()));
+ descriptor.toCompute.release();
}
//////////////////////////////////////////////////////////////////////
- /// Computation
- //////////////////////////////////////////////////////////////////////
- if(releasedAction.first == CHECK_PARTICLES){
- NeighborDescriptor& descriptor = neigDescriptors[releasedAction.second];
- assert(descriptor.toCompute != nullptr);
- assert(descriptor.isRecv);
- descriptor.positionsReceived = true;
- }
- //////////////////////////////////////////////////////////////////////
- /// Computation
+ /// Release memory that was sent back
//////////////////////////////////////////////////////////////////////
if(releasedAction.first == RELEASE_BUFFER_PARTICLES){
NeighborDescriptor& descriptor = neigDescriptors[releasedAction.second];
- assert(descriptor.toCompute != nullptr);
+ assert(descriptor.results != nullptr);
assert(descriptor.isRecv);
- descriptor.toCompute.release();
+ descriptor.results.release();
}
//////////////////////////////////////////////////////////////////////
/// Merge
--
GitLab
From 1cf4cdde4c27575221581b4fc4e421582553323a Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Mon, 23 Oct 2017 11:46:05 +0200
Subject: [PATCH 050/342] update hdf5 loading to use position size instead of 3
---
bfps/cpp/particles/particles_input_hdf5.hpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bfps/cpp/particles/particles_input_hdf5.hpp b/bfps/cpp/particles/particles_input_hdf5.hpp
index 32cfec05..92ac3138 100644
--- a/bfps/cpp/particles/particles_input_hdf5.hpp
+++ b/bfps/cpp/particles/particles_input_hdf5.hpp
@@ -157,7 +157,7 @@ public:
assert(rspace >= 0);
hsize_t offset[2] = {load_splitter.getMyOffset(), 0};
- hsize_t mem_dims[2] = {load_splitter.getMySize(), 3};
+ hsize_t mem_dims[2] = {load_splitter.getMySize(), size_particle_positions};
hid_t mspace = H5Screate_simple(2, &mem_dims[0], NULL);
assert(mspace >= 0);
--
GitLab
From b4bbc44b2ef0553cc6de755dbd137f75de2fcaeb Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 24 Oct 2017 13:01:45 +0200
Subject: [PATCH 051/342] deallocate particle data memory
---
bfps/cpp/full_code/NSVEparticles.cpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/bfps/cpp/full_code/NSVEparticles.cpp b/bfps/cpp/full_code/NSVEparticles.cpp
index 0b1adfdf..83f918b5 100644
--- a/bfps/cpp/full_code/NSVEparticles.cpp
+++ b/bfps/cpp/full_code/NSVEparticles.cpp
@@ -127,6 +127,7 @@ int NSVEparticles<rnumber>::do_stats()
this->ps->get_step_idx()-1);
// deallocate temporary data array
+ delete[] pdata.get();
pdata.release();
return EXIT_SUCCESS;
--
GitLab
From 6c803fa1081d9655be91e93a1269ef93a256cdef Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 24 Oct 2017 13:12:29 +0200
Subject: [PATCH 052/342] deallocate descriptor.toCompute
---
bfps/cpp/particles/particles_distr_mpi.hpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/bfps/cpp/particles/particles_distr_mpi.hpp b/bfps/cpp/particles/particles_distr_mpi.hpp
index ebc24750..e6babb79 100644
--- a/bfps/cpp/particles/particles_distr_mpi.hpp
+++ b/bfps/cpp/particles/particles_distr_mpi.hpp
@@ -417,6 +417,7 @@ public:
if(releasedAction.first == RELEASE_BUFFER_PARTICLES){
NeighborDescriptor& descriptor = neigDescriptors[releasedAction.second];
assert(descriptor.toCompute != nullptr);
+ delete[] descriptor.toCompute.get();
descriptor.toCompute.release();
}
//////////////////////////////////////////////////////////////////////
--
GitLab
From f843c813292ea33ad686cdaef67d870daf9a2617 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 24 Oct 2017 13:41:19 +0200
Subject: [PATCH 053/342] add some more deallocations
---
bfps/cpp/particles/abstract_particles_output.hpp | 6 ++++++
bfps/cpp/particles/particles_distr_mpi.hpp | 13 ++++++++++---
bfps/cpp/particles/particles_input_hdf5.hpp | 2 ++
bfps/cpp/particles/particles_system.hpp | 4 ++++
4 files changed, 22 insertions(+), 3 deletions(-)
diff --git a/bfps/cpp/particles/abstract_particles_output.hpp b/bfps/cpp/particles/abstract_particles_output.hpp
index 5285c90f..98ad52e9 100644
--- a/bfps/cpp/particles/abstract_particles_output.hpp
+++ b/bfps/cpp/particles/abstract_particles_output.hpp
@@ -133,14 +133,20 @@ public:
}
void releaseMemory(){
+ delete[] buffer_indexes_send.get();
buffer_indexes_send.release();
+ delete[] buffer_particles_positions_send.get();
buffer_particles_positions_send.release();
size_buffers_send = -1;
+ delete[] buffer_indexes_recv.get();
buffer_indexes_recv.release();
+ delete[] buffer_particles_positions_recv.get();
buffer_particles_positions_recv.release();
size_buffers_recv = -1;
for(int idx_rhs = 0 ; idx_rhs < nb_rhs ; ++idx_rhs){
+ delete[] buffer_particles_rhs_send.get();
buffer_particles_rhs_send[idx_rhs].release();
+ delete[] buffer_particles_rhs_recv.get();
buffer_particles_rhs_recv[idx_rhs].release();
}
}
diff --git a/bfps/cpp/particles/particles_distr_mpi.hpp b/bfps/cpp/particles/particles_distr_mpi.hpp
index e6babb79..917e540b 100644
--- a/bfps/cpp/particles/particles_distr_mpi.hpp
+++ b/bfps/cpp/particles/particles_distr_mpi.hpp
@@ -282,7 +282,7 @@ public:
if(descriptor.nbParticlesToSend){
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
- mpiRequests.emplace_back();
+ mpiRequests.emplace_back();
assert(descriptor.nbParticlesToSend*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(const_cast<real_number*>(&particles_positions[(current_offset_particles_for_partition[current_partition_size-descriptor.nbPartitionsToSend])*size_particle_positions]),
int(descriptor.nbParticlesToSend*size_particle_positions), particles_utils::GetMpiType(real_number()),
@@ -406,7 +406,7 @@ public:
const int destProc = descriptor.destProc;
whatNext.emplace_back(std::pair<Action,int>{RELEASE_BUFFER_PARTICLES, releasedAction.second});
mpiRequests.emplace_back();
- const int tag = descriptor.isLower? TAG_LOW_UP_RESULTS : TAG_UP_LOW_RESULTS;
+ const int tag = descriptor.isLower? TAG_LOW_UP_RESULTS : TAG_UP_LOW_RESULTS;
assert(NbParticlesToReceive*size_particle_rhs < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(descriptor.results.get(), int(NbParticlesToReceive*size_particle_rhs), particles_utils::GetMpiType(real_number()), destProc, tag,
current_com, &mpiRequests.back()));
@@ -430,6 +430,7 @@ public:
TIMEZONE("reduce");
assert(descriptor.toRecvAndMerge != nullptr);
in_computer.template reduce_particles_rhs<size_particle_rhs>(&particles_current_rhs[0], descriptor.toRecvAndMerge.get(), descriptor.nbParticlesToSend);
+ delete[] descriptor.toRecvAndMerge.get();
descriptor.toRecvAndMerge.release();
}
else {
@@ -437,6 +438,7 @@ public:
assert(descriptor.toRecvAndMerge != nullptr);
in_computer.template reduce_particles_rhs<size_particle_rhs>(&particles_current_rhs[(current_offset_particles_for_partition[current_partition_size]-descriptor.nbParticlesToSend)*size_particle_rhs],
descriptor.toRecvAndMerge.get(), descriptor.nbParticlesToSend);
+ delete[] descriptor.toRecvAndMerge.get();
descriptor.toRecvAndMerge.release();
}
}
@@ -604,7 +606,7 @@ public:
if(nbOutLower){
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
- mpiRequests.emplace_back();
+ mpiRequests.emplace_back();
assert(nbOutLower*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(&(*inout_positions_particles)[0], int(nbOutLower*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
@@ -794,6 +796,11 @@ public:
}
myTotalNbParticles = myTotalNewNbParticles;
+ // clean up
+ for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
+ delete[] newArrayRhs[idx_rhs].get();
+ newArrayRhs[idx_rhs].release();
+ }
}
// Partitions all particles
diff --git a/bfps/cpp/particles/particles_input_hdf5.hpp b/bfps/cpp/particles/particles_input_hdf5.hpp
index 32cfec05..9e65e1ae 100644
--- a/bfps/cpp/particles/particles_input_hdf5.hpp
+++ b/bfps/cpp/particles/particles_input_hdf5.hpp
@@ -256,10 +256,12 @@ public:
my_particles_positions.reset(new real_number[exchanger.getTotalToRecv()*size_particle_positions]);
exchanger.alltoallv<real_number>(split_particles_positions.get(), my_particles_positions.get(), size_particle_positions);
+ delete[] split_particles_positions.get();
split_particles_positions.release();
my_particles_indexes.reset(new partsize_t[exchanger.getTotalToRecv()]);
exchanger.alltoallv<partsize_t>(split_particles_indexes.get(), my_particles_indexes.get());
+ delete[] split_particles_indexes.get();
split_particles_indexes.release();
my_particles_rhs.resize(nb_rhs);
diff --git a/bfps/cpp/particles/particles_system.hpp b/bfps/cpp/particles/particles_system.hpp
index 081f4b4b..2a9b5065 100644
--- a/bfps/cpp/particles/particles_system.hpp
+++ b/bfps/cpp/particles/particles_system.hpp
@@ -76,6 +76,10 @@ public:
}
~particles_system(){
+ delete[] current_my_nb_particles_per_partition.get();
+ current_my_nb_particles_per_partition.release();
+ delete[] current_offset_particles_for_partition.get();
+ current_offset_particles_for_partition.release();
}
void init(abstract_particles_input<partsize_t, real_number>& particles_input) {
--
GitLab
From 0b071b9512e977e7b6c291542460be9fe702f23b Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 24 Oct 2017 15:21:54 +0200
Subject: [PATCH 054/342] Revert "add some more deallocations"
This reverts commit f843c813292ea33ad686cdaef67d870daf9a2617.
---
bfps/cpp/particles/abstract_particles_output.hpp | 6 ------
bfps/cpp/particles/particles_distr_mpi.hpp | 13 +++----------
bfps/cpp/particles/particles_input_hdf5.hpp | 2 --
bfps/cpp/particles/particles_system.hpp | 4 ----
4 files changed, 3 insertions(+), 22 deletions(-)
diff --git a/bfps/cpp/particles/abstract_particles_output.hpp b/bfps/cpp/particles/abstract_particles_output.hpp
index 98ad52e9..5285c90f 100644
--- a/bfps/cpp/particles/abstract_particles_output.hpp
+++ b/bfps/cpp/particles/abstract_particles_output.hpp
@@ -133,20 +133,14 @@ public:
}
void releaseMemory(){
- delete[] buffer_indexes_send.get();
buffer_indexes_send.release();
- delete[] buffer_particles_positions_send.get();
buffer_particles_positions_send.release();
size_buffers_send = -1;
- delete[] buffer_indexes_recv.get();
buffer_indexes_recv.release();
- delete[] buffer_particles_positions_recv.get();
buffer_particles_positions_recv.release();
size_buffers_recv = -1;
for(int idx_rhs = 0 ; idx_rhs < nb_rhs ; ++idx_rhs){
- delete[] buffer_particles_rhs_send.get();
buffer_particles_rhs_send[idx_rhs].release();
- delete[] buffer_particles_rhs_recv.get();
buffer_particles_rhs_recv[idx_rhs].release();
}
}
diff --git a/bfps/cpp/particles/particles_distr_mpi.hpp b/bfps/cpp/particles/particles_distr_mpi.hpp
index 917e540b..e6babb79 100644
--- a/bfps/cpp/particles/particles_distr_mpi.hpp
+++ b/bfps/cpp/particles/particles_distr_mpi.hpp
@@ -282,7 +282,7 @@ public:
if(descriptor.nbParticlesToSend){
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
- mpiRequests.emplace_back();
+ mpiRequests.emplace_back();
assert(descriptor.nbParticlesToSend*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(const_cast<real_number*>(&particles_positions[(current_offset_particles_for_partition[current_partition_size-descriptor.nbPartitionsToSend])*size_particle_positions]),
int(descriptor.nbParticlesToSend*size_particle_positions), particles_utils::GetMpiType(real_number()),
@@ -406,7 +406,7 @@ public:
const int destProc = descriptor.destProc;
whatNext.emplace_back(std::pair<Action,int>{RELEASE_BUFFER_PARTICLES, releasedAction.second});
mpiRequests.emplace_back();
- const int tag = descriptor.isLower? TAG_LOW_UP_RESULTS : TAG_UP_LOW_RESULTS;
+ const int tag = descriptor.isLower? TAG_LOW_UP_RESULTS : TAG_UP_LOW_RESULTS;
assert(NbParticlesToReceive*size_particle_rhs < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(descriptor.results.get(), int(NbParticlesToReceive*size_particle_rhs), particles_utils::GetMpiType(real_number()), destProc, tag,
current_com, &mpiRequests.back()));
@@ -430,7 +430,6 @@ public:
TIMEZONE("reduce");
assert(descriptor.toRecvAndMerge != nullptr);
in_computer.template reduce_particles_rhs<size_particle_rhs>(&particles_current_rhs[0], descriptor.toRecvAndMerge.get(), descriptor.nbParticlesToSend);
- delete[] descriptor.toRecvAndMerge.get();
descriptor.toRecvAndMerge.release();
}
else {
@@ -438,7 +437,6 @@ public:
assert(descriptor.toRecvAndMerge != nullptr);
in_computer.template reduce_particles_rhs<size_particle_rhs>(&particles_current_rhs[(current_offset_particles_for_partition[current_partition_size]-descriptor.nbParticlesToSend)*size_particle_rhs],
descriptor.toRecvAndMerge.get(), descriptor.nbParticlesToSend);
- delete[] descriptor.toRecvAndMerge.get();
descriptor.toRecvAndMerge.release();
}
}
@@ -606,7 +604,7 @@ public:
if(nbOutLower){
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
- mpiRequests.emplace_back();
+ mpiRequests.emplace_back();
assert(nbOutLower*size_particle_positions < std::numeric_limits<int>::max());
AssertMpi(MPI_Isend(&(*inout_positions_particles)[0], int(nbOutLower*size_particle_positions), particles_utils::GetMpiType(real_number()), (my_rank-1+nb_processes_involved)%nb_processes_involved, TAG_LOW_UP_MOVED_PARTICLES,
MPI_COMM_WORLD, &mpiRequests.back()));
@@ -796,11 +794,6 @@ public:
}
myTotalNbParticles = myTotalNewNbParticles;
- // clean up
- for(int idx_rhs = 0 ; idx_rhs < in_nb_rhs ; ++idx_rhs){
- delete[] newArrayRhs[idx_rhs].get();
- newArrayRhs[idx_rhs].release();
- }
}
// Partitions all particles
diff --git a/bfps/cpp/particles/particles_input_hdf5.hpp b/bfps/cpp/particles/particles_input_hdf5.hpp
index 9e65e1ae..32cfec05 100644
--- a/bfps/cpp/particles/particles_input_hdf5.hpp
+++ b/bfps/cpp/particles/particles_input_hdf5.hpp
@@ -256,12 +256,10 @@ public:
my_particles_positions.reset(new real_number[exchanger.getTotalToRecv()*size_particle_positions]);
exchanger.alltoallv<real_number>(split_particles_positions.get(), my_particles_positions.get(), size_particle_positions);
- delete[] split_particles_positions.get();
split_particles_positions.release();
my_particles_indexes.reset(new partsize_t[exchanger.getTotalToRecv()]);
exchanger.alltoallv<partsize_t>(split_particles_indexes.get(), my_particles_indexes.get());
- delete[] split_particles_indexes.get();
split_particles_indexes.release();
my_particles_rhs.resize(nb_rhs);
diff --git a/bfps/cpp/particles/particles_system.hpp b/bfps/cpp/particles/particles_system.hpp
index 2a9b5065..081f4b4b 100644
--- a/bfps/cpp/particles/particles_system.hpp
+++ b/bfps/cpp/particles/particles_system.hpp
@@ -76,10 +76,6 @@ public:
}
~particles_system(){
- delete[] current_my_nb_particles_per_partition.get();
- current_my_nb_particles_per_partition.release();
- delete[] current_offset_particles_for_partition.get();
- current_offset_particles_for_partition.release();
}
void init(abstract_particles_input<partsize_t, real_number>& particles_input) {
--
GitLab
From 5b61e5f87fde5bf25b8ce0bc69df583f281130b2 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 24 Oct 2017 15:21:58 +0200
Subject: [PATCH 055/342] Revert "deallocate descriptor.toCompute"
This reverts commit 6c803fa1081d9655be91e93a1269ef93a256cdef.
---
bfps/cpp/particles/particles_distr_mpi.hpp | 1 -
1 file changed, 1 deletion(-)
diff --git a/bfps/cpp/particles/particles_distr_mpi.hpp b/bfps/cpp/particles/particles_distr_mpi.hpp
index e6babb79..ebc24750 100644
--- a/bfps/cpp/particles/particles_distr_mpi.hpp
+++ b/bfps/cpp/particles/particles_distr_mpi.hpp
@@ -417,7 +417,6 @@ public:
if(releasedAction.first == RELEASE_BUFFER_PARTICLES){
NeighborDescriptor& descriptor = neigDescriptors[releasedAction.second];
assert(descriptor.toCompute != nullptr);
- delete[] descriptor.toCompute.get();
descriptor.toCompute.release();
}
//////////////////////////////////////////////////////////////////////
--
GitLab
From 2e89f009dc479695ab5cda402574714cdc05d75a Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 24 Oct 2017 15:22:00 +0200
Subject: [PATCH 056/342] Revert "deallocate particle data memory"
This reverts commit b4bbc44b2ef0553cc6de755dbd137f75de2fcaeb.
---
bfps/cpp/full_code/NSVEparticles.cpp | 1 -
1 file changed, 1 deletion(-)
diff --git a/bfps/cpp/full_code/NSVEparticles.cpp b/bfps/cpp/full_code/NSVEparticles.cpp
index 83f918b5..0b1adfdf 100644
--- a/bfps/cpp/full_code/NSVEparticles.cpp
+++ b/bfps/cpp/full_code/NSVEparticles.cpp
@@ -127,7 +127,6 @@ int NSVEparticles<rnumber>::do_stats()
this->ps->get_step_idx()-1);
// deallocate temporary data array
- delete[] pdata.get();
pdata.release();
return EXIT_SUCCESS;
--
GitLab
From c5eac43c9bccd06dbd959816103fdbfdee417132 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 24 Oct 2017 15:23:21 +0200
Subject: [PATCH 057/342] add comment about releasing unique_ptr
---
bfps/cpp/full_code/NSVEparticles.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/bfps/cpp/full_code/NSVEparticles.cpp b/bfps/cpp/full_code/NSVEparticles.cpp
index 0b1adfdf..72a67355 100644
--- a/bfps/cpp/full_code/NSVEparticles.cpp
+++ b/bfps/cpp/full_code/NSVEparticles.cpp
@@ -127,7 +127,8 @@ int NSVEparticles<rnumber>::do_stats()
this->ps->get_step_idx()-1);
// deallocate temporary data array
- pdata.release();
+ // TODO: is it required/safe to call the release method here?
+ //pdata.release();
return EXIT_SUCCESS;
}
--
GitLab
From 72088a65cd98566e7fcb18ef6a9299bbb6a6551c Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 25 Oct 2017 15:28:41 +0200
Subject: [PATCH 058/342] add new test executable
---
setup.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/setup.py b/setup.py
index be630674..27a2993f 100644
--- a/setup.py
+++ b/setup.py
@@ -280,7 +280,8 @@ setup(
'console_scripts': [
'bfps = bfps.__main__:main',
'bfps1 = bfps.__main__:main',
- 'bfps.test_NSVEparticles = bfps.test.test_bfps_NSVEparticles:main'],
+ 'bfps.test_NSVEparticles = bfps.test.test_bfps_NSVEparticles:main',
+ 'bfps.test_particles = bfps.test.test_particles:main'],
},
version = VERSION,
########################################################################
--
GitLab
From 05a30905bb2a121e6a56126c634a47e44ec65c29 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 25 Oct 2017 15:29:26 +0200
Subject: [PATCH 059/342] add new test script
---
bfps/test/test_particles.py | 59 +++++++++++++++++++++++++++++++++++++
1 file changed, 59 insertions(+)
create mode 100644 bfps/test/test_particles.py
diff --git a/bfps/test/test_particles.py b/bfps/test/test_particles.py
new file mode 100644
index 00000000..f0cda7db
--- /dev/null
+++ b/bfps/test/test_particles.py
@@ -0,0 +1,59 @@
+#! /usr/bin/env python
+
+import os
+import numpy as np
+import h5py
+import sys
+
+import bfps
+from bfps import DNS
+
+import matplotlib.pyplot as plt
+
+
+def main():
+ assert(sys.argv[1] in ['p2p_sampling']);
+ niterations = 32
+ nparticles = 100
+ njobs = 1
+ c = DNS()
+ c.launch(
+ ['NSVEparticlesP2P',
+ '-n', '32',
+ '--src-simname', 'B32p1e4',
+ '--src-wd', bfps.lib_dir + '/test',
+ '--src-iteration', '0',
+ '--np', '4',
+ '--ntpp', '1',
+ '--niter_todo', '{0}'.format(niterations),
+ '--niter_out', '{0}'.format(niterations),
+ '--niter_stat', '1',
+ '--checkpoints_per_file', '{0}'.format(3),
+ '--nparticles', '{0}'.format(nparticles),
+ '--particle-rand-seed', '2',
+ '--njobs', '{0}'.format(njobs),
+ '--wd', './'] +
+ sys.argv[2:])
+ if sys.argv[1] == 'p2p_sampling':
+ pf = h5py.File(
+ os.path.join(
+ c.work_dir,
+ c.simname + '_particles.h5'),
+ 'r')
+ # show a histogram of the positions
+ f = plt.figure()
+ a = f.add_subplot(111)
+ for iteration in [0, 16, 32, 48, 64]:
+ x = pf['tracers0/position/{0}'.format(iteration)].value
+ print(x.shape)
+ #bins, hist = np.histogram(
+# a.
+ f.tight_layout()
+ f.savefig('position_histogram.pdf')
+ plt.close(f)
+ # show a histogram of the orientations
+ return None
+
+if __name__ == '__main__':
+ main()
+
--
GitLab
From ef790303c627caa8b61bef6dfe75d834b93c6c52 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 25 Oct 2017 15:30:00 +0200
Subject: [PATCH 060/342] add empty line
---
bfps/cpp/full_code/NSVEparticles.hpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/bfps/cpp/full_code/NSVEparticles.hpp b/bfps/cpp/full_code/NSVEparticles.hpp
index 03d45aaa..97ea5c84 100644
--- a/bfps/cpp/full_code/NSVEparticles.hpp
+++ b/bfps/cpp/full_code/NSVEparticles.hpp
@@ -58,6 +58,7 @@ class NSVEparticles: public NSVE<rnumber>
/* other stuff */
std::unique_ptr<abstract_particles_system<long long int, double>> ps;
+
particles_output_hdf5<long long int, double,3,3> *particles_output_writer_mpi;
particles_output_sampling_hdf5<long long int, double, 3, 3> *particles_sample_writer_mpi;
--
GitLab
From ff4532ea21cc24d457c268a0e29feea01d05fbc2 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 25 Oct 2017 16:20:51 +0200
Subject: [PATCH 061/342] [broken] use new sampling class for P2P
Sampled position is different from checkpoint position, I still need to
fix that.
However, there's another issue that needs to be solved first
---
bfps/cpp/full_code/NSVEparticlesP2P.cpp | 71 ++++++++++++++++---------
bfps/cpp/full_code/NSVEparticlesP2P.hpp | 2 +
bfps/test/test_particles.py | 67 +++++++++++++----------
3 files changed, 89 insertions(+), 51 deletions(-)
diff --git a/bfps/cpp/full_code/NSVEparticlesP2P.cpp b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
index 3f648308..11b400d1 100644
--- a/bfps/cpp/full_code/NSVEparticlesP2P.cpp
+++ b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
@@ -16,6 +16,7 @@ int NSVEparticlesP2P<rnumber>::initialize(void)
particles_inner_computer<double, long long int> current_particles_inner_computer(inner_v0);
current_particles_inner_computer.setEnable(enable_inner);
+ this->cutoff = 5.0;
this->ps = particles_system_builder_with_p2p(
this->fs->cvelocity, // (field object)
@@ -39,6 +40,13 @@ int NSVEparticlesP2P<rnumber>::initialize(void)
"tracers0",
nparticles,
tracers0_integration_steps);
+ this->particles_sample_writer_mpi = new particles_output_sampling_hdf5<
+ long long int, double, 3, 3>(
+ MPI_COMM_WORLD,
+ this->ps->getGlobalNbParticles(),
+ (this->simname + "_particles.h5"),
+ "tracers0",
+ "position/0");
return EXIT_SUCCESS;
}
@@ -76,8 +84,8 @@ int NSVEparticlesP2P<rnumber>::write_checkpoint(void)
template <typename rnumber>
int NSVEparticlesP2P<rnumber>::finalize(void)
{
- this->ps.release();
delete this->particles_output_writer_mpi;
+ delete this->particles_sample_writer_mpi;
this->NSVE<rnumber>::finalize();
return EXIT_SUCCESS;
}
@@ -95,30 +103,45 @@ int NSVEparticlesP2P<rnumber>::do_stats()
if (!(this->iteration % this->niter_part == 0))
return EXIT_SUCCESS;
+ // allocate temporary data array
+ std::unique_ptr<double[]> pdata(new double[3*this->ps->getLocalNbParticles()]);
+
/// sample position
- sample_particles_system_position(
- this->ps,
- (this->simname + "_particles.h5"), // filename
- "tracers0", // hdf5 parent group
- "position" // dataset basename TODO
- );
-
- /// sample velocity
- sample_from_particles_system(*this->tmp_vec_field, // field to save
- this->ps,
- (this->simname + "_particles.h5"), // filename
- "tracers0", // hdf5 parent group
- "velocity" // dataset basename TODO
- );
-
- /// compute acceleration and sample it
- this->fs->compute_Lagrangian_acceleration(this->tmp_vec_field);
- this->tmp_vec_field->ift();
- sample_from_particles_system(*this->tmp_vec_field,
- this->ps,
- (this->simname + "_particles.h5"),
- "tracers0",
- "acceleration");
+ std::copy(this->ps->getParticlesPositions(),
+ this->ps->getParticlesPositions()+3*this->ps->getLocalNbParticles(),
+ pdata.get());
+ this->particles_sample_writer_mpi->save_dataset(
+ "tracers0",
+ "position",
+ this->ps->getParticlesPositions(),
+ &pdata,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+
+ ///// sample velocity
+ //this->ps->sample_compute_field(*this->tmp_vec_field, pdata.get());
+ //this->particles_sample_writer_mpi->save_dataset(
+ // "tracers0",
+ // "velocity",
+ // this->ps->getParticlesPositions(),
+ // &pdata,
+ // this->ps->getParticlesIndexes(),
+ // this->ps->getLocalNbParticles(),
+ // this->ps->get_step_idx()-1);
+
+ ///// compute acceleration and sample it
+ //this->fs->compute_Lagrangian_acceleration(this->tmp_vec_field);
+ //this->tmp_vec_field->ift();
+ //this->ps->sample_compute_field(*this->tmp_vec_field, pdata.get());
+ //this->particles_sample_writer_mpi->save_dataset(
+ // "tracers0",
+ // "acceleration",
+ // this->ps->getParticlesPositions(),
+ // &pdata,
+ // this->ps->getParticlesIndexes(),
+ // this->ps->getLocalNbParticles(),
+ // this->ps->get_step_idx()-1);
return EXIT_SUCCESS;
}
diff --git a/bfps/cpp/full_code/NSVEparticlesP2P.hpp b/bfps/cpp/full_code/NSVEparticlesP2P.hpp
index 62ae8165..8f435116 100644
--- a/bfps/cpp/full_code/NSVEparticlesP2P.hpp
+++ b/bfps/cpp/full_code/NSVEparticlesP2P.hpp
@@ -35,6 +35,7 @@
#include "full_code/NSVE.hpp"
#include "particles/particles_system_builder.hpp"
#include "particles/particles_output_hdf5.hpp"
+#include "particles/particles_sampling.hpp"
/** \brief Navier-Stokes solver that includes simple Lagrangian tracers.
*
@@ -65,6 +66,7 @@ class NSVEparticlesP2P: public NSVE<rnumber>
std::unique_ptr<abstract_particles_system<long long int, double>> ps;
// TODO P2P use a reader with particle data
particles_output_hdf5<long long int, double,6,6> *particles_output_writer_mpi;
+ particles_output_sampling_hdf5<long long int, double,3,3> *particles_sample_writer_mpi;
NSVEparticlesP2P(
diff --git a/bfps/test/test_particles.py b/bfps/test/test_particles.py
index f0cda7db..7e72eedf 100644
--- a/bfps/test/test_particles.py
+++ b/bfps/test/test_particles.py
@@ -12,45 +12,58 @@ import matplotlib.pyplot as plt
def main():
- assert(sys.argv[1] in ['p2p_sampling']);
+ assert(sys.argv[1] in ['p2p_sampling'])
+ assert(sys.argv[2] in ['on', 'off'])
niterations = 32
- nparticles = 100
+ nparticles = 1000
njobs = 1
- c = DNS()
- c.launch(
- ['NSVEparticlesP2P',
- '-n', '32',
- '--src-simname', 'B32p1e4',
- '--src-wd', bfps.lib_dir + '/test',
- '--src-iteration', '0',
- '--np', '4',
- '--ntpp', '1',
- '--niter_todo', '{0}'.format(niterations),
- '--niter_out', '{0}'.format(niterations),
- '--niter_stat', '1',
- '--checkpoints_per_file', '{0}'.format(3),
- '--nparticles', '{0}'.format(nparticles),
- '--particle-rand-seed', '2',
- '--njobs', '{0}'.format(njobs),
- '--wd', './'] +
- sys.argv[2:])
+ if sys.argv[2] == 'on':
+ c = DNS()
+ c.launch(
+ ['NSVEparticlesP2P',
+ '-n', '32',
+ '--src-simname', 'B32p1e4',
+ '--src-wd', bfps.lib_dir + '/test',
+ '--src-iteration', '0',
+ '--np', '4',
+ '--ntpp', '1',
+ '--niter_todo', '{0}'.format(niterations),
+ '--niter_out', '{0}'.format(niterations),
+ '--niter_stat', '1',
+ '--checkpoints_per_file', '{0}'.format(3),
+ '--nparticles', '{0}'.format(nparticles),
+ '--particle-rand-seed', '2',
+ '--njobs', '{0}'.format(njobs),
+ '--wd', './'] +
+ sys.argv[3:])
if sys.argv[1] == 'p2p_sampling':
+ cf = h5py.File(
+ 'test_checkpoint_0.h5',
+ 'r')
pf = h5py.File(
- os.path.join(
- c.work_dir,
- c.simname + '_particles.h5'),
+ 'test_particles.h5',
'r')
# show a histogram of the positions
f = plt.figure()
a = f.add_subplot(111)
- for iteration in [0, 16, 32, 48, 64]:
+ for iteration in range(0, niterations*njobs+1, niterations//2):
x = pf['tracers0/position/{0}'.format(iteration)].value
- print(x.shape)
- #bins, hist = np.histogram(
-# a.
+ hist, bins = np.histogram(
+ np.sum(x**2, axis = -1).flatten()**.5,
+ bins = 40)
+ bb = (bins[:-1] + bins[1:])/2
+ pp = hist.astype(np.float) / (np.sum(hist) * (bb[1] - bb[0]))
+ a.plot(bb, pp, label = '{0}'.format(iteration))
+ a.legend(loc = 'best')
f.tight_layout()
f.savefig('position_histogram.pdf')
plt.close(f)
+ # compared sampled positions with checkpoint positions
+ for iteration in range(0, niterations*njobs+1, niterations):
+ x = pf['tracers0/position/{0}'.format(iteration)].value
+ s = cf['tracers0/state/{0}'.format(iteration)].value
+ distance = np.max(np.abs(x - s[..., :3]))
+ print(iteration, distance)
# show a histogram of the orientations
return None
--
GitLab
From 6c5be0dfe9f0ade0330577ae4066f107095ecc7e Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Thu, 26 Oct 2017 12:38:05 +0200
Subject: [PATCH 062/342] Ensure that the limit will not be included in the
interval -- Add an assertion if a dead lock will appear
---
bfps/cpp/full_code/NSVEparticlesP2P.cpp | 2 +-
bfps/cpp/particles/p2p_distr_mpi.hpp | 46 +++++++++++++++++++++++--
2 files changed, 44 insertions(+), 4 deletions(-)
diff --git a/bfps/cpp/full_code/NSVEparticlesP2P.cpp b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
index 11b400d1..e8065e2c 100644
--- a/bfps/cpp/full_code/NSVEparticlesP2P.cpp
+++ b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
@@ -16,7 +16,7 @@ int NSVEparticlesP2P<rnumber>::initialize(void)
particles_inner_computer<double, long long int> current_particles_inner_computer(inner_v0);
current_particles_inner_computer.setEnable(enable_inner);
- this->cutoff = 5.0;
+ this->cutoff = 1.0;
this->ps = particles_system_builder_with_p2p(
this->fs->cvelocity, // (field object)
diff --git a/bfps/cpp/particles/p2p_distr_mpi.hpp b/bfps/cpp/particles/p2p_distr_mpi.hpp
index d607666a..ac69822a 100644
--- a/bfps/cpp/particles/p2p_distr_mpi.hpp
+++ b/bfps/cpp/particles/p2p_distr_mpi.hpp
@@ -193,8 +193,13 @@ public:
long int last_cell_level_proc(const int dest_proc) const{
const real_number field_section_width_z = spatial_box_width[IDX_Z]/real_number(field_grid_dim[IDX_Z]);
- return static_cast<long int>((field_section_width_z*real_number(partition_interval_offset_per_proc[dest_proc+1])
+ const long int limite = static_cast<long int>((field_section_width_z*real_number(partition_interval_offset_per_proc[dest_proc+1])
- std::numeric_limits<real_number>::epsilon())/cutoff_radius);
+ if(static_cast<real_number>(limite)*cutoff_radius
+ == field_section_width_z*real_number(partition_interval_offset_per_proc[dest_proc+1])){
+ return limite-1;
+ }
+ return limite;
}
real_number apply_pbc(real_number pos, IDXS_3D dim) const{
@@ -421,7 +426,10 @@ public:
assert(whatNext.size() == 0);
assert(mpiRequests.size() == 0);
-
+#ifndef NDEBUG // Just for assertion
+ std::vector<int> willsend(nb_processes_involved, 0);
+ std::vector<int> willrecv(nb_processes_involved, 0);
+#endif
for(int idxDescr = 0 ; idxDescr < int(neigDescriptors.size()) ; ++idxDescr){
NeighborDescriptor& descriptor = neigDescriptors[idxDescr];
@@ -433,7 +441,9 @@ public:
1, particles_utils::GetMpiType(partsize_t()),
descriptor.destProc, TAG_NB_PARTICLES,
current_com, &mpiRequests.back()));
-
+#ifndef NDEBUG // Just for assertion
+ willsend[descriptor.destProc] += 1;
+#endif
if(descriptor.nbParticlesToExchange){
whatNext.emplace_back(std::pair<Action,int>{NOTHING_TODO, -1});
mpiRequests.emplace_back();
@@ -454,6 +464,9 @@ public:
}
}
else{
+#ifndef NDEBUG // Just for assertion
+ willrecv[descriptor.destProc] += 1;
+#endif
whatNext.emplace_back(std::pair<Action,int>{RECV_PARTICLES, idxDescr});
mpiRequests.emplace_back();
AssertMpi(MPI_Irecv(&descriptor.nbParticlesToExchange,
@@ -462,6 +475,33 @@ public:
}
}
+#ifndef NDEBUG // Just for assertion
+ {
+ if(myrank == 0){
+ std::vector<int> willsendall(nb_processes_involved*nb_processes_involved, 0);// TODO debug
+ std::vector<int> willrecvall(nb_processes_involved*nb_processes_involved, 0);// TODO debug
+
+ MPI_Gather(willrecv.data(), nb_processes_involved, MPI_INT, willrecvall.data(),
+ nb_processes_involved, MPI_INT, 0, MPI_COMM_WORLD);
+ MPI_Gather(willsend.data(), nb_processes_involved, MPI_INT, willsendall.data(),
+ nb_processes_involved, MPI_INT, 0, MPI_COMM_WORLD);
+
+ for(int idxproc = 0 ; idxproc < nb_processes_involved ; ++idxproc){
+ for(int idxtest = 0 ; idxtest < nb_processes_involved ; ++idxtest){
+ assert(willsendall[idxproc*nb_processes_involved + idxtest]
+ == willrecvall[idxtest*nb_processes_involved + idxproc]);
+ }
+ }
+ }
+ else{
+ MPI_Gather(willrecv.data(), nb_processes_involved, MPI_INT, nullptr,
+ 0, MPI_INT, 0, MPI_COMM_WORLD);
+ MPI_Gather(willsend.data(), nb_processes_involved, MPI_INT, nullptr,
+ 0, MPI_INT, 0, MPI_COMM_WORLD);
+ }
+ }
+#endif
+
lock_free_bool_array cells_locker(512);
TIMEZONE_OMP_INIT_PREPARALLEL(omp_get_max_threads())
--
GitLab
From e07c9800af0419c3a1603ff6225ddcc372903158 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 26 Oct 2017 16:11:14 +0200
Subject: [PATCH 063/342] remove bad comment
---
bfps/cpp/particles/particles_input_hdf5.hpp | 2 --
1 file changed, 2 deletions(-)
diff --git a/bfps/cpp/particles/particles_input_hdf5.hpp b/bfps/cpp/particles/particles_input_hdf5.hpp
index 92ac3138..72f7a905 100644
--- a/bfps/cpp/particles/particles_input_hdf5.hpp
+++ b/bfps/cpp/particles/particles_input_hdf5.hpp
@@ -14,8 +14,6 @@
#include "scope_timer.hpp"
-// why is "size_particle_rhs" a template parameter?
-// I think it's safe to assume this will always be 3.
template <class partsize_t, class real_number, int size_particle_positions, int size_particle_rhs>
class particles_input_hdf5 : public abstract_particles_input<partsize_t, real_number> {
const std::string filename;
--
GitLab
From 5eaf4028aa3eb9f0f8ae896a828948771defeaaf Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 26 Oct 2017 16:31:34 +0200
Subject: [PATCH 064/342] rename getParticlePositions to getParticleState
---
bfps/cpp/full_code/NSVEparticles.cpp | 12 ++++++------
bfps/cpp/full_code/NSVEparticlesP2P.cpp | 12 ++++++------
bfps/cpp/particles/abstract_particles_system.hpp | 2 +-
bfps/cpp/particles/particles_sampling.hpp | 6 +++---
bfps/cpp/particles/particles_system.hpp | 2 +-
5 files changed, 17 insertions(+), 17 deletions(-)
diff --git a/bfps/cpp/full_code/NSVEparticles.cpp b/bfps/cpp/full_code/NSVEparticles.cpp
index 72a67355..953ad9f9 100644
--- a/bfps/cpp/full_code/NSVEparticles.cpp
+++ b/bfps/cpp/full_code/NSVEparticles.cpp
@@ -52,7 +52,7 @@ int NSVEparticles<rnumber>::write_checkpoint(void)
this->NSVE<rnumber>::write_checkpoint();
this->particles_output_writer_mpi->open_file(this->fs->get_current_fname());
this->particles_output_writer_mpi->save(
- this->ps->getParticlesPositions(),
+ this->ps->getParticlesState(),
this->ps->getParticlesRhs(),
this->ps->getParticlesIndexes(),
this->ps->getLocalNbParticles(),
@@ -90,13 +90,13 @@ int NSVEparticles<rnumber>::do_stats()
// copy position data
/// sample position
- std::copy(this->ps->getParticlesPositions(),
- this->ps->getParticlesPositions()+3*this->ps->getLocalNbParticles(),
+ std::copy(this->ps->getParticlesState(),
+ this->ps->getParticlesState()+3*this->ps->getLocalNbParticles(),
pdata.get());
this->particles_sample_writer_mpi->save_dataset(
"tracers0",
"position",
- this->ps->getParticlesPositions(),
+ this->ps->getParticlesState(),
&pdata,
this->ps->getParticlesIndexes(),
this->ps->getLocalNbParticles(),
@@ -107,7 +107,7 @@ int NSVEparticles<rnumber>::do_stats()
this->particles_sample_writer_mpi->save_dataset(
"tracers0",
"velocity",
- this->ps->getParticlesPositions(),
+ this->ps->getParticlesState(),
&pdata,
this->ps->getParticlesIndexes(),
this->ps->getLocalNbParticles(),
@@ -120,7 +120,7 @@ int NSVEparticles<rnumber>::do_stats()
this->particles_sample_writer_mpi->save_dataset(
"tracers0",
"acceleration",
- this->ps->getParticlesPositions(),
+ this->ps->getParticlesState(),
&pdata,
this->ps->getParticlesIndexes(),
this->ps->getLocalNbParticles(),
diff --git a/bfps/cpp/full_code/NSVEparticlesP2P.cpp b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
index e8065e2c..d9b529f2 100644
--- a/bfps/cpp/full_code/NSVEparticlesP2P.cpp
+++ b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
@@ -72,7 +72,7 @@ int NSVEparticlesP2P<rnumber>::write_checkpoint(void)
this->particles_output_writer_mpi->open_file(this->fs->get_current_fname());
// TODO P2P write particle data too
this->particles_output_writer_mpi->save(
- this->ps->getParticlesPositions(),
+ this->ps->getParticlesState(),
this->ps->getParticlesRhs(),
this->ps->getParticlesIndexes(),
this->ps->getLocalNbParticles(),
@@ -107,13 +107,13 @@ int NSVEparticlesP2P<rnumber>::do_stats()
std::unique_ptr<double[]> pdata(new double[3*this->ps->getLocalNbParticles()]);
/// sample position
- std::copy(this->ps->getParticlesPositions(),
- this->ps->getParticlesPositions()+3*this->ps->getLocalNbParticles(),
+ std::copy(this->ps->getParticlesState(),
+ this->ps->getParticlesState()+3*this->ps->getLocalNbParticles(),
pdata.get());
this->particles_sample_writer_mpi->save_dataset(
"tracers0",
"position",
- this->ps->getParticlesPositions(),
+ this->ps->getParticlesState(),
&pdata,
this->ps->getParticlesIndexes(),
this->ps->getLocalNbParticles(),
@@ -124,7 +124,7 @@ int NSVEparticlesP2P<rnumber>::do_stats()
//this->particles_sample_writer_mpi->save_dataset(
// "tracers0",
// "velocity",
- // this->ps->getParticlesPositions(),
+ // this->ps->getParticlesState(),
// &pdata,
// this->ps->getParticlesIndexes(),
// this->ps->getLocalNbParticles(),
@@ -137,7 +137,7 @@ int NSVEparticlesP2P<rnumber>::do_stats()
//this->particles_sample_writer_mpi->save_dataset(
// "tracers0",
// "acceleration",
- // this->ps->getParticlesPositions(),
+ // this->ps->getParticlesState(),
// &pdata,
// this->ps->getParticlesIndexes(),
// this->ps->getLocalNbParticles(),
diff --git a/bfps/cpp/particles/abstract_particles_system.hpp b/bfps/cpp/particles/abstract_particles_system.hpp
index b2d09566..6a2a01dc 100644
--- a/bfps/cpp/particles/abstract_particles_system.hpp
+++ b/bfps/cpp/particles/abstract_particles_system.hpp
@@ -33,7 +33,7 @@ public:
virtual void completeLoopWithVorticity(const real_number dt,
const real_number particle_extra_rhs[]) = 0;
- virtual const real_number* getParticlesPositions() const = 0;
+ virtual const real_number* getParticlesState() const = 0;
virtual const std::unique_ptr<real_number[]>* getParticlesRhs() const = 0;
diff --git a/bfps/cpp/particles/particles_sampling.hpp b/bfps/cpp/particles/particles_sampling.hpp
index c6b7e295..a8927591 100644
--- a/bfps/cpp/particles/particles_sampling.hpp
+++ b/bfps/cpp/particles/particles_sampling.hpp
@@ -41,7 +41,7 @@ void sample_from_particles_system(const field<rnumber, be, fc>& in_field, // a p
filename,
parent_groupname,
datasetname);
- outputclass.save(ps->getParticlesPositions(),
+ outputclass.save(ps->getParticlesState(),
&sample_rhs,
ps->getParticlesIndexes(),
ps->getLocalNbParticles(),
@@ -66,14 +66,14 @@ void sample_particles_system_position(
const partsize_t nb_particles = ps->getLocalNbParticles();
std::unique_ptr<particles_rnumber[]> sample_rhs(new particles_rnumber[3*nb_particles]);
- std::copy(ps->getParticlesPositions(), ps->getParticlesPositions() + 3*nb_particles, sample_rhs.get());
+ std::copy(ps->getParticlesState(), ps->getParticlesState() + 3*nb_particles, sample_rhs.get());
particles_output_sampling_hdf5<partsize_t, particles_rnumber, 3, 3> outputclass(MPI_COMM_WORLD,
ps->getGlobalNbParticles(),
filename,
parent_groupname,
datasetname);
- outputclass.save(ps->getParticlesPositions(),
+ outputclass.save(ps->getParticlesState(),
&sample_rhs,
ps->getParticlesIndexes(),
ps->getLocalNbParticles(),
diff --git a/bfps/cpp/particles/particles_system.hpp b/bfps/cpp/particles/particles_system.hpp
index 6623a671..1d99033e 100644
--- a/bfps/cpp/particles/particles_system.hpp
+++ b/bfps/cpp/particles/particles_system.hpp
@@ -268,7 +268,7 @@ public:
shift_rhs_vectors();
}
- const real_number* getParticlesPositions() const final {
+ const real_number* getParticlesState() const final {
return my_particles_positions.get();
}
--
GitLab
From fda03b1571d35f81d87cfdc9bd31fa36a316f441 Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Thu, 26 Oct 2017 16:38:49 +0200
Subject: [PATCH 065/342] Add a method to extract some states of the particles
-- extractParticlesState(0,3) should return the positions
---
bfps/cpp/particles/abstract_particles_system.hpp | 2 ++
bfps/cpp/particles/particles_system.hpp | 16 ++++++++++++++++
2 files changed, 18 insertions(+)
diff --git a/bfps/cpp/particles/abstract_particles_system.hpp b/bfps/cpp/particles/abstract_particles_system.hpp
index 6a2a01dc..456a7563 100644
--- a/bfps/cpp/particles/abstract_particles_system.hpp
+++ b/bfps/cpp/particles/abstract_particles_system.hpp
@@ -35,6 +35,8 @@ public:
virtual const real_number* getParticlesState() const = 0;
+ virtual std::unique_ptr<real_number[]> extractParticlesState(const int firstState, const int lastState) const = 0;
+
virtual const std::unique_ptr<real_number[]>* getParticlesRhs() const = 0;
virtual const partsize_t* getParticlesIndexes() const = 0;
diff --git a/bfps/cpp/particles/particles_system.hpp b/bfps/cpp/particles/particles_system.hpp
index 1d99033e..24bd228d 100644
--- a/bfps/cpp/particles/particles_system.hpp
+++ b/bfps/cpp/particles/particles_system.hpp
@@ -17,6 +17,8 @@
template <class partsize_t, class real_number, class field_rnumber, class field_class, class interpolator_class, int interp_neighbours,
int size_particle_positions, int size_particle_rhs, class p2p_computer_class, class particles_inner_computer_class>
class particles_system : public abstract_particles_system<partsize_t, real_number> {
+ static_assert(size_particle_positions >= 3, "There should be at least the positions X,Y,Z in the state");
+
MPI_Comm mpi_com;
const std::pair<int,int> current_partition_interval;
@@ -272,6 +274,20 @@ public:
return my_particles_positions.get();
}
+ std::unique_ptr<real_number[]> extractParticlesState(const int firstState, const int lastState) const final {
+ const int nbStates = std::max(0,(std::min(lastState,size_particle_positions)-firstState));
+
+ std::unique_ptr<real_number[]> stateExtract(new real_number[my_nb_particles*nbStates]);
+
+ for(partsize_t idx_part = 0 ; idx_part < my_nb_particles ; ++idx_part){
+ for(int idxState = 0 ; idxState < nbStates ; ++idxState){
+ stateExtract[idx_part*nbStates + idxState] = my_particles_positions[idx_part*size_particle_positions + idxState+firstState];
+ }
+ }
+
+ return stateExtract;
+ }
+
const std::unique_ptr<real_number[]>* getParticlesRhs() const final {
return my_particles_rhs.data();
}
--
GitLab
From 491e8bec0693896bb26dda653b2268ece92aa9c8 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 26 Oct 2017 17:07:23 +0200
Subject: [PATCH 066/342] use new state extractor
---
bfps/cpp/full_code/NSVEparticlesP2P.cpp | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
diff --git a/bfps/cpp/full_code/NSVEparticlesP2P.cpp b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
index d9b529f2..fd189ae4 100644
--- a/bfps/cpp/full_code/NSVEparticlesP2P.cpp
+++ b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
@@ -104,17 +104,15 @@ int NSVEparticlesP2P<rnumber>::do_stats()
return EXIT_SUCCESS;
// allocate temporary data array
- std::unique_ptr<double[]> pdata(new double[3*this->ps->getLocalNbParticles()]);
+ std::unique_ptr<double[]> pdata0 = this->ps->extractParticlesState(0, 3);
+ std::unique_ptr<double[]> pdata1 = this->ps->extractParticlesState(3, 6);
/// sample position
- std::copy(this->ps->getParticlesState(),
- this->ps->getParticlesState()+3*this->ps->getLocalNbParticles(),
- pdata.get());
this->particles_sample_writer_mpi->save_dataset(
"tracers0",
"position",
- this->ps->getParticlesState(),
- &pdata,
+ pdata0.get(),
+ &pdata0,
this->ps->getParticlesIndexes(),
this->ps->getLocalNbParticles(),
this->ps->get_step_idx()-1);
--
GitLab
From 114401c468aebd5fe49ce2b9fd3e9c4489d07f26 Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Thu, 26 Oct 2017 22:02:00 +0200
Subject: [PATCH 067/342] fix initialization of orientations
---
bfps/DNS.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index be973e17..3be9123b 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -746,7 +746,7 @@ class DNS(_code):
def get_random_versors(npoints):
bla = np.random.normal(
size = (npoints, 3))
- bla /= np.sum(bla**2, axis = 1)[:, None]
+ bla /= np.sum(bla**2, axis = 1)[:, None]**.5
return bla
while nn > 0:
if nn > batch_size:
@@ -908,6 +908,8 @@ class DNS(_code):
particle_file.create_group('tracers0/position')
particle_file.create_group('tracers0/velocity')
particle_file.create_group('tracers0/acceleration')
+ if self.dns_type in ['NSVEparticlesP2P']:
+ particle_file.create_group('tracers0/orientation')
return None
def launch_jobs(
self,
--
GitLab
From fb6eb88299e929892ae3420d26bb664ed7f47956 Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Thu, 26 Oct 2017 22:08:41 +0200
Subject: [PATCH 068/342] sample orientations, fix comparison with checkpoints
---
bfps/cpp/full_code/NSVEparticlesP2P.cpp | 17 ++++++++++++---
bfps/test/test_particles.py | 29 ++++++++++++++++++++++---
2 files changed, 40 insertions(+), 6 deletions(-)
diff --git a/bfps/cpp/full_code/NSVEparticlesP2P.cpp b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
index fd189ae4..a9e1493c 100644
--- a/bfps/cpp/full_code/NSVEparticlesP2P.cpp
+++ b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
@@ -96,14 +96,15 @@ int NSVEparticlesP2P<rnumber>::finalize(void)
template <typename rnumber>
int NSVEparticlesP2P<rnumber>::do_stats()
{
- /// fluid stats go here
+ /// perform fluid stats
this->NSVE<rnumber>::do_stats();
-
+ /// check if particle stats should be performed now;
+ /// if not, exit method.
if (!(this->iteration % this->niter_part == 0))
return EXIT_SUCCESS;
- // allocate temporary data array
+ /// allocate temporary data array
std::unique_ptr<double[]> pdata0 = this->ps->extractParticlesState(0, 3);
std::unique_ptr<double[]> pdata1 = this->ps->extractParticlesState(3, 6);
@@ -117,6 +118,16 @@ int NSVEparticlesP2P<rnumber>::do_stats()
this->ps->getLocalNbParticles(),
this->ps->get_step_idx()-1);
+ /// sample orientation
+ this->particles_sample_writer_mpi->save_dataset(
+ "tracers0",
+ "orientation",
+ pdata0.get(),
+ &pdata1,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+
///// sample velocity
//this->ps->sample_compute_field(*this->tmp_vec_field, pdata.get());
//this->particles_sample_writer_mpi->save_dataset(
diff --git a/bfps/test/test_particles.py b/bfps/test/test_particles.py
index 7e72eedf..d8728e2f 100644
--- a/bfps/test/test_particles.py
+++ b/bfps/test/test_particles.py
@@ -58,13 +58,36 @@ def main():
f.tight_layout()
f.savefig('position_histogram.pdf')
plt.close(f)
+ # show a histogram of the orientations
+ f = plt.figure()
+ a = f.add_subplot(111)
+ for iteration in range(0, niterations*njobs+1, niterations//2):
+ x = pf['tracers0/orientation/{0}'.format(iteration)].value
+ hist, bins = np.histogram(
+ np.sum(x**2, axis = -1).flatten()**.5,
+ bins = np.linspace(0, 2, 40))
+ bb = (bins[:-1] + bins[1:])/2
+ pp = hist.astype(np.float) / (np.sum(hist) * (bb[1] - bb[0]))
+ a.plot(bb, pp, label = '{0}'.format(iteration))
+ a.legend(loc = 'best')
+ f.tight_layout()
+ f.savefig('orientation_histogram.pdf')
+ plt.close(f)
# compared sampled positions with checkpoint positions
for iteration in range(0, niterations*njobs+1, niterations):
x = pf['tracers0/position/{0}'.format(iteration)].value
s = cf['tracers0/state/{0}'.format(iteration)].value
- distance = np.max(np.abs(x - s[..., :3]))
- print(iteration, distance)
- # show a histogram of the orientations
+ distance = (np.max(np.abs(x - s[..., :3]) /
+ np.maximum(np.ones(x.shape),
+ np.maximum(np.abs(x),
+ np.abs(s[..., :3])))))
+ assert(distance < 1e-14)
+ x = pf['tracers0/orientation/{0}'.format(iteration)].value
+ distance = (np.max(np.abs(x - s[..., 3:]) /
+ np.maximum(np.ones(x.shape),
+ np.maximum(np.abs(x),
+ np.abs(s[..., 3:])))))
+ assert(distance < 1e-14)
return None
if __name__ == '__main__':
--
GitLab
From 11f4c388175405e2e33b50b5b25a61b281345bde Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Thu, 26 Oct 2017 22:31:48 +0200
Subject: [PATCH 069/342] sample vorticity at particle locations
---
bfps/DNS.py | 1 +
bfps/cpp/full_code/NSVEparticlesP2P.cpp | 61 +++++++++++++++----------
2 files changed, 38 insertions(+), 24 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 3be9123b..488ff1b0 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -910,6 +910,7 @@ class DNS(_code):
particle_file.create_group('tracers0/acceleration')
if self.dns_type in ['NSVEparticlesP2P']:
particle_file.create_group('tracers0/orientation')
+ particle_file.create_group('tracers0/vorticity')
return None
def launch_jobs(
self,
diff --git a/bfps/cpp/full_code/NSVEparticlesP2P.cpp b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
index a9e1493c..85caafe0 100644
--- a/bfps/cpp/full_code/NSVEparticlesP2P.cpp
+++ b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
@@ -90,7 +90,7 @@ int NSVEparticlesP2P<rnumber>::finalize(void)
return EXIT_SUCCESS;
}
-/** \brief Compute fluid stats and sample fields at particle locations.
+/** \brief Compute fluid stats and sample particle data.
*/
template <typename rnumber>
@@ -128,29 +128,42 @@ int NSVEparticlesP2P<rnumber>::do_stats()
this->ps->getLocalNbParticles(),
this->ps->get_step_idx()-1);
- ///// sample velocity
- //this->ps->sample_compute_field(*this->tmp_vec_field, pdata.get());
- //this->particles_sample_writer_mpi->save_dataset(
- // "tracers0",
- // "velocity",
- // this->ps->getParticlesState(),
- // &pdata,
- // this->ps->getParticlesIndexes(),
- // this->ps->getLocalNbParticles(),
- // this->ps->get_step_idx()-1);
-
- ///// compute acceleration and sample it
- //this->fs->compute_Lagrangian_acceleration(this->tmp_vec_field);
- //this->tmp_vec_field->ift();
- //this->ps->sample_compute_field(*this->tmp_vec_field, pdata.get());
- //this->particles_sample_writer_mpi->save_dataset(
- // "tracers0",
- // "acceleration",
- // this->ps->getParticlesState(),
- // &pdata,
- // this->ps->getParticlesIndexes(),
- // this->ps->getLocalNbParticles(),
- // this->ps->get_step_idx()-1);
+ /// sample velocity
+ this->ps->sample_compute_field(*this->tmp_vec_field, pdata1.get());
+ this->particles_sample_writer_mpi->save_dataset(
+ "tracers0",
+ "velocity",
+ pdata0.get(),
+ &pdata1,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+
+ /// sample vorticity
+ *this->tmp_vec_field = this->fs->cvorticity->get_cdata();
+ this->tmp_vec_field->ift();
+ this->ps->sample_compute_field(*this->tmp_vec_field, pdata1.get());
+ this->particles_sample_writer_mpi->save_dataset(
+ "tracers0",
+ "vorticity",
+ pdata0.get(),
+ &pdata1,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+
+ /// compute acceleration and sample it
+ this->fs->compute_Lagrangian_acceleration(this->tmp_vec_field);
+ this->tmp_vec_field->ift();
+ this->ps->sample_compute_field(*this->tmp_vec_field, pdata1.get());
+ this->particles_sample_writer_mpi->save_dataset(
+ "tracers0",
+ "acceleration",
+ pdata0.get(),
+ &pdata1,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
return EXIT_SUCCESS;
}
--
GitLab
From 1e0c8cd2e6c83fa28c2c67abb4fd5a62da2c9aa2 Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Mon, 30 Oct 2017 09:36:33 +0100
Subject: [PATCH 070/342] Remove first index in output sampling hdf5
---
.../particles/particles_output_sampling_hdf5.hpp | 13 +++++--------
1 file changed, 5 insertions(+), 8 deletions(-)
diff --git a/bfps/cpp/particles/particles_output_sampling_hdf5.hpp b/bfps/cpp/particles/particles_output_sampling_hdf5.hpp
index 64faffdd..823754a5 100644
--- a/bfps/cpp/particles/particles_output_sampling_hdf5.hpp
+++ b/bfps/cpp/particles/particles_output_sampling_hdf5.hpp
@@ -187,10 +187,9 @@ public:
}
{
assert(size_particle_rhs >= 0);
- const hsize_t datacount[3] = {hsize_t(Parent::getNbRhs()),
- hsize_t(Parent::getTotalNbParticles()),
+ const hsize_t datacount[2] = {hsize_t(Parent::getTotalNbParticles()),
hsize_t(size_particle_rhs)};
- hid_t dataspace = H5Screate_simple(3, datacount, NULL);
+ hid_t dataspace = H5Screate_simple(2, datacount, NULL);
assert(dataspace >= 0);
hid_t dataset_id = H5Dcreate( pgroup_id,
@@ -203,15 +202,13 @@ public:
assert(dataset_id >= 0);
assert(particles_idx_offset >= 0);
- const hsize_t count[3] = {
- 1,
+ const hsize_t count[2] = {
hsize_t(nb_particles),
hsize_t(size_particle_rhs)};
- const hsize_t offset[3] = {
- 0,
+ const hsize_t offset[2] = {
hsize_t(particles_idx_offset),
0};
- hid_t memspace = H5Screate_simple(3, count, NULL);
+ hid_t memspace = H5Screate_simple(2, count, NULL);
assert(memspace >= 0);
hid_t filespace = H5Dget_space(dataset_id);
--
GitLab
From 92fe48855a72538bc4d115c7a9ca092991139e58 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 2 Nov 2017 09:33:41 +0100
Subject: [PATCH 071/342] change MaxNbRhs to 10
We will never use AdamsBashforth with more than 10 steps.
---
bfps/cpp/particles/p2p_distr_mpi.hpp | 2 +-
bfps/cpp/particles/particles_distr_mpi.hpp | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/bfps/cpp/particles/p2p_distr_mpi.hpp b/bfps/cpp/particles/p2p_distr_mpi.hpp
index ac69822a..31d5f4f8 100644
--- a/bfps/cpp/particles/p2p_distr_mpi.hpp
+++ b/bfps/cpp/particles/p2p_distr_mpi.hpp
@@ -19,7 +19,7 @@
template <class partsize_t, class real_number>
class p2p_distr_mpi {
protected:
- static const int MaxNbRhs = 100;
+ static const int MaxNbRhs = 10;
enum MpiTag{
TAG_NB_PARTICLES,
diff --git a/bfps/cpp/particles/particles_distr_mpi.hpp b/bfps/cpp/particles/particles_distr_mpi.hpp
index 889762d4..bd77162e 100644
--- a/bfps/cpp/particles/particles_distr_mpi.hpp
+++ b/bfps/cpp/particles/particles_distr_mpi.hpp
@@ -17,7 +17,7 @@
template <class partsize_t, class real_number>
class particles_distr_mpi {
protected:
- static const int MaxNbRhs = 100;
+ static const int MaxNbRhs = 10;
enum MpiTag{
TAG_LOW_UP_NB_PARTICLES,
--
GitLab
From 22fbefec516e6d72121429aa9c95bd2f956f042d Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 2 Nov 2017 09:43:53 +0100
Subject: [PATCH 072/342] specialize test for simple motion
---
bfps/cpp/full_code/NSVEparticlesP2P.cpp | 3 +++
bfps/test/test_particles.py | 24 ++++++++++++++++++++++++
2 files changed, 27 insertions(+)
diff --git a/bfps/cpp/full_code/NSVEparticlesP2P.cpp b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
index 85caafe0..12d77149 100644
--- a/bfps/cpp/full_code/NSVEparticlesP2P.cpp
+++ b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
@@ -47,6 +47,9 @@ int NSVEparticlesP2P<rnumber>::initialize(void)
(this->simname + "_particles.h5"),
"tracers0",
"position/0");
+ // TODO: remove the following testing initial condition, and use a proper
+ // way to initialize with 0 (i.e. generate a 0 field as the initial condition).
+ //*this->fs->cvorticity = 0.0;
return EXIT_SUCCESS;
}
diff --git a/bfps/test/test_particles.py b/bfps/test/test_particles.py
index d8728e2f..79595a23 100644
--- a/bfps/test/test_particles.py
+++ b/bfps/test/test_particles.py
@@ -88,6 +88,30 @@ def main():
np.maximum(np.abs(x),
np.abs(s[..., 3:])))))
assert(distance < 1e-14)
+ # print movement
+ x0 = pf['tracers0/position/0'].value
+ x1 = pf['tracers0/position/32'].value
+ # compute distance travelled by first particle
+ deltax = x1[0] - x0[0]
+ print('distance travelled by first particle is ', np.sum(deltax**2)**.5)
+ ## code relevant when velocity field is 0 everywhere.
+ ## we check to see what happens to the orientation of the particles
+ ## show a histogram of the orientations
+ #f = plt.figure()
+ #a = f.add_subplot(111)
+ #for iteration in range(0, niterations*njobs+1, niterations//4):
+ # x = pf['tracers0/orientation/{0}'.format(iteration)].value
+ # print(x)
+ # hist, bins = np.histogram(
+ # x.flatten(),
+ # bins = 100)
+ # bb = (bins[:-1] + bins[1:])/2
+ # pp = hist.astype(np.float) / (np.sum(hist) * (bb[1] - bb[0]))
+ # a.plot(bb, pp, label = '{0}'.format(iteration))
+ #a.legend(loc = 'best')
+ #f.tight_layout()
+ #f.savefig('full_orientation_histogram.pdf')
+ #plt.close(f)
return None
if __name__ == '__main__':
--
GitLab
From 44bcc1aa1421d681b9cf06b6d78f8324909efbff Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 2 Nov 2017 10:31:31 +0100
Subject: [PATCH 073/342] [broken] apply time stepping to complicated particles
as well.
This code fails for particles with interactions.
Most likely because the time step is too large.
But we need the commit
---
.../particles/particles_adams_bashforth.hpp | 112 +++++++++++++++++-
1 file changed, 111 insertions(+), 1 deletion(-)
diff --git a/bfps/cpp/particles/particles_adams_bashforth.hpp b/bfps/cpp/particles/particles_adams_bashforth.hpp
index 20c1ea50..abb47fbe 100644
--- a/bfps/cpp/particles/particles_adams_bashforth.hpp
+++ b/bfps/cpp/particles/particles_adams_bashforth.hpp
@@ -22,7 +22,117 @@ public:
const partsize_t nb_particles,
const std::unique_ptr<real_number[]> particles_rhs[],
const int nb_rhs, const real_number dt) const{
- // TODO
+ TIMEZONE("particles_adams_bashforth::move_particles");
+
+ if(Max_steps < nb_rhs){
+ throw std::runtime_error("Error, in bfps particles_adams_bashforth.\n"
+ "Step in particles_adams_bashforth is too large,"
+ "you must add formulation up this number or limit the number of steps.");
+ }
+
+ // Not needed: TIMEZONE_OMP_INIT_PREPARALLEL(omp_get_max_threads())
+#pragma omp parallel default(shared)
+ {
+ particles_utils::IntervalSplitter<partsize_t> interval(nb_particles,
+ omp_get_num_threads(),
+ omp_get_thread_num());
+
+ const partsize_t value_start = interval.getMyOffset()*size_particle_positions;
+ const partsize_t value_end = (interval.getMyOffset()+interval.getMySize())*size_particle_positions;
+
+ // TODO full unroll + blocking
+ switch (nb_rhs){
+ case 1:
+ {
+ const real_number* __restrict__ rhs_0 = particles_rhs[0].get();
+ for(partsize_t idx_value = value_start ; idx_value < value_end ; ++idx_value){
+ // dt × [0]
+ particles_positions[idx_value] += dt * rhs_0[idx_value];
+ }
+ }
+ break;
+ case 2:
+ {
+ const real_number* __restrict__ rhs_0 = particles_rhs[0].get();
+ const real_number* __restrict__ rhs_1 = particles_rhs[1].get();
+ for(partsize_t idx_value = value_start ; idx_value < value_end ; ++idx_value){
+ // dt × (3[0] - [1])/2
+ particles_positions[idx_value]
+ += dt * (3.*rhs_0[idx_value]
+ - rhs_1[idx_value])/2.;
+ }
+ }
+ break;
+ case 3:
+ {
+ const real_number* __restrict__ rhs_0 = particles_rhs[0].get();
+ const real_number* __restrict__ rhs_1 = particles_rhs[1].get();
+ const real_number* __restrict__ rhs_2 = particles_rhs[2].get();
+ for(partsize_t idx_value = value_start ; idx_value < value_end ; ++idx_value){
+ // dt × (23[0] - 16[1] + 5[2])/12
+ particles_positions[idx_value]
+ += dt * (23.*rhs_0[idx_value]
+ - 16.*rhs_1[idx_value]
+ + 5.*rhs_2[idx_value])/12.;
+ }
+ }
+ break;
+ case 4:
+ {
+ const real_number* __restrict__ rhs_0 = particles_rhs[0].get();
+ const real_number* __restrict__ rhs_1 = particles_rhs[1].get();
+ const real_number* __restrict__ rhs_2 = particles_rhs[2].get();
+ const real_number* __restrict__ rhs_3 = particles_rhs[3].get();
+ for(partsize_t idx_value = value_start ; idx_value < value_end ; ++idx_value){
+ // dt × (55[0] - 59[1] + 37[2] - 9[3])/24
+ particles_positions[idx_value]
+ += dt * (55.*rhs_0[idx_value]
+ - 59.*rhs_1[idx_value]
+ + 37.*rhs_2[idx_value]
+ - 9.*rhs_3[idx_value])/24.;
+ }
+ }
+ break;
+ case 5:
+ {
+ const real_number* __restrict__ rhs_0 = particles_rhs[0].get();
+ const real_number* __restrict__ rhs_1 = particles_rhs[1].get();
+ const real_number* __restrict__ rhs_2 = particles_rhs[2].get();
+ const real_number* __restrict__ rhs_3 = particles_rhs[3].get();
+ const real_number* __restrict__ rhs_4 = particles_rhs[4].get();
+ for(partsize_t idx_value = value_start ; idx_value < value_end ; ++idx_value){
+ // dt × (1901[0] - 2774[1] + 2616[2] - 1274[3] + 251[4])/720
+ particles_positions[idx_value]
+ += dt * (1901.*rhs_0[idx_value]
+ - 2774.*rhs_1[idx_value]
+ + 2616.*rhs_2[idx_value]
+ - 1274.*rhs_3[idx_value]
+ + 251.*rhs_4[idx_value])/720.;
+ }
+ }
+ break;
+ case 6:
+ {
+ const real_number* __restrict__ rhs_0 = particles_rhs[0].get();
+ const real_number* __restrict__ rhs_1 = particles_rhs[1].get();
+ const real_number* __restrict__ rhs_2 = particles_rhs[2].get();
+ const real_number* __restrict__ rhs_3 = particles_rhs[3].get();
+ const real_number* __restrict__ rhs_4 = particles_rhs[4].get();
+ const real_number* __restrict__ rhs_5 = particles_rhs[5].get();
+ for(partsize_t idx_value = value_start ; idx_value < value_end ; ++idx_value){
+ // dt × (4277[0] - 7923[1] + 9982[2] - 7298[3] + 2877[4] - 475[5])/1440
+ particles_positions[idx_value]
+ += dt * (4277.*rhs_0[idx_value]
+ - 7923.*rhs_1[idx_value]
+ + 9982.*rhs_2[idx_value]
+ - 7298.*rhs_3[idx_value]
+ + 2877.*rhs_4[idx_value]
+ - 475.*rhs_5[idx_value])/1440.;
+ }
+ }
+ break;
+ }
+ }
}
};
--
GitLab
From 06ada1bf697ef65368319431b469585a843e1224 Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Thu, 2 Nov 2017 10:42:01 +0100
Subject: [PATCH 074/342] Use the same Adams Bashforth with or without P2P
---
.../particles/particles_adams_bashforth.hpp | 134 +-----------------
1 file changed, 2 insertions(+), 132 deletions(-)
diff --git a/bfps/cpp/particles/particles_adams_bashforth.hpp b/bfps/cpp/particles/particles_adams_bashforth.hpp
index abb47fbe..e81e6022 100644
--- a/bfps/cpp/particles/particles_adams_bashforth.hpp
+++ b/bfps/cpp/particles/particles_adams_bashforth.hpp
@@ -8,139 +8,9 @@
#include "particles_utils.hpp"
template <class partsize_t, class real_number, int size_particle_positions, int size_particle_rhs>
-class particles_adams_bashforth;
+class particles_adams_bashforth{
+ static_assert(size_particle_positions == size_particle_rhs, "This class is designed for the same number of values in positions and rhs");
-
-template <class partsize_t, class real_number>
-class particles_adams_bashforth<partsize_t,real_number,6,6>{
- static const int size_particle_positions = 6;
- static const int size_particle_rhs = 6;
-public:
- static const int Max_steps = 6;
-
- void move_particles(real_number*__restrict__ particles_positions,
- const partsize_t nb_particles,
- const std::unique_ptr<real_number[]> particles_rhs[],
- const int nb_rhs, const real_number dt) const{
- TIMEZONE("particles_adams_bashforth::move_particles");
-
- if(Max_steps < nb_rhs){
- throw std::runtime_error("Error, in bfps particles_adams_bashforth.\n"
- "Step in particles_adams_bashforth is too large,"
- "you must add formulation up this number or limit the number of steps.");
- }
-
- // Not needed: TIMEZONE_OMP_INIT_PREPARALLEL(omp_get_max_threads())
-#pragma omp parallel default(shared)
- {
- particles_utils::IntervalSplitter<partsize_t> interval(nb_particles,
- omp_get_num_threads(),
- omp_get_thread_num());
-
- const partsize_t value_start = interval.getMyOffset()*size_particle_positions;
- const partsize_t value_end = (interval.getMyOffset()+interval.getMySize())*size_particle_positions;
-
- // TODO full unroll + blocking
- switch (nb_rhs){
- case 1:
- {
- const real_number* __restrict__ rhs_0 = particles_rhs[0].get();
- for(partsize_t idx_value = value_start ; idx_value < value_end ; ++idx_value){
- // dt × [0]
- particles_positions[idx_value] += dt * rhs_0[idx_value];
- }
- }
- break;
- case 2:
- {
- const real_number* __restrict__ rhs_0 = particles_rhs[0].get();
- const real_number* __restrict__ rhs_1 = particles_rhs[1].get();
- for(partsize_t idx_value = value_start ; idx_value < value_end ; ++idx_value){
- // dt × (3[0] - [1])/2
- particles_positions[idx_value]
- += dt * (3.*rhs_0[idx_value]
- - rhs_1[idx_value])/2.;
- }
- }
- break;
- case 3:
- {
- const real_number* __restrict__ rhs_0 = particles_rhs[0].get();
- const real_number* __restrict__ rhs_1 = particles_rhs[1].get();
- const real_number* __restrict__ rhs_2 = particles_rhs[2].get();
- for(partsize_t idx_value = value_start ; idx_value < value_end ; ++idx_value){
- // dt × (23[0] - 16[1] + 5[2])/12
- particles_positions[idx_value]
- += dt * (23.*rhs_0[idx_value]
- - 16.*rhs_1[idx_value]
- + 5.*rhs_2[idx_value])/12.;
- }
- }
- break;
- case 4:
- {
- const real_number* __restrict__ rhs_0 = particles_rhs[0].get();
- const real_number* __restrict__ rhs_1 = particles_rhs[1].get();
- const real_number* __restrict__ rhs_2 = particles_rhs[2].get();
- const real_number* __restrict__ rhs_3 = particles_rhs[3].get();
- for(partsize_t idx_value = value_start ; idx_value < value_end ; ++idx_value){
- // dt × (55[0] - 59[1] + 37[2] - 9[3])/24
- particles_positions[idx_value]
- += dt * (55.*rhs_0[idx_value]
- - 59.*rhs_1[idx_value]
- + 37.*rhs_2[idx_value]
- - 9.*rhs_3[idx_value])/24.;
- }
- }
- break;
- case 5:
- {
- const real_number* __restrict__ rhs_0 = particles_rhs[0].get();
- const real_number* __restrict__ rhs_1 = particles_rhs[1].get();
- const real_number* __restrict__ rhs_2 = particles_rhs[2].get();
- const real_number* __restrict__ rhs_3 = particles_rhs[3].get();
- const real_number* __restrict__ rhs_4 = particles_rhs[4].get();
- for(partsize_t idx_value = value_start ; idx_value < value_end ; ++idx_value){
- // dt × (1901[0] - 2774[1] + 2616[2] - 1274[3] + 251[4])/720
- particles_positions[idx_value]
- += dt * (1901.*rhs_0[idx_value]
- - 2774.*rhs_1[idx_value]
- + 2616.*rhs_2[idx_value]
- - 1274.*rhs_3[idx_value]
- + 251.*rhs_4[idx_value])/720.;
- }
- }
- break;
- case 6:
- {
- const real_number* __restrict__ rhs_0 = particles_rhs[0].get();
- const real_number* __restrict__ rhs_1 = particles_rhs[1].get();
- const real_number* __restrict__ rhs_2 = particles_rhs[2].get();
- const real_number* __restrict__ rhs_3 = particles_rhs[3].get();
- const real_number* __restrict__ rhs_4 = particles_rhs[4].get();
- const real_number* __restrict__ rhs_5 = particles_rhs[5].get();
- for(partsize_t idx_value = value_start ; idx_value < value_end ; ++idx_value){
- // dt × (4277[0] - 7923[1] + 9982[2] - 7298[3] + 2877[4] - 475[5])/1440
- particles_positions[idx_value]
- += dt * (4277.*rhs_0[idx_value]
- - 7923.*rhs_1[idx_value]
- + 9982.*rhs_2[idx_value]
- - 7298.*rhs_3[idx_value]
- + 2877.*rhs_4[idx_value]
- - 475.*rhs_5[idx_value])/1440.;
- }
- }
- break;
- }
- }
- }
-};
-
-
-template <class partsize_t, class real_number>
-class particles_adams_bashforth<partsize_t,real_number,3,3>{
- static const int size_particle_positions = 3;
- static const int size_particle_rhs = 3;
public:
static const int Max_steps = 6;
--
GitLab
From c1e9b6c2d994b75ee88912c4757edf6779d788c8 Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Thu, 2 Nov 2017 15:20:50 +0100
Subject: [PATCH 075/342] Test if the move are not too big, and simple assert
for now
---
bfps/cpp/particles/particles_distr_mpi.hpp | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/bfps/cpp/particles/particles_distr_mpi.hpp b/bfps/cpp/particles/particles_distr_mpi.hpp
index bd77162e..85f4f416 100644
--- a/bfps/cpp/particles/particles_distr_mpi.hpp
+++ b/bfps/cpp/particles/particles_distr_mpi.hpp
@@ -517,6 +517,19 @@ public:
return;
}
+ {// TODO remove
+ partsize_t partOffset = 0;
+ for(int idxPartition = 0 ; idxPartition < current_partition_size ; ++idxPartition){
+ for(partsize_t idx = 0 ; idx < current_my_nb_particles_per_partition[idxPartition] ; ++idx){
+ const int partition_level = in_computer.pbc_field_layer((*inout_positions_particles)[(idx+partOffset)*size_particle_positions+IDX_Z], IDX_Z);
+ assert(partition_level == current_partition_interval.first + idxPartition
+ || partition_level == (current_partition_interval.first + idxPartition-1+int(field_grid_dim[IDX_Z]))%int(field_grid_dim[IDX_Z])
+ || partition_level == (current_partition_interval.first + idxPartition+1)%int(field_grid_dim[IDX_Z]));
+ }
+ partOffset += current_my_nb_particles_per_partition[idxPartition];
+ }
+ }
+
current_offset_particles_for_partition[0] = 0;
partsize_t myTotalNbParticles = 0;
for(int idxPartition = 0 ; idxPartition < current_partition_size ; ++idxPartition){
--
GitLab
From 1f53da15d5297608d613620c1dbcae42cdb1fe86 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 2 Nov 2017 17:11:15 +0100
Subject: [PATCH 076/342] WP: fix dot product expression
still a work in progress because the code will only work with the Euler
integration scheme for now.
we need to switch `enforce_unit_orientation` to work on the orientation
itself, and to call it in a different place in the code.
---
bfps/cpp/full_code/NSVEparticlesP2P.cpp | 10 ++-
.../particles/abstract_particles_system.hpp | 2 +
.../particles/particles_inner_computer.hpp | 83 +++++++++++++++----
.../particles_inner_computer_empty.hpp | 4 +
bfps/cpp/particles/particles_system.hpp | 10 +++
bfps/test/test_particles.py | 41 ++++-----
setup.py | 6 +-
7 files changed, 112 insertions(+), 44 deletions(-)
diff --git a/bfps/cpp/full_code/NSVEparticlesP2P.cpp b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
index 12d77149..3a8312f1 100644
--- a/bfps/cpp/full_code/NSVEparticlesP2P.cpp
+++ b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
@@ -12,7 +12,10 @@ int NSVEparticlesP2P<rnumber>::initialize(void)
this->NSVE<rnumber>::initialize();
p2p_computer<double, long long int> current_p2p_computer;
+ // TODO: particle interactions are switched off manually for testing purposes.
+ // this needs to be fixed once particle interactions can be properly resolved.
current_p2p_computer.setEnable(enable_p2p);
+ //current_p2p_computer.setEnable(false);
particles_inner_computer<double, long long int> current_particles_inner_computer(inner_v0);
current_particles_inner_computer.setEnable(enable_inner);
@@ -59,7 +62,12 @@ int NSVEparticlesP2P<rnumber>::step(void)
this->fs->compute_velocity(this->fs->cvorticity);
this->fs->cvelocity->ift();
if(enable_vorticity_omega){
- this->ps->completeLoopWithVorticity(this->dt, *this->fs->cvorticity);
+ *this->tmp_vec_field = this->fs->cvorticity->get_cdata();
+ this->tmp_vec_field->ift();
+ std::unique_ptr<double[]> pdata;
+ pdata.reset(new double[ps->getLocalNbParticles()*3]);
+ this->ps->sample_compute_field(*this->tmp_vec_field, pdata.get());
+ this->ps->completeLoopWithVorticity(this->dt, pdata.get());
}
else{
this->ps->completeLoop(this->dt);
diff --git a/bfps/cpp/particles/abstract_particles_system.hpp b/bfps/cpp/particles/abstract_particles_system.hpp
index 456a7563..859432c0 100644
--- a/bfps/cpp/particles/abstract_particles_system.hpp
+++ b/bfps/cpp/particles/abstract_particles_system.hpp
@@ -18,6 +18,8 @@ public:
virtual void compute_particles_inner() = 0;
+ virtual void enforce_unit_orientation() = 0;
+
virtual void compute_particles_inner(const real_number particle_extra_rhs[]) = 0;
virtual void move(const real_number dt) = 0;
diff --git a/bfps/cpp/particles/particles_inner_computer.hpp b/bfps/cpp/particles/particles_inner_computer.hpp
index b20abfda..d58981e6 100644
--- a/bfps/cpp/particles/particles_inner_computer.hpp
+++ b/bfps/cpp/particles/particles_inner_computer.hpp
@@ -14,7 +14,7 @@ public:
}
template <int size_particle_positions, int size_particle_rhs>
- void compute_interaction(const partsize_t nb_particles, real_number pos_part[], real_number rhs_part[]) const{
+ void compute_interaction(const partsize_t nb_particles, const real_number pos_part[], real_number rhs_part[]) const{
static_assert(size_particle_positions == 6, "This kernel works only with 6 values for one particle's position");
static_assert(size_particle_rhs == 6, "This kernel works only with 6 values per particle's rhs");
@@ -24,34 +24,85 @@ public:
rhs_part[idx_part*size_particle_rhs + IDX_X] += pos_part[idx_part*size_particle_positions + 3+IDX_X]*v0;
rhs_part[idx_part*size_particle_rhs + IDX_Y] += pos_part[idx_part*size_particle_positions + 3+IDX_Y]*v0;
rhs_part[idx_part*size_particle_rhs + IDX_Z] += pos_part[idx_part*size_particle_positions + 3+IDX_Z]*v0;
+ }
+ }
+
+ template <int size_particle_positions, int size_particle_rhs>
+ void enforce_unit_orientation(const partsize_t nb_particles, const real_number pos_part[], real_number rhs_part[]) const{
+ static_assert(size_particle_positions == 6, "This kernel works only with 6 values for one particle's position");
+ static_assert(size_particle_rhs == 6, "This kernel works only with 6 values per particle's rhs");
- real_number alpha[3] = {0, 0, 0};
+ #pragma omp parallel for
+ for(partsize_t idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
+ const partsize_t idx0 = idx_part*size_particle_positions + 3;
+ const partsize_t idx1 = idx_part*size_particle_rhs + 3;
+ //real_number alpha[3] = {0, 0, 0};
+ // check that orientation is unit vector:
+ real_number orientation_size = sqrt(
+ pos_part[idx0+IDX_X]*pos_part[idx1+IDX_X] +
+ pos_part[idx0+IDX_Y]*pos_part[idx1+IDX_Y] +
+ pos_part[idx0+IDX_Z]*pos_part[idx1+IDX_Z]);
+ assert(orientation_size > 0.99);
+ assert(orientation_size < 1.01);
// I call "rotation" to be the right hand side of the orientation part of the ODE
// project rotation on orientation:
real_number projection = (
- pos_part[idx_part*size_particle_positions + 3+IDX_X]*rhs_part[idx_part*size_particle_rhs + 3+IDX_X] +
- pos_part[idx_part*size_particle_positions + 3+IDX_Y]*rhs_part[idx_part*size_particle_rhs + 3+IDX_Y] +
- pos_part[idx_part*size_particle_positions + 3+IDX_Z]*rhs_part[idx_part*size_particle_rhs + 3+IDX_Z]);
- // alpha is the vector that makes rotation perpendicular to orientation.
- // note that the following three lines assume the current orientation is a unit vector.
- alpha[IDX_X] = -pos_part[idx_part*size_particle_positions + 3+IDX_X]*projection;
- alpha[IDX_Y] = -pos_part[idx_part*size_particle_positions + 3+IDX_Z]*projection;
- alpha[IDX_Z] = -pos_part[idx_part*size_particle_positions + 3+IDX_Y]*projection;
-
- // now add alpha term to orientation ODE right-hand side.
- rhs_part[idx_part*size_particle_rhs + 3+IDX_X] += alpha[IDX_X];
- rhs_part[idx_part*size_particle_rhs + 3+IDX_Y] += alpha[IDX_Y];
- rhs_part[idx_part*size_particle_rhs + 3+IDX_Z] += alpha[IDX_Z];
+ pos_part[idx0+IDX_X]*rhs_part[idx1+IDX_X] +
+ pos_part[idx0+IDX_Y]*rhs_part[idx1+IDX_Y] +
+ pos_part[idx0+IDX_Z]*rhs_part[idx1+IDX_Z]);
+ //// alpha is the vector that makes rotation perpendicular to orientation.
+ //// note that the following three lines assume the current orientation is a unit vector.
+ //alpha[IDX_X] = -;
+ //alpha[IDX_Y] = -;
+ //alpha[IDX_Z] = -;
+ // DEBUG_MSG("projection = %g\n"
+ // "pos_part[%d] = %g, pos_part[%d] = %g, pos_part[%d] = %g\n"
+ // "rhs_part[%d] = %g, rhs_part[%d] = %g, rhs_part[%d] = %g\n",
+ // projection,
+ // IDX_X, pos_part[idx0 + IDX_X],
+ // IDX_Y, pos_part[idx0 + IDX_Y],
+ // IDX_Z, pos_part[idx0 + IDX_Z],
+ // IDX_X, rhs_part[idx1 + IDX_X],
+ // IDX_Y, rhs_part[idx1 + IDX_Y],
+ // IDX_Z, rhs_part[idx1 + IDX_Z]);
+
+ // now remove parallel bit.
+ rhs_part[idx1+IDX_X] -= pos_part[idx0+IDX_X]*projection;
+ rhs_part[idx1+IDX_Y] -= pos_part[idx0+IDX_Y]*projection;
+ rhs_part[idx1+IDX_Z] -= pos_part[idx0+IDX_Z]*projection;
+
+ // compute dot product between orientation and orientation change
+ real_number dotproduct = (
+ rhs_part[idx1 + IDX_X]*pos_part[idx0 + IDX_X] +
+ rhs_part[idx1 + IDX_Y]*pos_part[idx0 + IDX_Y] +
+ rhs_part[idx1 + IDX_Z]*pos_part[idx0 + IDX_Z]);
+ if (dotproduct > 0.1)
+ {
+ DEBUG_MSG("dotproduct = %g, projection = %g\n"
+ "pos_part[%d] = %g, pos_part[%d] = %g, pos_part[%d] = %g\n"
+ "rhs_part[%d] = %g, rhs_part[%d] = %g, rhs_part[%d] = %g\n",
+ dotproduct,
+ projection,
+ IDX_X, pos_part[idx0 + IDX_X],
+ IDX_Y, pos_part[idx0 + IDX_Y],
+ IDX_Z, pos_part[idx0 + IDX_Z],
+ IDX_X, rhs_part[idx1 + IDX_X],
+ IDX_Y, rhs_part[idx1 + IDX_Y],
+ IDX_Z, rhs_part[idx1 + IDX_Z]);
+ }
+ assert(dotproduct <= 0.1);
}
}
template <int size_particle_positions, int size_particle_rhs, int size_particle_rhs_extra>
- void compute_interaction_with_extra(const partsize_t nb_particles, real_number pos_part[], real_number rhs_part[],
+ void compute_interaction_with_extra(const partsize_t nb_particles, const real_number pos_part[], real_number rhs_part[],
const real_number rhs_part_extra[]) const{
static_assert(size_particle_rhs_extra == 3, "This kernel works only with 3 values for one particle's rhs extra");
+ // call plain compute_interaction first
compute_interaction<size_particle_positions, size_particle_rhs>(nb_particles, pos_part, rhs_part);
+ // now add vorticity term
#pragma omp parallel for
for(partsize_t idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
// Cross product vorticity/orientation
diff --git a/bfps/cpp/particles/particles_inner_computer_empty.hpp b/bfps/cpp/particles/particles_inner_computer_empty.hpp
index 1bd3b1ec..263d8b17 100644
--- a/bfps/cpp/particles/particles_inner_computer_empty.hpp
+++ b/bfps/cpp/particles/particles_inner_computer_empty.hpp
@@ -11,6 +11,10 @@ public:
void compute_interaction(const partsize_t /*nb_particles*/, real_number /*pos_part*/[], real_number /*rhs_part*/[]) const{
}
+ template <int size_particle_positions, int size_particle_rhs>
+ void enforce_unit_orientation(const partsize_t /*nb_particles*/, real_number /*pos_part*/[], real_number /*rhs_part*/[]) const{
+ }
+
template <int size_particle_positions, int size_particle_rhs, int size_particle_rhs_extra>
void compute_interaction_with_extra(const partsize_t /*nb_particles*/, real_number /*pos_part*/[], real_number /*rhs_part*/[],
const real_number /*rhs_part_extra*/[]) const{
diff --git a/bfps/cpp/particles/particles_system.hpp b/bfps/cpp/particles/particles_system.hpp
index 24bd228d..a5d7878f 100644
--- a/bfps/cpp/particles/particles_system.hpp
+++ b/bfps/cpp/particles/particles_system.hpp
@@ -162,6 +162,14 @@ public:
}
}
+ void enforce_unit_orientation() final {
+ if(computer_particules_inner.isEnable() == true){
+ TIMEZONE("particles_system::enforce_unit_orientation");
+ computer_particules_inner.template enforce_unit_orientation<size_particle_positions, size_particle_rhs>(
+ my_nb_particles, my_particles_positions.get(), my_particles_rhs.front().get());
+ }
+ }
+
void compute_particles_inner(const real_number particle_extra_rhs[]) final {
if(computer_particules_inner.isEnable() == true){
TIMEZONE("particles_system::compute_particles_inner");
@@ -252,6 +260,7 @@ public:
compute();
compute_p2p();
compute_particles_inner();
+ enforce_unit_orientation();
move(dt);
redistribute();
inc_step_idx();
@@ -264,6 +273,7 @@ public:
compute();
compute_p2p();
compute_particles_inner(particle_extra_rhs);
+ enforce_unit_orientation();
move(dt);
redistribute();
inc_step_idx();
diff --git a/bfps/test/test_particles.py b/bfps/test/test_particles.py
index 79595a23..c3d4c415 100644
--- a/bfps/test/test_particles.py
+++ b/bfps/test/test_particles.py
@@ -88,30 +88,23 @@ def main():
np.maximum(np.abs(x),
np.abs(s[..., 3:])))))
assert(distance < 1e-14)
- # print movement
- x0 = pf['tracers0/position/0'].value
- x1 = pf['tracers0/position/32'].value
- # compute distance travelled by first particle
- deltax = x1[0] - x0[0]
- print('distance travelled by first particle is ', np.sum(deltax**2)**.5)
- ## code relevant when velocity field is 0 everywhere.
- ## we check to see what happens to the orientation of the particles
- ## show a histogram of the orientations
- #f = plt.figure()
- #a = f.add_subplot(111)
- #for iteration in range(0, niterations*njobs+1, niterations//4):
- # x = pf['tracers0/orientation/{0}'.format(iteration)].value
- # print(x)
- # hist, bins = np.histogram(
- # x.flatten(),
- # bins = 100)
- # bb = (bins[:-1] + bins[1:])/2
- # pp = hist.astype(np.float) / (np.sum(hist) * (bb[1] - bb[0]))
- # a.plot(bb, pp, label = '{0}'.format(iteration))
- #a.legend(loc = 'best')
- #f.tight_layout()
- #f.savefig('full_orientation_histogram.pdf')
- #plt.close(f)
+ # code relevant when velocity field is 0 everywhere.
+ # we check to see what happens to the orientation of the particles
+ # show a histogram of the orientations
+ f = plt.figure()
+ a = f.add_subplot(111)
+ for iteration in range(0, niterations*njobs+1, niterations//4):
+ x = pf['tracers0/orientation/{0}'.format(iteration)].value
+ hist, bins = np.histogram(
+ x.flatten(),
+ bins = 100)
+ bb = (bins[:-1] + bins[1:])/2
+ pp = hist.astype(np.float) / (np.sum(hist) * (bb[1] - bb[0]))
+ a.plot(bb, pp, label = '{0}'.format(iteration))
+ a.legend(loc = 'best')
+ f.tight_layout()
+ f.savefig('full_orientation_histogram.pdf')
+ plt.close(f)
return None
if __name__ == '__main__':
diff --git a/setup.py b/setup.py
index 27a2993f..34b9e16a 100644
--- a/setup.py
+++ b/setup.py
@@ -88,7 +88,8 @@ print('This is bfps version ' + VERSION)
### lists of files and MANIFEST.in
-src_file_list = ['full_code/joint_acc_vel_stats',
+src_file_list = ['full_code/NSVEparticlesP2P',
+ 'full_code/joint_acc_vel_stats',
'full_code/test',
'full_code/filter_test',
'full_code/field_test',
@@ -128,8 +129,7 @@ src_file_list = ['full_code/joint_acc_vel_stats',
'spline_n10',
'Lagrange_polys',
'scope_timer',
- 'full_code/NSVEparticles',
- 'full_code/NSVEparticlesP2P']
+ 'full_code/NSVEparticles']
particle_headers = [
'cpp/particles/abstract_particles_input.hpp',
--
GitLab
From 7124c8152027c638c2d977729c1e690b59fd1ca1 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 14 Nov 2017 16:49:42 +0100
Subject: [PATCH 077/342] add resize operation to field class
---
bfps/cpp/field.cpp | 136 +++++++++++++++++++++++++++++++++++++++++++++
bfps/cpp/field.hpp | 21 +++++++
2 files changed, 157 insertions(+)
diff --git a/bfps/cpp/field.cpp b/bfps/cpp/field.cpp
index e20207c1..ba0656f8 100644
--- a/bfps/cpp/field.cpp
+++ b/bfps/cpp/field.cpp
@@ -1360,6 +1360,142 @@ int joint_rspace_PDF(
return EXIT_SUCCESS;
}
+template <typename rnumber,
+ field_backend be,
+ field_components fc>
+field<rnumber, be, fc> &field<rnumber, be, fc>::operator=(
+ const field<rnumber, be, fc> &src)
+{
+ TIMEZONE("field::operator=");
+ if (src.real_space_representation)
+ {
+ assert(this->get_nx() == src.get_nx());
+ assert(this->get_ny() == src.get_ny());
+ assert(this->get_nz() == src.get_nz());
+ this->real_space_representation = true;
+ std::copy(src.data,
+ src.data + this->rmemlayout->local_size,
+ this->data);
+ }
+ else
+ {
+ this->real_space_representation = false;
+ // simple copy
+ if (this->get_nx() == src.get_nx() &&
+ this->get_ny() == src.get_ny() &&
+ this->get_nz() == src.get_nz())
+ {
+ std::copy(src.data,
+ src.data + 2*this->clayout->local_size,
+ this->data);
+ }
+ // complicated resize
+ else
+ {
+ int64_t slice_size = src.clayout->local_size / src.clayout->subsizes[0];
+ // clean up
+ std::fill_n(this->data,
+ this->rmemlayout->local_size,
+ 0.0);
+ typename fftw_interface<rnumber>::complex *buffer;
+ buffer = fftw_interface<rnumber>::alloc_complex(slice_size*ncomp(fc));
+
+ int min_fast_dim =
+ (src.clayout->sizes[2] > this->clayout->sizes[2]) ?
+ this->clayout->sizes[2] : src.clayout->sizes[2];
+
+ int64_t ii0, ii1;
+ int64_t oi0, oi1;
+ int64_t delta1, delta0;
+ int irank, orank;
+ delta0 = (this->clayout->sizes[0] - src.clayout->sizes[0]);
+ delta1 = (this->clayout->sizes[1] - src.clayout->sizes[1]);
+ for (ii0=0; ii0 < int64_t(src.clayout->sizes[0]); ii0++)
+ {
+ if (ii0 <= int64_t(src.clayout->sizes[0]/2))
+ {
+ oi0 = ii0;
+ if (oi0 > int64_t(this->clayout->sizes[0]/2))
+ continue;
+ }
+ else
+ {
+ oi0 = ii0 + delta0;
+ if ((oi0 < 0) || ((int64_t(this->clayout->sizes[0]) - oi0) >= int64_t(this->clayout->sizes[0]/2)))
+ continue;
+ }
+ if (be == FFTW)
+ {
+ irank = src.clayout->rank[0][ii0];
+ orank = this->clayout->rank[0][oi0];
+ }
+ else
+ {// TODO: handle 2D layout here
+ }
+ if ((irank == orank) &&
+ (irank == src.clayout->myrank))
+ {
+ std::copy(
+ (rnumber*)(src.get_cdata() + (ii0 - src.clayout->starts[0] )*slice_size),
+ (rnumber*)(src.get_cdata() + (ii0 - src.clayout->starts[0] + 1)*slice_size),
+ (rnumber*)buffer);
+ }
+ else
+ {
+ if (src.clayout->myrank == irank)
+ {
+ MPI_Send(
+ (void*)(src.get_cdata() + (ii0-src.clayout->starts[0])*slice_size),
+ slice_size,
+ mpi_real_type<rnumber>::complex(),
+ orank,
+ ii0,
+ src.clayout->comm);
+ }
+ if (src.clayout->myrank == orank)
+ {
+ MPI_Recv(
+ (void*)(buffer),
+ slice_size,
+ mpi_real_type<rnumber>::complex(),
+ irank,
+ ii0,
+ src.clayout->comm,
+ MPI_STATUS_IGNORE);
+ }
+ }
+ if (src.clayout->myrank == orank)
+ {
+ for (ii1 = 0; ii1 < int64_t(src.clayout->sizes[1]); ii1++)
+ {
+ if (ii1 <= int64_t(src.clayout->sizes[1]/2))
+ {
+ oi1 = ii1;
+ if (oi1 > int64_t(this->clayout->sizes[1]/2))
+ continue;
+ }
+ else
+ {
+ oi1 = ii1 + delta1;
+ if ((oi1 < 0) || ((int64_t(this->clayout->sizes[1]) - oi1) >= int64_t(this->clayout->sizes[1]/2)))
+ continue;
+ }
+ std::copy(
+ (rnumber*)(buffer + (ii1*src.clayout->sizes[2]*ncomp(fc))),
+ (rnumber*)(buffer + (ii1*src.clayout->sizes[2] + min_fast_dim)*ncomp(fc)),
+ (rnumber*)(this->get_cdata() +
+ ((oi0 - this->clayout->starts[0])*this->clayout->sizes[1] +
+ oi1)*this->clayout->sizes[2]*ncomp(fc)));
+ }
+ }
+ }
+ fftw_interface<rnumber>::free(buffer);
+ MPI_Barrier(src.clayout->comm);
+ }
+ }
+ return *this;
+}
+
template class field<float, FFTW, ONE>;
template class field<float, FFTW, THREE>;
template class field<float, FFTW, THREExTHREE>;
diff --git a/bfps/cpp/field.hpp b/bfps/cpp/field.hpp
index d8038517..41609acd 100644
--- a/bfps/cpp/field.hpp
+++ b/bfps/cpp/field.hpp
@@ -129,6 +129,20 @@ class field
const hsize_t toffset,
const std::vector<double> max_estimate);
+ /* access sizes */
+ inline int get_nx() const
+ {
+ return this->rlayout->sizes[2];
+ }
+ inline int get_ny() const
+ {
+ return this->rlayout->sizes[1];
+ }
+ inline int get_nz() const
+ {
+ return this->rlayout->sizes[0];
+ }
+
/* acess data */
inline rnumber *__restrict__ get_rdata()
{
@@ -145,6 +159,11 @@ class field
return (typename fftw_interface<rnumber>::complex*__restrict__)this->data;
}
+ inline typename fftw_interface<rnumber>::complex *__restrict__ get_cdata() const
+ {
+ return (typename fftw_interface<rnumber>::complex*__restrict__)this->data;
+ }
+
inline rnumber &rval(ptrdiff_t rindex, unsigned int component = 0)
{
assert(fc == ONE || fc == THREE);
@@ -216,6 +235,8 @@ class field
return *this;
}
+ field<rnumber, be, fc>& operator=(const field<rnumber, be, fc> &src);
+
template <kspace_dealias_type dt>
void compute_stats(
kspace<be, dt> *kk,
--
GitLab
From b1b5a27f4e0ebeb9c5d8e2f0bfb682672f7155f9 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 14 Nov 2017 16:50:09 +0100
Subject: [PATCH 078/342] add read_value method
very simple functionality, but I gain two lines per value in the useful
file, which I consider a big deal.
---
bfps/cpp/hdf5_tools.cpp | 33 +++++++++++++++++++++++++++++++++
bfps/cpp/hdf5_tools.hpp | 5 +++++
2 files changed, 38 insertions(+)
diff --git a/bfps/cpp/hdf5_tools.cpp b/bfps/cpp/hdf5_tools.cpp
index 4328b287..baa94637 100644
--- a/bfps/cpp/hdf5_tools.cpp
+++ b/bfps/cpp/hdf5_tools.cpp
@@ -1,4 +1,6 @@
#include "hdf5_tools.hpp"
+#include <cfloat>
+#include <climits>
int hdf5_tools::require_size_single_dataset(hid_t dset, int tsize)
{
@@ -136,6 +138,37 @@ std::vector<number> hdf5_tools::read_vector(
return result;
}
+template <typename number>
+std::vector<number> hdf5_tools::read_value(
+ const hid_t group,
+ const std::string dset_name)
+{
+ number result;
+ hid_t dset;
+ hid_t mem_dtype;
+ if (typeid(number) == typeid(int))
+ mem_dtype = H5Tcopy(H5T_NATIVE_INT);
+ else if (typeid(number) == typeid(double))
+ mem_dtype = H5Tcopy(H5T_NATIVE_DOUBLE);
+ if (H5Lexists(group, dset_name.c_str(), H5P_DEFAULT))
+ {
+ dset = H5Dopen(group, dset_name.c_str(), H5P_DEFAULT);
+ H5Dread(dset, mem_dtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &result);
+ H5Dclose(dset);
+ }
+ else
+ {
+ DEBUG_MSG("attempted to read dataset %s which does not exist.\n",
+ dset_name.c_str());
+ if (typeid(number) == typeid(int))
+ result = INT_MAX;
+ else if (typeid(number) == typeid(double))
+ result = DBL_MAX;
+ }
+ H5Tclose(mem_dtype);
+ return result;
+}
+
template <typename dtype>
std::vector<dtype> hdf5_tools::read_vector_with_single_rank(
const int myrank,
diff --git a/bfps/cpp/hdf5_tools.hpp b/bfps/cpp/hdf5_tools.hpp
index 456beefe..99ba45a1 100644
--- a/bfps/cpp/hdf5_tools.hpp
+++ b/bfps/cpp/hdf5_tools.hpp
@@ -79,6 +79,11 @@ namespace hdf5_tools
std::string read_string(
const hid_t group,
const std::string dset_name);
+
+ template <typename number>
+ number read_value(
+ const hid_t group,
+ const std::string dset_name);
}
#endif//HDF5_TOOLS_HPP
--
GitLab
From ce1a45da0644e77048ae4f8999b8d98a0789a32c Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 14 Nov 2017 16:51:02 +0100
Subject: [PATCH 079/342] add resize cpp files
---
bfps/cpp/full_code/resize.cpp | 74 +++++++++++++++++++++++++++++++++++
bfps/cpp/full_code/resize.hpp | 67 +++++++++++++++++++++++++++++++
setup.py | 1 +
3 files changed, 142 insertions(+)
create mode 100644 bfps/cpp/full_code/resize.cpp
create mode 100644 bfps/cpp/full_code/resize.hpp
diff --git a/bfps/cpp/full_code/resize.cpp b/bfps/cpp/full_code/resize.cpp
new file mode 100644
index 00000000..87a45c03
--- /dev/null
+++ b/bfps/cpp/full_code/resize.cpp
@@ -0,0 +1,74 @@
+#include <string>
+#include <cmath>
+#include "resize.hpp"
+#include "scope_timer.hpp"
+
+
+template <typename rnumber>
+int resize<rnumber>::initialize(void)
+{
+ this->NSVE_field_stats<rnumber>::initialize();
+ DEBUG_MSG("after NSVE_field_stats::initialize\n");
+ hid_t parameter_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+
+ this->niter_out = hdf5_tools::read_value<int>(
+ parameter_file, "/parameters/niter_out");
+ H5Fclose(parameter_file);
+ parameter_file = H5Fopen(
+ (this->simname + std::string("_post.h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ DEBUG_MSG("before read_vector\n");
+ this->iteration_list = hdf5_tools::read_vector<int>(
+ parameter_file,
+ "/resize/parameters/iteration_list");
+
+ this->new_nx = hdf5_tools::read_value<int>(
+ parameter_file, "/resize/parameters/new_nx");
+ this->new_ny = hdf5_tools::read_value<int>(
+ parameter_file, "/resize/parameters/new_ny");
+ this->new_nz = hdf5_tools::read_value<int>(
+ parameter_file, "/resize/parameters/new_nz");
+ this->new_simname = hdf5_tools::read_string(
+ parameter_file, "/resize/parameters/new_simname");
+ H5Fclose(parameter_file);
+
+ this->new_field = new field<rnumber, FFTW, THREE>(
+ this->new_nx, this->new_ny, this->new_nz,
+ this->comm,
+ this->vorticity->fftw_plan_rigor);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int resize<rnumber>::work_on_current_iteration(void)
+{
+ DEBUG_MSG("entered resize::work_on_current_iteration\n");
+ this->read_current_cvorticity();
+
+ std::string fname = (
+ this->new_simname +
+ std::string("_fields.h5"));
+ this->new_field = this->vorticity;
+ this->new_field->io(
+ fname,
+ "vorticity",
+ this->iteration,
+ false);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int resize<rnumber>::finalize(void)
+{
+ delete this->new_field;
+ this->NSVE_field_stats<rnumber>::finalize();
+ return EXIT_SUCCESS;
+}
+
+template class resize<float>;
+template class resize<double>;
+
diff --git a/bfps/cpp/full_code/resize.hpp b/bfps/cpp/full_code/resize.hpp
new file mode 100644
index 00000000..de227c88
--- /dev/null
+++ b/bfps/cpp/full_code/resize.hpp
@@ -0,0 +1,67 @@
+/**********************************************************************
+* *
+* Copyright 2017 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef RESIZE_HPP
+#define RESIZE_HPP
+
+#include <cstdlib>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <vector>
+#include "base.hpp"
+#include "field.hpp"
+#include "field_binary_IO.hpp"
+#include "full_code/NSVE_field_stats.hpp"
+
+template <typename rnumber>
+class resize: public NSVE_field_stats<rnumber>
+{
+ public:
+ std::string new_simname;
+
+ int new_nx;
+ int new_ny;
+ int new_nz;
+
+ int niter_out;
+
+ field<rnumber, FFTW, THREE> *new_field;
+
+ resize(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ NSVE_field_stats<rnumber>(
+ COMMUNICATOR,
+ simulation_name){}
+ virtual ~resize(){}
+
+ int initialize(void);
+ int work_on_current_iteration(void);
+ int finalize(void);
+};
+
+#endif//RESIZE_HPP
+
diff --git a/setup.py b/setup.py
index b03bd4f4..d837670f 100644
--- a/setup.py
+++ b/setup.py
@@ -94,6 +94,7 @@ src_file_list = ['full_code/joint_acc_vel_stats',
'full_code/field_test',
'hdf5_tools',
'full_code/get_rfields',
+ 'full_code/resize',
'full_code/NSVE_field_stats',
'full_code/native_binary_to_hdf5',
'full_code/postprocess',
--
GitLab
From e2091a0511bdad824be345846717e3968897def3 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 15 Nov 2017 10:15:48 +0100
Subject: [PATCH 080/342] add resize to PP, fix linking issue
---
bfps/PP.py | 14 ++++++++++++++
bfps/cpp/hdf5_tools.cpp | 14 ++++++++++++--
2 files changed, 26 insertions(+), 2 deletions(-)
diff --git a/bfps/PP.py b/bfps/PP.py
index c93c9551..95c5f253 100644
--- a/bfps/PP.py
+++ b/bfps/PP.py
@@ -139,6 +139,11 @@ class PP(_code):
pars['max_acceleration_estimate'] = float(10)
pars['max_velocity_estimate'] = float(1)
pars['histogram_bins'] = int(129)
+ elif dns_type == 'resize':
+ pars['new_nx'] = int(32)
+ pars['new_ny'] = int(32)
+ pars['new_nz'] = int(32)
+ pars['new_simname'] = 'test_resized'
return pars
def get_data_file_name(self):
return os.path.join(self.work_dir, self.simname + '.h5')
@@ -444,6 +449,15 @@ class PP(_code):
self.parameters_to_parser_arguments(
parser_joint_acc_vel_stats,
parameters = self.extra_postprocessing_parameters('joint_acc_vel_stats'))
+ parser_resize = subparsers.add_parser(
+ 'resize',
+ help = 'get joint acceleration and velocity statistics')
+ self.simulation_parser_arguments(parser_resize)
+ self.job_parser_arguments(parser_resize)
+ self.parameters_to_parser_arguments(parser_resize)
+ self.parameters_to_parser_arguments(
+ parser_resize,
+ parameters = self.extra_postprocessing_parameters('resize'))
return None
def prepare_launch(
self,
diff --git a/bfps/cpp/hdf5_tools.cpp b/bfps/cpp/hdf5_tools.cpp
index baa94637..c2ef6aae 100644
--- a/bfps/cpp/hdf5_tools.cpp
+++ b/bfps/cpp/hdf5_tools.cpp
@@ -139,7 +139,7 @@ std::vector<number> hdf5_tools::read_vector(
}
template <typename number>
-std::vector<number> hdf5_tools::read_value(
+number hdf5_tools::read_value(
const hid_t group,
const std::string dset_name)
{
@@ -163,7 +163,7 @@ std::vector<number> hdf5_tools::read_value(
if (typeid(number) == typeid(int))
result = INT_MAX;
else if (typeid(number) == typeid(double))
- result = DBL_MAX;
+ result = number(DBL_MAX);
}
H5Tclose(mem_dtype);
return result;
@@ -247,3 +247,13 @@ std::vector<double> hdf5_tools::read_vector_with_single_rank<double>(
const hid_t file_id,
const std::string dset_name);
+template
+int hdf5_tools::read_value<int>(
+ const hid_t,
+ const std::string);
+
+template
+double hdf5_tools::read_value<double>(
+ const hid_t,
+ const std::string);
+
--
GitLab
From b50f048604de63a91dede66c89dae06332dfbd89 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 15 Nov 2017 10:42:37 +0100
Subject: [PATCH 081/342] bugfix: use proper syntax to call operator=
code runs without segfaults. I still need to test the correctness.
---
bfps/cpp/field.cpp | 4 +++-
bfps/cpp/full_code/resize.cpp | 2 +-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/bfps/cpp/field.cpp b/bfps/cpp/field.cpp
index ba0656f8..b18706da 100644
--- a/bfps/cpp/field.cpp
+++ b/bfps/cpp/field.cpp
@@ -1385,13 +1385,15 @@ field<rnumber, be, fc> &field<rnumber, be, fc>::operator=(
this->get_ny() == src.get_ny() &&
this->get_nz() == src.get_nz())
{
+ DEBUG_MSG("in operator=, doing simple copy\n");
std::copy(src.data,
- src.data + 2*this->clayout->local_size,
+ src.data + this->rmemlayout->local_size,
this->data);
}
// complicated resize
else
{
+ DEBUG_MSG("in operator=, doing complicated resize\n");
int64_t slice_size = src.clayout->local_size / src.clayout->subsizes[0];
// clean up
std::fill_n(this->data,
diff --git a/bfps/cpp/full_code/resize.cpp b/bfps/cpp/full_code/resize.cpp
index 87a45c03..41d68ef7 100644
--- a/bfps/cpp/full_code/resize.cpp
+++ b/bfps/cpp/full_code/resize.cpp
@@ -52,7 +52,7 @@ int resize<rnumber>::work_on_current_iteration(void)
std::string fname = (
this->new_simname +
std::string("_fields.h5"));
- this->new_field = this->vorticity;
+ *this->new_field = *this->vorticity;
this->new_field->io(
fname,
"vorticity",
--
GitLab
From 285dad17891082edd080e7a4f2e0fe807dac9dad Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 15 Nov 2017 13:48:19 +0100
Subject: [PATCH 082/342] add basic resize test
---
bfps/__init__.py | 1 +
bfps/test/test_bfps_resize.py | 113 ++++++++++++++++++++++++++++++++++
2 files changed, 114 insertions(+)
create mode 100644 bfps/test/test_bfps_resize.py
diff --git a/bfps/__init__.py b/bfps/__init__.py
index 6c220e69..d723b9ad 100644
--- a/bfps/__init__.py
+++ b/bfps/__init__.py
@@ -47,6 +47,7 @@ sys.path.append(bfpsfolder)
from host_information import host_info
from .DNS import DNS
+from .PP import PP
from .FluidConvert import FluidConvert
from .FluidResize import FluidResize
from .NavierStokes import NavierStokes
diff --git a/bfps/test/test_bfps_resize.py b/bfps/test/test_bfps_resize.py
new file mode 100644
index 00000000..ce0a051d
--- /dev/null
+++ b/bfps/test/test_bfps_resize.py
@@ -0,0 +1,113 @@
+#! /usr/bin/env python
+
+import os
+import numpy as np
+import h5py
+import sys
+
+import bfps
+from bfps import DNS
+from bfps import PP
+
+import matplotlib.pyplot as plt
+import pyfftw
+
+
+def main():
+ niterations = 2
+ c = DNS()
+ c.launch(
+ ['NSVE',
+ '-n', '32',
+ '--src-simname', 'B32p1e4',
+ '--src-wd', bfps.lib_dir + '/test',
+ '--src-iteration', '0',
+ '--simname', 'dns_test',
+ '--np', '4',
+ '--ntpp', '1',
+ '--niter_todo', '{0}'.format(niterations),
+ '--niter_out', '{0}'.format(niterations),
+ '--niter_stat', '1',
+ '--wd', './'] +
+ sys.argv[1:])
+ rr = PP()
+ rr.launch(
+ ['resize',
+ '--simname', 'dns_test',
+ '--new_nx', '64',
+ '--new_ny', '64',
+ '--new_nz', '64',
+ '--new_simname', 'pp_resize_test',
+ '--np', '4',
+ '--ntpp', '1',
+ '--iter0', '0',
+ '--iter1', '{0}'.format(niterations),
+ '--wd', './'] +
+ sys.argv[1:])
+ f0 = h5py.File(c.get_checkpoint_0_fname(), 'r')
+ f1 = h5py.File('pp_resize_test_fields.h5', 'r')
+ d0 = f0['vorticity/complex/0'].value
+ d1 = f1['vorticity/complex/0'].value
+ small_kdata = pyfftw.n_byte_align_empty(
+ (32, 32, 17, 3),
+ pyfftw.simd_alignment,
+ dtype = c.ctype)
+ small_rdata = pyfftw.n_byte_align_empty(
+ (32, 32, 32, 3),
+ pyfftw.simd_alignment,
+ dtype = c.rtype)
+ small_plan = pyfftw.FFTW(
+ small_kdata.transpose((1, 0, 2, 3)),
+ small_rdata,
+ axes = (0, 1, 2),
+ direction = 'FFTW_BACKWARD',
+ threads = 4)
+ big_kdata = pyfftw.n_byte_align_empty(
+ (64, 64, 33, 3),
+ pyfftw.simd_alignment,
+ dtype = c.ctype)
+ big_rdata = pyfftw.n_byte_align_empty(
+ (64, 64, 64, 3),
+ pyfftw.simd_alignment,
+ dtype = c.rtype)
+ big_plan = pyfftw.FFTW(
+ big_kdata.transpose((1, 0, 2, 3)),
+ big_rdata,
+ axes = (0, 1, 2),
+ direction = 'FFTW_BACKWARD',
+ threads = 4)
+ small_kdata[:] = d0
+ big_kdata[:] = d1
+ small_plan.execute()
+ big_plan.execute()
+
+ se = np.mean(small_rdata**2, axis = 3)**.5
+ be = np.mean(big_rdata**2, axis = 3)**.5
+
+ f = plt.figure(figsize = (6, 4))
+ a = f.add_subplot(231)
+ a.set_axis_off()
+ a.imshow(se[0])
+ a = f.add_subplot(234)
+ a.set_axis_off()
+ a.imshow(be[0])
+ a = f.add_subplot(232)
+ a.set_axis_off()
+ a.imshow(se[:, 0])
+ a = f.add_subplot(235)
+ a.set_axis_off()
+ a.imshow(be[:, 0])
+ a = f.add_subplot(233)
+ a.set_axis_off()
+ a.imshow(se[:, :, 0])
+ a = f.add_subplot(236)
+ a.set_axis_off()
+ a.imshow(be[:, :, 0])
+ f.tight_layout()
+ f.savefig('resize_test.pdf')
+ plt.close(f)
+ return None
+
+if __name__ == '__main__':
+ main()
+
--
GitLab
From c2584708063978664f70920d5d22568f608cc437 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 15 Nov 2017 14:24:28 +0100
Subject: [PATCH 083/342] remove old FluidResize
---
bfps/FluidResize.py | 156 --------------------------------------------
bfps/__init__.py | 1 -
bfps/__main__.py | 9 ---
3 files changed, 166 deletions(-)
delete mode 100644 bfps/FluidResize.py
diff --git a/bfps/FluidResize.py b/bfps/FluidResize.py
deleted file mode 100644
index fb5e2620..00000000
--- a/bfps/FluidResize.py
+++ /dev/null
@@ -1,156 +0,0 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
-
-
-
-import os
-
-import bfps
-from ._fluid_base import _fluid_particle_base
-
-class FluidResize(_fluid_particle_base):
- """This class is meant to resize snapshots of DNS states to new grids.
- Typical stuff for DNS of turbulence.
- It will become superfluous when HDF5 is used for field I/O.
- """
- def __init__(
- self,
- name = 'FluidResize-v' + bfps.__version__,
- work_dir = './',
- simname = 'test',
- fluid_precision = 'single',
- use_fftw_wisdom = False):
- _fluid_particle_base.__init__(
- self,
- name = name + '-' + fluid_precision,
- work_dir = work_dir,
- simname = simname,
- dtype = fluid_precision,
- use_fftw_wisdom = use_fftw_wisdom)
- self.parameters['src_simname'] = 'test'
- self.parameters['dst_iter'] = 0
- self.parameters['dst_nx'] = 32
- self.parameters['dst_ny'] = 32
- self.parameters['dst_nz'] = 32
- self.parameters['dst_simname'] = 'new_test'
- self.parameters['dst_dkx'] = 1.0
- self.parameters['dst_dky'] = 1.0
- self.parameters['dst_dkz'] = 1.0
- self.fill_up_fluid_code()
- self.finalize_code()
- return None
- def fill_up_fluid_code(self):
- self.fluid_includes += '#include <cstring>\n'
- self.fluid_includes += '#include "fftw_tools.hpp"\n'
- self.fluid_variables += ('double t;\n' +
- 'fluid_solver<' + self.C_dtype + '> *fs0, *fs1;\n')
- self.fluid_start += """
- //begincpp
- char fname[512];
- fs0 = new fluid_solver<{0}>(
- src_simname,
- nx, ny, nz,
- dkx, dky, dkz);
- fs1 = new fluid_solver<{0}>(
- dst_simname,
- dst_nx, dst_ny, dst_nz,
- dst_dkx, dst_dky, dst_dkz);
- fs0->iteration = iteration;
- fs1->iteration = 0;
- DEBUG_MSG("about to read field\\n");
- fs0->read('v', 'c');
- DEBUG_MSG("field read, about to copy data\\n");
- double a, b;
- fs0->compute_velocity(fs0->cvorticity);
- a = 0.5*fs0->autocorrel(fs0->cvelocity);
- b = 0.5*fs0->autocorrel(fs0->cvorticity);
- DEBUG_MSG("old field %d %g %g\\n", fs0->iteration, a, b);
- copy_complex_array<{0}>(fs0->cd, fs0->cvorticity,
- fs1->cd, fs1->cvorticity,
- 3);
- DEBUG_MSG("data copied, about to write new field\\n");
- fs1->write('v', 'c');
- DEBUG_MSG("finished writing\\n");
- fs1->compute_velocity(fs1->cvorticity);
- a = 0.5*fs1->autocorrel(fs1->cvelocity);
- b = 0.5*fs1->autocorrel(fs1->cvorticity);
- DEBUG_MSG("new field %d %g %g\\n", fs1->iteration, a, b);
- //endcpp
- """.format(self.C_dtype)
- self.fluid_end += """
- //begincpp
- delete fs0;
- delete fs1;
- //endcpp
- """
- return None
- def specific_parser_arguments(
- self,
- parser):
- _fluid_particle_base.specific_parser_arguments(self, parser)
- parser.add_argument(
- '-m',
- type = int,
- dest = 'm',
- default = 32,
- metavar = 'M',
- help = 'resize from N to M')
- parser.add_argument(
- '--src_wd',
- type = str,
- dest = 'src_work_dir',
- required = True)
- parser.add_argument(
- '--src_iteration',
- type = int,
- dest = 'src_iteration',
- required = True)
- return None
- def launch(
- self,
- args = [],
- **kwargs):
- opt = self.prepare_launch(args)
- cmd_line_pars = vars(opt)
- for k in ['dst_nx', 'dst_ny', 'dst_nz']:
- if type(cmd_line_pars[k]) == type(None):
- cmd_line_pars[k] = opt.m
- # the 3 dst_ni have been updated in opt itself at this point
- # I'm not sure if this code is future-proof...
- self.parameters['niter_todo'] = 0
- self.pars_from_namespace(opt)
- src_file = os.path.join(
- os.path.realpath(opt.src_work_dir),
- opt.src_simname + '_cvorticity_i{0:0>5x}'.format(opt.src_iteration))
- read_file = os.path.join(
- self.work_dir,
- opt.src_simname + '_cvorticity_i{0:0>5x}'.format(opt.src_iteration))
- self.write_par(iter0 = opt.src_iteration)
- if not os.path.exists(read_file):
- os.symlink(src_file, read_file)
- self.run(ncpu = opt.ncpu,
- hours = opt.minutes // 60,
- minutes = opt.minutes % 60)
- return None
-
diff --git a/bfps/__init__.py b/bfps/__init__.py
index d723b9ad..bc9b5a05 100644
--- a/bfps/__init__.py
+++ b/bfps/__init__.py
@@ -49,7 +49,6 @@ from host_information import host_info
from .DNS import DNS
from .PP import PP
from .FluidConvert import FluidConvert
-from .FluidResize import FluidResize
from .NavierStokes import NavierStokes
from .NSVorticityEquation import NSVorticityEquation
diff --git a/bfps/__main__.py b/bfps/__main__.py
index c41a6ffb..0df362fc 100644
--- a/bfps/__main__.py
+++ b/bfps/__main__.py
@@ -33,7 +33,6 @@ from .PP import PP
from .TEST import TEST
from .NavierStokes import NavierStokes
from .NSVorticityEquation import NSVorticityEquation
-from .FluidResize import FluidResize
from .FluidConvert import FluidConvert
from .NSManyParticles import NSManyParticles
@@ -55,12 +54,6 @@ def main():
'NSVE',
'NSVE-single',
'NSVE-double']
- FRoptions = ['FluidResize',
- 'FluidResize-single',
- 'FluidResize-double',
- 'FR',
- 'FR-single',
- 'FR-double']
FCoptions = ['FluidConvert']
NSMPopt = ['NSManyParticles',
'NSManyParticles-single',
@@ -99,8 +92,6 @@ def main():
base_class = NavierStokes
if opt.base_class in NSVEoptions:
base_class = NSVorticityEquation
- elif opt.base_class in FRoptions:
- base_class = FluidResize
elif opt.base_class in FCoptions:
base_class = FluidConvert
elif opt.base_class in NSMPopt:
--
GitLab
From d7a3c33981a5aa190e076796c9f5e6a02e1b1e7f Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 15 Nov 2017 14:41:27 +0100
Subject: [PATCH 084/342] remove commented out lines
---
bfps/cpp/vorticity_equation.cpp | 46 ---------------------------------
1 file changed, 46 deletions(-)
diff --git a/bfps/cpp/vorticity_equation.cpp b/bfps/cpp/vorticity_equation.cpp
index 737db2c4..62660505 100644
--- a/bfps/cpp/vorticity_equation.cpp
+++ b/bfps/cpp/vorticity_equation.cpp
@@ -188,13 +188,6 @@ void vorticity_equation<rnumber, be>::compute_vorticity()
this->cvorticity->cval(cindex,1,1) = (this->kk->kz[zindex]*this->u->cval(cindex,0,0) - this->kk->kx[xindex]*this->u->cval(cindex,2,0));
this->cvorticity->cval(cindex,2,0) = -(this->kk->kx[xindex]*this->u->cval(cindex,1,1) - this->kk->ky[yindex]*this->u->cval(cindex,0,1));
this->cvorticity->cval(cindex,2,1) = (this->kk->kx[xindex]*this->u->cval(cindex,1,0) - this->kk->ky[yindex]*this->u->cval(cindex,0,0));
- //ptrdiff_t tindex = 3*cindex;
- //this->cvorticity->get_cdata()[tindex+0][0] = -(this->kk->ky[yindex]*this->u->get_cdata()[tindex+2][1] - this->kk->kz[zindex]*this->u->get_cdata()[tindex+1][1]);
- //this->cvorticity->get_cdata()[tindex+1][0] = -(this->kk->kz[zindex]*this->u->get_cdata()[tindex+0][1] - this->kk->kx[xindex]*this->u->get_cdata()[tindex+2][1]);
- //this->cvorticity->get_cdata()[tindex+2][0] = -(this->kk->kx[xindex]*this->u->get_cdata()[tindex+1][1] - this->kk->ky[yindex]*this->u->get_cdata()[tindex+0][1]);
- //this->cvorticity->get_cdata()[tindex+0][1] = (this->kk->ky[yindex]*this->u->get_cdata()[tindex+2][0] - this->kk->kz[zindex]*this->u->get_cdata()[tindex+1][0]);
- //this->cvorticity->get_cdata()[tindex+1][1] = (this->kk->kz[zindex]*this->u->get_cdata()[tindex+0][0] - this->kk->kx[xindex]*this->u->get_cdata()[tindex+2][0]);
- //this->cvorticity->get_cdata()[tindex+2][1] = (this->kk->kx[xindex]*this->u->get_cdata()[tindex+1][0] - this->kk->ky[yindex]*this->u->get_cdata()[tindex+0][0]);
}
else
std::fill_n((rnumber*)(this->cvorticity->get_cdata()+3*cindex), 6, 0.0);
@@ -223,13 +216,6 @@ void vorticity_equation<rnumber, be>::compute_velocity(field<rnumber, be, THREE>
this->u->cval(cindex,1,1) = (this->kk->kz[zindex]*vorticity->cval(cindex,0,0) - this->kk->kx[xindex]*vorticity->cval(cindex,2,0)) / k2;
this->u->cval(cindex,2,0) = -(this->kk->kx[xindex]*vorticity->cval(cindex,1,1) - this->kk->ky[yindex]*vorticity->cval(cindex,0,1)) / k2;
this->u->cval(cindex,2,1) = (this->kk->kx[xindex]*vorticity->cval(cindex,1,0) - this->kk->ky[yindex]*vorticity->cval(cindex,0,0)) / k2;
- //ptrdiff_t tindex = 3*cindex;
- //this->u->get_cdata()[tindex+0][0] = -(this->kk->ky[yindex]*vorticity->get_cdata()[tindex+2][1] - this->kk->kz[zindex]*vorticity->get_cdata()[tindex+1][1]) / k2;
- //this->u->get_cdata()[tindex+0][1] = (this->kk->ky[yindex]*vorticity->get_cdata()[tindex+2][0] - this->kk->kz[zindex]*vorticity->get_cdata()[tindex+1][0]) / k2;
- //this->u->get_cdata()[tindex+1][0] = -(this->kk->kz[zindex]*vorticity->get_cdata()[tindex+0][1] - this->kk->kx[xindex]*vorticity->get_cdata()[tindex+2][1]) / k2;
- //this->u->get_cdata()[tindex+1][1] = (this->kk->kz[zindex]*vorticity->get_cdata()[tindex+0][0] - this->kk->kx[xindex]*vorticity->get_cdata()[tindex+2][0]) / k2;
- //this->u->get_cdata()[tindex+2][0] = -(this->kk->kx[xindex]*vorticity->get_cdata()[tindex+1][1] - this->kk->ky[yindex]*vorticity->get_cdata()[tindex+0][1]) / k2;
- //this->u->get_cdata()[tindex+2][1] = (this->kk->kx[xindex]*vorticity->get_cdata()[tindex+1][0] - this->kk->ky[yindex]*vorticity->get_cdata()[tindex+0][0]) / k2;
}
else
std::fill_n((rnumber*)(this->u->get_cdata()+3*cindex), 6, 0.0);
@@ -255,13 +241,11 @@ void vorticity_equation<rnumber, be>::add_forcing(
{
cindex = ((this->fmode - this->cvorticity->clayout->starts[0]) * this->cvorticity->clayout->sizes[1])*this->cvorticity->clayout->sizes[2];
dst->cval(cindex,2, 0) -= this->famplitude*factor/2;
- //dst->get_cdata()[cindex*3+2][0] -= this->famplitude*factor/2;
}
if (this->cvorticity->clayout->myrank == this->cvorticity->clayout->rank[0][this->cvorticity->clayout->sizes[0] - this->fmode])
{
cindex = ((this->cvorticity->clayout->sizes[0] - this->fmode - this->cvorticity->clayout->starts[0]) * this->cvorticity->clayout->sizes[1])*this->cvorticity->clayout->sizes[2];
dst->cval(cindex, 2, 0) -= this->famplitude*factor/2;
- //dst->get_cdata()[cindex*3+2][0] -= this->famplitude*factor/2;
}
return;
}
@@ -280,7 +264,6 @@ void vorticity_equation<rnumber, be>::add_forcing(
for (int c=0; c<3; c++)
for (int i=0; i<2; i++)
dst->cval(cindex,c,i) += this->famplitude*vort_field->cval(cindex,c,i)*factor;
- //dst->get_cdata()[cindex*3+c][i] += this->famplitude*vort_field->get_cdata()[cindex*3+c][i]*factor;
}
);
return;
@@ -306,16 +289,12 @@ void vorticity_equation<rnumber, be>::omega_nonlin(
ptrdiff_t xindex,
ptrdiff_t yindex,
ptrdiff_t zindex){
- //ptrdiff_t tindex = 3*rindex;
rnumber tmp[3];
for (int cc=0; cc<3; cc++)
tmp[cc] = (this->u->rval(rindex,(cc+1)%3)*this->rvorticity->rval(rindex,(cc+2)%3) -
this->u->rval(rindex,(cc+2)%3)*this->rvorticity->rval(rindex,(cc+1)%3));
- //tmp[cc][0] = (this->u->get_rdata()[tindex+(cc+1)%3]*this->rvorticity->get_rdata()[tindex+(cc+2)%3] -
- // this->u->get_rdata()[tindex+(cc+2)%3]*this->rvorticity->get_rdata()[tindex+(cc+1)%3]);
for (int cc=0; cc<3; cc++)
this->u->rval(rindex,cc) = tmp[cc] / this->u->npoints;
- //this->u->get_rdata()[(3*rindex)+cc] = tmp[cc][0] / this->u->npoints;
}
);
/* go back to Fourier space */
@@ -337,18 +316,8 @@ void vorticity_equation<rnumber, be>::omega_nonlin(
tmp[1][1] = (this->kk->kz[zindex]*this->u->cval(cindex,0,0) - this->kk->kx[xindex]*this->u->cval(cindex,2,0));
tmp[2][1] = (this->kk->kx[xindex]*this->u->cval(cindex,1,0) - this->kk->ky[yindex]*this->u->cval(cindex,0,0));
}
- //ptrdiff_t tindex = 3*cindex;
- //{
- // tmp[0][0] = -(this->kk->ky[yindex]*this->u->get_cdata()[tindex+2][1] - this->kk->kz[zindex]*this->u->get_cdata()[tindex+1][1]);
- // tmp[1][0] = -(this->kk->kz[zindex]*this->u->get_cdata()[tindex+0][1] - this->kk->kx[xindex]*this->u->get_cdata()[tindex+2][1]);
- // tmp[2][0] = -(this->kk->kx[xindex]*this->u->get_cdata()[tindex+1][1] - this->kk->ky[yindex]*this->u->get_cdata()[tindex+0][1]);
- // tmp[0][1] = (this->kk->ky[yindex]*this->u->get_cdata()[tindex+2][0] - this->kk->kz[zindex]*this->u->get_cdata()[tindex+1][0]);
- // tmp[1][1] = (this->kk->kz[zindex]*this->u->get_cdata()[tindex+0][0] - this->kk->kx[xindex]*this->u->get_cdata()[tindex+2][0]);
- // tmp[2][1] = (this->kk->kx[xindex]*this->u->get_cdata()[tindex+1][0] - this->kk->ky[yindex]*this->u->get_cdata()[tindex+0][0]);
- //}
for (int cc=0; cc<3; cc++) for (int i=0; i<2; i++)
this->u->cval(cindex, cc, i) = tmp[cc][i];
- //this->u->get_cdata()[3*cindex+cc][i] = tmp[cc][i];
}
);
this->add_forcing(this->u, this->v[src], 1.0);
@@ -377,9 +346,6 @@ void vorticity_equation<rnumber, be>::step(double dt)
this->v[1]->cval(cindex,cc,i) = (
this->v[0]->cval(cindex,cc,i) +
dt*this->u->cval(cindex,cc,i))*factor0;
- //this->v[1]->get_cdata()[3*cindex+cc][i] = (
- // this->v[0]->get_cdata()[3*cindex+cc][i] +
- // dt*this->u->get_cdata()[3*cindex+cc][i])*factor0;
}
}
);
@@ -401,10 +367,6 @@ void vorticity_equation<rnumber, be>::step(double dt)
3*this->v[0]->cval(cindex,cc,i)*factor0 +
( this->v[1]->cval(cindex,cc,i) +
dt*this->u->cval(cindex,cc,i))*factor1)*0.25;
- //this->v[2]->get_cdata()[3*cindex+cc][i] = (
- // 3*this->v[0]->get_cdata()[3*cindex+cc][i]*factor0 +
- // (this->v[1]->get_cdata()[3*cindex+cc][i] +
- // dt*this->u->get_cdata()[3*cindex+cc][i])*factor1)*0.25;
}
}
);
@@ -425,10 +387,6 @@ void vorticity_equation<rnumber, be>::step(double dt)
this->v[0]->cval(cindex,cc,i)*factor0 +
2*(this->v[2]->cval(cindex,cc,i) +
dt*this->u->cval(cindex,cc,i)))*factor0/3;
- //this->v[3]->get_cdata()[3*cindex+cc][i] = (
- // this->v[0]->get_cdata()[3*cindex+cc][i]*factor0 +
- // 2*(this->v[2]->get_cdata()[3*cindex+cc][i] +
- // dt*this->u->get_cdata()[3*cindex+cc][i]))*factor0/3;
}
}
);
@@ -456,7 +414,6 @@ void vorticity_equation<rnumber, be>::compute_pressure(field<rnumber, be, ONE> *
//ptrdiff_t tindex = 3*rindex;
for (int cc=0; cc<3; cc++)
this->v[1]->rval(rindex,cc) = this->u->rval(rindex,cc)*this->u->rval(rindex,cc);
- //this->v[1]->get_rdata()[tindex+cc] = this->u->get_rdata()[tindex+cc]*this->u->get_rdata()[tindex+cc];
}
);
//this->clean_up_real_space(this->rv[1], 3);
@@ -493,7 +450,6 @@ void vorticity_equation<rnumber, be>::compute_pressure(field<rnumber, be, ONE> *
//ptrdiff_t tindex = 3*rindex;
for (int cc=0; cc<3; cc++)
this->v[1]->rval(rindex,cc) = this->u->rval(rindex,cc)*this->u->rval(rindex,(cc+1)%3);
- //this->v[1]->get_rdata()[tindex+cc] = this->u->get_rdata()[tindex+cc]*this->u->get_rdata()[tindex+(cc+1)%3];
}
);
//this->clean_up_real_space(this->rv[1], 3);
@@ -626,7 +582,6 @@ void vorticity_equation<rnumber, be>::compute_Eulerian_acceleration(
for (int cc=0; cc<3; cc++)
this->v[1]->rval(rindex,cc) = \
this->cvelocity->rval(rindex,cc)*this->cvelocity->rval(rindex,cc) / this->cvelocity->npoints;
- //this->v[1]->get_rdata()[tindex+cc] = this->cvelocity->get_rdata()[tindex+cc]*this->cvelocity->get_rdata()[tindex+cc] / this->cvelocity->npoints;
}
);
this->v[1]->dft();
@@ -666,7 +621,6 @@ void vorticity_equation<rnumber, be>::compute_Eulerian_acceleration(
for (int cc=0; cc<3; cc++)
this->v[1]->rval(rindex,cc) = \
this->cvelocity->rval(rindex,cc)*this->cvelocity->rval(rindex,(cc+1)%3) / this->cvelocity->npoints;
- //this->v[1]->get_rdata()[tindex+cc] = this->cvelocity->get_rdata()[tindex+cc]*this->cvelocity->get_rdata()[tindex+(cc+1)%3] / this->cvelocity->npoints;
}
);
this->v[1]->dft();
--
GitLab
From d808db24581c8b5c6d4335b1e4dfc5aae45fb08c Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 15 Nov 2017 16:37:10 +0100
Subject: [PATCH 085/342] add new forcing schemes
not tested yet.
---
bfps/DNS.py | 4 +-
bfps/cpp/full_code/NSVE.cpp | 2 +
bfps/cpp/full_code/NSVE.hpp | 2 +
bfps/cpp/vorticity_equation.cpp | 104 ++++++++++++++++++++++++---
bfps/cpp/vorticity_equation.hpp | 11 +--
bfps/test/test_bfps_NSVEparticles.py | 1 +
6 files changed, 110 insertions(+), 14 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 8cbe8d9c..7a97e2f1 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -151,9 +151,11 @@ class DNS(_code):
self.parameters['nu'] = float(0.1)
self.parameters['fmode'] = int(1)
self.parameters['famplitude'] = float(0.5)
+ self.parameters['energy'] = float(0.5)
+ self.parameters['injection_rate'] = float(0.4)
self.parameters['fk0'] = float(2.0)
self.parameters['fk1'] = float(4.0)
- self.parameters['forcing_type'] = 'linear'
+ self.parameters['forcing_type'] = 'fixed_energy_injection_rate'
self.parameters['histogram_bins'] = int(256)
self.parameters['max_velocity_estimate'] = float(1)
self.parameters['max_vorticity_estimate'] = float(1)
diff --git a/bfps/cpp/full_code/NSVE.cpp b/bfps/cpp/full_code/NSVE.cpp
index 1e24c7af..ba8a3ed6 100644
--- a/bfps/cpp/full_code/NSVE.cpp
+++ b/bfps/cpp/full_code/NSVE.cpp
@@ -47,6 +47,8 @@ int NSVE<rnumber>::initialize(void)
this->fs->nu = nu;
this->fs->fmode = fmode;
this->fs->famplitude = famplitude;
+ this->fs->energy = energy;
+ this->fs->injection_rate = injection_rate;
this->fs->fk0 = fk0;
this->fs->fk1 = fk1;
strncpy(this->fs->forcing_type, forcing_type, 128);
diff --git a/bfps/cpp/full_code/NSVE.hpp b/bfps/cpp/full_code/NSVE.hpp
index d444b71c..e3f6b276 100644
--- a/bfps/cpp/full_code/NSVE.hpp
+++ b/bfps/cpp/full_code/NSVE.hpp
@@ -44,6 +44,8 @@ class NSVE: public direct_numerical_simulation
double famplitude;
double fk0;
double fk1;
+ double energy;
+ double injection_rate;
int fmode;
char forcing_type[512];
int histogram_bins;
diff --git a/bfps/cpp/vorticity_equation.cpp b/bfps/cpp/vorticity_equation.cpp
index 62660505..86e4a97e 100644
--- a/bfps/cpp/vorticity_equation.cpp
+++ b/bfps/cpp/vorticity_equation.cpp
@@ -228,8 +228,7 @@ template <class rnumber,
field_backend be>
void vorticity_equation<rnumber, be>::add_forcing(
field<rnumber, be, THREE> *dst,
- field<rnumber, be, THREE> *vort_field,
- rnumber factor)
+ field<rnumber, be, THREE> *vort_field)
{
TIMEZONE("vorticity_equation::add_forcing");
if (strcmp(this->forcing_type, "none") == 0)
@@ -239,13 +238,13 @@ void vorticity_equation<rnumber, be>::add_forcing(
ptrdiff_t cindex;
if (this->cvorticity->clayout->myrank == this->cvorticity->clayout->rank[0][this->fmode])
{
- cindex = ((this->fmode - this->cvorticity->clayout->starts[0]) * this->cvorticity->clayout->sizes[1])*this->cvorticity->clayout->sizes[2];
- dst->cval(cindex,2, 0) -= this->famplitude*factor/2;
+ cindex = dst->get_cindex(0, (this->fmode - this->cvorticity->clayout->starts[0]), 0);
+ dst->cval(cindex,2, 0) -= this->famplitude/2;
}
if (this->cvorticity->clayout->myrank == this->cvorticity->clayout->rank[0][this->cvorticity->clayout->sizes[0] - this->fmode])
{
- cindex = ((this->cvorticity->clayout->sizes[0] - this->fmode - this->cvorticity->clayout->starts[0]) * this->cvorticity->clayout->sizes[1])*this->cvorticity->clayout->sizes[2];
- dst->cval(cindex, 2, 0) -= this->famplitude*factor/2;
+ cindex = dst->get_cindex(0, (this->cvorticity->clayout->sizes[0] - this->fmode - this->cvorticity->clayout->starts[0]), 0);
+ dst->cval(cindex, 2, 0) -= this->famplitude/2;
}
return;
}
@@ -260,10 +259,99 @@ void vorticity_equation<rnumber, be>::add_forcing(
this->kk->ky[yindex]*this->kk->ky[yindex] +
this->kk->kz[zindex]*this->kk->kz[zindex]);
if ((this->fk0 <= knorm) &&
+ (this->fk1 >= knorm))
+ for (int c=0; c<3; c++)
+ for (int i=0; i<2; i++)
+ dst->cval(cindex,c,i) += this->famplitude*vort_field->cval(cindex,c,i);
+ }
+ );
+ return;
+ }
+ if (strcmp(this->forcing_type, "fixed_energy_injection_rate") == 0)
+ {
+ // first, compute energy in shell
+ double energy_in_shell = 0;
+ this->kk->CLOOP_K2(
+ [&](ptrdiff_t cindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex,
+ double k2){
+ double knorm = sqrt(k2);
+ if ((k2 > 0) &&
+ (this->fk0 <= knorm) &&
+ (this->fk1 >= knorm))
+ energy_in_shell += (
+ vort_field->cval(cindex, 0, 0)*vort_field->cval(cindex, 0, 0) + vort_field->cval(cindex, 0, 1)*vort_field->cval(cindex, 0, 1) +
+ vort_field->cval(cindex, 1, 0)*vort_field->cval(cindex, 1, 0) + vort_field->cval(cindex, 1, 1)*vort_field->cval(cindex, 1, 1) +
+ vort_field->cval(cindex, 2, 0)*vort_field->cval(cindex, 2, 0) + vort_field->cval(cindex, 2, 1)*vort_field->cval(cindex, 2, 1)
+ ) / k2;
+ }
+ );
+ // divide by 2, because we want energy
+ energy_in_shell /= 2;
+ // now, add forcing term
+ double temp_famplitude = this->injection_rate / energy_in_shell;
+ this->kk->CLOOP_K2(
+ [&](ptrdiff_t cindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex,
+ double k2){
+ double knorm = sqrt(k2);
+ if ((this->fk0 <= knorm) &&
+ (this->fk1 >= knorm))
+ for (int c=0; c<3; c++)
+ for (int i=0; i<2; i++)
+ dst->cval(cindex,c,i) += temp_famplitude*vort_field->cval(cindex,c,i);
+ }
+ );
+ return;
+ }
+ if (strcmp(this->forcing_type, "fixed_energy") == 0)
+ {
+ // first, compute energy in shell
+ double energy_in_shell = 0;
+ double total_energy = 0;
+ this->kk->CLOOP_K2(
+ [&](ptrdiff_t cindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex,
+ double k2){
+ if (k2 > 0)
+ {
+ double local_energy = (
+ vort_field->cval(cindex, 0, 0)*vort_field->cval(cindex, 0, 0) + vort_field->cval(cindex, 0, 1)*vort_field->cval(cindex, 0, 1) +
+ vort_field->cval(cindex, 1, 0)*vort_field->cval(cindex, 1, 0) + vort_field->cval(cindex, 1, 1)*vort_field->cval(cindex, 1, 1) +
+ vort_field->cval(cindex, 2, 0)*vort_field->cval(cindex, 2, 0) + vort_field->cval(cindex, 2, 1)*vort_field->cval(cindex, 2, 1)
+ ) / k2;
+ total_energy += local_energy;
+ double knorm = sqrt(k2);
+ if ((this->fk0 <= knorm) &&
(this->fk1 >= knorm))
+ energy_in_shell += local_energy;
+ }
+ }
+ );
+ // divide by 2, because we want energy
+ total_energy /= 2;
+ energy_in_shell /= 2;
+ // now, add forcing term
+ // see Michael's thesis, page 38
+ double temp_famplitude = sqrt((this->energy - total_energy + energy_in_shell) / energy_in_shell);
+ this->kk->CLOOP_K2(
+ [&](ptrdiff_t cindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex,
+ double k2){
+ double knorm = sqrt(k2);
+ if ((this->fk0 <= knorm) &&
+ (this->fk1 >= knorm))
for (int c=0; c<3; c++)
for (int i=0; i<2; i++)
- dst->cval(cindex,c,i) += this->famplitude*vort_field->cval(cindex,c,i)*factor;
+ dst->cval(cindex,c,i) += temp_famplitude*vort_field->cval(cindex,c,i);
}
);
return;
@@ -320,7 +408,7 @@ void vorticity_equation<rnumber, be>::omega_nonlin(
this->u->cval(cindex, cc, i) = tmp[cc][i];
}
);
- this->add_forcing(this->u, this->v[src], 1.0);
+ this->add_forcing(this->u, this->v[src]);
this->kk->template force_divfree<rnumber>(this->u->get_cdata());
}
diff --git a/bfps/cpp/vorticity_equation.hpp b/bfps/cpp/vorticity_equation.hpp
index e8bd1d84..21a5f039 100644
--- a/bfps/cpp/vorticity_equation.hpp
+++ b/bfps/cpp/vorticity_equation.hpp
@@ -67,9 +67,11 @@ class vorticity_equation
/* physical parameters */
double nu;
- int fmode; // for Kolmogorov flow
- double famplitude; // both for Kflow and band forcing
- double fk0, fk1; // for band forcing
+ int fmode; // for Kolmogorov flow
+ double famplitude; // both for Kflow and band forcing
+ double fk0, fk1; // for band forcing
+ double injection_rate; // for fixed energy injection rate
+ double energy; // for fixed energy
char forcing_type[128];
/* constructor, destructor */
@@ -89,8 +91,7 @@ class vorticity_equation
void step(double dt);
void impose_zero_modes(void);
void add_forcing(field<rnumber, be, THREE> *dst,
- field<rnumber, be, THREE> *src_vorticity,
- rnumber factor);
+ field<rnumber, be, THREE> *src_vorticity);
void compute_vorticity(void);
void compute_velocity(field<rnumber, be, THREE> *vorticity);
diff --git a/bfps/test/test_bfps_NSVEparticles.py b/bfps/test/test_bfps_NSVEparticles.py
index ab77e210..33212e76 100644
--- a/bfps/test/test_bfps_NSVEparticles.py
+++ b/bfps/test/test_bfps_NSVEparticles.py
@@ -18,6 +18,7 @@ def main():
['NSVEparticles',
'-n', '32',
'--src-simname', 'B32p1e4',
+ '--forcing_type', 'linear',
'--src-wd', bfps.lib_dir + '/test',
'--src-iteration', '0',
'--simname', 'dns_nsveparticles',
--
GitLab
From 13bd22ab8ba6dab32653664bc8c4a38cdae32390 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 15 Nov 2017 17:07:53 +0100
Subject: [PATCH 086/342] bugfix: remove non-existing variable
---
bfps/__main__.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/bfps/__main__.py b/bfps/__main__.py
index 0df362fc..03f68a97 100644
--- a/bfps/__main__.py
+++ b/bfps/__main__.py
@@ -63,7 +63,6 @@ def main():
choices = ['DNS', 'PP', 'TEST'] +
NSoptions +
NSVEoptions +
- FRoptions +
FCoptions +
NSMPopt,
type = str)
--
GitLab
From 699e60d32ce0cc517d4e8304c0a480df75250f29 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 16 Nov 2017 10:35:22 +0100
Subject: [PATCH 087/342] tweak initialization for different forcing
---
bfps/DNS.py | 21 +++++++++++++++------
1 file changed, 15 insertions(+), 6 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 7a97e2f1..b9e960f7 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -665,13 +665,7 @@ class DNS(_code):
if self.dns_type in extra_parameters.keys():
for k in extra_parameters[self.dns_type].keys():
self.parameters[k] = extra_parameters[self.dns_type][k]
- self.parameters['nu'] = (opt.kMeta * 2 / opt.n)**(4./3)
self.parameters['dt'] = (opt.dtfactor / opt.n)
- # custom famplitude for 288 and 576
- if opt.n == 288:
- self.parameters['famplitude'] = 0.45
- elif opt.n == 576:
- self.parameters['famplitude'] = 0.47
if ((self.parameters['niter_todo'] % self.parameters['niter_out']) != 0):
self.parameters['niter_out'] = self.parameters['niter_todo']
if len(opt.src_work_dir) == 0:
@@ -688,6 +682,21 @@ class DNS(_code):
opt.ny = opt.n
if type(opt.nz) == type(None):
opt.nz = opt.n
+ self.parameters['nu'] = (opt.kMeta * 2 / opt.n)**(4./3)
+ if self.parameters['forcing_type'] == 'linear':
+ # custom famplitude for 288 and 576
+ if opt.n == 288:
+ self.parameters['famplitude'] = 0.45
+ elif opt.n == 576:
+ self.parameters['famplitude'] = 0.47
+ if self.parameters['forcing_type'] == 'fixed_energy_injection_rate':
+ kM = opt.n * 0.5
+ if self.parameters['dealias_type'] == 1:
+ kM *= 0.8
+ # use the fact that mean dissipation rate is equal to injection rate
+ self.parameters['nu'] = (
+ self.parameters['injection_rate'] *
+ (opt.kMeta / kM)**4)**(1./3)
if type(opt.checkpoints_per_file) == type(None):
# hardcoded FFTW complex representation size
field_size = 3*(opt.nx+2)*opt.ny*opt.nz*self.fluid_dtype.itemsize
--
GitLab
From 181282d0b69ccdb396418fca921fd8441342c196 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 16 Nov 2017 10:54:18 +0100
Subject: [PATCH 088/342] fix parameter issue
---
bfps/DNS.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index b9e960f7..3207ccce 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -683,13 +683,13 @@ class DNS(_code):
if type(opt.nz) == type(None):
opt.nz = opt.n
self.parameters['nu'] = (opt.kMeta * 2 / opt.n)**(4./3)
- if self.parameters['forcing_type'] == 'linear':
+ if opt.forcing_type == 'linear':
# custom famplitude for 288 and 576
if opt.n == 288:
self.parameters['famplitude'] = 0.45
elif opt.n == 576:
self.parameters['famplitude'] = 0.47
- if self.parameters['forcing_type'] == 'fixed_energy_injection_rate':
+ elif opt.forcing_type == 'fixed_energy_injection_rate':
kM = opt.n * 0.5
if self.parameters['dealias_type'] == 1:
kM *= 0.8
--
GitLab
From bb634369a4bf4b55806bef3259f7e1541fa9da14 Mon Sep 17 00:00:00 2001
From: Cristian Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 16 Nov 2017 11:28:40 +0100
Subject: [PATCH 089/342] move split-fftw-many definition to compile command
---
bfps/cpp/fftw_interface.hpp | 3 ++-
setup.py | 5 +++++
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/bfps/cpp/fftw_interface.hpp b/bfps/cpp/fftw_interface.hpp
index 00c71d27..7f5e0144 100644
--- a/bfps/cpp/fftw_interface.hpp
+++ b/bfps/cpp/fftw_interface.hpp
@@ -35,7 +35,8 @@
#endif
// To have multiple calls to c2r/r2c
-#define SPLIT_FFTW_MANY
+// you must define SPLIT_FFTW_MANY
+// by calling setup.py --split-fftw-many
#ifdef SPLIT_FFTW_MANY
#include <vector>
#include <memory>
diff --git a/setup.py b/setup.py
index b03bd4f4..7d7e6ff2 100644
--- a/setup.py
+++ b/setup.py
@@ -183,15 +183,18 @@ class CompileLibCommand(distutils.cmd.Command):
user_options = [
('timing-output=', None, 'Toggle timing output.'),
('fftw-estimate=', None, 'Use FFTW ESTIMATE.'),
+ ('split-fftw-many=', None, 'Turn on SPLIT_FFTW_MANY.'),
('disable-fftw-omp=', None, 'Turn Off FFTW OpenMP.'),
]
def initialize_options(self):
self.timing_output = 0
self.fftw_estimate = 0
self.disable_fftw_omp = 0
+ self.split_fftw_many = 0
return None
def finalize_options(self):
self.timing_output = (int(self.timing_output) == 1)
+ self.split_fftw_many = (int(self.split_fftw_many) == 1)
self.fftw_estimate = (int(self.fftw_estimate) == 1)
self.disable_fftw_omp = (int(self.disable_fftw_omp) == 1)
return None
@@ -216,6 +219,8 @@ class CompileLibCommand(distutils.cmd.Command):
eca += ['-fPIC']
if self.timing_output:
eca += ['-DUSE_TIMINGOUTPUT']
+ if self.split_fftw_many:
+ eca += ['-DSPLIT_FFTW_MANY']
if self.fftw_estimate:
eca += ['-DUSE_FFTWESTIMATE']
if self.disable_fftw_omp:
--
GitLab
From f19dcf09cb4564a5cd56ed70f0592e2e3388db47 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 16 Nov 2017 13:17:15 +0100
Subject: [PATCH 090/342] fix fixed energy forcing
---
bfps/cpp/vorticity_equation.cpp | 54 ++++++++++++++++++++++++---------
bfps/cpp/vorticity_equation.hpp | 19 ++++++++++++
2 files changed, 59 insertions(+), 14 deletions(-)
diff --git a/bfps/cpp/vorticity_equation.cpp b/bfps/cpp/vorticity_equation.cpp
index 86e4a97e..bdcf43e4 100644
--- a/bfps/cpp/vorticity_equation.cpp
+++ b/bfps/cpp/vorticity_equation.cpp
@@ -267,9 +267,29 @@ void vorticity_equation<rnumber, be>::add_forcing(
);
return;
}
+ if (strcmp(this->forcing_type, "fixed_energy_injection_rate") == 0)
+ return;
+ if (strcmp(this->forcing_type, "fixed_energy") == 0)
+ return;
+}
+
+template <class rnumber,
+ field_backend be>
+void vorticity_equation<rnumber, be>::impose_forcing(
+ field<rnumber, be, THREE> *onew,
+ field<rnumber, be, THREE> *oold)
+{
+ TIMEZONE("vorticity_equation::impose_forcing");
+ if (strcmp(this->forcing_type, "none") == 0)
+ return;
+ if (strcmp(this->forcing_type, "Kolmogorov") == 0)
+ return;
+ if (strcmp(this->forcing_type, "linear") == 0)
+ return;
if (strcmp(this->forcing_type, "fixed_energy_injection_rate") == 0)
{
// first, compute energy in shell
+ double local_energy_in_shell = 0;
double energy_in_shell = 0;
this->kk->CLOOP_K2(
[&](ptrdiff_t cindex,
@@ -282,15 +302,15 @@ void vorticity_equation<rnumber, be>::add_forcing(
(this->fk0 <= knorm) &&
(this->fk1 >= knorm))
energy_in_shell += (
- vort_field->cval(cindex, 0, 0)*vort_field->cval(cindex, 0, 0) + vort_field->cval(cindex, 0, 1)*vort_field->cval(cindex, 0, 1) +
- vort_field->cval(cindex, 1, 0)*vort_field->cval(cindex, 1, 0) + vort_field->cval(cindex, 1, 1)*vort_field->cval(cindex, 1, 1) +
- vort_field->cval(cindex, 2, 0)*vort_field->cval(cindex, 2, 0) + vort_field->cval(cindex, 2, 1)*vort_field->cval(cindex, 2, 1)
+ onew->cval(cindex, 0, 0)*onew->cval(cindex, 0, 0) + onew->cval(cindex, 0, 1)*onew->cval(cindex, 0, 1) +
+ onew->cval(cindex, 1, 0)*onew->cval(cindex, 1, 0) + onew->cval(cindex, 1, 1)*onew->cval(cindex, 1, 1) +
+ onew->cval(cindex, 2, 0)*onew->cval(cindex, 2, 0) + onew->cval(cindex, 2, 1)*onew->cval(cindex, 2, 1)
) / k2;
}
);
// divide by 2, because we want energy
energy_in_shell /= 2;
- // now, add forcing term
+ // now, modify amplitudes
double temp_famplitude = this->injection_rate / energy_in_shell;
this->kk->CLOOP_K2(
[&](ptrdiff_t cindex,
@@ -303,7 +323,7 @@ void vorticity_equation<rnumber, be>::add_forcing(
(this->fk1 >= knorm))
for (int c=0; c<3; c++)
for (int i=0; i<2; i++)
- dst->cval(cindex,c,i) += temp_famplitude*vort_field->cval(cindex,c,i);
+ onew->cval(cindex,c,i) *= temp_famplitude;
}
);
return;
@@ -313,23 +333,23 @@ void vorticity_equation<rnumber, be>::add_forcing(
// first, compute energy in shell
double energy_in_shell = 0;
double total_energy = 0;
- this->kk->CLOOP_K2(
+ this->kk->CLOOP_K2_NXMODES(
[&](ptrdiff_t cindex,
ptrdiff_t xindex,
ptrdiff_t yindex,
ptrdiff_t zindex,
- double k2){
+ double k2,
+ int nxmodes){
if (k2 > 0)
{
- double local_energy = (
- vort_field->cval(cindex, 0, 0)*vort_field->cval(cindex, 0, 0) + vort_field->cval(cindex, 0, 1)*vort_field->cval(cindex, 0, 1) +
- vort_field->cval(cindex, 1, 0)*vort_field->cval(cindex, 1, 0) + vort_field->cval(cindex, 1, 1)*vort_field->cval(cindex, 1, 1) +
- vort_field->cval(cindex, 2, 0)*vort_field->cval(cindex, 2, 0) + vort_field->cval(cindex, 2, 1)*vort_field->cval(cindex, 2, 1)
+ double local_energy = nxmodes*(
+ onew->cval(cindex, 0, 0)*onew->cval(cindex, 0, 0) + onew->cval(cindex, 0, 1)*onew->cval(cindex, 0, 1) +
+ onew->cval(cindex, 1, 0)*onew->cval(cindex, 1, 0) + onew->cval(cindex, 1, 1)*onew->cval(cindex, 1, 1) +
+ onew->cval(cindex, 2, 0)*onew->cval(cindex, 2, 0) + onew->cval(cindex, 2, 1)*onew->cval(cindex, 2, 1)
) / k2;
total_energy += local_energy;
double knorm = sqrt(k2);
- if ((this->fk0 <= knorm) &&
- (this->fk1 >= knorm))
+ if ((this->fk0 <= knorm) && (this->fk1 >= knorm))
energy_in_shell += local_energy;
}
}
@@ -351,7 +371,7 @@ void vorticity_equation<rnumber, be>::add_forcing(
(this->fk1 >= knorm))
for (int c=0; c<3; c++)
for (int i=0; i<2; i++)
- dst->cval(cindex,c,i) += temp_famplitude*vort_field->cval(cindex,c,i);
+ onew->cval(cindex,c,i) *= temp_famplitude;
}
);
return;
@@ -410,6 +430,7 @@ void vorticity_equation<rnumber, be>::omega_nonlin(
);
this->add_forcing(this->u, this->v[src]);
this->kk->template force_divfree<rnumber>(this->u->get_cdata());
+ this->u->symmetrize();
}
template <class rnumber,
@@ -437,6 +458,7 @@ void vorticity_equation<rnumber, be>::step(double dt)
}
}
);
+ this->impose_forcing(this->v[1], this->v[0]);
this->omega_nonlin(1);
this->kk->CLOOP_K2(
@@ -458,8 +480,11 @@ void vorticity_equation<rnumber, be>::step(double dt)
}
}
);
+ this->impose_forcing(this->v[2], this->v[0]);
this->omega_nonlin(2);
+ // store old vorticity
+ *this->v[1] = *this->v[0];
this->kk->CLOOP_K2(
[&](ptrdiff_t cindex,
ptrdiff_t xindex,
@@ -478,6 +503,7 @@ void vorticity_equation<rnumber, be>::step(double dt)
}
}
);
+ this->impose_forcing(this->v[0], this->v[1]);
this->kk->template force_divfree<rnumber>(this->cvorticity->get_cdata());
this->cvorticity->symmetrize();
diff --git a/bfps/cpp/vorticity_equation.hpp b/bfps/cpp/vorticity_equation.hpp
index 21a5f039..7ce31071 100644
--- a/bfps/cpp/vorticity_equation.hpp
+++ b/bfps/cpp/vorticity_equation.hpp
@@ -90,8 +90,27 @@ class vorticity_equation
void omega_nonlin(int src);
void step(double dt);
void impose_zero_modes(void);
+
+ /** \brief Method that computes force and adds it to the right hand side of the NS equations.
+ *
+ * If the force has an explicit expression, as for instance in the case of Kolmogorov forcing,
+ * the term should be added to the nonlinear term for the purposes of time-stepping, since
+ * otherwise a custom time-stepping scheme would need to be implemented for each forcing type.
+ *
+ */
void add_forcing(field<rnumber, be, THREE> *dst,
field<rnumber, be, THREE> *src_vorticity);
+
+ /** \brief Method that imposes action of forcing on new vorticity field.
+ *
+ * If the force is implicit, in the sense that kinetic energy must be
+ * preserved or something similar, then the action must be imposed
+ * after the non-linear term has been added.
+ *
+ */
+ void impose_forcing(
+ field<rnumber, be, THREE> *omega_new,
+ field<rnumber, be, THREE> *omega_old);
void compute_vorticity(void);
void compute_velocity(field<rnumber, be, THREE> *vorticity);
--
GitLab
From bfd186ea5ee716a2745e780a1660c0a095b05d56 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 16 Nov 2017 14:24:43 +0100
Subject: [PATCH 091/342] make forcing work for MPI+threads
---
bfps/cpp/vorticity_equation.cpp | 91 +++++++++++++++++++++------------
1 file changed, 59 insertions(+), 32 deletions(-)
diff --git a/bfps/cpp/vorticity_equation.cpp b/bfps/cpp/vorticity_equation.cpp
index bdcf43e4..6b15bafc 100644
--- a/bfps/cpp/vorticity_equation.cpp
+++ b/bfps/cpp/vorticity_equation.cpp
@@ -32,6 +32,7 @@
#include "fftw_tools.hpp"
#include "vorticity_equation.hpp"
#include "scope_timer.hpp"
+#include "shared_array.hpp"
@@ -267,47 +268,37 @@ void vorticity_equation<rnumber, be>::add_forcing(
);
return;
}
- if (strcmp(this->forcing_type, "fixed_energy_injection_rate") == 0)
- return;
- if (strcmp(this->forcing_type, "fixed_energy") == 0)
- return;
-}
-
-template <class rnumber,
- field_backend be>
-void vorticity_equation<rnumber, be>::impose_forcing(
- field<rnumber, be, THREE> *onew,
- field<rnumber, be, THREE> *oold)
-{
- TIMEZONE("vorticity_equation::impose_forcing");
- if (strcmp(this->forcing_type, "none") == 0)
- return;
- if (strcmp(this->forcing_type, "Kolmogorov") == 0)
- return;
- if (strcmp(this->forcing_type, "linear") == 0)
- return;
if (strcmp(this->forcing_type, "fixed_energy_injection_rate") == 0)
{
// first, compute energy in shell
- double local_energy_in_shell = 0;
+ shared_array<double> local_energy_in_shell(1);
double energy_in_shell = 0;
- this->kk->CLOOP_K2(
+ this->kk->CLOOP_K2_NXMODES(
[&](ptrdiff_t cindex,
ptrdiff_t xindex,
ptrdiff_t yindex,
ptrdiff_t zindex,
- double k2){
+ double k2,
+ int nxmodes){
double knorm = sqrt(k2);
if ((k2 > 0) &&
(this->fk0 <= knorm) &&
(this->fk1 >= knorm))
- energy_in_shell += (
- onew->cval(cindex, 0, 0)*onew->cval(cindex, 0, 0) + onew->cval(cindex, 0, 1)*onew->cval(cindex, 0, 1) +
- onew->cval(cindex, 1, 0)*onew->cval(cindex, 1, 0) + onew->cval(cindex, 1, 1)*onew->cval(cindex, 1, 1) +
- onew->cval(cindex, 2, 0)*onew->cval(cindex, 2, 0) + onew->cval(cindex, 2, 1)*onew->cval(cindex, 2, 1)
+ *local_energy_in_shell.getMine() += nxmodes*(
+ vort_field->cval(cindex, 0, 0)*vort_field->cval(cindex, 0, 0) + vort_field->cval(cindex, 0, 1)*vort_field->cval(cindex, 0, 1) +
+ vort_field->cval(cindex, 1, 0)*vort_field->cval(cindex, 1, 0) + vort_field->cval(cindex, 1, 1)*vort_field->cval(cindex, 1, 1) +
+ vort_field->cval(cindex, 2, 0)*vort_field->cval(cindex, 2, 0) + vort_field->cval(cindex, 2, 1)*vort_field->cval(cindex, 2, 1)
) / k2;
}
);
+ local_energy_in_shell.mergeParallel();
+ MPI_Allreduce(
+ local_energy_in_shell.getMasterData(),
+ &energy_in_shell,
+ 1,
+ MPI_DOUBLE,
+ MPI_SUM,
+ vort_field->comm);
// divide by 2, because we want energy
energy_in_shell /= 2;
// now, modify amplitudes
@@ -323,16 +314,36 @@ void vorticity_equation<rnumber, be>::impose_forcing(
(this->fk1 >= knorm))
for (int c=0; c<3; c++)
for (int i=0; i<2; i++)
- onew->cval(cindex,c,i) *= temp_famplitude;
+ dst->cval(cindex,c,i) += temp_famplitude*vort_field->cval(cindex, c, i);
}
);
return;
}
+ if (strcmp(this->forcing_type, "fixed_energy") == 0)
+ return;
+}
+
+template <class rnumber,
+ field_backend be>
+void vorticity_equation<rnumber, be>::impose_forcing(
+ field<rnumber, be, THREE> *onew,
+ field<rnumber, be, THREE> *oold)
+{
+ TIMEZONE("vorticity_equation::impose_forcing");
+ if (strcmp(this->forcing_type, "none") == 0)
+ return;
+ if (strcmp(this->forcing_type, "Kolmogorov") == 0)
+ return;
+ if (strcmp(this->forcing_type, "linear") == 0)
+ return;
+ if (strcmp(this->forcing_type, "fixed_energy_injection_rate") == 0)
+ return;
if (strcmp(this->forcing_type, "fixed_energy") == 0)
{
// first, compute energy in shell
- double energy_in_shell = 0;
- double total_energy = 0;
+ shared_array<double> local_energy_in_shell(1);
+ shared_array<double> local_total_energy(1);
+ double energy_in_shell, total_energy;
this->kk->CLOOP_K2_NXMODES(
[&](ptrdiff_t cindex,
ptrdiff_t xindex,
@@ -342,18 +353,34 @@ void vorticity_equation<rnumber, be>::impose_forcing(
int nxmodes){
if (k2 > 0)
{
- double local_energy = nxmodes*(
+ double mode_energy = nxmodes*(
onew->cval(cindex, 0, 0)*onew->cval(cindex, 0, 0) + onew->cval(cindex, 0, 1)*onew->cval(cindex, 0, 1) +
onew->cval(cindex, 1, 0)*onew->cval(cindex, 1, 0) + onew->cval(cindex, 1, 1)*onew->cval(cindex, 1, 1) +
onew->cval(cindex, 2, 0)*onew->cval(cindex, 2, 0) + onew->cval(cindex, 2, 1)*onew->cval(cindex, 2, 1)
) / k2;
- total_energy += local_energy;
+ *local_total_energy.getMine() += mode_energy;
double knorm = sqrt(k2);
if ((this->fk0 <= knorm) && (this->fk1 >= knorm))
- energy_in_shell += local_energy;
+ *local_energy_in_shell.getMine() += mode_energy;
}
}
);
+ local_total_energy.mergeParallel();
+ local_energy_in_shell.mergeParallel();
+ MPI_Allreduce(
+ local_energy_in_shell.getMasterData(),
+ &energy_in_shell,
+ 1,
+ MPI_DOUBLE,
+ MPI_SUM,
+ onew->comm);
+ MPI_Allreduce(
+ local_total_energy.getMasterData(),
+ &total_energy,
+ 1,
+ MPI_DOUBLE,
+ MPI_SUM,
+ onew->comm);
// divide by 2, because we want energy
total_energy /= 2;
energy_in_shell /= 2;
--
GitLab
From 2755402c070e1a7804379649683286bfa075b6fe Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 16 Nov 2017 14:35:10 +0100
Subject: [PATCH 092/342] fix injection rate prefactor
---
bfps/cpp/vorticity_equation.cpp | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/bfps/cpp/vorticity_equation.cpp b/bfps/cpp/vorticity_equation.cpp
index 6b15bafc..ce32ef2b 100644
--- a/bfps/cpp/vorticity_equation.cpp
+++ b/bfps/cpp/vorticity_equation.cpp
@@ -299,8 +299,11 @@ void vorticity_equation<rnumber, be>::add_forcing(
MPI_DOUBLE,
MPI_SUM,
vort_field->comm);
- // divide by 2, because we want energy
- energy_in_shell /= 2;
+ // we should divide by 2, if we wanted energy;
+ // but then we would need to multiply the amplitude by 2 anyway,
+ // because what we really care about is force dotted into velocity,
+ // without the division by 2.
+
// now, modify amplitudes
double temp_famplitude = this->injection_rate / energy_in_shell;
this->kk->CLOOP_K2(
--
GitLab
From d129eb9b74a88f54a0d6cad8d388a22b48b47676 Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Sat, 18 Nov 2017 06:15:23 -0700
Subject: [PATCH 093/342] remove debug messages
---
bfps/cpp/field.cpp | 3 ---
1 file changed, 3 deletions(-)
diff --git a/bfps/cpp/field.cpp b/bfps/cpp/field.cpp
index b18706da..8fc6f9ff 100644
--- a/bfps/cpp/field.cpp
+++ b/bfps/cpp/field.cpp
@@ -1180,7 +1180,6 @@ int joint_rspace_PDF(
H5P_DEFAULT);
wspace = H5Dget_space(dset);
ndims = H5Sget_simple_extent_dims(wspace, dims, NULL);
- DEBUG_MSG("number of dimensions is %d\n", ndims);
assert(ndims == 5);
assert(dims[3] == 3);
assert(dims[4] == 3);
@@ -1385,7 +1384,6 @@ field<rnumber, be, fc> &field<rnumber, be, fc>::operator=(
this->get_ny() == src.get_ny() &&
this->get_nz() == src.get_nz())
{
- DEBUG_MSG("in operator=, doing simple copy\n");
std::copy(src.data,
src.data + this->rmemlayout->local_size,
this->data);
@@ -1393,7 +1391,6 @@ field<rnumber, be, fc> &field<rnumber, be, fc>::operator=(
// complicated resize
else
{
- DEBUG_MSG("in operator=, doing complicated resize\n");
int64_t slice_size = src.clayout->local_size / src.clayout->subsizes[0];
// clean up
std::fill_n(this->data,
--
GitLab
From cea91d307d9012cdfb8926892fe495e5ba8e1d10 Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Sat, 18 Nov 2017 09:34:43 -0700
Subject: [PATCH 094/342] fix infinity issue
---
bfps/DNS.py | 3 +++
bfps/NavierStokes.py | 3 +++
2 files changed, 6 insertions(+)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 3207ccce..9d57338d 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -236,6 +236,9 @@ class DNS(_code):
ii0 = iter0 // self.parameters['niter_stat']
ii1 = iter1 // self.parameters['niter_stat']
self.statistics['kshell'] = data_file['kspace/kshell'].value
+ for kk in [-1, -2]:
+ if (self.statistics['kshell'][kk] == 0):
+ self.statistics['kshell'][kk] = np.nan
self.statistics['kM'] = data_file['kspace/kM'].value
self.statistics['dk'] = data_file['kspace/dk'].value
computation_needed = True
diff --git a/bfps/NavierStokes.py b/bfps/NavierStokes.py
index 59fb907c..b7bf4ff8 100644
--- a/bfps/NavierStokes.py
+++ b/bfps/NavierStokes.py
@@ -598,6 +598,9 @@ class NavierStokes(_fluid_particle_base):
ii0 = iter0 // self.parameters['niter_stat']
ii1 = iter1 // self.parameters['niter_stat']
self.statistics['kshell'] = data_file['kspace/kshell'].value
+ for kk in [-1, -2]:
+ if (self.statistics['kshell'][kk] == 0):
+ self.statistics['kshell'][kk] = np.nan
self.statistics['kM'] = data_file['kspace/kM'].value
self.statistics['dk'] = data_file['kspace/dk'].value
computation_needed = True
--
GitLab
From 18ac43d7f28b39b0d0ef6d9bcd73aa38c6fa478f Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Thu, 23 Nov 2017 13:12:13 +0100
Subject: [PATCH 095/342] add mode counter
---
meta/count_nmodes.py | 27 +++++++++++++++++++++++++++
1 file changed, 27 insertions(+)
create mode 100644 meta/count_nmodes.py
diff --git a/meta/count_nmodes.py b/meta/count_nmodes.py
new file mode 100644
index 00000000..daf9d0fc
--- /dev/null
+++ b/meta/count_nmodes.py
@@ -0,0 +1,27 @@
+import numpy as np
+
+def count_expensive(fk0, fk1):
+ kcomponent = np.arange(-fk1-1, fk1+2, 1).astype(np.float)
+ ksize = (kcomponent[:, None, None]**2 +
+ kcomponent[None, :, None]**2 +
+ kcomponent[None, None, :]**2)**.5
+
+ good_indices = np.where(np.logical_and(
+ ksize >= fk0,
+ ksize <= fk1))
+ #print(ksize[good_indices])
+ #print(good_indices[0].shape)
+ return good_indices[0].shape[0]
+
+def main():
+ for ff in [[2, 4],
+ [1.5, 3],
+ [1, 2],
+ [1.5, 2.5],
+ [1.5, 2.3]]:
+ print(1 / ff[1], ff, count_expensive(ff[0], ff[1]))
+ return None
+
+if __name__ == '__main__':
+ main()
+
--
GitLab
From 6f7a769e60cbdde12be55e3fb0e9cde93535aec9 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 23 Nov 2017 15:12:43 +0100
Subject: [PATCH 096/342] update mode counter
---
meta/count_nmodes.py | 23 +++++++++++++++--------
1 file changed, 15 insertions(+), 8 deletions(-)
diff --git a/meta/count_nmodes.py b/meta/count_nmodes.py
index daf9d0fc..19af4ab3 100644
--- a/meta/count_nmodes.py
+++ b/meta/count_nmodes.py
@@ -1,25 +1,32 @@
import numpy as np
def count_expensive(fk0, fk1):
- kcomponent = np.arange(-fk1-1, fk1+2, 1).astype(np.float)
+ kcomponent = np.arange(-np.floor(fk1)-1, np.floor(fk1)+2, 1).astype(np.float)
ksize = (kcomponent[:, None, None]**2 +
kcomponent[None, :, None]**2 +
kcomponent[None, None, :]**2)**.5
+ #print(ksize[0])
good_indices = np.where(np.logical_and(
ksize >= fk0,
ksize <= fk1))
#print(ksize[good_indices])
#print(good_indices[0].shape)
- return good_indices[0].shape[0]
+ return np.unique(ksize[good_indices].flatten(), return_counts = True)
def main():
- for ff in [[2, 4],
- [1.5, 3],
- [1, 2],
- [1.5, 2.5],
- [1.5, 2.3]]:
- print(1 / ff[1], ff, count_expensive(ff[0], ff[1]))
+ for ff in [[1, 2],
+ [1.4, 2.3],
+ [1.4, 2.2]]:
+ modes, counts = count_expensive(ff[0], ff[1])
+ nmodes = np.sum(counts)
+ print(1 / ff[1], ff, nmodes)
+ modes_str = ''
+ counts_str = ''
+ for ii in range(counts.shape[0]):
+ modes_str += '{0:>5g}\t'.format(modes[ii])
+ counts_str += '{0:>5g}\t'.format(counts[ii])
+ print(modes_str + '\n' + counts_str + '\n')
return None
if __name__ == '__main__':
--
GitLab
From 737d2ae228ca8e0fd52c7018bc4a84ceb5554c85 Mon Sep 17 00:00:00 2001
From: Cristian Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 24 Nov 2017 17:20:01 +0100
Subject: [PATCH 097/342] use dependencies for load leveler
---
bfps/_code.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/bfps/_code.py b/bfps/_code.py
index 9064f972..473e2a04 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -512,7 +512,9 @@ class _code(_base):
first_node_tasks = int(nb_mpi_processes - (nb_nodes-1)*nb_processes_per_node)
for job in range(njobs):
- script_file.write('# @ step_name = {0}.$(stepid)\n'.format(self.simname))
+ script_file.write('# @ step_name = {0}.{1}\n'.format(self.simname, job))
+ if job > 0:
+ script_file.write('# @ dependency = {0}.{1} == 0\n'.format(self.simname, job - 1))
script_file.write('# @ resources = ConsumableCpus({})\n'.format(nb_threads_per_process))
script_file.write('# @ network.MPI = sn_all,not_shared,us\n')
script_file.write('# @ wall_clock_limit = {0}:{1:0>2d}:00\n'.format(hours, minutes))
--
GitLab
From 7112e71beb433cb90d0e4203be66fa5d80a3220a Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 30 Nov 2017 12:28:35 +0100
Subject: [PATCH 098/342] estimate dissipation for fixed kinetic energy
---
bfps/DNS.py | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 9d57338d..6f46fb21 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -685,6 +685,12 @@ class DNS(_code):
opt.ny = opt.n
if type(opt.nz) == type(None):
opt.nz = opt.n
+ if type(opt.fk0) == type(None):
+ opt.fk0 = self.parameters['fk0']
+ if type(opt.fk1) == type(None):
+ opt.fk1 = self.parameters['fk1']
+ if type(opt.injection_rate) == type(None):
+ opt.injection_rate = self.parameters['injection_rate']
self.parameters['nu'] = (opt.kMeta * 2 / opt.n)**(4./3)
if opt.forcing_type == 'linear':
# custom famplitude for 288 and 576
@@ -700,6 +706,16 @@ class DNS(_code):
self.parameters['nu'] = (
self.parameters['injection_rate'] *
(opt.kMeta / kM)**4)**(1./3)
+ elif opt.forcing_type == 'fixed_energy':
+ kM = opt.n * 0.5
+ if self.parameters['dealias_type'] == 1:
+ kM *= 0.8
+ kf = 1. / (1./self.parameters['fk0'] +
+ 1./self.parameters['fk1'])
+ self.parameters['nu'] = (
+ (opt.kMeta / kM)**(4./3) *
+ (np.pi / kf)**(1./3) *
+ (2*self.parameters['energy'] / 3)**0.5)
if type(opt.checkpoints_per_file) == type(None):
# hardcoded FFTW complex representation size
field_size = 3*(opt.nx+2)*opt.ny*opt.nz*self.fluid_dtype.itemsize
--
GitLab
From f6be18373b439be1434d5ea5038538f3e65021a9 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 30 Nov 2017 14:02:57 +0100
Subject: [PATCH 099/342] fix parameter construction
---
bfps/DNS.py | 17 ++++++++---------
1 file changed, 8 insertions(+), 9 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 6f46fb21..5e250bd3 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -691,6 +691,8 @@ class DNS(_code):
opt.fk1 = self.parameters['fk1']
if type(opt.injection_rate) == type(None):
opt.injection_rate = self.parameters['injection_rate']
+ if type(opt.dealias_type) == type(None):
+ opt.fk1 = self.parameters['dealias_type']
self.parameters['nu'] = (opt.kMeta * 2 / opt.n)**(4./3)
if opt.forcing_type == 'linear':
# custom famplitude for 288 and 576
@@ -698,20 +700,17 @@ class DNS(_code):
self.parameters['famplitude'] = 0.45
elif opt.n == 576:
self.parameters['famplitude'] = 0.47
+ kM = opt.n * 0.5
+ if opt.dealias_type == 1:
+ kM *= 0.8
elif opt.forcing_type == 'fixed_energy_injection_rate':
- kM = opt.n * 0.5
- if self.parameters['dealias_type'] == 1:
- kM *= 0.8
# use the fact that mean dissipation rate is equal to injection rate
self.parameters['nu'] = (
- self.parameters['injection_rate'] *
+ opt.injection_rate *
(opt.kMeta / kM)**4)**(1./3)
elif opt.forcing_type == 'fixed_energy':
- kM = opt.n * 0.5
- if self.parameters['dealias_type'] == 1:
- kM *= 0.8
- kf = 1. / (1./self.parameters['fk0'] +
- 1./self.parameters['fk1'])
+ kf = 1. / (1./opt.fk0 +
+ 1./opt.fk1)
self.parameters['nu'] = (
(opt.kMeta / kM)**(4./3) *
(np.pi / kf)**(1./3) *
--
GitLab
From e94fa37b15da2b5dc92393780e39d399ca5663fd Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 30 Nov 2017 14:14:44 +0100
Subject: [PATCH 100/342] bugfix --- fix copy/paste blunder
---
bfps/DNS.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 5e250bd3..bbc354d0 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -692,7 +692,7 @@ class DNS(_code):
if type(opt.injection_rate) == type(None):
opt.injection_rate = self.parameters['injection_rate']
if type(opt.dealias_type) == type(None):
- opt.fk1 = self.parameters['dealias_type']
+ opt.dealias_type = self.parameters['dealias_type']
self.parameters['nu'] = (opt.kMeta * 2 / opt.n)**(4./3)
if opt.forcing_type == 'linear':
# custom famplitude for 288 and 576
--
GitLab
From ec8e16d253ad6ab9d1d4621de4c134aeb409bffd Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 1 Dec 2017 11:29:36 +0100
Subject: [PATCH 101/342] fix parameter tweaking
---
bfps/DNS.py | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index bbc354d0..48d5c455 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -694,15 +694,17 @@ class DNS(_code):
if type(opt.dealias_type) == type(None):
opt.dealias_type = self.parameters['dealias_type']
self.parameters['nu'] = (opt.kMeta * 2 / opt.n)**(4./3)
+ # check value of kMax
+ kM = opt.n * 0.5
+ if opt.dealias_type == 1:
+ kM *= 0.8
+ # tweak forcing/viscosity based on forcint type
if opt.forcing_type == 'linear':
# custom famplitude for 288 and 576
if opt.n == 288:
self.parameters['famplitude'] = 0.45
elif opt.n == 576:
self.parameters['famplitude'] = 0.47
- kM = opt.n * 0.5
- if opt.dealias_type == 1:
- kM *= 0.8
elif opt.forcing_type == 'fixed_energy_injection_rate':
# use the fact that mean dissipation rate is equal to injection rate
self.parameters['nu'] = (
--
GitLab
From 913d5d14902d372e0e97a9b9d26b07e310912cbf Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 6 Dec 2017 14:37:26 +0100
Subject: [PATCH 102/342] fix computation of orientation size
---
bfps/cpp/particles/particles_inner_computer.hpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/bfps/cpp/particles/particles_inner_computer.hpp b/bfps/cpp/particles/particles_inner_computer.hpp
index d58981e6..b9568bd1 100644
--- a/bfps/cpp/particles/particles_inner_computer.hpp
+++ b/bfps/cpp/particles/particles_inner_computer.hpp
@@ -39,9 +39,9 @@ public:
//real_number alpha[3] = {0, 0, 0};
// check that orientation is unit vector:
real_number orientation_size = sqrt(
- pos_part[idx0+IDX_X]*pos_part[idx1+IDX_X] +
- pos_part[idx0+IDX_Y]*pos_part[idx1+IDX_Y] +
- pos_part[idx0+IDX_Z]*pos_part[idx1+IDX_Z]);
+ pos_part[idx0+IDX_X]*pos_part[idx0+IDX_X] +
+ pos_part[idx0+IDX_Y]*pos_part[idx0+IDX_Y] +
+ pos_part[idx0+IDX_Z]*pos_part[idx0+IDX_Z]);
assert(orientation_size > 0.99);
assert(orientation_size < 1.01);
// I call "rotation" to be the right hand side of the orientation part of the ODE
--
GitLab
From e7d42ead9d0e014673212a7aca3eca56823dce15 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 15 Dec 2017 11:30:06 +0100
Subject: [PATCH 103/342] fix particle vorticity data access
particle system needs vorticity field, then does its own interpolation.
---
bfps/cpp/full_code/NSVEparticlesP2P.cpp | 9 ++---
.../particles/abstract_particles_system.hpp | 2 +
.../particles/particles_inner_computer.hpp | 37 ++++++++++++++++++-
.../particles_inner_computer_empty.hpp | 6 ++-
bfps/cpp/particles/particles_system.hpp | 18 +++++++--
bfps/test/test_particles.py | 16 ++++++++
6 files changed, 75 insertions(+), 13 deletions(-)
diff --git a/bfps/cpp/full_code/NSVEparticlesP2P.cpp b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
index 3a8312f1..c858b38c 100644
--- a/bfps/cpp/full_code/NSVEparticlesP2P.cpp
+++ b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
@@ -14,8 +14,8 @@ int NSVEparticlesP2P<rnumber>::initialize(void)
p2p_computer<double, long long int> current_p2p_computer;
// TODO: particle interactions are switched off manually for testing purposes.
// this needs to be fixed once particle interactions can be properly resolved.
- current_p2p_computer.setEnable(enable_p2p);
- //current_p2p_computer.setEnable(false);
+ //current_p2p_computer.setEnable(enable_p2p);
+ current_p2p_computer.setEnable(false);
particles_inner_computer<double, long long int> current_particles_inner_computer(inner_v0);
current_particles_inner_computer.setEnable(enable_inner);
@@ -64,10 +64,7 @@ int NSVEparticlesP2P<rnumber>::step(void)
if(enable_vorticity_omega){
*this->tmp_vec_field = this->fs->cvorticity->get_cdata();
this->tmp_vec_field->ift();
- std::unique_ptr<double[]> pdata;
- pdata.reset(new double[ps->getLocalNbParticles()*3]);
- this->ps->sample_compute_field(*this->tmp_vec_field, pdata.get());
- this->ps->completeLoopWithVorticity(this->dt, pdata.get());
+ this->ps->completeLoopWithVorticity(this->dt, *this->tmp_vec_field);
}
else{
this->ps->completeLoop(this->dt);
diff --git a/bfps/cpp/particles/abstract_particles_system.hpp b/bfps/cpp/particles/abstract_particles_system.hpp
index 859432c0..91d40507 100644
--- a/bfps/cpp/particles/abstract_particles_system.hpp
+++ b/bfps/cpp/particles/abstract_particles_system.hpp
@@ -20,6 +20,8 @@ public:
virtual void enforce_unit_orientation() = 0;
+ virtual void add_Lagrange_multipliers() = 0;
+
virtual void compute_particles_inner(const real_number particle_extra_rhs[]) = 0;
virtual void move(const real_number dt) = 0;
diff --git a/bfps/cpp/particles/particles_inner_computer.hpp b/bfps/cpp/particles/particles_inner_computer.hpp
index b9568bd1..442e5220 100644
--- a/bfps/cpp/particles/particles_inner_computer.hpp
+++ b/bfps/cpp/particles/particles_inner_computer.hpp
@@ -3,6 +3,7 @@
#include <cstring>
#include <cassert>
+#include <iostream>
template <class real_number, class partsize_t>
class particles_inner_computer{
@@ -27,8 +28,12 @@ public:
}
}
+ // for given orientation and right-hand-side, recompute right-hand-side such
+ // that it is perpendicular to the current orientation.
+ // this is the job of the Lagrange multiplier terms, hence the
+ // "add_Lagrange_multipliers" name of the method.
template <int size_particle_positions, int size_particle_rhs>
- void enforce_unit_orientation(const partsize_t nb_particles, const real_number pos_part[], real_number rhs_part[]) const{
+ void add_Lagrange_multipliers(const partsize_t nb_particles, const real_number pos_part[], real_number rhs_part[]) const{
static_assert(size_particle_positions == 6, "This kernel works only with 6 values for one particle's position");
static_assert(size_particle_rhs == 6, "This kernel works only with 6 values per particle's rhs");
@@ -42,6 +47,12 @@ public:
pos_part[idx0+IDX_X]*pos_part[idx0+IDX_X] +
pos_part[idx0+IDX_Y]*pos_part[idx0+IDX_Y] +
pos_part[idx0+IDX_Z]*pos_part[idx0+IDX_Z]);
+ //DEBUG_MSG("particle ID %d\n"
+ // "pos_part[%d] = %g, pos_part[%d] = %g, pos_part[%d] = %g\n",
+ // idx_part,
+ // IDX_X, pos_part[idx0 + IDX_X],
+ // IDX_Y, pos_part[idx0 + IDX_Y],
+ // IDX_Z, pos_part[idx0 + IDX_Z]);
assert(orientation_size > 0.99);
assert(orientation_size < 1.01);
// I call "rotation" to be the right hand side of the orientation part of the ODE
@@ -89,8 +100,9 @@ public:
IDX_X, rhs_part[idx1 + IDX_X],
IDX_Y, rhs_part[idx1 + IDX_Y],
IDX_Z, rhs_part[idx1 + IDX_Z]);
+ assert(false);
}
- assert(dotproduct <= 0.1);
+ //assert(dotproduct <= 0.1);
}
}
@@ -115,6 +127,27 @@ public:
}
}
+ // meant to be called AFTER executing the time-stepping operation.
+ // once the particles have been moved, ensure that the orientation is a unit vector.
+ template <int size_particle_positions>
+ void enforce_unit_orientation(const partsize_t nb_particles, real_number pos_part[]) const{
+ static_assert(size_particle_positions == 6, "This kernel works only with 6 values for one particle's position");
+
+ #pragma omp parallel for
+ for(partsize_t idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
+ const partsize_t idx0 = idx_part*size_particle_positions + 3;
+ // compute orientation size:
+ real_number orientation_size = sqrt(
+ pos_part[idx0+IDX_X]*pos_part[idx0+IDX_X] +
+ pos_part[idx0+IDX_Y]*pos_part[idx0+IDX_Y] +
+ pos_part[idx0+IDX_Z]*pos_part[idx0+IDX_Z]);
+ // now renormalize
+ pos_part[idx0 + IDX_X] /= orientation_size;
+ pos_part[idx0 + IDX_Y] /= orientation_size;
+ pos_part[idx0 + IDX_Z] /= orientation_size;
+ }
+ }
+
bool isEnable() const {
return isActive;
diff --git a/bfps/cpp/particles/particles_inner_computer_empty.hpp b/bfps/cpp/particles/particles_inner_computer_empty.hpp
index 263d8b17..514bcba1 100644
--- a/bfps/cpp/particles/particles_inner_computer_empty.hpp
+++ b/bfps/cpp/particles/particles_inner_computer_empty.hpp
@@ -11,8 +11,12 @@ public:
void compute_interaction(const partsize_t /*nb_particles*/, real_number /*pos_part*/[], real_number /*rhs_part*/[]) const{
}
+ template <int size_particle_positions>
+ void enforce_unit_orientation(const partsize_t /*nb_particles*/, real_number /*pos_part*/[]) const{
+ }
+
template <int size_particle_positions, int size_particle_rhs>
- void enforce_unit_orientation(const partsize_t /*nb_particles*/, real_number /*pos_part*/[], real_number /*rhs_part*/[]) const{
+ void add_Lagrange_multipliers(const partsize_t /*nb_particles*/, real_number /*pos_part*/[], real_number /*rhs_part*/[]) const{
}
template <int size_particle_positions, int size_particle_rhs, int size_particle_rhs_extra>
diff --git a/bfps/cpp/particles/particles_system.hpp b/bfps/cpp/particles/particles_system.hpp
index a5d7878f..db2f33fa 100644
--- a/bfps/cpp/particles/particles_system.hpp
+++ b/bfps/cpp/particles/particles_system.hpp
@@ -162,11 +162,19 @@ public:
}
}
+ void add_Lagrange_multipliers() final {
+ if(computer_particules_inner.isEnable() == true){
+ TIMEZONE("particles_system::add_Lagrange_multipliers");
+ computer_particules_inner.template add_Lagrange_multipliers<size_particle_positions, size_particle_rhs>(
+ my_nb_particles, my_particles_positions.get(), my_particles_rhs.front().get());
+ }
+ }
+
void enforce_unit_orientation() final {
if(computer_particules_inner.isEnable() == true){
TIMEZONE("particles_system::enforce_unit_orientation");
- computer_particules_inner.template enforce_unit_orientation<size_particle_positions, size_particle_rhs>(
- my_nb_particles, my_particles_positions.get(), my_particles_rhs.front().get());
+ computer_particules_inner.template enforce_unit_orientation<size_particle_positions>(
+ my_nb_particles, my_particles_positions.get());
}
}
@@ -260,8 +268,9 @@ public:
compute();
compute_p2p();
compute_particles_inner();
- enforce_unit_orientation();
+ add_Lagrange_multipliers();
move(dt);
+ enforce_unit_orientation();
redistribute();
inc_step_idx();
shift_rhs_vectors();
@@ -273,8 +282,9 @@ public:
compute();
compute_p2p();
compute_particles_inner(particle_extra_rhs);
- enforce_unit_orientation();
+ add_Lagrange_multipliers();
move(dt);
+ enforce_unit_orientation();
redistribute();
inc_step_idx();
shift_rhs_vectors();
diff --git a/bfps/test/test_particles.py b/bfps/test/test_particles.py
index c3d4c415..40c428bd 100644
--- a/bfps/test/test_particles.py
+++ b/bfps/test/test_particles.py
@@ -43,6 +43,22 @@ def main():
pf = h5py.File(
'test_particles.h5',
'r')
+ # initial condition:
+ # show a histogram of the orientations
+ f = plt.figure()
+ a = f.add_subplot(111)
+ for iteration in range(1):
+ x = cf['tracers0/state/{0}'.format(iteration)][:, 3:]
+ hist, bins = np.histogram(
+ np.sum(x**2, axis = -1).flatten()**.5,
+ bins = np.linspace(0, 2, 40))
+ bb = (bins[:-1] + bins[1:])/2
+ pp = hist.astype(np.float) / (np.sum(hist) * (bb[1] - bb[0]))
+ a.plot(bb, pp, label = '{0}'.format(iteration))
+ a.legend(loc = 'best')
+ f.tight_layout()
+ f.savefig('orientation_histogram.pdf')
+ plt.close(f)
# show a histogram of the positions
f = plt.figure()
a = f.add_subplot(111)
--
GitLab
From 495be37630d84c717457d29f9ea85cf843177dce Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 15 Dec 2017 13:18:58 +0100
Subject: [PATCH 104/342] partial code clean up
---
bfps/DNS.py | 30 +++-------
bfps/cpp/full_code/NSVEparticlesP2P.cpp | 4 +-
.../particles/particles_inner_computer.hpp | 58 ++++++-------------
3 files changed, 30 insertions(+), 62 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 886f973c..44c1cca1 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -611,35 +611,23 @@ class DNS(_code):
parser_NSVEparticles_no_output = subparsers.add_parser(
'NSVEparticles_no_output',
help = 'plain Navier-Stokes vorticity formulation, with basic fluid tracers, checkpoints are NOT SAVED')
- self.simulation_parser_arguments(parser_NSVEparticles_no_output)
- self.job_parser_arguments(parser_NSVEparticles_no_output)
- self.particle_parser_arguments(parser_NSVEparticles_no_output)
- self.parameters_to_parser_arguments(parser_NSVEparticles_no_output)
- self.parameters_to_parser_arguments(
- parser_NSVEparticles_no_output,
- self.NSVEp_extra_parameters)
parser_NSVEp2 = subparsers.add_parser(
'NSVEparticles',
help = 'plain Navier-Stokes vorticity formulation, with basic fluid tracers')
- self.simulation_parser_arguments(parser_NSVEp2)
- self.job_parser_arguments(parser_NSVEp2)
- self.particle_parser_arguments(parser_NSVEp2)
- self.parameters_to_parser_arguments(parser_NSVEp2)
- self.parameters_to_parser_arguments(
- parser_NSVEp2,
- self.NSVEp_extra_parameters)
parser_NSVEp2p = subparsers.add_parser(
'NSVEparticlesP2P',
help = 'plain Navier-Stokes vorticity formulation, with basic fluid tracers')
- self.simulation_parser_arguments(parser_NSVEp2p)
- self.job_parser_arguments(parser_NSVEp2p)
- self.particle_parser_arguments(parser_NSVEp2p)
- self.parameters_to_parser_arguments(parser_NSVEp2p)
- self.parameters_to_parser_arguments(
- parser_NSVEp2p,
- self.NSVEp_extra_parameters)
+
+ for parser in ['NSVEparticles_no_output', 'NSVEp2', 'NSVEp2p']:
+ eval('self.simulation_parser_arguments({0})'.format('parser_' + parser))
+ eval('self.job_parser_arguments({0})'.format('parser_' + parser))
+ eval('self.particle_parser_arguments({0})'.format('parser_' + parser))
+ eval('self.parameters_to_parser_arguments({0})'.format('parser_' + parser))
+ eval('self.parameters_to_parser_arguments('
+ 'parser_{0},'
+ 'self.NSVEp_extra_parameters)'.format(parser))
return None
def prepare_launch(
self,
diff --git a/bfps/cpp/full_code/NSVEparticlesP2P.cpp b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
index c858b38c..ff1a594f 100644
--- a/bfps/cpp/full_code/NSVEparticlesP2P.cpp
+++ b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
@@ -14,8 +14,8 @@ int NSVEparticlesP2P<rnumber>::initialize(void)
p2p_computer<double, long long int> current_p2p_computer;
// TODO: particle interactions are switched off manually for testing purposes.
// this needs to be fixed once particle interactions can be properly resolved.
- //current_p2p_computer.setEnable(enable_p2p);
- current_p2p_computer.setEnable(false);
+ current_p2p_computer.setEnable(enable_p2p);
+ //current_p2p_computer.setEnable(false);
particles_inner_computer<double, long long int> current_particles_inner_computer(inner_v0);
current_particles_inner_computer.setEnable(enable_inner);
diff --git a/bfps/cpp/particles/particles_inner_computer.hpp b/bfps/cpp/particles/particles_inner_computer.hpp
index 442e5220..5c855cf5 100644
--- a/bfps/cpp/particles/particles_inner_computer.hpp
+++ b/bfps/cpp/particles/particles_inner_computer.hpp
@@ -41,18 +41,11 @@ public:
for(partsize_t idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
const partsize_t idx0 = idx_part*size_particle_positions + 3;
const partsize_t idx1 = idx_part*size_particle_rhs + 3;
- //real_number alpha[3] = {0, 0, 0};
// check that orientation is unit vector:
real_number orientation_size = sqrt(
pos_part[idx0+IDX_X]*pos_part[idx0+IDX_X] +
pos_part[idx0+IDX_Y]*pos_part[idx0+IDX_Y] +
pos_part[idx0+IDX_Z]*pos_part[idx0+IDX_Z]);
- //DEBUG_MSG("particle ID %d\n"
- // "pos_part[%d] = %g, pos_part[%d] = %g, pos_part[%d] = %g\n",
- // idx_part,
- // IDX_X, pos_part[idx0 + IDX_X],
- // IDX_Y, pos_part[idx0 + IDX_Y],
- // IDX_Z, pos_part[idx0 + IDX_Z]);
assert(orientation_size > 0.99);
assert(orientation_size < 1.01);
// I call "rotation" to be the right hand side of the orientation part of the ODE
@@ -61,14 +54,25 @@ public:
pos_part[idx0+IDX_X]*rhs_part[idx1+IDX_X] +
pos_part[idx0+IDX_Y]*rhs_part[idx1+IDX_Y] +
pos_part[idx0+IDX_Z]*rhs_part[idx1+IDX_Z]);
- //// alpha is the vector that makes rotation perpendicular to orientation.
- //// note that the following three lines assume the current orientation is a unit vector.
- //alpha[IDX_X] = -;
- //alpha[IDX_Y] = -;
- //alpha[IDX_Z] = -;
- // DEBUG_MSG("projection = %g\n"
+
+ // now remove parallel bit.
+ rhs_part[idx1+IDX_X] -= pos_part[idx0+IDX_X]*projection;
+ rhs_part[idx1+IDX_Y] -= pos_part[idx0+IDX_Y]*projection;
+ rhs_part[idx1+IDX_Z] -= pos_part[idx0+IDX_Z]*projection;
+
+ // DEBUG
+ // sanity check, for debugging purposes
+ // compute dot product between orientation and orientation change
+ //real_number dotproduct = (
+ // rhs_part[idx1 + IDX_X]*pos_part[idx0 + IDX_X] +
+ // rhs_part[idx1 + IDX_Y]*pos_part[idx0 + IDX_Y] +
+ // rhs_part[idx1 + IDX_Z]*pos_part[idx0 + IDX_Z]);
+ //if (dotproduct > 0.1)
+ //{
+ // DEBUG_MSG("dotproduct = %g, projection = %g\n"
// "pos_part[%d] = %g, pos_part[%d] = %g, pos_part[%d] = %g\n"
// "rhs_part[%d] = %g, rhs_part[%d] = %g, rhs_part[%d] = %g\n",
+ // dotproduct,
// projection,
// IDX_X, pos_part[idx0 + IDX_X],
// IDX_Y, pos_part[idx0 + IDX_Y],
@@ -76,32 +80,8 @@ public:
// IDX_X, rhs_part[idx1 + IDX_X],
// IDX_Y, rhs_part[idx1 + IDX_Y],
// IDX_Z, rhs_part[idx1 + IDX_Z]);
-
- // now remove parallel bit.
- rhs_part[idx1+IDX_X] -= pos_part[idx0+IDX_X]*projection;
- rhs_part[idx1+IDX_Y] -= pos_part[idx0+IDX_Y]*projection;
- rhs_part[idx1+IDX_Z] -= pos_part[idx0+IDX_Z]*projection;
-
- // compute dot product between orientation and orientation change
- real_number dotproduct = (
- rhs_part[idx1 + IDX_X]*pos_part[idx0 + IDX_X] +
- rhs_part[idx1 + IDX_Y]*pos_part[idx0 + IDX_Y] +
- rhs_part[idx1 + IDX_Z]*pos_part[idx0 + IDX_Z]);
- if (dotproduct > 0.1)
- {
- DEBUG_MSG("dotproduct = %g, projection = %g\n"
- "pos_part[%d] = %g, pos_part[%d] = %g, pos_part[%d] = %g\n"
- "rhs_part[%d] = %g, rhs_part[%d] = %g, rhs_part[%d] = %g\n",
- dotproduct,
- projection,
- IDX_X, pos_part[idx0 + IDX_X],
- IDX_Y, pos_part[idx0 + IDX_Y],
- IDX_Z, pos_part[idx0 + IDX_Z],
- IDX_X, rhs_part[idx1 + IDX_X],
- IDX_Y, rhs_part[idx1 + IDX_Y],
- IDX_Z, rhs_part[idx1 + IDX_Z]);
- assert(false);
- }
+ // assert(false);
+ //}
//assert(dotproduct <= 0.1);
}
}
--
GitLab
From 90740f4bff33063c3098d77c2bc1fa125c4ad769 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 15 Jan 2018 16:40:38 +0100
Subject: [PATCH 105/342] use subprocess.check_call instead of subprocess.call
---
bfps/_code.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/bfps/_code.py b/bfps/_code.py
index 473e2a04..78302109 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -223,7 +223,7 @@ class _code(_base):
self.write_src()
print('compiling code with command\n' + ' '.join(command_strings))
- return subprocess.call(command_strings)
+ return subprocess.check_call(command_strings)
def set_host_info(
self,
host_info = {}):
@@ -282,7 +282,7 @@ class _code(_base):
qsub_atoms = ['qsub']
if len(job_name_list) >= 1:
qsub_atoms += ['-hold_jid', job_name_list[-1]]
- subprocess.call(qsub_atoms + [qsub_script_name])
+ subprocess.check_call(qsub_atoms + [qsub_script_name])
os.chdir(current_dir)
job_name_list.append(suffix)
if self.host_info['type'] == 'SLURM':
@@ -342,7 +342,7 @@ class _code(_base):
submit_atoms = ['llsubmit']
if not no_submit:
- subprocess.call(submit_atoms + [os.path.join(self.work_dir, job_script_name)])
+ subprocess.check_call(submit_atoms + [os.path.join(self.work_dir, job_script_name)])
elif self.host_info['type'] == 'pc':
os.chdir(self.work_dir)
@@ -352,7 +352,7 @@ class _code(_base):
for j in range(njobs):
suffix = self.simname + '_{0}'.format(iter0 + j*self.parameters['niter_todo'])
print('running code with command\n' + ' '.join(command_atoms))
- subprocess.call(command_atoms,
+ subprocess.check_call(command_atoms,
stdout = open(out_file + '_' + suffix, 'w'),
stderr = open(err_file + '_' + suffix, 'w'))
os.chdir(current_dir)
--
GitLab
From 9ac326414af5bd3a7c785b77c9021091dd456ede Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 16 Jan 2018 14:25:02 +0100
Subject: [PATCH 106/342] add double Kolmogorov forcing
add forcing with the following formula:
$f_x = (1 - A) \sin k_0 y + (k_1 / k_0)^3 A \sin k_1 y$
The prefactor is there to (somewhat) ensure that the Reynolds number
doesn't change drastically when modifying A.
---
bfps/cpp/vorticity_equation.cpp | 35 +++++++++++++++++++++++++++++++++
1 file changed, 35 insertions(+)
diff --git a/bfps/cpp/vorticity_equation.cpp b/bfps/cpp/vorticity_equation.cpp
index ce32ef2b..5c1cc056 100644
--- a/bfps/cpp/vorticity_equation.cpp
+++ b/bfps/cpp/vorticity_equation.cpp
@@ -249,6 +249,39 @@ void vorticity_equation<rnumber, be>::add_forcing(
}
return;
}
+ if (strcmp(this->forcing_type, "2Kolmogorov") == 0)
+ {
+ // 2 Kolmogorov forces
+ // first one wavenumber fk0, amplitude 1 - A
+ DEBUG_MSG("famplitude = %g\n", this->famplitude);
+ ptrdiff_t cindex;
+ double amplitude = 1 - this->famplitude;
+ int fmode = int(this->fk0 / this->kk->dky);
+ if (this->cvorticity->clayout->myrank == this->cvorticity->clayout->rank[0][fmode])
+ {
+ cindex = dst->get_cindex(0, (fmode - this->cvorticity->clayout->starts[0]), 0);
+ dst->cval(cindex,2, 0) -= amplitude/2;
+ }
+ if (this->cvorticity->clayout->myrank == this->cvorticity->clayout->rank[0][this->cvorticity->clayout->sizes[0] - fmode])
+ {
+ cindex = dst->get_cindex(0, (this->cvorticity->clayout->sizes[0] - fmode - this->cvorticity->clayout->starts[0]), 0);
+ dst->cval(cindex, 2, 0) -= amplitude/2;
+ }
+ // second one wavenumber fk1, amplitude A
+ amplitude = this->famplitude * pow(int(this->fk1) / double(int(this->fk0)), 3);
+ fmode = int(this->fk1 / this->kk->dky);
+ if (this->cvorticity->clayout->myrank == this->cvorticity->clayout->rank[0][fmode])
+ {
+ cindex = dst->get_cindex(0, (fmode - this->cvorticity->clayout->starts[0]), 0);
+ dst->cval(cindex,2, 0) -= amplitude/2;
+ }
+ if (this->cvorticity->clayout->myrank == this->cvorticity->clayout->rank[0][this->cvorticity->clayout->sizes[0] - fmode])
+ {
+ cindex = dst->get_cindex(0, (this->cvorticity->clayout->sizes[0] - fmode - this->cvorticity->clayout->starts[0]), 0);
+ dst->cval(cindex, 2, 0) -= amplitude/2;
+ }
+ return;
+ }
if (strcmp(this->forcing_type, "linear") == 0)
{
this->kk->CLOOP(
@@ -337,6 +370,8 @@ void vorticity_equation<rnumber, be>::impose_forcing(
return;
if (strcmp(this->forcing_type, "Kolmogorov") == 0)
return;
+ if (strcmp(this->forcing_type, "2Kolmogorov") == 0)
+ return;
if (strcmp(this->forcing_type, "linear") == 0)
return;
if (strcmp(this->forcing_type, "fixed_energy_injection_rate") == 0)
--
GitLab
From db01eb7d749a42ca33af49f9b6bb06c61d3228f6 Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Fri, 19 Jan 2018 22:23:51 +0100
Subject: [PATCH 107/342] add new force
---
bfps/DNS.py | 1 +
bfps/cpp/full_code/NSVE.cpp | 1 +
bfps/cpp/full_code/NSVE.hpp | 1 +
bfps/cpp/full_code/postprocess.cpp | 3 +
bfps/cpp/full_code/postprocess.hpp | 1 +
bfps/cpp/vorticity_equation.cpp | 130 +++++++++++++++--------------
bfps/cpp/vorticity_equation.hpp | 20 +++--
7 files changed, 90 insertions(+), 67 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 48d5c455..32b559e1 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -151,6 +151,7 @@ class DNS(_code):
self.parameters['nu'] = float(0.1)
self.parameters['fmode'] = int(1)
self.parameters['famplitude'] = float(0.5)
+ self.parameters['friction_coefficient'] = float(0.5)
self.parameters['energy'] = float(0.5)
self.parameters['injection_rate'] = float(0.4)
self.parameters['fk0'] = float(2.0)
diff --git a/bfps/cpp/full_code/NSVE.cpp b/bfps/cpp/full_code/NSVE.cpp
index ba8a3ed6..e8bf9fd2 100644
--- a/bfps/cpp/full_code/NSVE.cpp
+++ b/bfps/cpp/full_code/NSVE.cpp
@@ -47,6 +47,7 @@ int NSVE<rnumber>::initialize(void)
this->fs->nu = nu;
this->fs->fmode = fmode;
this->fs->famplitude = famplitude;
+ this->fs->friction_coefficient = friction_coefficient;
this->fs->energy = energy;
this->fs->injection_rate = injection_rate;
this->fs->fk0 = fk0;
diff --git a/bfps/cpp/full_code/NSVE.hpp b/bfps/cpp/full_code/NSVE.hpp
index e3f6b276..062627fd 100644
--- a/bfps/cpp/full_code/NSVE.hpp
+++ b/bfps/cpp/full_code/NSVE.hpp
@@ -42,6 +42,7 @@ class NSVE: public direct_numerical_simulation
/* parameters that are read in read_parameters */
double dt;
double famplitude;
+ double friction_coefficient;
double fk0;
double fk1;
double energy;
diff --git a/bfps/cpp/full_code/postprocess.cpp b/bfps/cpp/full_code/postprocess.cpp
index edb5929f..c48bcdb8 100644
--- a/bfps/cpp/full_code/postprocess.cpp
+++ b/bfps/cpp/full_code/postprocess.cpp
@@ -57,6 +57,9 @@ int postprocess::read_parameters()
dset = H5Dopen(parameter_file, "/parameters/famplitude", H5P_DEFAULT);
H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->famplitude);
H5Dclose(dset);
+ dset = H5Dopen(parameter_file, "/parameters/friction_coefficient", H5P_DEFAULT);
+ H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->friction_coefficient);
+ H5Dclose(dset);
dset = H5Dopen(parameter_file, "/parameters/fk0", H5P_DEFAULT);
H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->fk0);
H5Dclose(dset);
diff --git a/bfps/cpp/full_code/postprocess.hpp b/bfps/cpp/full_code/postprocess.hpp
index c80fc3f2..660e561a 100644
--- a/bfps/cpp/full_code/postprocess.hpp
+++ b/bfps/cpp/full_code/postprocess.hpp
@@ -43,6 +43,7 @@ class postprocess: public code_base
/* parameters that are read in read_parameters */
double dt;
double famplitude;
+ double friction_coefficient;
double fk0;
double fk1;
int fmode;
diff --git a/bfps/cpp/vorticity_equation.cpp b/bfps/cpp/vorticity_equation.cpp
index 5c1cc056..47fb0c68 100644
--- a/bfps/cpp/vorticity_equation.cpp
+++ b/bfps/cpp/vorticity_equation.cpp
@@ -152,6 +152,7 @@ vorticity_equation<rnumber, be>::vorticity_equation(
this->nu = 0.1;
this->fmode = 1;
this->famplitude = 1.0;
+ this->friction_coefficient = 1.0;
this->fk0 = 2.0;
this->fk1 = 4.0;
}
@@ -225,6 +226,53 @@ void vorticity_equation<rnumber, be>::compute_velocity(field<rnumber, be, THREE>
this->u->symmetrize();
}
+template <class rnumber,
+ field_backend be>
+void vorticity_equation<rnumber, be>::add_Kolmogorov_forcing(
+ field<rnumber, be, THREE> *dst,
+ int fmode,
+ double famplitude)
+{
+ TIMEZONE("vorticity_equation::add_Kolmogorov_forcing");
+ ptrdiff_t cindex;
+ if (dst->clayout->myrank == dst->clayout->rank[0][fmode])
+ {
+ cindex = dst->get_cindex(0, (fmode - dst->clayout->starts[0]), 0);
+ dst->cval(cindex,2, 0) -= famplitude/2;
+ }
+ if (dst->clayout->myrank == dst->clayout->rank[0][dst->clayout->sizes[0] - fmode])
+ {
+ cindex = dst->get_cindex(0, (dst->clayout->sizes[0] - fmode - dst->clayout->starts[0]), 0);
+ dst->cval(cindex, 2, 0) -= famplitude/2;
+ }
+}
+
+template <class rnumber,
+ field_backend be>
+void vorticity_equation<rnumber, be>::add_field_band(
+ field<rnumber, be, THREE> *dst,
+ field<rnumber, be, THREE> *src,
+ double k0, double k1,
+ double prefactor)
+{
+ TIMEZONE("vorticity_equation::add_field_band");
+ this->kk->CLOOP(
+ [&](ptrdiff_t cindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex){
+ double knorm = sqrt(this->kk->kx[xindex]*this->kk->kx[xindex] +
+ this->kk->ky[yindex]*this->kk->ky[yindex] +
+ this->kk->kz[zindex]*this->kk->kz[zindex]);
+ if ((k0 <= knorm) &&
+ (k1 >= knorm))
+ for (int c=0; c<3; c++)
+ for (int i=0; i<2; i++)
+ dst->cval(cindex,c,i) += prefactor*src->cval(cindex,c,i);
+ }
+ );
+}
+
template <class rnumber,
field_backend be>
void vorticity_equation<rnumber, be>::add_forcing(
@@ -236,69 +284,37 @@ void vorticity_equation<rnumber, be>::add_forcing(
return;
if (strcmp(this->forcing_type, "Kolmogorov") == 0)
{
- ptrdiff_t cindex;
- if (this->cvorticity->clayout->myrank == this->cvorticity->clayout->rank[0][this->fmode])
- {
- cindex = dst->get_cindex(0, (this->fmode - this->cvorticity->clayout->starts[0]), 0);
- dst->cval(cindex,2, 0) -= this->famplitude/2;
- }
- if (this->cvorticity->clayout->myrank == this->cvorticity->clayout->rank[0][this->cvorticity->clayout->sizes[0] - this->fmode])
- {
- cindex = dst->get_cindex(0, (this->cvorticity->clayout->sizes[0] - this->fmode - this->cvorticity->clayout->starts[0]), 0);
- dst->cval(cindex, 2, 0) -= this->famplitude/2;
- }
+ this->add_Kolmogorov_forcing(dst, this->fmode, this->famplitude);
return;
}
if (strcmp(this->forcing_type, "2Kolmogorov") == 0)
{
// 2 Kolmogorov forces
// first one wavenumber fk0, amplitude 1 - A
- DEBUG_MSG("famplitude = %g\n", this->famplitude);
- ptrdiff_t cindex;
double amplitude = 1 - this->famplitude;
int fmode = int(this->fk0 / this->kk->dky);
- if (this->cvorticity->clayout->myrank == this->cvorticity->clayout->rank[0][fmode])
- {
- cindex = dst->get_cindex(0, (fmode - this->cvorticity->clayout->starts[0]), 0);
- dst->cval(cindex,2, 0) -= amplitude/2;
- }
- if (this->cvorticity->clayout->myrank == this->cvorticity->clayout->rank[0][this->cvorticity->clayout->sizes[0] - fmode])
- {
- cindex = dst->get_cindex(0, (this->cvorticity->clayout->sizes[0] - fmode - this->cvorticity->clayout->starts[0]), 0);
- dst->cval(cindex, 2, 0) -= amplitude/2;
- }
+ this->add_Kolmogorov_forcing(dst, fmode, amplitude);
// second one wavenumber fk1, amplitude A
amplitude = this->famplitude * pow(int(this->fk1) / double(int(this->fk0)), 3);
fmode = int(this->fk1 / this->kk->dky);
- if (this->cvorticity->clayout->myrank == this->cvorticity->clayout->rank[0][fmode])
- {
- cindex = dst->get_cindex(0, (fmode - this->cvorticity->clayout->starts[0]), 0);
- dst->cval(cindex,2, 0) -= amplitude/2;
- }
- if (this->cvorticity->clayout->myrank == this->cvorticity->clayout->rank[0][this->cvorticity->clayout->sizes[0] - fmode])
- {
- cindex = dst->get_cindex(0, (this->cvorticity->clayout->sizes[0] - fmode - this->cvorticity->clayout->starts[0]), 0);
- dst->cval(cindex, 2, 0) -= amplitude/2;
- }
+ this->add_Kolmogorov_forcing(dst, fmode, amplitude);
+ return;
+ }
+ if (strcmp(this->forcing_type, "Kolmogorov_and_drag") == 0)
+ {
+ this->add_Kolmogorov_forcing(dst, this->fmode, this->famplitude);
+ this->add_field_band(
+ dst, vort_field,
+ 0, this->fmode,
+ -this->friction_coefficient);
return;
}
if (strcmp(this->forcing_type, "linear") == 0)
{
- this->kk->CLOOP(
- [&](ptrdiff_t cindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex){
- double knorm = sqrt(this->kk->kx[xindex]*this->kk->kx[xindex] +
- this->kk->ky[yindex]*this->kk->ky[yindex] +
- this->kk->kz[zindex]*this->kk->kz[zindex]);
- if ((this->fk0 <= knorm) &&
- (this->fk1 >= knorm))
- for (int c=0; c<3; c++)
- for (int i=0; i<2; i++)
- dst->cval(cindex,c,i) += this->famplitude*vort_field->cval(cindex,c,i);
- }
- );
+ this->add_field_band(
+ dst, vort_field,
+ this->fk0, this->fk1,
+ this->famplitude);
return;
}
if (strcmp(this->forcing_type, "fixed_energy_injection_rate") == 0)
@@ -339,20 +355,10 @@ void vorticity_equation<rnumber, be>::add_forcing(
// now, modify amplitudes
double temp_famplitude = this->injection_rate / energy_in_shell;
- this->kk->CLOOP_K2(
- [&](ptrdiff_t cindex,
- ptrdiff_t xindex,
- ptrdiff_t yindex,
- ptrdiff_t zindex,
- double k2){
- double knorm = sqrt(k2);
- if ((this->fk0 <= knorm) &&
- (this->fk1 >= knorm))
- for (int c=0; c<3; c++)
- for (int i=0; i<2; i++)
- dst->cval(cindex,c,i) += temp_famplitude*vort_field->cval(cindex, c, i);
- }
- );
+ this->add_field_band(
+ dst, vort_field,
+ this->fk0, this->fk1,
+ temp_famplitude);
return;
}
if (strcmp(this->forcing_type, "fixed_energy") == 0)
diff --git a/bfps/cpp/vorticity_equation.hpp b/bfps/cpp/vorticity_equation.hpp
index 7ce31071..81f0cb66 100644
--- a/bfps/cpp/vorticity_equation.hpp
+++ b/bfps/cpp/vorticity_equation.hpp
@@ -67,11 +67,12 @@ class vorticity_equation
/* physical parameters */
double nu;
- int fmode; // for Kolmogorov flow
- double famplitude; // both for Kflow and band forcing
- double fk0, fk1; // for band forcing
- double injection_rate; // for fixed energy injection rate
- double energy; // for fixed energy
+ int fmode; // for Kolmogorov flow
+ double famplitude; // both for Kflow and band forcing
+ double fk0, fk1; // for band forcing
+ double injection_rate; // for fixed energy injection rate
+ double energy; // for fixed energy
+ double friction_coefficient; // for Kolmogorov_and_drag
char forcing_type[128];
/* constructor, destructor */
@@ -101,6 +102,15 @@ class vorticity_equation
void add_forcing(field<rnumber, be, THREE> *dst,
field<rnumber, be, THREE> *src_vorticity);
+ void add_Kolmogorov_forcing(field<rnumber, be, THREE> *dst,
+ int fmode,
+ double famplitude);
+ void add_field_band(
+ field<rnumber, be, THREE> *dst,
+ field<rnumber, be, THREE> *src,
+ double k0, double k1,
+ double prefactor);
+
/** \brief Method that imposes action of forcing on new vorticity field.
*
* If the force is implicit, in the sense that kinetic energy must be
--
GitLab
From 6211f520929d9940d7d47c1607970237e4b1d11e Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Sun, 21 Jan 2018 13:43:06 +0100
Subject: [PATCH 108/342] add warning about nu computation
---
bfps/DNS.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 32b559e1..dc2d4c5f 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -694,6 +694,11 @@ class DNS(_code):
opt.injection_rate = self.parameters['injection_rate']
if type(opt.dealias_type) == type(None):
opt.dealias_type = self.parameters['dealias_type']
+ if (opt.nx > opt.n or
+ opt.ny > opt.n or
+ opt.nz > opt.n):
+ opt.n = min(opt.nx, opt.ny, opt.nz)
+ print("Warning: '-n' parameter changed to minimum of nx, ny, nz. This affects the computation of nu.")
self.parameters['nu'] = (opt.kMeta * 2 / opt.n)**(4./3)
# check value of kMax
kM = opt.n * 0.5
--
GitLab
From 9e61abc97b055bc3af64736eaa924b4362326581 Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Sun, 21 Jan 2018 13:47:57 +0100
Subject: [PATCH 109/342] try to keep energy input independent of friction
---
bfps/cpp/vorticity_equation.cpp | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/bfps/cpp/vorticity_equation.cpp b/bfps/cpp/vorticity_equation.cpp
index 47fb0c68..def6b4a4 100644
--- a/bfps/cpp/vorticity_equation.cpp
+++ b/bfps/cpp/vorticity_equation.cpp
@@ -302,7 +302,9 @@ void vorticity_equation<rnumber, be>::add_forcing(
}
if (strcmp(this->forcing_type, "Kolmogorov_and_drag") == 0)
{
- this->add_Kolmogorov_forcing(dst, this->fmode, this->famplitude);
+ double amplitude = this->famplitude * (
+ 1 + this->friction_coefficient / sqrt(this->fmode * this->famplitude));
+ this->add_Kolmogorov_forcing(dst, this->fmode, amplitude);
this->add_field_band(
dst, vort_field,
0, this->fmode,
--
GitLab
From cc7c8e8ae3726e435a6d419675f736723b99a262 Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Sun, 21 Jan 2018 16:32:10 +0100
Subject: [PATCH 110/342] tweak drag interval
---
bfps/DNS.py | 2 +-
bfps/cpp/vorticity_equation.cpp | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index dc2d4c5f..3d531005 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -669,7 +669,6 @@ class DNS(_code):
if self.dns_type in extra_parameters.keys():
for k in extra_parameters[self.dns_type].keys():
self.parameters[k] = extra_parameters[self.dns_type][k]
- self.parameters['dt'] = (opt.dtfactor / opt.n)
if ((self.parameters['niter_todo'] % self.parameters['niter_out']) != 0):
self.parameters['niter_out'] = self.parameters['niter_todo']
if len(opt.src_work_dir) == 0:
@@ -699,6 +698,7 @@ class DNS(_code):
opt.nz > opt.n):
opt.n = min(opt.nx, opt.ny, opt.nz)
print("Warning: '-n' parameter changed to minimum of nx, ny, nz. This affects the computation of nu.")
+ self.parameters['dt'] = (opt.dtfactor / opt.n)
self.parameters['nu'] = (opt.kMeta * 2 / opt.n)**(4./3)
# check value of kMax
kM = opt.n * 0.5
diff --git a/bfps/cpp/vorticity_equation.cpp b/bfps/cpp/vorticity_equation.cpp
index def6b4a4..9535ad88 100644
--- a/bfps/cpp/vorticity_equation.cpp
+++ b/bfps/cpp/vorticity_equation.cpp
@@ -307,7 +307,7 @@ void vorticity_equation<rnumber, be>::add_forcing(
this->add_Kolmogorov_forcing(dst, this->fmode, amplitude);
this->add_field_band(
dst, vort_field,
- 0, this->fmode,
+ this->fk0, this->fk1,
-this->friction_coefficient);
return;
}
--
GitLab
From 3b78ac4f72286cac2268cd1b54776faa79444876 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 24 Jan 2018 17:02:17 +0100
Subject: [PATCH 111/342] change Kolmogorov_and_drag naming convention
---
bfps/cpp/vorticity_equation.cpp | 21 +++++++++------------
1 file changed, 9 insertions(+), 12 deletions(-)
diff --git a/bfps/cpp/vorticity_equation.cpp b/bfps/cpp/vorticity_equation.cpp
index 9535ad88..248aaa2b 100644
--- a/bfps/cpp/vorticity_equation.cpp
+++ b/bfps/cpp/vorticity_equation.cpp
@@ -280,8 +280,6 @@ void vorticity_equation<rnumber, be>::add_forcing(
field<rnumber, be, THREE> *vort_field)
{
TIMEZONE("vorticity_equation::add_forcing");
- if (strcmp(this->forcing_type, "none") == 0)
- return;
if (strcmp(this->forcing_type, "Kolmogorov") == 0)
{
this->add_Kolmogorov_forcing(dst, this->fmode, this->famplitude);
@@ -301,6 +299,15 @@ void vorticity_equation<rnumber, be>::add_forcing(
return;
}
if (strcmp(this->forcing_type, "Kolmogorov_and_drag") == 0)
+ {
+ this->add_Kolmogorov_forcing(dst, this->fmode, this->famplitude);
+ this->add_field_band(
+ dst, vort_field,
+ this->fk0, this->fk1,
+ -this->friction_coefficient);
+ return;
+ }
+ if (strcmp(this->forcing_type, "Kolmogorov_and_compensated_drag") == 0)
{
double amplitude = this->famplitude * (
1 + this->friction_coefficient / sqrt(this->fmode * this->famplitude));
@@ -374,16 +381,6 @@ void vorticity_equation<rnumber, be>::impose_forcing(
field<rnumber, be, THREE> *oold)
{
TIMEZONE("vorticity_equation::impose_forcing");
- if (strcmp(this->forcing_type, "none") == 0)
- return;
- if (strcmp(this->forcing_type, "Kolmogorov") == 0)
- return;
- if (strcmp(this->forcing_type, "2Kolmogorov") == 0)
- return;
- if (strcmp(this->forcing_type, "linear") == 0)
- return;
- if (strcmp(this->forcing_type, "fixed_energy_injection_rate") == 0)
- return;
if (strcmp(this->forcing_type, "fixed_energy") == 0)
{
// first, compute energy in shell
--
GitLab
From 0c6176d829bee62673769c2f5a54a1a4acf3b25f Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 6 Feb 2018 16:45:47 +0100
Subject: [PATCH 112/342] add a simple check that dset exists before reading
---
bfps/_base.py | 17 ++++++++++++-----
1 file changed, 12 insertions(+), 5 deletions(-)
diff --git a/bfps/_base.py b/bfps/_base.py
index 037261d3..4edc7f64 100644
--- a/bfps/_base.py
+++ b/bfps/_base.py
@@ -106,18 +106,24 @@ class _base(object):
src_txt += 'dset = H5Dopen(parameter_file, "/{0}/{1}", H5P_DEFAULT);\n'.format(
file_group, key[i])
if (type(parameters[key[i]]) == int and parameters[key[i]] >= 1<<30):
- src_txt += 'H5Dread(dset, H5T_NATIVE_LLONG, H5S_ALL, H5S_ALL, H5P_DEFAULT, &{0});\n'.format(key_prefix + key[i])
+ src_txt += ('if (dset > 0) H5Dread(dset, H5T_NATIVE_LLONG, H5S_ALL, H5S_ALL, H5P_DEFAULT, &{0});\n'
+ + 'else {0} = 0;\n').format(key_prefix + key[i])
elif type(parameters[key[i]]) == int:
- src_txt += 'H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &{0});\n'.format(key_prefix + key[i])
+ src_txt += ('if (dset > 0) H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &{0});\n'
+ + 'else {0} = 0;\n').format(key_prefix + key[i])
elif type(parameters[key[i]]) == str:
- src_txt += ('space = H5Dget_space(dset);\n' +
+ src_txt += ('if (dset > 0)\n' +
+ '{\n'
+ 'space = H5Dget_space(dset);\n' +
'memtype = H5Dget_type(dset);\n' +
'string_data = (char*)malloc(256);\n' +
'H5Dread(dset, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &string_data);\n' +
'sprintf({0}, "%s", string_data);\n'.format(key_prefix + key[i]) +
'free(string_data);\n'
'H5Sclose(space);\n' +
- 'H5Tclose(memtype);\n')
+ 'H5Tclose(memtype);\n' +
+ '}\n' +
+ 'else printf({0}, "NULL");\n'.format(key_prefix + key[i]))
elif type(parameters[key[i]]) == np.ndarray:
if parameters[key[i]].dtype in [np.int, np.int64, np.int32]:
template_par = 'int'
@@ -126,7 +132,8 @@ class _base(object):
src_txt += '{0} = hdf5_tools::read_vector<{1}>(parameter_file, "/{2}/{0}");\n'.format(
key_prefix + key[i], template_par, file_group)
else:
- src_txt += 'H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &{0});\n'.format(key_prefix + key[i])
+ src_txt += ('if (dset > 0) H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &{0});\n' +
+ 'else {0} = 0.0;\n').format(key_prefix + key[i])
src_txt += 'H5Dclose(dset);\n'
src_txt += 'H5Fclose(parameter_file);\n'
src_txt += 'return 0;\n}\n' # finishing read_parameters
--
GitLab
From dd3f7a7bd56058dc406f74a6ea586875756988f1 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 7 Feb 2018 17:18:54 +0100
Subject: [PATCH 113/342] add zaverage method
---
bfps/cpp/field.cpp | 115 ++++++++++++++++++++++++++++++++++++++++++++-
bfps/cpp/field.hpp | 5 ++
2 files changed, 119 insertions(+), 1 deletion(-)
diff --git a/bfps/cpp/field.cpp b/bfps/cpp/field.cpp
index 8fc6f9ff..a4e4dad0 100644
--- a/bfps/cpp/field.cpp
+++ b/bfps/cpp/field.cpp
@@ -576,7 +576,7 @@ int field<rnumber, be, fc>::write_0slice(
count[1] = this->rmemlayout->sizes[1];
count[2] = this->rmemlayout->sizes[2];
count[3] = 3;
- count[3] = 3;
+ count[4] = 3;
mspace = H5Screate_simple(ndims, count, NULL);
// array in file should not have the extra 2 points
count[1] = this->rlayout->sizes[1];
@@ -888,6 +888,119 @@ void field<rnumber, be, fc>::compute_rspace_stats(
delete[] hist;
}
+
+
+template <typename rnumber,
+ field_backend be,
+ field_components fc>
+void field<rnumber, be, fc>::compute_rspace_zaverage(
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset)
+{
+ TIMEZONE("field::compute_rspace_zaverage");
+ assert(this->real_space_representation);
+ const hsize_t slice_size = this->rlayout->local_size / this->rlayout->subsizes[0];
+
+ // initial arrays MUST be 0, because I'm just adding to them afterwards.
+ shared_array<double> local_zaverage_threaded(
+ slice_size, [&](double* local_zaverage){
+ std::fill_n(local_zaverage, slice_size, 0);
+ });
+
+ // sum along z direction
+ {
+ TIMEZONE("field::RLOOP");
+ this->RLOOP(
+ [&](ptrdiff_t rindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex){
+
+ double *local_zaverage = local_zaverage_threaded.getMine();
+ ptrdiff_t zaverage_index = (yindex*this->rlayout->subsizes[2]+xindex)*ncomp(fc);
+
+ for (unsigned int i=0; i<ncomp(fc); i++)
+ {
+ local_zaverage[zaverage_index + i] += this->rval(rindex, i);
+ }
+ });
+
+ TIMEZONE("FIELD_RLOOP::Merge");
+ local_zaverage_threaded.mergeParallel();
+ }
+ // sum along MPI processes
+ double *zaverage = new double[slice_size];
+ {
+ TIMEZONE("MPI_Allreduce");
+ MPI_Allreduce(
+ (void*)local_zaverage_threaded.getMasterData(),
+ (void*)zaverage,
+ slice_size,
+ MPI_DOUBLE, MPI_SUM, this->comm);
+ }
+ // divide by total number of slices
+ for (ptrdiff_t n=1; n < slice_size; n++)
+ zaverage[n] /= this->rlayout->sizes[0];
+
+ if (this->myrank == 0)
+ {
+ TIMEZONE("root-work");
+ hid_t dset, wspace, mspace;
+ int ndims;
+ hsize_t count[5], offset[5], dims[5];
+ offset[0] = toffset;
+ offset[1] = 0;
+ offset[2] = 0;
+ offset[3] = 0;
+ offset[4] = 0;
+ dset = H5Dopen(
+ group,
+ ("zaverage/" + dset_name).c_str(),
+ H5P_DEFAULT);
+ wspace = H5Dget_space(dset);
+ ndims = H5Sget_simple_extent_dims(wspace, dims, NULL);
+ // array in memory has 2 extra x points, because FFTW
+ count[0] = 1;
+ count[1] = this->rmemlayout->sizes[1];
+ count[2] = this->rmemlayout->sizes[2];
+ count[3] = 3;
+ count[4] = 3;
+ mspace = H5Screate_simple(ndims, count, NULL);
+ // array in file should not have the extra 2 points
+ count[1] = this->rlayout->sizes[1];
+ count[2] = this->rlayout->sizes[2];
+ // select right slice in file
+ H5Sselect_hyperslab(
+ wspace,
+ H5S_SELECT_SET,
+ offset,
+ NULL,
+ count,
+ NULL);
+ offset[0] = 0;
+ // select proper regions of memory
+ H5Sselect_hyperslab(
+ mspace,
+ H5S_SELECT_SET,
+ offset,
+ NULL,
+ count,
+ NULL);
+ H5Dwrite(
+ dset,
+ this->rnumber_H5T,
+ mspace,
+ wspace,
+ H5P_DEFAULT,
+ this->data);
+ H5Dclose(dset);
+ H5Sclose(mspace);
+ H5Sclose(wspace);
+ }
+ delete[] zaverage;
+}
+
template <typename rnumber,
field_backend be,
field_components fc>
diff --git a/bfps/cpp/field.hpp b/bfps/cpp/field.hpp
index 41609acd..da03fc32 100644
--- a/bfps/cpp/field.hpp
+++ b/bfps/cpp/field.hpp
@@ -129,6 +129,11 @@ class field
const hsize_t toffset,
const std::vector<double> max_estimate);
+ void compute_rspace_zaverage(
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset);
+
/* access sizes */
inline int get_nx() const
{
--
GitLab
From 559d4b8df907f31c70603e121051a129b4067d8d Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 8 Feb 2018 16:22:44 +0100
Subject: [PATCH 114/342] use EXIT_SUCCESS instead of 0
---
bfps/_base.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bfps/_base.py b/bfps/_base.py
index 4edc7f64..2ea8fbbb 100644
--- a/bfps/_base.py
+++ b/bfps/_base.py
@@ -136,7 +136,7 @@ class _base(object):
'else {0} = 0.0;\n').format(key_prefix + key[i])
src_txt += 'H5Dclose(dset);\n'
src_txt += 'H5Fclose(parameter_file);\n'
- src_txt += 'return 0;\n}\n' # finishing read_parameters
+ src_txt += 'return EXIT_SUCCESS;\n}\n' # finishing read_parameters
return src_txt
def cprint_pars(self):
key = sorted(list(self.parameters.keys()))
--
GitLab
From 3f9d4b6940382d3e0f208f565b40c741ea8becc0 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 8 Feb 2018 17:13:44 +0100
Subject: [PATCH 115/342] add better control of dset generation
---
bfps/DNS.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 3d531005..d1ffb1eb 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -379,7 +379,8 @@ class DNS(_code):
def write_par(
self,
iter0 = 0,
- particle_ic = None):
+ particle_ic = None,
+ particles_off = False):
assert (self.parameters['niter_todo'] % self.parameters['niter_stat'] == 0)
assert (self.parameters['niter_todo'] % self.parameters['niter_out'] == 0)
assert (self.parameters['niter_out'] % self.parameters['niter_stat'] == 0)
@@ -426,7 +427,7 @@ class DNS(_code):
4),
dtype = np.int64)
ofile['checkpoint'] = int(0)
- if self.dns_type in ['NSVE', 'NSVE_no_output']:
+ if (self.dns_type in ['NSVE', 'NSVE_no_output']) or particles_off:
return None
if type(particle_ic) == type(None):
--
GitLab
From 9bde4598034a7211a1e66fa7d5d3072ab275f448 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 8 Feb 2018 17:14:03 +0100
Subject: [PATCH 116/342] make explicit switch based on field_components
---
bfps/cpp/field.cpp | 36 ++++++++++++++++++++----------------
1 file changed, 20 insertions(+), 16 deletions(-)
diff --git a/bfps/cpp/field.cpp b/bfps/cpp/field.cpp
index a4e4dad0..00855514 100644
--- a/bfps/cpp/field.cpp
+++ b/bfps/cpp/field.cpp
@@ -869,20 +869,6 @@ void field<rnumber, be, fc>::compute_rspace_stats(
H5Sclose(wspace);
H5Sclose(mspace);
H5Dclose(dset);
- if (H5Lexists(
- group,
- "0slices",
- H5P_DEFAULT))
- {
- if (H5Lexists(
- group,
- (std::string("0slices/") + dset_name).c_str(),
- H5P_DEFAULT))
- this->write_0slice(
- group,
- dset_name,
- toffset);
- }
}
delete[] moments;
delete[] hist;
@@ -920,9 +906,27 @@ void field<rnumber, be, fc>::compute_rspace_zaverage(
double *local_zaverage = local_zaverage_threaded.getMine();
ptrdiff_t zaverage_index = (yindex*this->rlayout->subsizes[2]+xindex)*ncomp(fc);
- for (unsigned int i=0; i<ncomp(fc); i++)
+ switch(fc)
{
- local_zaverage[zaverage_index + i] += this->rval(rindex, i);
+ case ONE:
+ local_zaverage[zaverage_index] += this->rval(rindex);
+ break;
+ case THREE:
+ local_zaverage[zaverage_index+0] += this->rval(rindex, 0);
+ local_zaverage[zaverage_index+1] += this->rval(rindex, 1);
+ local_zaverage[zaverage_index+2] += this->rval(rindex, 2);
+ break;
+ case THREExTHREE:
+ local_zaverage[zaverage_index+0 + 0] += this->rval(rindex, 0, 0);
+ local_zaverage[zaverage_index+0 + 1] += this->rval(rindex, 0, 1);
+ local_zaverage[zaverage_index+0 + 2] += this->rval(rindex, 0, 2);
+ local_zaverage[zaverage_index+3 + 0] += this->rval(rindex, 1, 0);
+ local_zaverage[zaverage_index+3 + 1] += this->rval(rindex, 1, 1);
+ local_zaverage[zaverage_index+3 + 2] += this->rval(rindex, 1, 2);
+ local_zaverage[zaverage_index+6 + 0] += this->rval(rindex, 2, 0);
+ local_zaverage[zaverage_index+6 + 1] += this->rval(rindex, 2, 1);
+ local_zaverage[zaverage_index+6 + 2] += this->rval(rindex, 2, 2);
+ break;
}
});
--
GitLab
From 3c69782fac5500eeeb8d64c009b250a9bcd1fde3 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 9 Feb 2018 10:25:47 +0100
Subject: [PATCH 117/342] fix zaverage output
---
bfps/cpp/field.cpp | 20 ++++++++------------
1 file changed, 8 insertions(+), 12 deletions(-)
diff --git a/bfps/cpp/field.cpp b/bfps/cpp/field.cpp
index 00855514..a9135a89 100644
--- a/bfps/cpp/field.cpp
+++ b/bfps/cpp/field.cpp
@@ -944,7 +944,7 @@ void field<rnumber, be, fc>::compute_rspace_zaverage(
MPI_DOUBLE, MPI_SUM, this->comm);
}
// divide by total number of slices
- for (ptrdiff_t n=1; n < slice_size; n++)
+ for (ptrdiff_t n=0; n < slice_size; n++)
zaverage[n] /= this->rlayout->sizes[0];
if (this->myrank == 0)
@@ -964,16 +964,11 @@ void field<rnumber, be, fc>::compute_rspace_zaverage(
H5P_DEFAULT);
wspace = H5Dget_space(dset);
ndims = H5Sget_simple_extent_dims(wspace, dims, NULL);
- // array in memory has 2 extra x points, because FFTW
count[0] = 1;
- count[1] = this->rmemlayout->sizes[1];
- count[2] = this->rmemlayout->sizes[2];
- count[3] = 3;
- count[4] = 3;
- mspace = H5Screate_simple(ndims, count, NULL);
- // array in file should not have the extra 2 points
count[1] = this->rlayout->sizes[1];
count[2] = this->rlayout->sizes[2];
+ count[3] = 3;
+ count[4] = 3;
// select right slice in file
H5Sselect_hyperslab(
wspace,
@@ -984,20 +979,21 @@ void field<rnumber, be, fc>::compute_rspace_zaverage(
NULL);
offset[0] = 0;
// select proper regions of memory
+ mspace = H5Screate_simple(ndims-1, count+1, NULL);
H5Sselect_hyperslab(
mspace,
H5S_SELECT_SET,
- offset,
+ offset+1,
NULL,
- count,
+ count+1,
NULL);
H5Dwrite(
dset,
- this->rnumber_H5T,
+ H5T_NATIVE_DOUBLE,
mspace,
wspace,
H5P_DEFAULT,
- this->data);
+ zaverage);
H5Dclose(dset);
H5Sclose(mspace);
H5Sclose(wspace);
--
GitLab
From 10486066c9ee546ff503aeedb18d071c3f2f86e0 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 26 Feb 2018 12:40:24 +0100
Subject: [PATCH 118/342] add forcing for quasi2D runs
---
bfps/cpp/vorticity_equation.cpp | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
diff --git a/bfps/cpp/vorticity_equation.cpp b/bfps/cpp/vorticity_equation.cpp
index 248aaa2b..7f1e3203 100644
--- a/bfps/cpp/vorticity_equation.cpp
+++ b/bfps/cpp/vorticity_equation.cpp
@@ -26,6 +26,7 @@
#define NDEBUG
+#include <limits>
#include <cassert>
#include <cmath>
#include <cstring>
@@ -326,7 +327,8 @@ void vorticity_equation<rnumber, be>::add_forcing(
this->famplitude);
return;
}
- if (strcmp(this->forcing_type, "fixed_energy_injection_rate") == 0)
+ if ((strcmp(this->forcing_type, "fixed_energy_injection_rate") == 0) ||
+ (strcmp(this->forcing_type, "fixed_energy_injection_rate_and_drag") == 0))
{
// first, compute energy in shell
shared_array<double> local_energy_in_shell(1);
@@ -363,11 +365,19 @@ void vorticity_equation<rnumber, be>::add_forcing(
// without the division by 2.
// now, modify amplitudes
+ if (energy_in_shell < 10*std::numeric_limits<rnumber>::epsilon())
+ energy_in_shell = 1;
double temp_famplitude = this->injection_rate / energy_in_shell;
this->add_field_band(
dst, vort_field,
this->fk0, this->fk1,
temp_famplitude);
+ // and add drag if desired
+ if (strcmp(this->forcing_type, "fixed_energy_injection_rate_and_drag") == 0)
+ this->add_field_band(
+ dst, vort_field,
+ this->fmode, this->fmode + (this->fk0 - this->fk1),
+ -this->friction_coefficient);
return;
}
if (strcmp(this->forcing_type, "fixed_energy") == 0)
--
GitLab
From b656f4ca6d6797da0eddc7bfa9109c5ded38c496 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 26 Feb 2018 20:33:25 +0100
Subject: [PATCH 119/342] add code for future
---
bfps/DNS.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 44c1cca1..90988019 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -166,6 +166,11 @@ class DNS(_code):
self.NSVEp_extra_parameters['tracers0_integration_steps'] = int(4)
self.NSVEp_extra_parameters['tracers0_neighbours'] = int(1)
self.NSVEp_extra_parameters['tracers0_smoothness'] = int(1)
+ #self.extra_parameters = {}
+ #for key in ['NSVE', 'NSVE_no_output', 'NSVEparticles', 'NSVEparticles_no_output', 'NSVEparticlesP2P']:
+ # self.extra_parameters[key] = {}
+ #for key in ['NSVEparticles', 'NSVEparticles_no_output', 'NSVEparticlesP2P']:
+ # self.extra_parameters[key].update(self.NSVEp_extra_parameters)
return None
def get_kspace(self):
kspace = {}
--
GitLab
From e9cb95cf97a40f4a3b40d5876c34fd998cc15f0f Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 26 Feb 2018 20:33:40 +0100
Subject: [PATCH 120/342] tweak template parameters for particle output
---
bfps/cpp/full_code/NSVEparticles.cpp | 12 +++---
bfps/cpp/full_code/NSVEparticles.hpp | 4 +-
bfps/cpp/full_code/NSVEparticlesP2P.cpp | 16 ++++----
bfps/cpp/full_code/NSVEparticlesP2P.hpp | 4 +-
.../particles/abstract_particles_output.hpp | 7 ++--
bfps/cpp/particles/particles_output_hdf5.hpp | 15 +++----
bfps/cpp/particles/particles_output_mpiio.hpp | 6 +--
.../particles_output_sampling_hdf5.hpp | 15 ++++---
bfps/cpp/particles/particles_sampling.hpp | 40 +++++++++----------
9 files changed, 58 insertions(+), 61 deletions(-)
diff --git a/bfps/cpp/full_code/NSVEparticles.cpp b/bfps/cpp/full_code/NSVEparticles.cpp
index 953ad9f9..5f9f480d 100644
--- a/bfps/cpp/full_code/NSVEparticles.cpp
+++ b/bfps/cpp/full_code/NSVEparticles.cpp
@@ -21,13 +21,13 @@ int NSVEparticles<rnumber>::initialize(void)
this->comm,
this->fs->iteration+1);
this->particles_output_writer_mpi = new particles_output_hdf5<
- long long int, double, 3, 3>(
+ long long int, double, 3>(
MPI_COMM_WORLD,
"tracers0",
nparticles,
tracers0_integration_steps);
this->particles_sample_writer_mpi = new particles_output_sampling_hdf5<
- long long int, double, 3, 3>(
+ long long int, double, 3>(
MPI_COMM_WORLD,
this->ps->getGlobalNbParticles(),
(this->simname + "_particles.h5"),
@@ -51,7 +51,7 @@ int NSVEparticles<rnumber>::write_checkpoint(void)
{
this->NSVE<rnumber>::write_checkpoint();
this->particles_output_writer_mpi->open_file(this->fs->get_current_fname());
- this->particles_output_writer_mpi->save(
+ this->particles_output_writer_mpi->template save<3>(
this->ps->getParticlesState(),
this->ps->getParticlesRhs(),
this->ps->getParticlesIndexes(),
@@ -93,7 +93,7 @@ int NSVEparticles<rnumber>::do_stats()
std::copy(this->ps->getParticlesState(),
this->ps->getParticlesState()+3*this->ps->getLocalNbParticles(),
pdata.get());
- this->particles_sample_writer_mpi->save_dataset(
+ this->particles_sample_writer_mpi->template save_dataset<3>(
"tracers0",
"position",
this->ps->getParticlesState(),
@@ -104,7 +104,7 @@ int NSVEparticles<rnumber>::do_stats()
/// sample velocity
this->ps->sample_compute_field(*this->tmp_vec_field, pdata.get());
- this->particles_sample_writer_mpi->save_dataset(
+ this->particles_sample_writer_mpi->template save_dataset<3>(
"tracers0",
"velocity",
this->ps->getParticlesState(),
@@ -117,7 +117,7 @@ int NSVEparticles<rnumber>::do_stats()
this->fs->compute_Lagrangian_acceleration(this->tmp_vec_field);
this->tmp_vec_field->ift();
this->ps->sample_compute_field(*this->tmp_vec_field, pdata.get());
- this->particles_sample_writer_mpi->save_dataset(
+ this->particles_sample_writer_mpi->template save_dataset<3>(
"tracers0",
"acceleration",
this->ps->getParticlesState(),
diff --git a/bfps/cpp/full_code/NSVEparticles.hpp b/bfps/cpp/full_code/NSVEparticles.hpp
index 97ea5c84..1a2a6535 100644
--- a/bfps/cpp/full_code/NSVEparticles.hpp
+++ b/bfps/cpp/full_code/NSVEparticles.hpp
@@ -59,8 +59,8 @@ class NSVEparticles: public NSVE<rnumber>
/* other stuff */
std::unique_ptr<abstract_particles_system<long long int, double>> ps;
- particles_output_hdf5<long long int, double,3,3> *particles_output_writer_mpi;
- particles_output_sampling_hdf5<long long int, double, 3, 3> *particles_sample_writer_mpi;
+ particles_output_hdf5<long long int, double,3> *particles_output_writer_mpi;
+ particles_output_sampling_hdf5<long long int, double, 3> *particles_sample_writer_mpi;
NSVEparticles(
diff --git a/bfps/cpp/full_code/NSVEparticlesP2P.cpp b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
index ff1a594f..a6b7082e 100644
--- a/bfps/cpp/full_code/NSVEparticlesP2P.cpp
+++ b/bfps/cpp/full_code/NSVEparticlesP2P.cpp
@@ -38,13 +38,13 @@ int NSVEparticlesP2P<rnumber>::initialize(void)
cutoff);
this->particles_output_writer_mpi = new particles_output_hdf5<
- long long int, double, 6, 6>(
+ long long int, double, 6>(
MPI_COMM_WORLD,
"tracers0",
nparticles,
tracers0_integration_steps);
this->particles_sample_writer_mpi = new particles_output_sampling_hdf5<
- long long int, double, 3, 3>(
+ long long int, double, 3>(
MPI_COMM_WORLD,
this->ps->getGlobalNbParticles(),
(this->simname + "_particles.h5"),
@@ -79,7 +79,7 @@ int NSVEparticlesP2P<rnumber>::write_checkpoint(void)
this->NSVE<rnumber>::write_checkpoint();
this->particles_output_writer_mpi->open_file(this->fs->get_current_fname());
// TODO P2P write particle data too
- this->particles_output_writer_mpi->save(
+ this->particles_output_writer_mpi->template save<6>(
this->ps->getParticlesState(),
this->ps->getParticlesRhs(),
this->ps->getParticlesIndexes(),
@@ -117,7 +117,7 @@ int NSVEparticlesP2P<rnumber>::do_stats()
std::unique_ptr<double[]> pdata1 = this->ps->extractParticlesState(3, 6);
/// sample position
- this->particles_sample_writer_mpi->save_dataset(
+ this->particles_sample_writer_mpi->template save_dataset<3>(
"tracers0",
"position",
pdata0.get(),
@@ -127,7 +127,7 @@ int NSVEparticlesP2P<rnumber>::do_stats()
this->ps->get_step_idx()-1);
/// sample orientation
- this->particles_sample_writer_mpi->save_dataset(
+ this->particles_sample_writer_mpi->template save_dataset<3>(
"tracers0",
"orientation",
pdata0.get(),
@@ -138,7 +138,7 @@ int NSVEparticlesP2P<rnumber>::do_stats()
/// sample velocity
this->ps->sample_compute_field(*this->tmp_vec_field, pdata1.get());
- this->particles_sample_writer_mpi->save_dataset(
+ this->particles_sample_writer_mpi->template save_dataset<3>(
"tracers0",
"velocity",
pdata0.get(),
@@ -151,7 +151,7 @@ int NSVEparticlesP2P<rnumber>::do_stats()
*this->tmp_vec_field = this->fs->cvorticity->get_cdata();
this->tmp_vec_field->ift();
this->ps->sample_compute_field(*this->tmp_vec_field, pdata1.get());
- this->particles_sample_writer_mpi->save_dataset(
+ this->particles_sample_writer_mpi->template save_dataset<3>(
"tracers0",
"vorticity",
pdata0.get(),
@@ -164,7 +164,7 @@ int NSVEparticlesP2P<rnumber>::do_stats()
this->fs->compute_Lagrangian_acceleration(this->tmp_vec_field);
this->tmp_vec_field->ift();
this->ps->sample_compute_field(*this->tmp_vec_field, pdata1.get());
- this->particles_sample_writer_mpi->save_dataset(
+ this->particles_sample_writer_mpi->template save_dataset<3>(
"tracers0",
"acceleration",
pdata0.get(),
diff --git a/bfps/cpp/full_code/NSVEparticlesP2P.hpp b/bfps/cpp/full_code/NSVEparticlesP2P.hpp
index 8f435116..b74169f5 100644
--- a/bfps/cpp/full_code/NSVEparticlesP2P.hpp
+++ b/bfps/cpp/full_code/NSVEparticlesP2P.hpp
@@ -65,8 +65,8 @@ class NSVEparticlesP2P: public NSVE<rnumber>
/* other stuff */
std::unique_ptr<abstract_particles_system<long long int, double>> ps;
// TODO P2P use a reader with particle data
- particles_output_hdf5<long long int, double,6,6> *particles_output_writer_mpi;
- particles_output_sampling_hdf5<long long int, double,3,3> *particles_sample_writer_mpi;
+ particles_output_hdf5<long long int, double,6> *particles_output_writer_mpi;
+ particles_output_sampling_hdf5<long long int, double, 3> *particles_sample_writer_mpi;
NSVEparticlesP2P(
diff --git a/bfps/cpp/particles/abstract_particles_output.hpp b/bfps/cpp/particles/abstract_particles_output.hpp
index 5285c90f..7510bc6e 100644
--- a/bfps/cpp/particles/abstract_particles_output.hpp
+++ b/bfps/cpp/particles/abstract_particles_output.hpp
@@ -13,7 +13,7 @@
#include "scope_timer.hpp"
#include "env_utils.hpp"
-template <class partsize_t, class real_number, int size_particle_positions, int size_particle_rhs>
+template <class partsize_t, class real_number, int size_particle_positions>
class abstract_particles_output {
MPI_Comm mpi_com;
MPI_Comm mpi_com_writer;
@@ -145,6 +145,7 @@ public:
}
}
+ template <int size_particle_rhs>
void save(
const real_number input_particles_positions[],
const std::unique_ptr<real_number[]> input_particles_rhs[],
@@ -264,11 +265,11 @@ public:
}
write(idx_time_step, buffer_particles_positions_send.get(), buffer_particles_rhs_send.data(),
- nb_to_receive, particles_chunk_current_offset);
+ nb_to_receive, particles_chunk_current_offset, size_particle_rhs);
}
virtual void write(const int idx_time_step, const real_number* positions, const std::unique_ptr<real_number[]>* rhs,
- const partsize_t nb_particles, const partsize_t particles_idx_offset) = 0;
+ const partsize_t nb_particles, const partsize_t particles_idx_offset, const int size_particle_rhs) = 0;
};
#endif
diff --git a/bfps/cpp/particles/particles_output_hdf5.hpp b/bfps/cpp/particles/particles_output_hdf5.hpp
index 647103ca..22d2fa85 100644
--- a/bfps/cpp/particles/particles_output_hdf5.hpp
+++ b/bfps/cpp/particles/particles_output_hdf5.hpp
@@ -10,16 +10,13 @@
template <class partsize_t,
class real_number,
- int size_particle_positions,
- int size_particle_rhs>
+ int size_particle_positions>
class particles_output_hdf5 : public abstract_particles_output<partsize_t,
real_number,
- size_particle_positions,
- size_particle_rhs>{
+ size_particle_positions>{
using Parent = abstract_particles_output<partsize_t,
real_number,
- size_particle_positions,
- size_particle_rhs>;
+ size_particle_positions>;
std::string particle_species_name;
@@ -39,8 +36,7 @@ public:
const bool in_use_collective_io = false)
: abstract_particles_output<partsize_t,
real_number,
- size_particle_positions,
- size_particle_rhs>(
+ size_particle_positions>(
in_mpi_com,
inTotalNbParticles,
in_nb_rhs),
@@ -183,7 +179,8 @@ public:
const real_number* particles_positions,
const std::unique_ptr<real_number[]>* particles_rhs,
const partsize_t nb_particles,
- const partsize_t particles_idx_offset) final{
+ const partsize_t particles_idx_offset,
+ const int size_particle_rhs) final{
assert(Parent::isInvolved());
TIMEZONE("particles_output_hdf5::write");
diff --git a/bfps/cpp/particles/particles_output_mpiio.hpp b/bfps/cpp/particles/particles_output_mpiio.hpp
index 77dae6ca..5810c4a0 100644
--- a/bfps/cpp/particles/particles_output_mpiio.hpp
+++ b/bfps/cpp/particles/particles_output_mpiio.hpp
@@ -11,8 +11,8 @@
#include "particles_utils.hpp"
template <class partsize_t, class real_number, int size_particle_positions, int size_particle_rhs>
-class particles_output_mpiio : public abstract_particles_output<partsize_t, real_number, size_particle_positions, size_particle_rhs>{
- using Parent = abstract_particles_output<partsize_t, real_number, size_particle_positions, size_particle_rhs>;
+class particles_output_mpiio : public abstract_particles_output<partsize_t, real_number, size_particle_positions>{
+ using Parent = abstract_particles_output<partsize_t, real_number, size_particle_positions>;
const std::string filename;
const int nb_step_prealloc;
@@ -24,7 +24,7 @@ class particles_output_mpiio : public abstract_particles_output<partsize_t, real
public:
particles_output_mpiio(MPI_Comm in_mpi_com, const std::string in_filename, const partsize_t inTotalNbParticles,
const int in_nb_rhs, const int in_nb_step_prealloc = -1)
- : abstract_particles_output<partsize_t, real_number, size_particle_positions, size_particle_rhs>(in_mpi_com, inTotalNbParticles, in_nb_rhs),
+ : abstract_particles_output<partsize_t, real_number, size_particle_positions>(in_mpi_com, inTotalNbParticles, in_nb_rhs),
filename(in_filename), nb_step_prealloc(in_nb_step_prealloc), current_step_in_file(0){
if(Parent::isInvolved()){
{
diff --git a/bfps/cpp/particles/particles_output_sampling_hdf5.hpp b/bfps/cpp/particles/particles_output_sampling_hdf5.hpp
index 823754a5..dc213226 100644
--- a/bfps/cpp/particles/particles_output_sampling_hdf5.hpp
+++ b/bfps/cpp/particles/particles_output_sampling_hdf5.hpp
@@ -7,17 +7,14 @@
template <class partsize_t,
class real_number,
- int size_particle_positions,
- int size_particle_rhs>
+ int size_particle_positions>
class particles_output_sampling_hdf5 : public abstract_particles_output<
partsize_t,
real_number,
- size_particle_positions,
- size_particle_rhs>{
+ size_particle_positions>{
using Parent = abstract_particles_output<partsize_t,
real_number,
- size_particle_positions,
- size_particle_rhs>;
+ size_particle_positions>;
hid_t file_id, pgroup_id;
@@ -121,6 +118,7 @@ public:
return EXIT_SUCCESS;
}
+ template <int size_particle_rhs>
int save_dataset(
const std::string& in_groupname,
const std::string& in_dataset_name,
@@ -144,7 +142,7 @@ public:
H5P_DEFAULT);
AssertMpi(MPI_Bcast(&dataset_exists, 1, MPI_INT, 0, this->getCom()));
if (dataset_exists == 0)
- this->save(
+ this->template save<size_particle_rhs>(
input_particles_positions,
input_particles_rhs,
index_particles,
@@ -158,7 +156,8 @@ public:
const real_number* /*particles_positions*/,
const std::unique_ptr<real_number[]>* particles_rhs,
const partsize_t nb_particles,
- const partsize_t particles_idx_offset) final{
+ const partsize_t particles_idx_offset,
+ const int size_particle_rhs) final{
assert(Parent::isInvolved());
TIMEZONE("particles_output_hdf5::write");
diff --git a/bfps/cpp/particles/particles_sampling.hpp b/bfps/cpp/particles/particles_sampling.hpp
index a8927591..8baff633 100644
--- a/bfps/cpp/particles/particles_sampling.hpp
+++ b/bfps/cpp/particles/particles_sampling.hpp
@@ -21,10 +21,10 @@ void sample_from_particles_system(const field<rnumber, be, fc>& in_field, // a p
const int size_particle_rhs = ncomp(fc);
// Stop here if already exists
- if(particles_output_sampling_hdf5<partsize_t, particles_rnumber, 3, size_particle_rhs>::DatasetExistsCol(MPI_COMM_WORLD,
- filename,
- parent_groupname,
- datasetname)){
+ if(particles_output_sampling_hdf5<partsize_t, particles_rnumber, 3>::DatasetExistsCol(MPI_COMM_WORLD,
+ filename,
+ parent_groupname,
+ datasetname)){
return;
}
@@ -36,12 +36,12 @@ void sample_from_particles_system(const field<rnumber, be, fc>& in_field, // a p
- particles_output_sampling_hdf5<partsize_t, particles_rnumber, 3, size_particle_rhs> outputclass(MPI_COMM_WORLD,
- ps->getGlobalNbParticles(),
- filename,
- parent_groupname,
- datasetname);
- outputclass.save(ps->getParticlesState(),
+ particles_output_sampling_hdf5<partsize_t, particles_rnumber, 3> outputclass(MPI_COMM_WORLD,
+ ps->getGlobalNbParticles(),
+ filename,
+ parent_groupname,
+ datasetname);
+ outputclass.template save<size_particle_rhs>(ps->getParticlesState(),
&sample_rhs,
ps->getParticlesIndexes(),
ps->getLocalNbParticles(),
@@ -57,10 +57,10 @@ void sample_particles_system_position(
const std::string datasetname = fname + std::string("/") + std::to_string(ps->get_step_idx());
// Stop here if already exists
- if(particles_output_sampling_hdf5<partsize_t, particles_rnumber, 3, 3>::DatasetExistsCol(MPI_COMM_WORLD,
- filename,
- parent_groupname,
- datasetname)){
+ if(particles_output_sampling_hdf5<partsize_t, particles_rnumber, 3>::DatasetExistsCol(MPI_COMM_WORLD,
+ filename,
+ parent_groupname,
+ datasetname)){
return;
}
@@ -68,12 +68,12 @@ void sample_particles_system_position(
std::unique_ptr<particles_rnumber[]> sample_rhs(new particles_rnumber[3*nb_particles]);
std::copy(ps->getParticlesState(), ps->getParticlesState() + 3*nb_particles, sample_rhs.get());
- particles_output_sampling_hdf5<partsize_t, particles_rnumber, 3, 3> outputclass(MPI_COMM_WORLD,
- ps->getGlobalNbParticles(),
- filename,
- parent_groupname,
- datasetname);
- outputclass.save(ps->getParticlesState(),
+ particles_output_sampling_hdf5<partsize_t, particles_rnumber, 3> outputclass(MPI_COMM_WORLD,
+ ps->getGlobalNbParticles(),
+ filename,
+ parent_groupname,
+ datasetname);
+ outputclass.template save<3>(ps->getParticlesState(),
&sample_rhs,
ps->getParticlesIndexes(),
ps->getLocalNbParticles(),
--
GitLab
From 0bb5654609551e17225d2e1d90083de3c192e0c6 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 27 Feb 2018 10:13:05 +0100
Subject: [PATCH 121/342] rename NSVE complex particles source files
---
.../full_code/{NSVEparticlesP2P.cpp => NSVEcomplex_particles.cpp} | 0
.../full_code/{NSVEparticlesP2P.hpp => NSVEcomplex_particles.hpp} | 0
2 files changed, 0 insertions(+), 0 deletions(-)
rename bfps/cpp/full_code/{NSVEparticlesP2P.cpp => NSVEcomplex_particles.cpp} (100%)
rename bfps/cpp/full_code/{NSVEparticlesP2P.hpp => NSVEcomplex_particles.hpp} (100%)
diff --git a/bfps/cpp/full_code/NSVEparticlesP2P.cpp b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
similarity index 100%
rename from bfps/cpp/full_code/NSVEparticlesP2P.cpp
rename to bfps/cpp/full_code/NSVEcomplex_particles.cpp
diff --git a/bfps/cpp/full_code/NSVEparticlesP2P.hpp b/bfps/cpp/full_code/NSVEcomplex_particles.hpp
similarity index 100%
rename from bfps/cpp/full_code/NSVEparticlesP2P.hpp
rename to bfps/cpp/full_code/NSVEcomplex_particles.hpp
--
GitLab
From 3655957b6da5e720264c4ebd086f2b024021bc79 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 27 Feb 2018 10:51:54 +0100
Subject: [PATCH 122/342] [broken] sample velocity gradient for complex
particles
---
bfps/DNS.py | 24 +++++-----
bfps/cpp/full_code/NSVEcomplex_particles.cpp | 47 ++++++++++++--------
bfps/cpp/full_code/NSVEcomplex_particles.hpp | 21 +++++----
bfps/test/test_particles.py | 2 +-
setup.py | 2 +-
5 files changed, 56 insertions(+), 40 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 90988019..7eb15af6 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -127,7 +127,7 @@ class DNS(_code):
template_class = '{0}<{1}>::'.format(self.dns_type, rnumber),
template_prefix = 'template '.format(rnumber),
just_declaration = True) + '\n\n')
- if self.dns_type in ['NSVEparticles', 'NSVE_no_output', 'NSVEparticles_no_output', 'NSVEparticlesP2P']:
+ if self.dns_type in ['NSVEparticles', 'NSVE_no_output', 'NSVEparticles_no_output', 'NSVEcomplex_particles']:
outfile.write('template <typename rnumber> int NSVE<rnumber>::read_parameters(){return EXIT_SUCCESS;}\n')
outfile.write('template int NSVE<float>::read_parameters();\n')
outfile.write('template int NSVE<double>::read_parameters();\n\n')
@@ -167,9 +167,9 @@ class DNS(_code):
self.NSVEp_extra_parameters['tracers0_neighbours'] = int(1)
self.NSVEp_extra_parameters['tracers0_smoothness'] = int(1)
#self.extra_parameters = {}
- #for key in ['NSVE', 'NSVE_no_output', 'NSVEparticles', 'NSVEparticles_no_output', 'NSVEparticlesP2P']:
+ #for key in ['NSVE', 'NSVE_no_output', 'NSVEparticles', 'NSVEparticles_no_output', 'NSVEcomplex_particles']:
# self.extra_parameters[key] = {}
- #for key in ['NSVEparticles', 'NSVEparticles_no_output', 'NSVEparticlesP2P']:
+ #for key in ['NSVEparticles', 'NSVEparticles_no_output', 'NSVEcomplex_particles']:
# self.extra_parameters[key].update(self.NSVEp_extra_parameters)
return None
def get_kspace(self):
@@ -387,7 +387,7 @@ class DNS(_code):
assert (self.parameters['niter_todo'] % self.parameters['niter_stat'] == 0)
assert (self.parameters['niter_todo'] % self.parameters['niter_out'] == 0)
assert (self.parameters['niter_out'] % self.parameters['niter_stat'] == 0)
- if self.dns_type in ['NSVEparticles_no_output', 'NSVEparticlesP2P', 'NSVEparticles']:
+ if self.dns_type in ['NSVEparticles_no_output', 'NSVEcomplex_particles', 'NSVEparticles']:
assert (self.parameters['niter_todo'] % self.parameters['niter_part'] == 0)
assert (self.parameters['niter_out'] % self.parameters['niter_part'] == 0)
_code.write_par(self, iter0 = iter0)
@@ -443,7 +443,7 @@ class DNS(_code):
for val in pbase_shape[1:]:
number_of_particles *= val
ncomponents = 3
- if self.dns_type in ['NSVEparticlesP2P']:
+ if self.dns_type in ['NSVEcomplex_particles']:
ncomponents = 6
with h5py.File(self.get_checkpoint_0_fname(), 'a') as ofile:
s = 0
@@ -622,8 +622,8 @@ class DNS(_code):
help = 'plain Navier-Stokes vorticity formulation, with basic fluid tracers')
parser_NSVEp2p = subparsers.add_parser(
- 'NSVEparticlesP2P',
- help = 'plain Navier-Stokes vorticity formulation, with basic fluid tracers')
+ 'NSVEcomplex_particles',
+ help = 'plain Navier-Stokes vorticity formulation, with oriented active particles')
for parser in ['NSVEparticles_no_output', 'NSVEp2', 'NSVEp2p']:
eval('self.simulation_parser_arguments({0})'.format('parser_' + parser))
@@ -668,7 +668,7 @@ class DNS(_code):
self.dns_type = opt.DNS_class
self.name = self.dns_type + '-' + self.fluid_precision + '-v' + bfps.__version__
# merge parameters if needed
- if self.dns_type in ['NSVEparticles', 'NSVEparticlesP2P', 'NSVEparticles_no_output']:
+ if self.dns_type in ['NSVEparticles', 'NSVEcomplex_particles', 'NSVEparticles_no_output']:
for k in self.NSVEp_extra_parameters.keys():
self.parameters[k] = self.NSVEp_extra_parameters[k]
if type(extra_parameters) != type(None):
@@ -728,7 +728,7 @@ class DNS(_code):
# hardcoded FFTW complex representation size
field_size = 3*(opt.nx+2)*opt.ny*opt.nz*self.fluid_dtype.itemsize
checkpoint_size = field_size
- if self.dns_type in ['NSVEparticles', 'NSVEparticlesP2P', 'NSVEparticles_no_output']:
+ if self.dns_type in ['NSVEparticles', 'NSVEcomplex_particles', 'NSVEparticles_no_output']:
rhs_size = self.parameters['tracers0_integration_steps']
if type(opt.tracers0_integration_steps) != type(None):
rhs_size = opt.tracers0_integration_steps
@@ -932,9 +932,9 @@ class DNS(_code):
particle_file.create_group('tracers0/position')
particle_file.create_group('tracers0/velocity')
particle_file.create_group('tracers0/acceleration')
- if self.dns_type in ['NSVEparticlesP2P']:
+ if self.dns_type in ['NSVEcomplex_particles']:
particle_file.create_group('tracers0/orientation')
- particle_file.create_group('tracers0/vorticity')
+ particle_file.create_group('tracers0/velocity_gradient')
return None
def launch_jobs(
self,
@@ -994,7 +994,7 @@ class DNS(_code):
# particle_initial_condition[..., 2] += onedarray[None, :, None, None]
self.write_par(
particle_ic = None)
- if self.dns_type in ['NSVEparticles', 'NSVEparticlesP2P', 'NSVEparticles_no_output']:
+ if self.dns_type in ['NSVEparticles', 'NSVEcomplex_particles', 'NSVEparticles_no_output']:
self.generate_particle_data(opt = opt)
self.run(
nb_processes = opt.nb_processes,
diff --git a/bfps/cpp/full_code/NSVEcomplex_particles.cpp b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
index a6b7082e..93d0edc1 100644
--- a/bfps/cpp/full_code/NSVEcomplex_particles.cpp
+++ b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
@@ -1,13 +1,13 @@
#include <string>
#include <cmath>
-#include "NSVEparticlesP2P.hpp"
+#include "NSVEcomplex_particles.hpp"
#include "scope_timer.hpp"
#include "particles/particles_sampling.hpp"
#include "particles/p2p_computer.hpp"
#include "particles/particles_inner_computer.hpp"
template <typename rnumber>
-int NSVEparticlesP2P<rnumber>::initialize(void)
+int NSVEcomplex_particles<rnumber>::initialize(void)
{
this->NSVE<rnumber>::initialize();
@@ -50,14 +50,18 @@ int NSVEparticlesP2P<rnumber>::initialize(void)
(this->simname + "_particles.h5"),
"tracers0",
"position/0");
- // TODO: remove the following testing initial condition, and use a proper
- // way to initialize with 0 (i.e. generate a 0 field as the initial condition).
- //*this->fs->cvorticity = 0.0;
+
+
+ /// allocate grad vel field
+ this->nabla_u = new field<rnumber, FFTW, THREExTHREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ this->fs->cvorticity->fftw_plan_rigor);
return EXIT_SUCCESS;
}
template <typename rnumber>
-int NSVEparticlesP2P<rnumber>::step(void)
+int NSVEcomplex_particles<rnumber>::step(void)
{
this->fs->compute_velocity(this->fs->cvorticity);
this->fs->cvelocity->ift();
@@ -74,7 +78,7 @@ int NSVEparticlesP2P<rnumber>::step(void)
}
template <typename rnumber>
-int NSVEparticlesP2P<rnumber>::write_checkpoint(void)
+int NSVEcomplex_particles<rnumber>::write_checkpoint(void)
{
this->NSVE<rnumber>::write_checkpoint();
this->particles_output_writer_mpi->open_file(this->fs->get_current_fname());
@@ -90,8 +94,9 @@ int NSVEparticlesP2P<rnumber>::write_checkpoint(void)
}
template <typename rnumber>
-int NSVEparticlesP2P<rnumber>::finalize(void)
+int NSVEcomplex_particles<rnumber>::finalize(void)
{
+ delete this->nabla_u;
delete this->particles_output_writer_mpi;
delete this->particles_sample_writer_mpi;
this->NSVE<rnumber>::finalize();
@@ -102,7 +107,7 @@ int NSVEparticlesP2P<rnumber>::finalize(void)
*/
template <typename rnumber>
-int NSVEparticlesP2P<rnumber>::do_stats()
+int NSVEcomplex_particles<rnumber>::do_stats()
{
/// perform fluid stats
this->NSVE<rnumber>::do_stats();
@@ -115,6 +120,7 @@ int NSVEparticlesP2P<rnumber>::do_stats()
/// allocate temporary data array
std::unique_ptr<double[]> pdata0 = this->ps->extractParticlesState(0, 3);
std::unique_ptr<double[]> pdata1 = this->ps->extractParticlesState(3, 6);
+ std::unique_ptr<double[]> pdata2(new double[9*this->ps->getLocalNbParticles()]);
/// sample position
this->particles_sample_writer_mpi->template save_dataset<3>(
@@ -147,15 +153,20 @@ int NSVEparticlesP2P<rnumber>::do_stats()
this->ps->getLocalNbParticles(),
this->ps->get_step_idx()-1);
- /// sample vorticity
- *this->tmp_vec_field = this->fs->cvorticity->get_cdata();
- this->tmp_vec_field->ift();
- this->ps->sample_compute_field(*this->tmp_vec_field, pdata1.get());
- this->particles_sample_writer_mpi->template save_dataset<3>(
+ /// sample velocity gradient
+ /// fs->cvelocity should contain the velocity in Fourier space
+ this->fs->compute_velocity(this->fs->cvorticity);
+ compute_gradient(
+ this->fs->kk,
+ this->fs->cvelocity,
+ this->nabla_u);
+ this->nabla_u->ift();
+ this->ps->sample_compute_field(*this->nabla_u, pdata2.get());
+ this->particles_sample_writer_mpi->template save_dataset<9>(
"tracers0",
- "vorticity",
+ "velocity_gradient",
pdata0.get(),
- &pdata1,
+ &pdata2,
this->ps->getParticlesIndexes(),
this->ps->getLocalNbParticles(),
this->ps->get_step_idx()-1);
@@ -176,6 +187,6 @@ int NSVEparticlesP2P<rnumber>::do_stats()
return EXIT_SUCCESS;
}
-template class NSVEparticlesP2P<float>;
-template class NSVEparticlesP2P<double>;
+template class NSVEcomplex_particles<float>;
+template class NSVEcomplex_particles<double>;
diff --git a/bfps/cpp/full_code/NSVEcomplex_particles.hpp b/bfps/cpp/full_code/NSVEcomplex_particles.hpp
index b74169f5..2015ec5b 100644
--- a/bfps/cpp/full_code/NSVEcomplex_particles.hpp
+++ b/bfps/cpp/full_code/NSVEcomplex_particles.hpp
@@ -24,8 +24,8 @@
-#ifndef NSVEPARTICLESP2P_HPP
-#define NSVEPARTICLESP2P_HPP
+#ifndef NSVECOMPLEX_PARTICLES_HPP
+#define NSVECOMPLEX_PARTICLES_HPP
@@ -37,15 +37,18 @@
#include "particles/particles_output_hdf5.hpp"
#include "particles/particles_sampling.hpp"
-/** \brief Navier-Stokes solver that includes simple Lagrangian tracers.
+/** \brief Navier-Stokes solver that includes complex particles.
*
* Child of Navier Stokes vorticity equation solver, this class calls all the
- * methods from `NSVE`, and in addition integrates simple Lagrangian tracers
+ * methods from `NSVE`, and in addition integrates `complex particles`
* in the resulting velocity field.
+ * By `complex particles` we mean neutrally buoyant, very small particles,
+ * which have an orientation and actively swim in that direction, and they may
+ * also interact with each other, trying to reorient to a common orientation.
*/
template <typename rnumber>
-class NSVEparticlesP2P: public NSVE<rnumber>
+class NSVEcomplex_particles: public NSVE<rnumber>
{
public:
@@ -67,16 +70,18 @@ class NSVEparticlesP2P: public NSVE<rnumber>
// TODO P2P use a reader with particle data
particles_output_hdf5<long long int, double,6> *particles_output_writer_mpi;
particles_output_sampling_hdf5<long long int, double, 3> *particles_sample_writer_mpi;
+ // field for sampling velocity gradient
+ field<rnumber, FFTW, THREExTHREE> *nabla_u;
- NSVEparticlesP2P(
+ NSVEcomplex_particles(
const MPI_Comm COMMUNICATOR,
const std::string &simulation_name):
NSVE<rnumber>(
COMMUNICATOR,
simulation_name),
cutoff(10), inner_v0(1), enable_p2p(true), enable_inner(true), enable_vorticity_omega(true){}
- ~NSVEparticlesP2P(){}
+ ~NSVEcomplex_particles(){}
int initialize(void);
int step(void);
@@ -87,5 +92,5 @@ class NSVEparticlesP2P: public NSVE<rnumber>
int do_stats(void);
};
-#endif//NSVEPARTICLESP2P_HPP
+#endif//NSVECOMPLEX_PARTICLES_HPP
diff --git a/bfps/test/test_particles.py b/bfps/test/test_particles.py
index 40c428bd..6c12fac1 100644
--- a/bfps/test/test_particles.py
+++ b/bfps/test/test_particles.py
@@ -20,7 +20,7 @@ def main():
if sys.argv[2] == 'on':
c = DNS()
c.launch(
- ['NSVEparticlesP2P',
+ ['NSVEcomplex_particles',
'-n', '32',
'--src-simname', 'B32p1e4',
'--src-wd', bfps.lib_dir + '/test',
diff --git a/setup.py b/setup.py
index a160ce4b..ff73b945 100644
--- a/setup.py
+++ b/setup.py
@@ -88,7 +88,7 @@ print('This is bfps version ' + VERSION)
### lists of files and MANIFEST.in
-src_file_list = ['full_code/NSVEparticlesP2P',
+src_file_list = ['full_code/NSVEcomplex_particles',
'full_code/joint_acc_vel_stats',
'full_code/test',
'full_code/filter_test',
--
GitLab
From 33103bb86a9370eab731f00fdefa51ee4653969a Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 27 Feb 2018 12:19:41 +0100
Subject: [PATCH 123/342] update test to work without matplotlib
---
bfps/test/test_particles.py | 163 +++++++++++++++++++-----------------
1 file changed, 84 insertions(+), 79 deletions(-)
diff --git a/bfps/test/test_particles.py b/bfps/test/test_particles.py
index 6c12fac1..0179c86f 100644
--- a/bfps/test/test_particles.py
+++ b/bfps/test/test_particles.py
@@ -8,7 +8,11 @@ import sys
import bfps
from bfps import DNS
-import matplotlib.pyplot as plt
+try:
+ import matplotlib.pyplot as plt
+ matplotlib_on = True
+except: ImportError:
+ matplotlib_on = False
def main():
@@ -43,84 +47,85 @@ def main():
pf = h5py.File(
'test_particles.h5',
'r')
- # initial condition:
- # show a histogram of the orientations
- f = plt.figure()
- a = f.add_subplot(111)
- for iteration in range(1):
- x = cf['tracers0/state/{0}'.format(iteration)][:, 3:]
- hist, bins = np.histogram(
- np.sum(x**2, axis = -1).flatten()**.5,
- bins = np.linspace(0, 2, 40))
- bb = (bins[:-1] + bins[1:])/2
- pp = hist.astype(np.float) / (np.sum(hist) * (bb[1] - bb[0]))
- a.plot(bb, pp, label = '{0}'.format(iteration))
- a.legend(loc = 'best')
- f.tight_layout()
- f.savefig('orientation_histogram.pdf')
- plt.close(f)
- # show a histogram of the positions
- f = plt.figure()
- a = f.add_subplot(111)
- for iteration in range(0, niterations*njobs+1, niterations//2):
- x = pf['tracers0/position/{0}'.format(iteration)].value
- hist, bins = np.histogram(
- np.sum(x**2, axis = -1).flatten()**.5,
- bins = 40)
- bb = (bins[:-1] + bins[1:])/2
- pp = hist.astype(np.float) / (np.sum(hist) * (bb[1] - bb[0]))
- a.plot(bb, pp, label = '{0}'.format(iteration))
- a.legend(loc = 'best')
- f.tight_layout()
- f.savefig('position_histogram.pdf')
- plt.close(f)
- # show a histogram of the orientations
- f = plt.figure()
- a = f.add_subplot(111)
- for iteration in range(0, niterations*njobs+1, niterations//2):
- x = pf['tracers0/orientation/{0}'.format(iteration)].value
- hist, bins = np.histogram(
- np.sum(x**2, axis = -1).flatten()**.5,
- bins = np.linspace(0, 2, 40))
- bb = (bins[:-1] + bins[1:])/2
- pp = hist.astype(np.float) / (np.sum(hist) * (bb[1] - bb[0]))
- a.plot(bb, pp, label = '{0}'.format(iteration))
- a.legend(loc = 'best')
- f.tight_layout()
- f.savefig('orientation_histogram.pdf')
- plt.close(f)
- # compared sampled positions with checkpoint positions
- for iteration in range(0, niterations*njobs+1, niterations):
- x = pf['tracers0/position/{0}'.format(iteration)].value
- s = cf['tracers0/state/{0}'.format(iteration)].value
- distance = (np.max(np.abs(x - s[..., :3]) /
- np.maximum(np.ones(x.shape),
- np.maximum(np.abs(x),
- np.abs(s[..., :3])))))
- assert(distance < 1e-14)
- x = pf['tracers0/orientation/{0}'.format(iteration)].value
- distance = (np.max(np.abs(x - s[..., 3:]) /
- np.maximum(np.ones(x.shape),
- np.maximum(np.abs(x),
- np.abs(s[..., 3:])))))
- assert(distance < 1e-14)
- # code relevant when velocity field is 0 everywhere.
- # we check to see what happens to the orientation of the particles
- # show a histogram of the orientations
- f = plt.figure()
- a = f.add_subplot(111)
- for iteration in range(0, niterations*njobs+1, niterations//4):
- x = pf['tracers0/orientation/{0}'.format(iteration)].value
- hist, bins = np.histogram(
- x.flatten(),
- bins = 100)
- bb = (bins[:-1] + bins[1:])/2
- pp = hist.astype(np.float) / (np.sum(hist) * (bb[1] - bb[0]))
- a.plot(bb, pp, label = '{0}'.format(iteration))
- a.legend(loc = 'best')
- f.tight_layout()
- f.savefig('full_orientation_histogram.pdf')
- plt.close(f)
+ if matplotlib_on:
+ # initial condition:
+ # show a histogram of the orientations
+ f = plt.figure()
+ a = f.add_subplot(111)
+ for iteration in range(1):
+ x = cf['tracers0/state/{0}'.format(iteration)][:, 3:]
+ hist, bins = np.histogram(
+ np.sum(x**2, axis = -1).flatten()**.5,
+ bins = np.linspace(0, 2, 40))
+ bb = (bins[:-1] + bins[1:])/2
+ pp = hist.astype(np.float) / (np.sum(hist) * (bb[1] - bb[0]))
+ a.plot(bb, pp, label = '{0}'.format(iteration))
+ a.legend(loc = 'best')
+ f.tight_layout()
+ f.savefig('orientation_histogram.pdf')
+ plt.close(f)
+ # show a histogram of the positions
+ f = plt.figure()
+ a = f.add_subplot(111)
+ for iteration in range(0, niterations*njobs+1, niterations//2):
+ x = pf['tracers0/position/{0}'.format(iteration)].value
+ hist, bins = np.histogram(
+ np.sum(x**2, axis = -1).flatten()**.5,
+ bins = 40)
+ bb = (bins[:-1] + bins[1:])/2
+ pp = hist.astype(np.float) / (np.sum(hist) * (bb[1] - bb[0]))
+ a.plot(bb, pp, label = '{0}'.format(iteration))
+ a.legend(loc = 'best')
+ f.tight_layout()
+ f.savefig('position_histogram.pdf')
+ plt.close(f)
+ # show a histogram of the orientations
+ f = plt.figure()
+ a = f.add_subplot(111)
+ for iteration in range(0, niterations*njobs+1, niterations//2):
+ x = pf['tracers0/orientation/{0}'.format(iteration)].value
+ hist, bins = np.histogram(
+ np.sum(x**2, axis = -1).flatten()**.5,
+ bins = np.linspace(0, 2, 40))
+ bb = (bins[:-1] + bins[1:])/2
+ pp = hist.astype(np.float) / (np.sum(hist) * (bb[1] - bb[0]))
+ a.plot(bb, pp, label = '{0}'.format(iteration))
+ a.legend(loc = 'best')
+ f.tight_layout()
+ f.savefig('orientation_histogram.pdf')
+ plt.close(f)
+ # compared sampled positions with checkpoint positions
+ for iteration in range(0, niterations*njobs+1, niterations):
+ x = pf['tracers0/position/{0}'.format(iteration)].value
+ s = cf['tracers0/state/{0}'.format(iteration)].value
+ distance = (np.max(np.abs(x - s[..., :3]) /
+ np.maximum(np.ones(x.shape),
+ np.maximum(np.abs(x),
+ np.abs(s[..., :3])))))
+ assert(distance < 1e-14)
+ x = pf['tracers0/orientation/{0}'.format(iteration)].value
+ distance = (np.max(np.abs(x - s[..., 3:]) /
+ np.maximum(np.ones(x.shape),
+ np.maximum(np.abs(x),
+ np.abs(s[..., 3:])))))
+ assert(distance < 1e-14)
+ # code relevant when velocity field is 0 everywhere.
+ # we check to see what happens to the orientation of the particles
+ # show a histogram of the orientations
+ f = plt.figure()
+ a = f.add_subplot(111)
+ for iteration in range(0, niterations*njobs+1, niterations//4):
+ x = pf['tracers0/orientation/{0}'.format(iteration)].value
+ hist, bins = np.histogram(
+ x.flatten(),
+ bins = 100)
+ bb = (bins[:-1] + bins[1:])/2
+ pp = hist.astype(np.float) / (np.sum(hist) * (bb[1] - bb[0]))
+ a.plot(bb, pp, label = '{0}'.format(iteration))
+ a.legend(loc = 'best')
+ f.tight_layout()
+ f.savefig('full_orientation_histogram.pdf')
+ plt.close(f)
return None
if __name__ == '__main__':
--
GitLab
From f547b5fa6b6acecdb0e11146ce07bd061c7039b8 Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Tue, 27 Feb 2018 13:03:15 +0100
Subject: [PATCH 124/342] Ensure that the pre-allocated buffers are enlarged
when the size of rhs increase in particle output
---
.../particles/abstract_particles_output.hpp | 52 ++++++++++++++-----
1 file changed, 39 insertions(+), 13 deletions(-)
diff --git a/bfps/cpp/particles/abstract_particles_output.hpp b/bfps/cpp/particles/abstract_particles_output.hpp
index 7510bc6e..cf71c99e 100644
--- a/bfps/cpp/particles/abstract_particles_output.hpp
+++ b/bfps/cpp/particles/abstract_particles_output.hpp
@@ -28,11 +28,13 @@ class abstract_particles_output {
std::unique_ptr<real_number[]> buffer_particles_positions_send;
std::vector<std::unique_ptr<real_number[]>> buffer_particles_rhs_send;
partsize_t size_buffers_send;
+ int buffers_size_particle_rhs_send;
std::unique_ptr<real_number[]> buffer_particles_positions_recv;
std::vector<std::unique_ptr<real_number[]>> buffer_particles_rhs_recv;
std::unique_ptr<partsize_t[]> buffer_indexes_recv;
- partsize_t size_buffers_recv;
+ partsize_t size_buffers_recv;
+ int buffers_size_particle_rhs_recv;
int nb_processes_involved;
bool current_is_involved;
@@ -66,7 +68,9 @@ public:
: mpi_com(in_mpi_com), my_rank(-1), nb_processes(-1),
total_nb_particles(inTotalNbParticles), nb_rhs(in_nb_rhs),
buffer_particles_rhs_send(in_nb_rhs), size_buffers_send(-1),
+ buffers_size_particle_rhs_send(-1),
buffer_particles_rhs_recv(in_nb_rhs), size_buffers_recv(-1),
+ buffers_size_particle_rhs_recv(-1),
nb_processes_involved(0), current_is_involved(true), particles_chunk_per_process(0),
particles_chunk_current_size(0), particles_chunk_current_offset(0) {
@@ -143,6 +147,8 @@ public:
buffer_particles_rhs_send[idx_rhs].release();
buffer_particles_rhs_recv[idx_rhs].release();
}
+ buffers_size_particle_rhs_send = -1;
+ buffers_size_particle_rhs_recv = -1;
}
template <int size_particle_rhs>
@@ -159,12 +165,22 @@ public:
TIMEZONE("sort-to-distribute");
if(size_buffers_send < nb_particles && nb_particles){
- buffer_indexes_send.reset(new std::pair<partsize_t,partsize_t>[nb_particles]);
- buffer_particles_positions_send.reset(new real_number[nb_particles*size_particle_positions]);
+ size_buffers_send = nb_particles;
+ buffer_indexes_send.reset(new std::pair<partsize_t,partsize_t>[size_buffers_send]);
+ buffer_particles_positions_send.reset(new real_number[size_buffers_send*size_particle_positions]);
+
+ if(buffers_size_particle_rhs_send < size_particle_rhs){
+ buffers_size_particle_rhs_send = size_particle_rhs;
+ }
for(int idx_rhs = 0 ; idx_rhs < nb_rhs ; ++idx_rhs){
- buffer_particles_rhs_send[idx_rhs].reset(new real_number[nb_particles*size_particle_rhs]);
+ buffer_particles_rhs_send[idx_rhs].reset(new real_number[size_buffers_send*buffers_size_particle_rhs_send]);
+ }
+ }
+ else if(buffers_size_particle_rhs_send < size_particle_rhs){
+ buffers_size_particle_rhs_send = size_particle_rhs;
+ for(int idx_rhs = 0 ; idx_rhs < nb_rhs ; ++idx_rhs){
+ buffer_particles_rhs_send[idx_rhs].reset(new real_number[size_buffers_send*buffers_size_particle_rhs_send]);
}
- size_buffers_send = nb_particles;
}
for(partsize_t idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
@@ -210,12 +226,22 @@ public:
assert(nb_to_receive == particles_chunk_current_size);
if(size_buffers_recv < nb_to_receive && nb_to_receive){
- buffer_indexes_recv.reset(new partsize_t[nb_to_receive]);
- buffer_particles_positions_recv.reset(new real_number[nb_to_receive*size_particle_positions]);
+ size_buffers_recv = nb_to_receive;
+ buffer_indexes_recv.reset(new partsize_t[size_buffers_recv]);
+ buffer_particles_positions_recv.reset(new real_number[size_buffers_recv*size_particle_positions]);
+
+ if(buffers_size_particle_rhs_recv < size_particle_rhs){
+ buffers_size_particle_rhs_recv = size_particle_rhs;
+ }
for(int idx_rhs = 0 ; idx_rhs < nb_rhs ; ++idx_rhs){
- buffer_particles_rhs_recv[idx_rhs].reset(new real_number[nb_to_receive*size_particle_rhs]);
+ buffer_particles_rhs_recv[idx_rhs].reset(new real_number[size_buffers_recv*buffers_size_particle_rhs_recv]);
}
- size_buffers_recv = nb_to_receive;
+ }
+ else if(buffers_size_particle_rhs_recv < size_particle_rhs){
+ buffers_size_particle_rhs_recv = size_particle_rhs;
+ for(int idx_rhs = 0 ; idx_rhs < nb_rhs ; ++idx_rhs){
+ buffer_particles_rhs_recv[idx_rhs].reset(new real_number[size_buffers_recv*buffers_size_particle_rhs_recv]);
+ }
}
{
@@ -235,12 +261,12 @@ public:
}
if(size_buffers_send < nb_to_receive && nb_to_receive){
- buffer_indexes_send.reset(new std::pair<partsize_t,partsize_t>[nb_to_receive]);
- buffer_particles_positions_send.reset(new real_number[nb_to_receive*size_particle_positions]);
+ size_buffers_send = nb_to_receive;
+ buffer_indexes_send.reset(new std::pair<partsize_t,partsize_t>[size_buffers_send]);
+ buffer_particles_positions_send.reset(new real_number[size_buffers_send*size_particle_positions]);
for(int idx_rhs = 0 ; idx_rhs < nb_rhs ; ++idx_rhs){
- buffer_particles_rhs_send[idx_rhs].reset(new real_number[nb_to_receive*size_particle_rhs]);
+ buffer_particles_rhs_send[idx_rhs].reset(new real_number[size_buffers_send*buffers_size_particle_rhs_send]);
}
- size_buffers_send = nb_to_receive;
}
{
--
GitLab
From 757482a3ad2d66b4ec916076c1f133a9680107f2 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 27 Feb 2018 13:07:18 +0100
Subject: [PATCH 125/342] fix except syntax error
---
bfps/test/test_particles.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bfps/test/test_particles.py b/bfps/test/test_particles.py
index 0179c86f..6d3abec1 100644
--- a/bfps/test/test_particles.py
+++ b/bfps/test/test_particles.py
@@ -11,7 +11,7 @@ from bfps import DNS
try:
import matplotlib.pyplot as plt
matplotlib_on = True
-except: ImportError:
+except ImportError:
matplotlib_on = False
--
GitLab
From bfa2e9e69449b9e734cd138a50a7165e12b205c6 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 27 Feb 2018 13:27:54 +0100
Subject: [PATCH 126/342] add alternate random field generator
---
bfps/tools.py | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/bfps/tools.py b/bfps/tools.py
index 69756ec6..9d39e845 100644
--- a/bfps/tools.py
+++ b/bfps/tools.py
@@ -143,6 +143,19 @@ def generate_data_3D(
a[ii] = 0
return a
+
+def generate_random_discontinuous_data_3D(
+ n0, n1, n2,
+ dtype = np.complex128,
+ p = 1.5,
+ amplitude = 0.5):
+ """returns the Fourier representation of a random field.
+ """
+ assert(n0 % 2 == 0 and n1 % 2 == 0 and n2 % 2 == 0)
+ a = np.random.randn(n1, n0, n2)
+ b = np.fft.rfftn(a).astype(dtype)
+ return b
+
def randomize_phases(v):
"""randomize the phases of an FFTW complex field.
--
GitLab
From 859a9d873acb4e2f71bc75f214b89ffbeb862242 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 27 Feb 2018 13:28:38 +0100
Subject: [PATCH 127/342] fix drag term interval
---
bfps/cpp/vorticity_equation.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bfps/cpp/vorticity_equation.cpp b/bfps/cpp/vorticity_equation.cpp
index 7f1e3203..cfffc26c 100644
--- a/bfps/cpp/vorticity_equation.cpp
+++ b/bfps/cpp/vorticity_equation.cpp
@@ -376,7 +376,7 @@ void vorticity_equation<rnumber, be>::add_forcing(
if (strcmp(this->forcing_type, "fixed_energy_injection_rate_and_drag") == 0)
this->add_field_band(
dst, vort_field,
- this->fmode, this->fmode + (this->fk0 - this->fk1),
+ this->fmode, this->fmode + (this->fk1 - this->fk0),
-this->friction_coefficient);
return;
}
--
GitLab
From 463f16fe5d7ff7ed3fed8b57643f566a23d1c975 Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Tue, 27 Feb 2018 13:42:38 +0100
Subject: [PATCH 128/342] Update the allocation of the buffer in parallel
---
bfps/cpp/particles/abstract_particles_output.hpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/bfps/cpp/particles/abstract_particles_output.hpp b/bfps/cpp/particles/abstract_particles_output.hpp
index cf71c99e..7c25502c 100644
--- a/bfps/cpp/particles/abstract_particles_output.hpp
+++ b/bfps/cpp/particles/abstract_particles_output.hpp
@@ -176,7 +176,7 @@ public:
buffer_particles_rhs_send[idx_rhs].reset(new real_number[size_buffers_send*buffers_size_particle_rhs_send]);
}
}
- else if(buffers_size_particle_rhs_send < size_particle_rhs){
+ else if(buffers_size_particle_rhs_send < size_particle_rhs && size_particle_rhs > 0){
buffers_size_particle_rhs_send = size_particle_rhs;
for(int idx_rhs = 0 ; idx_rhs < nb_rhs ; ++idx_rhs){
buffer_particles_rhs_send[idx_rhs].reset(new real_number[size_buffers_send*buffers_size_particle_rhs_send]);
@@ -237,7 +237,7 @@ public:
buffer_particles_rhs_recv[idx_rhs].reset(new real_number[size_buffers_recv*buffers_size_particle_rhs_recv]);
}
}
- else if(buffers_size_particle_rhs_recv < size_particle_rhs){
+ else if(buffers_size_particle_rhs_recv < size_particle_rhs && size_buffers_recv > 0){
buffers_size_particle_rhs_recv = size_particle_rhs;
for(int idx_rhs = 0 ; idx_rhs < nb_rhs ; ++idx_rhs){
buffer_particles_rhs_recv[idx_rhs].reset(new real_number[size_buffers_recv*buffers_size_particle_rhs_recv]);
--
GitLab
From 04b2c5a8eb091a70c33856b651823b6e2ba39504 Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Thu, 8 Mar 2018 11:44:53 +0100
Subject: [PATCH 129/342] update scaling test for draco
---
tests/DNS/test_scaling.py | 29 ++++++++++++++++++-----------
1 file changed, 18 insertions(+), 11 deletions(-)
diff --git a/tests/DNS/test_scaling.py b/tests/DNS/test_scaling.py
index 1d4b12a5..99ad5e33 100644
--- a/tests/DNS/test_scaling.py
+++ b/tests/DNS/test_scaling.py
@@ -12,7 +12,11 @@ def get_DNS_parameters(
nprocesses = 1,
output_on = False,
cores_per_node = 16,
- nparticles = int(1e5)):
+ nparticles = int(1e5),
+ environment = 'express',
+ minutes = '29',
+ no_submit = True):
+ assert (N in [1024, 2048, 4096])
simname = (DNS_type + '{0:0>4d}'.format(N))
if output_on:
simname = DNS_type + simname
@@ -25,15 +29,14 @@ def get_DNS_parameters(
work_dir = 'nn{0:0>4d}np{1}'.format(nnodes, nprocesses)
if not output_on:
class_name += '_no_output'
- src_simname = 'N{0:0>4d}_kMeta2'.format(N)
- src_iteration = -1
- if N == 512:
- src_iteration = 3072
+ src_dirname = '/draco/ptmp/clalescu/scaling'
+ src_simname = 'fb3_N{0:0>4d}_kMeta1.5'.format(N)
if N == 1024:
- src_iteration = 0x4000
+ src_iteration = 32*1024
if N == 2048:
- src_iteration = 0x6000
+ src_iteration = 20*1024
if N == 4096:
+ src_simname = 'fb3_N2048x2_kMeta1.5'
src_iteration = 0
DNS_parameters = [
class_name,
@@ -45,9 +48,8 @@ def get_DNS_parameters(
'--niter_todo', '12',
'--niter_out', '12',
'--niter_stat', '3']
- if src_iteration >= 0:
- DNS_parameters += [
- '--src-wd', 'database',
+ DNS_parameters += [
+ '--src-wd', src_dirname,
'--src-simname', src_simname,
'--src-iteration', '{0}'.format(src_iteration)]
if DNS_type != 'A':
@@ -63,6 +65,8 @@ def get_DNS_parameters(
'--tracers0_neighbours', '{0}'.format(nneighbours),
'--tracers0_smoothness', '{0}'.format(smoothness),
'--particle-rand-seed', '2']
+ if no_submit:
+ DNS_parameters += ['--no-submit']
return simname, work_dir, DNS_parameters
def main():
@@ -86,18 +90,21 @@ def main():
parser.add_argument(
'--nnodes',
type = int,
+ help = 'how many nodes to use',
dest = 'nnodes',
default = 1)
parser.add_argument(
'--nprocesses',
type = int,
+ help = 'how many MPI processes to use',
dest = 'nprocesses',
default = 1)
parser.add_argument(
'--ncores',
type = int,
+ help = 'how many cores there are per node',
dest = 'ncores',
- default = 4)
+ default = 40)
parser.add_argument(
'--output-on',
action = 'store_true',
--
GitLab
From 3796d47998063f2e59a3659bdb03dba935f43938 Mon Sep 17 00:00:00 2001
From: Cristian Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 8 Mar 2018 11:54:29 +0100
Subject: [PATCH 130/342] further tweaks to scaling test
---
tests/DNS/test_scaling.py | 22 +++++++++++++++++++++-
1 file changed, 21 insertions(+), 1 deletion(-)
diff --git a/tests/DNS/test_scaling.py b/tests/DNS/test_scaling.py
index 99ad5e33..e24957d4 100644
--- a/tests/DNS/test_scaling.py
+++ b/tests/DNS/test_scaling.py
@@ -67,6 +67,8 @@ def get_DNS_parameters(
'--particle-rand-seed', '2']
if no_submit:
DNS_parameters += ['--no-submit']
+ DNS_parameters += ['--environment', environment,
+ '--minutes', '{0}'.format(minutes)]
return simname, work_dir, DNS_parameters
def main():
@@ -109,11 +111,26 @@ def main():
'--output-on',
action = 'store_true',
dest = 'output_on')
+ parser.add_argument(
+ '--submit',
+ action = 'store_true',
+ dest = 'submit')
parser.add_argument(
'--nparticles',
type = int,
dest = 'nparticles',
default = int(1e5))
+ parser.add_argument(
+ '--environment',
+ type = str,
+ dest = 'environment',
+ default = 'express')
+ parser.add_argument(
+ '--minutes',
+ type = int,
+ dest = 'minutes',
+ default = 29,
+ help = 'If environment supports it, this is the requested wall-clock-limit.')
opt = parser.parse_args(sys.argv[1:])
simname, work_dir, params = get_DNS_parameters(
DNS_type = opt.DNS_setup,
@@ -122,7 +139,10 @@ def main():
nprocesses = opt.nprocesses,
output_on = opt.output_on,
nparticles = opt.nparticles,
- cores_per_node = opt.ncores)
+ cores_per_node = opt.ncores,
+ no_submit = not opt.submit,
+ minutes = opt.minutes,
+ environment = opt.environment)
print(work_dir + '/' + simname)
print(' '.join(params))
# these following 2 lines actually launch something
--
GitLab
From 8c77e4413541b524d8acc46c2a9917eb83dab80b Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Fri, 9 Mar 2018 10:40:34 +0100
Subject: [PATCH 131/342] Add param to test scaling to pass database directory
---
tests/DNS/test_scaling.py | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
diff --git a/tests/DNS/test_scaling.py b/tests/DNS/test_scaling.py
index e24957d4..3ae1d299 100644
--- a/tests/DNS/test_scaling.py
+++ b/tests/DNS/test_scaling.py
@@ -15,7 +15,8 @@ def get_DNS_parameters(
nparticles = int(1e5),
environment = 'express',
minutes = '29',
- no_submit = True):
+ no_submit = True,
+ src_dirname = '/draco/ptmp/clalescu/scaling'):
assert (N in [1024, 2048, 4096])
simname = (DNS_type + '{0:0>4d}'.format(N))
if output_on:
@@ -29,7 +30,6 @@ def get_DNS_parameters(
work_dir = 'nn{0:0>4d}np{1}'.format(nnodes, nprocesses)
if not output_on:
class_name += '_no_output'
- src_dirname = '/draco/ptmp/clalescu/scaling'
src_simname = 'fb3_N{0:0>4d}_kMeta1.5'.format(N)
if N == 1024:
src_iteration = 32*1024
@@ -131,6 +131,11 @@ def main():
dest = 'minutes',
default = 29,
help = 'If environment supports it, this is the requested wall-clock-limit.')
+ parser.add_argument(
+ '--src-wd',
+ type = str,
+ dest = 'src_dirname',
+ default = '/draco/ptmp/clalescu/scaling')
opt = parser.parse_args(sys.argv[1:])
simname, work_dir, params = get_DNS_parameters(
DNS_type = opt.DNS_setup,
@@ -142,7 +147,8 @@ def main():
cores_per_node = opt.ncores,
no_submit = not opt.submit,
minutes = opt.minutes,
- environment = opt.environment)
+ environment = opt.environment,
+ src_dirname = opt.src_dirname)
print(work_dir + '/' + simname)
print(' '.join(params))
# these following 2 lines actually launch something
--
GitLab
From 15b61bf5a3430f61f6f4df0a66ea859f381537d9 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 19 Mar 2018 17:07:05 +0100
Subject: [PATCH 132/342] use read_value in postprocess
---
bfps/cpp/full_code/postprocess.cpp | 82 +++++++++++-------------------
1 file changed, 30 insertions(+), 52 deletions(-)
diff --git a/bfps/cpp/full_code/postprocess.cpp b/bfps/cpp/full_code/postprocess.cpp
index c48bcdb8..b7193bee 100644
--- a/bfps/cpp/full_code/postprocess.cpp
+++ b/bfps/cpp/full_code/postprocess.cpp
@@ -39,58 +39,36 @@ int postprocess::read_parameters()
char *string_data;
sprintf(fname, "%s.h5", this->simname.c_str());
parameter_file = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT);
- dset = H5Dopen(parameter_file, "/parameters/dealias_type", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->dealias_type);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/dkx", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->dkx);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/dky", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->dky);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/dkz", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->dkz);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/dt", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->dt);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/famplitude", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->famplitude);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/friction_coefficient", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->friction_coefficient);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/fk0", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->fk0);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/fk1", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->fk1);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/fmode", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->fmode);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/forcing_type", H5P_DEFAULT);
- space = H5Dget_space(dset);
- memtype = H5Dget_type(dset);
- string_data = (char*)malloc(256);
- H5Dread(dset, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &string_data);
- sprintf(this->forcing_type, "%s", string_data);
- free(string_data);
- H5Sclose(space);
- H5Tclose(memtype);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/nu", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->nu);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/nx", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->nx);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/ny", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->ny);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/nz", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->nz);
- H5Dclose(dset);
+ this->dealias_type = hdf5_tools::read_value<int>(
+ parameter_file, "/parameters/dealias_type");
+ this->dkx = hdf5_tools::read_value<double>(
+ parameter_file, "/parameters/dkx");
+ this->dky = hdf5_tools::read_value<double>(
+ parameter_file, "/parameters/dky");
+ this->dkz = hdf5_tools::read_value<double>(
+ parameter_file, "/parameters/dkz");
+ this->dt = hdf5_tools::read_value<double>(
+ parameter_file, "/parameters/dt");
+ this->famplitude = hdf5_tools::read_value<double>(
+ parameter_file, "/parameters/famplitude");
+ this->friction_coefficient = hdf5_tools::read_value<double>(
+ parameter_file, "/parameters/friction_coefficient");
+ this->fk0 = hdf5_tools::read_value<double>(
+ parameter_file, "/parameters/fk0");
+ this->fk1 = hdf5_tools::read_value<double>(
+ parameter_file, "/parameters/fk1");
+ this->fmode = hdf5_tools::read_value<int>(
+ parameter_file, "/parameters/fmode");
+ sprintf(this->forcing_type, "%s",
+ hdf5_tools::read_string(parameter_file, "/parameters/forcing_type").c_str());
+ this->nu = hdf5_tools::read_value<double>(
+ parameter_file, "/parameters/nu");
+ this->nx = hdf5_tools::read_value<int>(
+ parameter_file, "/parameters/nx");
+ this->ny = hdf5_tools::read_value<int>(
+ parameter_file, "/parameters/ny");
+ this->nz = hdf5_tools::read_value<int>(
+ parameter_file, "/parameters/nz");
H5Fclose(parameter_file);
return 0;
}
--
GitLab
From c4605fbfba3da7d5320e2a7fc9d2445d0e2b894c Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 19 Mar 2018 17:10:21 +0100
Subject: [PATCH 133/342] remove unused variables
---
bfps/cpp/full_code/postprocess.cpp | 2 --
1 file changed, 2 deletions(-)
diff --git a/bfps/cpp/full_code/postprocess.cpp b/bfps/cpp/full_code/postprocess.cpp
index b7193bee..cb04a030 100644
--- a/bfps/cpp/full_code/postprocess.cpp
+++ b/bfps/cpp/full_code/postprocess.cpp
@@ -34,9 +34,7 @@ int postprocess::main_loop(void)
int postprocess::read_parameters()
{
hid_t parameter_file;
- hid_t dset, memtype, space;
char fname[256];
- char *string_data;
sprintf(fname, "%s.h5", this->simname.c_str());
parameter_file = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT);
this->dealias_type = hdf5_tools::read_value<int>(
--
GitLab
From 3291590d1a1cf8a47a48d2343a0839d6d0554a8a Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Tue, 27 Mar 2018 22:24:41 +0200
Subject: [PATCH 134/342] add field output test
---
bfps/TEST.py | 6 +++
bfps/cpp/full_code/field_output_test.cpp | 63 ++++++++++++++++++++++++
bfps/cpp/full_code/field_output_test.hpp | 60 ++++++++++++++++++++++
setup.py | 1 +
4 files changed, 130 insertions(+)
create mode 100644 bfps/cpp/full_code/field_output_test.cpp
create mode 100644 bfps/cpp/full_code/field_output_test.hpp
diff --git a/bfps/TEST.py b/bfps/TEST.py
index f7e8e24e..43b2d813 100644
--- a/bfps/TEST.py
+++ b/bfps/TEST.py
@@ -263,6 +263,12 @@ class TEST(_code):
self.simulation_parser_arguments(parser_field_test)
self.job_parser_arguments(parser_field_test)
self.parameters_to_parser_arguments(parser_field_test)
+ parser_field_output_test = subparsers.add_parser(
+ 'field_output_test',
+ help = 'plain field output test')
+ self.simulation_parser_arguments(parser_field_output_test)
+ self.job_parser_arguments(parser_field_output_test)
+ self.parameters_to_parser_arguments(parser_field_output_test)
return None
def prepare_launch(
self,
diff --git a/bfps/cpp/full_code/field_output_test.cpp b/bfps/cpp/full_code/field_output_test.cpp
new file mode 100644
index 00000000..f5440064
--- /dev/null
+++ b/bfps/cpp/full_code/field_output_test.cpp
@@ -0,0 +1,63 @@
+#include <string>
+#include <cmath>
+#include <random>
+#include "field_output_test.hpp"
+#include "scope_timer.hpp"
+
+
+template <typename rnumber>
+int field_output_test<rnumber>::initialize(void)
+{
+ this->read_parameters();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int field_output_test<rnumber>::finalize(void)
+{
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int field_output_test<rnumber>::read_parameters()
+{
+ this->test::read_parameters();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int field_output_test<rnumber>::do_work(void)
+{
+ // allocate
+ field<rnumber, FFTW, ONE> *scal_field = new field<rnumber, FFTW, ONE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ DEFAULT_FFTW_FLAG);
+ std::default_random_engine rgen;
+ std::normal_distribution<rnumber> rdist;
+ rgen.seed(1);
+
+ // fill up scal_field
+ scal_field->real_space_representation = true;
+ scal_field->RLOOP(
+ [&](ptrdiff_t rindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex){
+ scal_field->rval(rindex) = rdist(rgen);
+ });
+
+ scal_field->io(
+ this->simname + std::string("_fields.h5"),
+ "scal_field",
+ 0,
+ false);
+
+ // deallocate
+ delete scal_field;
+ return EXIT_SUCCESS;
+}
+
+template class field_output_test<float>;
+template class field_output_test<double>;
+
diff --git a/bfps/cpp/full_code/field_output_test.hpp b/bfps/cpp/full_code/field_output_test.hpp
new file mode 100644
index 00000000..3662e4b1
--- /dev/null
+++ b/bfps/cpp/full_code/field_output_test.hpp
@@ -0,0 +1,60 @@
+/**********************************************************************
+* *
+* Copyright 2017 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef FILTER_OUTPUT_TEST_HPP
+#define FILTER_OUTPUT_TEST_HPP
+
+
+
+#include <cstdlib>
+#include "base.hpp"
+#include "kspace.hpp"
+#include "field.hpp"
+#include "full_code/test.hpp"
+
+/** \brief A class for testing basic field class functionality.
+ */
+
+template <typename rnumber>
+class field_output_test: public test
+{
+ public:
+ field_output_test(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ test(
+ COMMUNICATOR,
+ simulation_name){}
+ ~field_output_test(){}
+
+ int initialize(void);
+ int do_work(void);
+ int finalize(void);
+ int read_parameters(void);
+};
+
+#endif//FILTER_OUTPUT_TEST_HPP
+
diff --git a/setup.py b/setup.py
index ff73b945..9511a92d 100644
--- a/setup.py
+++ b/setup.py
@@ -93,6 +93,7 @@ src_file_list = ['full_code/NSVEcomplex_particles',
'full_code/test',
'full_code/filter_test',
'full_code/field_test',
+ 'full_code/field_output_test',
'hdf5_tools',
'full_code/get_rfields',
'full_code/resize',
--
GitLab
From 6df4f5ac20ebef5c315a701e2088e32edd677508 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 3 Apr 2018 17:45:35 +0200
Subject: [PATCH 135/342] add test interpolation code that compiles
---
bfps/cpp/full_code/test_interpolation.cpp | 162 ++++++++++++++++++++++
bfps/cpp/full_code/test_interpolation.hpp | 53 +++++++
setup.py | 1 +
3 files changed, 216 insertions(+)
create mode 100644 bfps/cpp/full_code/test_interpolation.cpp
create mode 100644 bfps/cpp/full_code/test_interpolation.hpp
diff --git a/bfps/cpp/full_code/test_interpolation.cpp b/bfps/cpp/full_code/test_interpolation.cpp
new file mode 100644
index 00000000..e93cd3eb
--- /dev/null
+++ b/bfps/cpp/full_code/test_interpolation.cpp
@@ -0,0 +1,162 @@
+#include "full_code/test_interpolation.hpp"
+
+
+template <typename rnumber>
+int test_interpolation<rnumber>::read_parameters(void)
+{
+ this->test::read_parameters();
+ hid_t parameter_file;
+ hid_t dset, memtype, space;
+ char fname[256];
+ hsize_t dims[1];
+ char *string_data;
+ sprintf(fname, "%s.h5", this->simname.c_str());
+ this->nparticles = hdf5_tools::read_value<int>(
+ parameter_file, "/parameters/nparticles");
+ this->tracers0_integration_steps = hdf5_tools::read_value<int>(
+ parameter_file, "/parameters/tracers0_integration_steps");
+ this->tracers0_neighbours = hdf5_tools::read_value<int>(
+ parameter_file, "/parameters/tracers0_neighbours");
+ this->tracers0_smoothness = hdf5_tools::read_value<int>(
+ parameter_file, "/parameters/tracers0_smoothness");
+ H5Fclose(parameter_file);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int test_interpolation<rnumber>::initialize(void)
+{
+ this->test::initialize();
+
+ this->vorticity = new field<rnumber, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ DEFAULT_FFTW_FLAG);
+
+ this->velocity = new field<rnumber, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ DEFAULT_FFTW_FLAG);
+
+ this->nabla_u = new field<rnumber, FFTW, THREExTHREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ DEFAULT_FFTW_FLAG);
+
+ this->kk = new kspace<FFTW, SMOOTH>(
+ this->vorticity->clayout, this->dkx, this->dky, this->dkz);
+
+ if (this->myrank == 0)
+ {
+ hid_t stat_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDWR,
+ H5P_DEFAULT);
+ this->kk->store(stat_file);
+ H5Fclose(stat_file);
+ }
+
+ this->ps = particles_system_builder(
+ this->velocity, // (field object)
+ this->kk, // (kspace object, contains dkx, dky, dkz)
+ this->tracers0_integration_steps, // to check coherency between parameters and hdf input file (nb rhs)
+ (long long int)nparticles, // to check coherency between parameters and hdf input file
+ this->simname + "_input.h5", // particles input filename
+ std::string("/tracers0/state/0"), // dataset name for initial input
+ std::string("/tracers0/rhs/0") , // dataset name for initial input
+ this->tracers0_neighbours, // parameter (interpolation no neighbours)
+ this->tracers0_smoothness, // parameter
+ this->comm,
+ 1);
+ this->particles_output_writer_mpi = new particles_output_hdf5<
+ long long int, double, 3>(
+ MPI_COMM_WORLD,
+ "tracers0",
+ nparticles,
+ this->tracers0_integration_steps);
+ this->particles_sample_writer_mpi = new particles_output_sampling_hdf5<
+ long long int, double, 3>(
+ MPI_COMM_WORLD,
+ this->ps->getGlobalNbParticles(),
+ (this->simname + "_output.h5"),
+ "tracers0",
+ "position/0");
+
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int test_interpolation<rnumber>::finalize(void)
+{
+ delete this->nabla_u;
+ delete this->velocity;
+ delete this->vorticity;
+ this->ps.release();
+ this->test::finalize();
+ delete this->kk;
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int test_interpolation<rnumber>::do_work()
+{
+ std::string fname = this->simname + std::string("_input.h5");
+ // read vorticity field
+ this->vorticity->io(
+ fname,
+ "vorticity",
+ 0, true);
+
+ // compute velocity
+ invert_curl(this->kk, this->vorticity, this->velocity);
+
+ // compute velocity gradient
+ compute_gradient(this->kk, this->velocity, this->nabla_u);
+
+ // go to real space
+ this->vorticity->ift();
+ this->velocity->ift();
+ this->nabla_u->ift();
+
+ // allocate interpolation arrays
+ std::unique_ptr<double[]> p3data(new double[3*this->ps->getLocalNbParticles()]);
+ std::unique_ptr<double[]> p9data(new double[9*this->ps->getLocalNbParticles()]);
+
+ /// sample velocity at particles' position
+ this->ps->sample_compute_field(*this->velocity, p3data.get());
+ this->particles_sample_writer_mpi->template save_dataset<3>(
+ "tracers0",
+ "velocity",
+ this->ps->getParticlesState(),
+ &p9data,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+ /// sample vorticity at particles' position
+ this->ps->sample_compute_field(*this->vorticity, p3data.get());
+ this->particles_sample_writer_mpi->template save_dataset<3>(
+ "tracers0",
+ "vorticity",
+ this->ps->getParticlesState(),
+ &p9data,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+ /// sample velocity gradient at particles' position
+ this->ps->sample_compute_field(*this->nabla_u, p9data.get());
+ this->particles_sample_writer_mpi->template save_dataset<9>(
+ "tracers0",
+ "velocity_gradient",
+ this->ps->getParticlesState(),
+ &p9data,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+
+ // no need to deallocate because we used "unique_ptr"
+ return EXIT_SUCCESS;
+}
+
+template class test_interpolation<float>;
+template class test_interpolation<double>;
+
diff --git a/bfps/cpp/full_code/test_interpolation.hpp b/bfps/cpp/full_code/test_interpolation.hpp
new file mode 100644
index 00000000..e8be247c
--- /dev/null
+++ b/bfps/cpp/full_code/test_interpolation.hpp
@@ -0,0 +1,53 @@
+#ifndef TEST_INTERPOLATION_HPP
+#define TEST_INTERPOLATION_HPP
+
+
+
+#include <cstdlib>
+#include "base.hpp"
+#include "kspace.hpp"
+#include "full_code/test.hpp"
+#include "particles/particles_system_builder.hpp"
+#include "particles/particles_output_hdf5.hpp"
+#include "particles/particles_sampling.hpp"
+
+/** \brief Interpolation tester.
+ *
+ */
+
+template <typename rnumber>
+class test_interpolation: public test
+{
+ public:
+ int nparticles;
+ int tracers0_integration_steps;
+ int tracers0_neighbours;
+ int tracers0_smoothness;
+
+ std::unique_ptr<abstract_particles_system<long long int, double>> ps;
+
+ particles_output_hdf5<long long int, double,3> *particles_output_writer_mpi;
+ particles_output_sampling_hdf5<long long int, double, 3> *particles_sample_writer_mpi;
+
+ field<rnumber, FFTW, THREE> *velocity, *vorticity;
+ field<rnumber, FFTW, THREExTHREE> *nabla_u;
+
+ kspace<FFTW, SMOOTH> *kk;
+
+ test_interpolation(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ test(
+ COMMUNICATOR,
+ simulation_name){}
+ ~test_interpolation(){}
+
+ int initialize(void);
+ int do_work(void);
+ int finalize(void);
+
+ int read_parameters(void);
+};
+
+#endif//TEST_INTERPOLATION_HPP
+
diff --git a/setup.py b/setup.py
index 9511a92d..350bccf2 100644
--- a/setup.py
+++ b/setup.py
@@ -94,6 +94,7 @@ src_file_list = ['full_code/NSVEcomplex_particles',
'full_code/filter_test',
'full_code/field_test',
'full_code/field_output_test',
+ 'full_code/test_interpolation',
'hdf5_tools',
'full_code/get_rfields',
'full_code/resize',
--
GitLab
From 831dc42e15e48dc18d05cf2afb527c6b194d183c Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 19 Apr 2018 10:46:51 +0200
Subject: [PATCH 136/342] use less data in cache
---
bfps/NavierStokes.py | 59 +++++++++++++++++++++++++-------------------
1 file changed, 34 insertions(+), 25 deletions(-)
diff --git a/bfps/NavierStokes.py b/bfps/NavierStokes.py
index b7bf4ff8..598e331d 100644
--- a/bfps/NavierStokes.py
+++ b/bfps/NavierStokes.py
@@ -610,7 +610,9 @@ class NavierStokes(_fluid_particle_base):
ii1 == pp_file['ii1'].value)
if computation_needed:
for k in ['t', 'vel_max(t)', 'renergy(t)',
- 'energy(t, k)', 'enstrophy(t, k)',
+ 'energy(t)', 'enstrophy(t)',
+ 'energy(k)', 'enstrophy(k)',
+ 'R_ij(t)',
'ii0', 'ii1', 'iter0', 'iter1']:
del pp_file[k]
if computation_needed:
@@ -621,21 +623,32 @@ class NavierStokes(_fluid_particle_base):
pp_file['t'] = (self.parameters['dt']*
self.parameters['niter_stat']*
(np.arange(ii0, ii1+1).astype(np.float)))
- pp_file['energy(t, k)'] = (
- data_file['statistics/spectra/velocity_velocity'][ii0:ii1+1, :, 0, 0] +
- data_file['statistics/spectra/velocity_velocity'][ii0:ii1+1, :, 1, 1] +
- data_file['statistics/spectra/velocity_velocity'][ii0:ii1+1, :, 2, 2])/2
- pp_file['enstrophy(t, k)'] = (
+ phi_ij = data_file['statistics/spectra/velocity_velocity'][ii0:ii1+1]
+ pp_file['R_ij(t)'] = self.statistics['dk']*np.sum(phi_ij, axis = 1)
+ energy_tk = (
+ phi_ij[:, :, 0, 0] +
+ phi_ij[:, :, 1, 1] +
+ phi_ij[:, :, 2, 2])/2
+ pp_file['energy(t)'] = (self.statistics['dk'] *
+ np.sum(energy_tk, axis = 1))
+ pp_file['energy(k)'] = np.mean(energy_tk, axis = 0)
+ enstrophy_tk = (
data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 0, 0] +
data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 1, 1] +
data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 2, 2])/2
- pp_file['vel_max(t)'] = data_file['statistics/moments/velocity'] [ii0:ii1+1, 9, 3]
+ pp_file['enstrophy(t)'] = (self.statistics['dk'] *
+ np.sum(enstrophy_tk, axis = 1))
+ pp_file['enstrophy(k)'] = np.mean(enstrophy_tk, axis = 0)
+ pp_file['vel_max(t)'] = data_file['statistics/moments/velocity'][ii0:ii1+1, 9, 3]
pp_file['renergy(t)'] = data_file['statistics/moments/velocity'][ii0:ii1+1, 2, 3]/2
if 'trS2_Q_R' in data_file['statistics/moments'].keys():
pp_file['mean_trS2(t)'] = data_file['statistics/moments/trS2_Q_R'][:, 1, 0]
for k in ['t',
- 'energy(t, k)',
- 'enstrophy(t, k)',
+ 'energy(t)',
+ 'energy(k)',
+ 'enstrophy(t)',
+ 'enstrophy(k)',
+ 'R_ij(t)',
'vel_max(t)',
'renergy(t)',
'mean_trS2(t)']:
@@ -654,9 +667,9 @@ class NavierStokes(_fluid_particle_base):
.. math::
U_{\\textrm{int}}(t) = \\sqrt{\\frac{2E(t)}{3}}, \\hskip .5cm
- L_{\\textrm{int}}(t) = \\frac{\pi}{2U_{int}^2(t)} \\int \\frac{dk}{k} E(t, k), \\hskip .5cm
- T_{\\textrm{int}}(t) =
- \\frac{L_{\\textrm{int}}(t)}{U_{\\textrm{int}}(t)}
+ L_{\\textrm{int}} = \\frac{\pi}{2U_{int}^2} \\int \\frac{dk}{k} E(k), \\hskip .5cm
+ T_{\\textrm{int}} =
+ \\frac{L_{\\textrm{int}}}{U_{\\textrm{int}}}
\\eta_K = \\left(\\frac{\\nu^3}{\\varepsilon}\\right)^{1/4}, \\hskip .5cm
\\tau_K = \\left(\\frac{\\nu}{\\varepsilon}\\right)^{1/2}, \\hskip .5cm
@@ -673,22 +686,14 @@ class NavierStokes(_fluid_particle_base):
J. Fluid Mech.,
**592**, 335-366, 2007
"""
- for key in ['energy', 'enstrophy']:
- self.statistics[key + '(t)'] = (self.statistics['dk'] *
- np.sum(self.statistics[key + '(t, k)'], axis = 1))
self.statistics['Uint(t)'] = np.sqrt(2*self.statistics['energy(t)'] / 3)
- self.statistics['Lint(t)'] = ((self.statistics['dk']*np.pi /
- (2*self.statistics['Uint(t)']**2)) *
- np.nansum(self.statistics['energy(t, k)'] /
- self.statistics['kshell'][None, :], axis = 1))
for key in ['energy',
'enstrophy',
- 'vel_max',
'mean_trS2',
- 'Uint',
- 'Lint']:
+ 'Uint']:
if key + '(t)' in self.statistics.keys():
self.statistics[key] = np.average(self.statistics[key + '(t)'], axis = 0)
+ self.statistics['vel_max'] = np.max(self.statistics['vel_max(t)'])
for suffix in ['', '(t)']:
self.statistics['diss' + suffix] = (self.parameters['nu'] *
self.statistics['enstrophy' + suffix]*2)
@@ -696,9 +701,6 @@ class NavierStokes(_fluid_particle_base):
self.statistics['diss' + suffix])**.25
self.statistics['tauK' + suffix] = (self.parameters['nu'] /
self.statistics['diss' + suffix])**.5
- self.statistics['Re' + suffix] = (self.statistics['Uint' + suffix] *
- self.statistics['Lint' + suffix] /
- self.parameters['nu'])
self.statistics['lambda' + suffix] = (15 * self.parameters['nu'] *
self.statistics['Uint' + suffix]**2 /
self.statistics['diss' + suffix])**.5
@@ -709,6 +711,13 @@ class NavierStokes(_fluid_particle_base):
self.statistics['etaK' + suffix])
if self.parameters['dealias_type'] == 1:
self.statistics['kMeta' + suffix] *= 0.8
+ self.statistics['Lint'] = ((self.statistics['dk']*np.pi /
+ (2*self.statistics['Uint']**2)) *
+ np.nansum(self.statistics['energy(k)'] /
+ self.statistics['kshell']))
+ self.statistics['Re'] = (self.statistics['Uint'] *
+ self.statistics['Lint'] /
+ self.parameters['nu'])
self.statistics['Tint'] = self.statistics['Lint'] / self.statistics['Uint']
self.statistics['Taylor_microscale'] = self.statistics['lambda']
return None
--
GitLab
From f6178cb1ff5c586a56ff16acbdbd261023c59c58 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 19 Apr 2018 10:48:30 +0200
Subject: [PATCH 137/342] add method to compute Reynolds stress invariants
---
bfps/NavierStokes.py | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/bfps/NavierStokes.py b/bfps/NavierStokes.py
index 598e331d..cbd881a7 100644
--- a/bfps/NavierStokes.py
+++ b/bfps/NavierStokes.py
@@ -656,6 +656,16 @@ class NavierStokes(_fluid_particle_base):
self.statistics[k] = pp_file[k].value
self.compute_time_averages()
return None
+ def compute_Reynolds_stress_invariants(
+ self):
+ Rij = self.statistics['R_ij(t)']
+ Rij /= (2*self.statistics['energy(t)'][:, None, None])
+ Rij[:, 0, 0] -= 1./3
+ Rij[:, 1, 1] -= 1./3
+ Rij[:, 2, 2] -= 1./3
+ self.statistics['I2(t)'] = np.sqrt(np.einsum('...ij,...ij', Rij, Rij, optimize = True) / 6)
+ self.statistics['I3(t)'] = np.cbrt(np.einsum('...ij,...jk,...ki', Rij, Rij, Rij, optimize = True) / 6)
+ return None
def compute_time_averages(self):
"""Compute easy stats.
--
GitLab
From 520a53567dca02bca33cea9feab88fddfdb73bca Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 20 Apr 2018 10:39:53 +0200
Subject: [PATCH 138/342] BROKEN - code compiles and segfaults
---
bfps/TEST.py | 65 +++++++++++++++++++----
bfps/cpp/full_code/test_interpolation.cpp | 13 ++---
2 files changed, 59 insertions(+), 19 deletions(-)
diff --git a/bfps/TEST.py b/bfps/TEST.py
index 43b2d813..9a36cbfc 100644
--- a/bfps/TEST.py
+++ b/bfps/TEST.py
@@ -124,6 +124,16 @@ class TEST(_code):
self.parameters['dkz'] = float(1.0)
self.parameters['filter_length'] = float(1.0)
return None
+ def generate_extra_parameters(
+ self,
+ dns_type = None):
+ pars = {}
+ if dns_type == 'test_interpolation':
+ pars['nparticles'] = 10
+ pars['tracers0_integration_steps'] = int(4)
+ pars['tracers0_neighbours'] = int(1)
+ pars['tracers0_smoothness'] = int(1)
+ return pars
def get_kspace(self):
kspace = {}
if self.parameters['dealias_type'] == 1:
@@ -254,21 +264,25 @@ class TEST(_code):
parser_filter_test = subparsers.add_parser(
'filter_test',
help = 'plain filter test')
- self.simulation_parser_arguments(parser_filter_test)
- self.job_parser_arguments(parser_filter_test)
- self.parameters_to_parser_arguments(parser_filter_test)
parser_field_test = subparsers.add_parser(
'field_test',
help = 'plain field test')
- self.simulation_parser_arguments(parser_field_test)
- self.job_parser_arguments(parser_field_test)
- self.parameters_to_parser_arguments(parser_field_test)
parser_field_output_test = subparsers.add_parser(
'field_output_test',
help = 'plain field output test')
- self.simulation_parser_arguments(parser_field_output_test)
- self.job_parser_arguments(parser_field_output_test)
- self.parameters_to_parser_arguments(parser_field_output_test)
+ parser_test_interpolation = subparsers.add_parser(
+ 'test_interpolation',
+ help = 'test velocity gradient interpolation')
+ for parser in ['parser_filter_test',
+ 'parser_field_test',
+ 'parser_field_output_test',
+ 'parser_test_interpolation']:
+ eval('self.simulation_parser_arguments(' + parser + ')')
+ eval('self.job_parser_arguments(' + parser + ')')
+ eval('self.parameters_to_parser_arguments(' + parser + ')')
+ eval('self.parameters_to_parser_arguments(' + parser + ',' +
+ 'parameters = self.generate_extra_parameters(dns_type = \'' + parser + '\'))')
+ print(self.parameters.keys())
return None
def prepare_launch(
self,
@@ -285,6 +299,8 @@ class TEST(_code):
args = [],
**kwargs):
opt = self.prepare_launch(args = args)
+ self.parameters.update(
+ self.generate_extra_parameters(dns_type = self.dns_type))
self.launch_jobs(opt = opt, **kwargs)
return None
def launch_jobs(
@@ -293,7 +309,36 @@ class TEST(_code):
particle_initial_condition = None):
if not os.path.exists(os.path.join(self.work_dir, self.simname + '.h5')):
self.write_par(
- particle_ic = None)
+ particle_ic = particle_initial_condition)
+ if self.dns_type == 'test_interpolation':
+ if type(particle_initial_condition) == type(None):
+ pbase_shape = (self.parameters['nparticles'],)
+ number_of_particles = self.parameters['nparticles']
+ else:
+ pbase_shape = particle_initial_condition.shape[:-1]
+ assert(particle_initial_condition.shape[-1] == 3)
+ number_of_particles = 1
+ for val in pbase_shape[1:]:
+ number_of_particles *= val
+ ncomponents = 3
+ with h5py.File(os.path.join(self.work_dir, self.simname + '_input.h5'), 'a') as ofile:
+ s = 0
+ ofile.create_group('tracers{0}'.format(s))
+ ofile.create_group('tracers{0}/rhs'.format(s))
+ ofile.create_group('tracers{0}/state'.format(s))
+ ofile['tracers{0}/rhs'.format(s)].create_dataset(
+ '0',
+ shape = (
+ (self.parameters['tracers{0}_integration_steps'.format(s)],) +
+ pbase_shape +
+ (ncomponents,)),
+ dtype = np.float)
+ ofile['tracers{0}/state'.format(s)].create_dataset(
+ '0',
+ shape = (
+ pbase_shape +
+ (ncomponents,)),
+ dtype = np.float)
self.run(
nb_processes = opt.nb_processes,
nb_threads_per_process = opt.nb_threads_per_process,
diff --git a/bfps/cpp/full_code/test_interpolation.cpp b/bfps/cpp/full_code/test_interpolation.cpp
index e93cd3eb..023b40f8 100644
--- a/bfps/cpp/full_code/test_interpolation.cpp
+++ b/bfps/cpp/full_code/test_interpolation.cpp
@@ -5,12 +5,10 @@ template <typename rnumber>
int test_interpolation<rnumber>::read_parameters(void)
{
this->test::read_parameters();
- hid_t parameter_file;
- hid_t dset, memtype, space;
- char fname[256];
- hsize_t dims[1];
- char *string_data;
- sprintf(fname, "%s.h5", this->simname.c_str());
+ hid_t parameter_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
this->nparticles = hdf5_tools::read_value<int>(
parameter_file, "/parameters/nparticles");
this->tracers0_integration_steps = hdf5_tools::read_value<int>(
@@ -26,8 +24,6 @@ int test_interpolation<rnumber>::read_parameters(void)
template <typename rnumber>
int test_interpolation<rnumber>::initialize(void)
{
- this->test::initialize();
-
this->vorticity = new field<rnumber, FFTW, THREE>(
this->nx, this->ny, this->nz,
this->comm,
@@ -92,7 +88,6 @@ int test_interpolation<rnumber>::finalize(void)
delete this->velocity;
delete this->vorticity;
this->ps.release();
- this->test::finalize();
delete this->kk;
return EXIT_SUCCESS;
}
--
GitLab
From 738102b68efc05ff25a337fd3597742aa20f23e7 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 20 Apr 2018 12:45:05 +0200
Subject: [PATCH 139/342] code runs with no obvious errors
---
bfps/TEST.py | 15 +++++++++++++++
bfps/cpp/full_code/test_interpolation.cpp | 3 +++
2 files changed, 18 insertions(+)
diff --git a/bfps/TEST.py b/bfps/TEST.py
index 9a36cbfc..a95756e1 100644
--- a/bfps/TEST.py
+++ b/bfps/TEST.py
@@ -37,6 +37,7 @@ import warnings
import bfps
from ._code import _code
from bfps import tools
+from bfps import DNS
class TEST(_code):
"""This class is meant to stitch together the C++ code into a final source file,
@@ -339,6 +340,20 @@ class TEST(_code):
pbase_shape +
(ncomponents,)),
dtype = np.float)
+ if type(particle_initial_condition) == type(None):
+ ofile['tracers0/state/0'][:] = np.random.random(pbase_shape + (ncomponents,))*2*np.pi
+ else:
+ ofile['tracers0/state/0'][:] = particle_initial_condition
+ with h5py.File(os.path.join(self.work_dir, self.simname + '_input.h5'), 'a') as ofile:
+ data = DNS.generate_vector_field(self,
+ write_to_file = False,
+ spectra_slope = 1.0,
+ amplitude = 0.05)
+ ofile['vorticity/complex/{0}'.format(0)] = data
+ with h5py.File(os.path.join(self.work_dir, self.simname + '_output.h5'), 'a') as ofile:
+ ofile.require_group('tracers0')
+ for kk in ['velocity', 'vorticity', 'velocity_gradient']:
+ ofile['tracers0'].require_group(kk)
self.run(
nb_processes = opt.nb_processes,
nb_threads_per_process = opt.nb_threads_per_process,
diff --git a/bfps/cpp/full_code/test_interpolation.cpp b/bfps/cpp/full_code/test_interpolation.cpp
index 023b40f8..2150d51f 100644
--- a/bfps/cpp/full_code/test_interpolation.cpp
+++ b/bfps/cpp/full_code/test_interpolation.cpp
@@ -24,10 +24,12 @@ int test_interpolation<rnumber>::read_parameters(void)
template <typename rnumber>
int test_interpolation<rnumber>::initialize(void)
{
+ this->read_parameters();
this->vorticity = new field<rnumber, FFTW, THREE>(
this->nx, this->ny, this->nz,
this->comm,
DEFAULT_FFTW_FLAG);
+ this->vorticity->real_space_representation = false;
this->velocity = new field<rnumber, FFTW, THREE>(
this->nx, this->ny, this->nz,
@@ -97,6 +99,7 @@ int test_interpolation<rnumber>::do_work()
{
std::string fname = this->simname + std::string("_input.h5");
// read vorticity field
+ this->vorticity->real_space_representation = false;
this->vorticity->io(
fname,
"vorticity",
--
GitLab
From d7eff61a65dae516bf1300c49597916e2c081d56 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 20 Apr 2018 13:14:14 +0200
Subject: [PATCH 140/342] fix output bug, sample positions as well
---
bfps/TEST.py | 2 +-
bfps/cpp/full_code/test_interpolation.cpp | 17 +++++++++++++++--
2 files changed, 16 insertions(+), 3 deletions(-)
diff --git a/bfps/TEST.py b/bfps/TEST.py
index a95756e1..1ce0322b 100644
--- a/bfps/TEST.py
+++ b/bfps/TEST.py
@@ -352,7 +352,7 @@ class TEST(_code):
ofile['vorticity/complex/{0}'.format(0)] = data
with h5py.File(os.path.join(self.work_dir, self.simname + '_output.h5'), 'a') as ofile:
ofile.require_group('tracers0')
- for kk in ['velocity', 'vorticity', 'velocity_gradient']:
+ for kk in ['position', 'velocity', 'vorticity', 'velocity_gradient']:
ofile['tracers0'].require_group(kk)
self.run(
nb_processes = opt.nb_processes,
diff --git a/bfps/cpp/full_code/test_interpolation.cpp b/bfps/cpp/full_code/test_interpolation.cpp
index 2150d51f..9c4cf62d 100644
--- a/bfps/cpp/full_code/test_interpolation.cpp
+++ b/bfps/cpp/full_code/test_interpolation.cpp
@@ -120,13 +120,26 @@ int test_interpolation<rnumber>::do_work()
std::unique_ptr<double[]> p3data(new double[3*this->ps->getLocalNbParticles()]);
std::unique_ptr<double[]> p9data(new double[9*this->ps->getLocalNbParticles()]);
+ /// sample position
+ std::copy(this->ps->getParticlesState(),
+ this->ps->getParticlesState()+3*this->ps->getLocalNbParticles(),
+ p3data.get());
+ this->particles_sample_writer_mpi->template save_dataset<3>(
+ "tracers0",
+ "position",
+ this->ps->getParticlesState(),
+ &p3data,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+
/// sample velocity at particles' position
this->ps->sample_compute_field(*this->velocity, p3data.get());
this->particles_sample_writer_mpi->template save_dataset<3>(
"tracers0",
"velocity",
this->ps->getParticlesState(),
- &p9data,
+ &p3data,
this->ps->getParticlesIndexes(),
this->ps->getLocalNbParticles(),
this->ps->get_step_idx()-1);
@@ -136,7 +149,7 @@ int test_interpolation<rnumber>::do_work()
"tracers0",
"vorticity",
this->ps->getParticlesState(),
- &p9data,
+ &p3data,
this->ps->getParticlesIndexes(),
this->ps->getLocalNbParticles(),
this->ps->get_step_idx()-1);
--
GitLab
From 548d9941eb0456bec335f7e15ec884cfa5ddbbb8 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 20 Apr 2018 13:40:49 +0200
Subject: [PATCH 141/342] add script for interpolation test
---
bfps/__init__.py | 1 +
bfps/test/test_interpolation.py | 46 +++++++++++++++++++++++++++++++++
2 files changed, 47 insertions(+)
create mode 100644 bfps/test/test_interpolation.py
diff --git a/bfps/__init__.py b/bfps/__init__.py
index bc9b5a05..29dc62a1 100644
--- a/bfps/__init__.py
+++ b/bfps/__init__.py
@@ -48,6 +48,7 @@ from host_information import host_info
from .DNS import DNS
from .PP import PP
+from .TEST import TEST
from .FluidConvert import FluidConvert
from .NavierStokes import NavierStokes
from .NSVorticityEquation import NSVorticityEquation
diff --git a/bfps/test/test_interpolation.py b/bfps/test/test_interpolation.py
new file mode 100644
index 00000000..6a8cf0fa
--- /dev/null
+++ b/bfps/test/test_interpolation.py
@@ -0,0 +1,46 @@
+#! /usr/bin/env python
+
+import os
+import numpy as np
+import h5py
+import sys
+
+import bfps
+from bfps import TEST
+
+try:
+ import matplotlib.pyplot as plt
+ matplotlib_on = True
+except ImportError:
+ matplotlib_on = False
+
+
+def main():
+ nparticles = 10
+ c = TEST()
+ c.launch(
+ ['test_interpolation',
+ '-n', '32',
+ '--np', '4',
+ '--ntpp', '1',
+ #'--nparticles', '{0}'.format(nparticles),
+ '--wd', './'] +
+ sys.argv[3:])
+ ifile = h5py.File(
+ 'test_input.h5',
+ 'r')
+ ofile = h5py.File(
+ 'test_output.h5',
+ 'r')
+ vort0 = ofile['tracers0/vorticity/0'].value
+ vel_gradient = ofile['tracers0/velocity_gradient/0'].value
+ vort1 = vort0.copy()
+ vort1[:, 0] = vel_gradient[:, 7] - vel_gradient[:, 5]
+ vort1[:, 1] = vel_gradient[:, 2] - vel_gradient[:, 6]
+ vort1[:, 2] = vel_gradient[:, 3] - vel_gradient[:, 1]
+ print(np.abs(vort0-vort1) / np.abs(vort0))
+ return None
+
+if __name__ == '__main__':
+ main()
+
--
GitLab
From 6b1f34872ee3069a1b46c7906da537caa674ea93 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 20 Apr 2018 14:47:35 +0200
Subject: [PATCH 142/342] add some more debug messages
---
bfps/cpp/full_code/test_interpolation.cpp | 20 ++++++++++++++++++++
bfps/test/test_interpolation.py | 7 ++++++-
2 files changed, 26 insertions(+), 1 deletion(-)
diff --git a/bfps/cpp/full_code/test_interpolation.cpp b/bfps/cpp/full_code/test_interpolation.cpp
index 9c4cf62d..b7efdebd 100644
--- a/bfps/cpp/full_code/test_interpolation.cpp
+++ b/bfps/cpp/full_code/test_interpolation.cpp
@@ -116,6 +116,23 @@ int test_interpolation<rnumber>::do_work()
this->velocity->ift();
this->nabla_u->ift();
+ *this->vorticity = 1.0;
+ *this->velocity = 2.0;
+ *this->nabla_u = 3.0;
+ DEBUG_MSG("changed values\n");
+ DEBUG_MSG("some velocity values: %g %g %g\n",
+ this->velocity->rval(20, 1),
+ this->velocity->rval(200, 2),
+ this->velocity->rval(741, 0));
+ DEBUG_MSG("some vorticity values: %g %g %g\n",
+ this->vorticity->rval(20, 1),
+ this->vorticity->rval(200, 2),
+ this->vorticity->rval(741, 0));
+ DEBUG_MSG("some velocity gradient values: %g %g %g\n",
+ this->nabla_u->rval(20, 1, 0),
+ this->nabla_u->rval(200, 2, 1),
+ this->nabla_u->rval(741, 0, 1));
+
// allocate interpolation arrays
std::unique_ptr<double[]> p3data(new double[3*this->ps->getLocalNbParticles()]);
std::unique_ptr<double[]> p9data(new double[9*this->ps->getLocalNbParticles()]);
@@ -135,6 +152,7 @@ int test_interpolation<rnumber>::do_work()
/// sample velocity at particles' position
this->ps->sample_compute_field(*this->velocity, p3data.get());
+ DEBUG_MSG("first vel value is %g\n", p3data.get()[0]);
this->particles_sample_writer_mpi->template save_dataset<3>(
"tracers0",
"velocity",
@@ -145,6 +163,7 @@ int test_interpolation<rnumber>::do_work()
this->ps->get_step_idx()-1);
/// sample vorticity at particles' position
this->ps->sample_compute_field(*this->vorticity, p3data.get());
+ DEBUG_MSG("first vort value is %g\n", p3data.get()[0]);
this->particles_sample_writer_mpi->template save_dataset<3>(
"tracers0",
"vorticity",
@@ -155,6 +174,7 @@ int test_interpolation<rnumber>::do_work()
this->ps->get_step_idx()-1);
/// sample velocity gradient at particles' position
this->ps->sample_compute_field(*this->nabla_u, p9data.get());
+ DEBUG_MSG("first vel gradient value is %g\n", p9data.get()[0]);
this->particles_sample_writer_mpi->template save_dataset<9>(
"tracers0",
"velocity_gradient",
diff --git a/bfps/test/test_interpolation.py b/bfps/test/test_interpolation.py
index 6a8cf0fa..a3991a1c 100644
--- a/bfps/test/test_interpolation.py
+++ b/bfps/test/test_interpolation.py
@@ -16,7 +16,7 @@ except ImportError:
def main():
- nparticles = 10
+ nparticles = 100
c = TEST()
c.launch(
['test_interpolation',
@@ -32,8 +32,13 @@ def main():
ofile = h5py.File(
'test_output.h5',
'r')
+ pos0 = ifile['tracers0/state/0'].value
+ pos1 = ofile['tracers0/position/0'].value
+ print('maximum position error is ', np.max(np.abs(pos0-pos1) / np.abs(pos0)))
vort0 = ofile['tracers0/vorticity/0'].value
+ print(vort0)
vel_gradient = ofile['tracers0/velocity_gradient/0'].value
+ print(vel_gradient)
vort1 = vort0.copy()
vort1[:, 0] = vel_gradient[:, 7] - vel_gradient[:, 5]
vort1[:, 1] = vel_gradient[:, 2] - vel_gradient[:, 6]
--
GitLab
From 8cc702b01b7e7c2804ec2d9313fa8fcf8a81011a Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Mon, 23 Apr 2018 13:50:07 +0200
Subject: [PATCH 143/342] Clean the hdf files in the test class to remove MPI
complains
---
bfps/cpp/full_code/test_interpolation.cpp | 2 ++
bfps/cpp/full_code/test_interpolation.hpp | 8 +++++++-
2 files changed, 9 insertions(+), 1 deletion(-)
diff --git a/bfps/cpp/full_code/test_interpolation.cpp b/bfps/cpp/full_code/test_interpolation.cpp
index b7efdebd..97e2982a 100644
--- a/bfps/cpp/full_code/test_interpolation.cpp
+++ b/bfps/cpp/full_code/test_interpolation.cpp
@@ -91,6 +91,8 @@ int test_interpolation<rnumber>::finalize(void)
delete this->vorticity;
this->ps.release();
delete this->kk;
+ delete particles_output_writer_mpi;
+ delete particles_sample_writer_mpi;
return EXIT_SUCCESS;
}
diff --git a/bfps/cpp/full_code/test_interpolation.hpp b/bfps/cpp/full_code/test_interpolation.hpp
index e8be247c..2d149802 100644
--- a/bfps/cpp/full_code/test_interpolation.hpp
+++ b/bfps/cpp/full_code/test_interpolation.hpp
@@ -39,7 +39,13 @@ class test_interpolation: public test
const std::string &simulation_name):
test(
COMMUNICATOR,
- simulation_name){}
+ simulation_name),
+ particles_output_writer_mpi(nullptr),
+ particles_sample_writer_mpi(nullptr),
+ velocity(nullptr),
+ vorticity(nullptr),
+ nabla_u(nullptr),
+ kk(nullptr) {}
~test_interpolation(){}
int initialize(void);
--
GitLab
From e11ff1b16ed3abf6d353d3757aa671fc3c25e7c9 Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Mon, 23 Apr 2018 14:28:50 +0200
Subject: [PATCH 144/342] Add virtual destructor for abstract_particles_system
---
bfps/cpp/particles/abstract_particles_system.hpp | 2 ++
1 file changed, 2 insertions(+)
diff --git a/bfps/cpp/particles/abstract_particles_system.hpp b/bfps/cpp/particles/abstract_particles_system.hpp
index 91d40507..871f0459 100644
--- a/bfps/cpp/particles/abstract_particles_system.hpp
+++ b/bfps/cpp/particles/abstract_particles_system.hpp
@@ -12,6 +12,8 @@
template <class partsize_t, class real_number>
class abstract_particles_system {
public:
+ virtual ~abstract_particles_system(){}
+
virtual void compute() = 0;
virtual void compute_p2p() = 0;
--
GitLab
From 7a9a9024f07b99fb7a6017afd0da446e0517f782 Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Mon, 23 Apr 2018 15:58:06 +0200
Subject: [PATCH 145/342] reset array before using it to store rhs (it
contained positions before that)
---
bfps/cpp/full_code/test_interpolation.cpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/bfps/cpp/full_code/test_interpolation.cpp b/bfps/cpp/full_code/test_interpolation.cpp
index 97e2982a..772fa773 100644
--- a/bfps/cpp/full_code/test_interpolation.cpp
+++ b/bfps/cpp/full_code/test_interpolation.cpp
@@ -153,6 +153,7 @@ int test_interpolation<rnumber>::do_work()
this->ps->get_step_idx()-1);
/// sample velocity at particles' position
+ std::fill_n(p3data.get(), 3*this->ps->getLocalNbParticles(), 0);
this->ps->sample_compute_field(*this->velocity, p3data.get());
DEBUG_MSG("first vel value is %g\n", p3data.get()[0]);
this->particles_sample_writer_mpi->template save_dataset<3>(
--
GitLab
From 7551955175c7bbb67b3b3dd80119bb193b9a7354 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 23 Apr 2018 17:37:58 +0200
Subject: [PATCH 146/342] clean up arrays before use
---
bfps/cpp/full_code/test_interpolation.cpp | 26 ++++++++++-------------
bfps/test/test_interpolation.py | 4 ++++
2 files changed, 15 insertions(+), 15 deletions(-)
diff --git a/bfps/cpp/full_code/test_interpolation.cpp b/bfps/cpp/full_code/test_interpolation.cpp
index 772fa773..4cfdca66 100644
--- a/bfps/cpp/full_code/test_interpolation.cpp
+++ b/bfps/cpp/full_code/test_interpolation.cpp
@@ -99,11 +99,13 @@ int test_interpolation<rnumber>::finalize(void)
template <typename rnumber>
int test_interpolation<rnumber>::do_work()
{
- std::string fname = this->simname + std::string("_input.h5");
- // read vorticity field
+ *this->nabla_u = 0.0;
+ this->velocity->real_space_representation = false;
this->vorticity->real_space_representation = false;
+ this->nabla_u->real_space_representation = false;
+ // read vorticity field
this->vorticity->io(
- fname,
+ this->simname + std::string("_input.h5"),
"vorticity",
0, true);
@@ -118,22 +120,14 @@ int test_interpolation<rnumber>::do_work()
this->velocity->ift();
this->nabla_u->ift();
- *this->vorticity = 1.0;
- *this->velocity = 2.0;
- *this->nabla_u = 3.0;
- DEBUG_MSG("changed values\n");
- DEBUG_MSG("some velocity values: %g %g %g\n",
- this->velocity->rval(20, 1),
- this->velocity->rval(200, 2),
- this->velocity->rval(741, 0));
DEBUG_MSG("some vorticity values: %g %g %g\n",
this->vorticity->rval(20, 1),
this->vorticity->rval(200, 2),
this->vorticity->rval(741, 0));
- DEBUG_MSG("some velocity gradient values: %g %g %g\n",
- this->nabla_u->rval(20, 1, 0),
- this->nabla_u->rval(200, 2, 1),
- this->nabla_u->rval(741, 0, 1));
+ DEBUG_MSG("corresponding velocity gradient to vorticity values: %g %g %g\n",
+ this->nabla_u->rval( 20, 2, 0) - this->nabla_u->rval( 20, 0, 2),
+ this->nabla_u->rval(200, 1, 0) - this->nabla_u->rval(200, 0, 1),
+ this->nabla_u->rval(741, 1, 2) - this->nabla_u->rval(741, 2, 1));
// allocate interpolation arrays
std::unique_ptr<double[]> p3data(new double[3*this->ps->getLocalNbParticles()]);
@@ -165,6 +159,7 @@ int test_interpolation<rnumber>::do_work()
this->ps->getLocalNbParticles(),
this->ps->get_step_idx()-1);
/// sample vorticity at particles' position
+ std::fill_n(p3data.get(), 3*this->ps->getLocalNbParticles(), 0);
this->ps->sample_compute_field(*this->vorticity, p3data.get());
DEBUG_MSG("first vort value is %g\n", p3data.get()[0]);
this->particles_sample_writer_mpi->template save_dataset<3>(
@@ -176,6 +171,7 @@ int test_interpolation<rnumber>::do_work()
this->ps->getLocalNbParticles(),
this->ps->get_step_idx()-1);
/// sample velocity gradient at particles' position
+ std::fill_n(p9data.get(), 9*this->ps->getLocalNbParticles(), 0);
this->ps->sample_compute_field(*this->nabla_u, p9data.get());
DEBUG_MSG("first vel gradient value is %g\n", p9data.get()[0]);
this->particles_sample_writer_mpi->template save_dataset<9>(
diff --git a/bfps/test/test_interpolation.py b/bfps/test/test_interpolation.py
index a3991a1c..06e8ee05 100644
--- a/bfps/test/test_interpolation.py
+++ b/bfps/test/test_interpolation.py
@@ -44,6 +44,10 @@ def main():
vort1[:, 1] = vel_gradient[:, 2] - vel_gradient[:, 6]
vort1[:, 2] = vel_gradient[:, 3] - vel_gradient[:, 1]
print(np.abs(vort0-vort1) / np.abs(vort0))
+ vort1[:, 0] = vel_gradient[:, 5] - vel_gradient[:, 7]
+ vort1[:, 1] = vel_gradient[:, 6] - vel_gradient[:, 2]
+ vort1[:, 2] = vel_gradient[:, 1] - vel_gradient[:, 3]
+ print(np.abs(vort0-vort1) / np.abs(vort0))
return None
if __name__ == '__main__':
--
GitLab
From 9dc6cfc5d5f4dba2aba9ef965b3e9380f7fef52f Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 23 Apr 2018 17:44:50 +0200
Subject: [PATCH 147/342] temporary array for sampling needs to be cleaned up
---
bfps/cpp/full_code/NSVEparticles.cpp | 2 ++
1 file changed, 2 insertions(+)
diff --git a/bfps/cpp/full_code/NSVEparticles.cpp b/bfps/cpp/full_code/NSVEparticles.cpp
index 5f9f480d..c4db9ee7 100644
--- a/bfps/cpp/full_code/NSVEparticles.cpp
+++ b/bfps/cpp/full_code/NSVEparticles.cpp
@@ -103,6 +103,7 @@ int NSVEparticles<rnumber>::do_stats()
this->ps->get_step_idx()-1);
/// sample velocity
+ std::fill_n(pdata.get(), 3*this->ps->getLocalNbParticles(), 0);
this->ps->sample_compute_field(*this->tmp_vec_field, pdata.get());
this->particles_sample_writer_mpi->template save_dataset<3>(
"tracers0",
@@ -116,6 +117,7 @@ int NSVEparticles<rnumber>::do_stats()
/// compute acceleration and sample it
this->fs->compute_Lagrangian_acceleration(this->tmp_vec_field);
this->tmp_vec_field->ift();
+ std::fill_n(pdata.get(), 3*this->ps->getLocalNbParticles(), 0);
this->ps->sample_compute_field(*this->tmp_vec_field, pdata.get());
this->particles_sample_writer_mpi->template save_dataset<3>(
"tracers0",
--
GitLab
From 1a191c1e834d690c16ae45eaff8d422deb28a516 Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Tue, 24 Apr 2018 10:49:42 +0200
Subject: [PATCH 148/342] Ensure to allocate only if not zero
---
bfps/cpp/particles/particles_input_hdf5.hpp | 29 ++++++++++++++++-----
1 file changed, 22 insertions(+), 7 deletions(-)
diff --git a/bfps/cpp/particles/particles_input_hdf5.hpp b/bfps/cpp/particles/particles_input_hdf5.hpp
index 72f7a905..090c8092 100644
--- a/bfps/cpp/particles/particles_input_hdf5.hpp
+++ b/bfps/cpp/particles/particles_input_hdf5.hpp
@@ -145,7 +145,11 @@ public:
const hid_t type_id = (sizeof(real_number) == 8?H5T_NATIVE_DOUBLE:H5T_NATIVE_FLOAT);
/// Load the data
- std::unique_ptr<real_number[]> split_particles_positions(new real_number[load_splitter.getMySize()*size_particle_positions]);
+ std::unique_ptr<real_number[]> split_particles_positions;
+ if(load_splitter.getMySize()){
+ split_particles_positions.reset(new real_number[load_splitter.getMySize()*size_particle_positions]);
+ }
+
{
TIMEZONE("state-read");
hid_t dset = H5Dopen(particle_file, inDatanameState.c_str(), H5P_DEFAULT);
@@ -181,7 +185,9 @@ public:
hid_t rspace = H5Dget_space(dset);
assert(rspace >= 0);
- split_particles_rhs[idx_rhs].reset(new real_number[load_splitter.getMySize()*size_particle_rhs]);
+ if(load_splitter.getMySize()){
+ split_particles_rhs[idx_rhs].reset(new real_number[load_splitter.getMySize()*size_particle_rhs]);
+ }
hsize_t offset[3] = {idx_rhs, load_splitter.getMyOffset(), 0};
hsize_t mem_dims[3] = {1, load_splitter.getMySize(), size_particle_rhs};
@@ -205,7 +211,10 @@ public:
assert(rethdf >= 0);
}
- std::unique_ptr<partsize_t[]> split_particles_indexes(new partsize_t[load_splitter.getMySize()]);
+ std::unique_ptr<partsize_t[]> split_particles_indexes;
+ if(load_splitter.getMySize()){
+ split_particles_indexes.reset(new partsize_t[load_splitter.getMySize()]);
+ }
for(partsize_t idx_part = 0 ; idx_part < partsize_t(load_splitter.getMySize()) ; ++idx_part){
split_particles_indexes[idx_part] = idx_part + partsize_t(load_splitter.getMyOffset());
}
@@ -252,17 +261,23 @@ public:
// nb_particles_per_processes cannot be used after due to move
nb_particles_for_me = exchanger.getTotalToRecv();
- my_particles_positions.reset(new real_number[exchanger.getTotalToRecv()*size_particle_positions]);
+ if(nb_particles_for_me){
+ my_particles_positions.reset(new real_number[exchanger.getTotalToRecv()*size_particle_positions]);
+ }
exchanger.alltoallv<real_number>(split_particles_positions.get(), my_particles_positions.get(), size_particle_positions);
split_particles_positions.release();
- my_particles_indexes.reset(new partsize_t[exchanger.getTotalToRecv()]);
+ if(nb_particles_for_me){
+ my_particles_indexes.reset(new partsize_t[exchanger.getTotalToRecv()]);
+ }
exchanger.alltoallv<partsize_t>(split_particles_indexes.get(), my_particles_indexes.get());
split_particles_indexes.release();
my_particles_rhs.resize(nb_rhs);
- for(int idx_rhs = 0 ; idx_rhs < int(nb_rhs) ; ++idx_rhs){
- my_particles_rhs[idx_rhs].reset(new real_number[exchanger.getTotalToRecv()*size_particle_rhs]);
+ for(int idx_rhs = 0 ; idx_rhs < int(nb_rhs) ; ++idx_rhs){
+ if(nb_particles_for_me){
+ my_particles_rhs[idx_rhs].reset(new real_number[exchanger.getTotalToRecv()*size_particle_rhs]);
+ }
exchanger.alltoallv<real_number>(split_particles_rhs[idx_rhs].get(), my_particles_rhs[idx_rhs].get(), size_particle_rhs);
}
}
--
GitLab
From 7413756fe2a535758d41e9ce4e3f54157e92e91b Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 30 Apr 2018 17:22:34 +0200
Subject: [PATCH 149/342] try fixed values for vorticity
---
bfps/TEST.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/bfps/TEST.py b/bfps/TEST.py
index 1ce0322b..1a49112b 100644
--- a/bfps/TEST.py
+++ b/bfps/TEST.py
@@ -349,6 +349,8 @@ class TEST(_code):
write_to_file = False,
spectra_slope = 1.0,
amplitude = 0.05)
+ data[:] = 0.0
+ data[0, 0, 1, 0] = 1.0
ofile['vorticity/complex/{0}'.format(0)] = data
with h5py.File(os.path.join(self.work_dir, self.simname + '_output.h5'), 'a') as ofile:
ofile.require_group('tracers0')
--
GitLab
From 9c5b58cdf103944e47deceafd667adf7728bb01c Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 30 Apr 2018 17:36:15 +0200
Subject: [PATCH 150/342] add ABC flow
---
bfps/TEST.py | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
diff --git a/bfps/TEST.py b/bfps/TEST.py
index 1a49112b..96090076 100644
--- a/bfps/TEST.py
+++ b/bfps/TEST.py
@@ -350,7 +350,17 @@ class TEST(_code):
spectra_slope = 1.0,
amplitude = 0.05)
data[:] = 0.0
- data[0, 0, 1, 0] = 1.0
+ # ABC
+ data[0, 0, 1, 1] = -0.5*(1j)
+ data[0, 0, 1, 2] = 0.5*(1j)
+ data[0, 1, 0, 0] = -0.5*(1j)
+ data[0, self.parameters['nz'] - 1, 0, 0] = 0.5*(1j)
+ data[0, 1, 0, 1] = 0.5
+ data[0, self.parameters['nz'] - 1, 0, 1] = 0.5
+ data[1, 0, 0, 0] = 0.5
+ data[self.parameters['ny'] - 1, 0, 0, 0] = 0.5
+ data[1, 0, 0, 2] = -0.5*(1j)
+ data[self.parameters['ny'] - 1, 0, 0, 2] = 0.5*(1j)
ofile['vorticity/complex/{0}'.format(0)] = data
with h5py.File(os.path.join(self.work_dir, self.simname + '_output.h5'), 'a') as ofile:
ofile.require_group('tracers0')
--
GitLab
From cf1d40f4ae22d8b2cf66e83eb3ed7c6ec75f2787 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 30 Apr 2018 17:40:09 +0200
Subject: [PATCH 151/342] force divfree on test vorticity
---
bfps/TEST.py | 24 +++++++++++------------
bfps/cpp/full_code/test_interpolation.cpp | 1 +
2 files changed, 13 insertions(+), 12 deletions(-)
diff --git a/bfps/TEST.py b/bfps/TEST.py
index 96090076..d9b3dedd 100644
--- a/bfps/TEST.py
+++ b/bfps/TEST.py
@@ -349,18 +349,18 @@ class TEST(_code):
write_to_file = False,
spectra_slope = 1.0,
amplitude = 0.05)
- data[:] = 0.0
- # ABC
- data[0, 0, 1, 1] = -0.5*(1j)
- data[0, 0, 1, 2] = 0.5*(1j)
- data[0, 1, 0, 0] = -0.5*(1j)
- data[0, self.parameters['nz'] - 1, 0, 0] = 0.5*(1j)
- data[0, 1, 0, 1] = 0.5
- data[0, self.parameters['nz'] - 1, 0, 1] = 0.5
- data[1, 0, 0, 0] = 0.5
- data[self.parameters['ny'] - 1, 0, 0, 0] = 0.5
- data[1, 0, 0, 2] = -0.5*(1j)
- data[self.parameters['ny'] - 1, 0, 0, 2] = 0.5*(1j)
+ #data[:] = 0.0
+ ## ABC
+ #data[0, 0, 1, 1] = -0.5*(1j)
+ #data[0, 0, 1, 2] = 0.5*(1j)
+ #data[0, 1, 0, 0] = -0.5*(1j)
+ #data[0, self.parameters['nz'] - 1, 0, 0] = 0.5*(1j)
+ #data[0, 1, 0, 1] = 0.5
+ #data[0, self.parameters['nz'] - 1, 0, 1] = 0.5
+ #data[1, 0, 0, 0] = 0.5
+ #data[self.parameters['ny'] - 1, 0, 0, 0] = 0.5
+ #data[1, 0, 0, 2] = -0.5*(1j)
+ #data[self.parameters['ny'] - 1, 0, 0, 2] = 0.5*(1j)
ofile['vorticity/complex/{0}'.format(0)] = data
with h5py.File(os.path.join(self.work_dir, self.simname + '_output.h5'), 'a') as ofile:
ofile.require_group('tracers0')
diff --git a/bfps/cpp/full_code/test_interpolation.cpp b/bfps/cpp/full_code/test_interpolation.cpp
index 4cfdca66..e702e708 100644
--- a/bfps/cpp/full_code/test_interpolation.cpp
+++ b/bfps/cpp/full_code/test_interpolation.cpp
@@ -108,6 +108,7 @@ int test_interpolation<rnumber>::do_work()
this->simname + std::string("_input.h5"),
"vorticity",
0, true);
+ this->kk->template force_divfree<rnumber>(this->vorticity->get_cdata());
// compute velocity
invert_curl(this->kk, this->vorticity, this->velocity);
--
GitLab
From 4cd6b3420cf459e2c8a7fa26e5759df6c4ed7b9f Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 1 May 2018 10:39:08 +0200
Subject: [PATCH 152/342] finalize velocity gradient interpolation test
---
bfps/test/test_interpolation.py | 11 +++--------
1 file changed, 3 insertions(+), 8 deletions(-)
diff --git a/bfps/test/test_interpolation.py b/bfps/test/test_interpolation.py
index 06e8ee05..d5cda159 100644
--- a/bfps/test/test_interpolation.py
+++ b/bfps/test/test_interpolation.py
@@ -34,20 +34,15 @@ def main():
'r')
pos0 = ifile['tracers0/state/0'].value
pos1 = ofile['tracers0/position/0'].value
- print('maximum position error is ', np.max(np.abs(pos0-pos1) / np.abs(pos0)))
+ assert(np.max(np.abs(pos0-pos1) / np.abs(pos0)) <= 1e-5)
vort0 = ofile['tracers0/vorticity/0'].value
- print(vort0)
vel_gradient = ofile['tracers0/velocity_gradient/0'].value
- print(vel_gradient)
vort1 = vort0.copy()
- vort1[:, 0] = vel_gradient[:, 7] - vel_gradient[:, 5]
- vort1[:, 1] = vel_gradient[:, 2] - vel_gradient[:, 6]
- vort1[:, 2] = vel_gradient[:, 3] - vel_gradient[:, 1]
- print(np.abs(vort0-vort1) / np.abs(vort0))
vort1[:, 0] = vel_gradient[:, 5] - vel_gradient[:, 7]
vort1[:, 1] = vel_gradient[:, 6] - vel_gradient[:, 2]
vort1[:, 2] = vel_gradient[:, 1] - vel_gradient[:, 3]
- print(np.abs(vort0-vort1) / np.abs(vort0))
+ assert(np.max(np.abs(vort0-vort1) / np.abs(vort0)) <= 1e-5)
+ print('SUCCESS! Interpolated vorticity agrees with vorticity from interpolated velocity gradient.')
return None
if __name__ == '__main__':
--
GitLab
From 35a9419be021ef8c9421dcf601e0ffe8d6196eb4 Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Wed, 2 May 2018 10:08:41 +0200
Subject: [PATCH 153/342] Update assert when nb particles is zero
---
bfps/cpp/particles/particles_input_hdf5.hpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bfps/cpp/particles/particles_input_hdf5.hpp b/bfps/cpp/particles/particles_input_hdf5.hpp
index 090c8092..c093cd85 100644
--- a/bfps/cpp/particles/particles_input_hdf5.hpp
+++ b/bfps/cpp/particles/particles_input_hdf5.hpp
@@ -307,7 +307,7 @@ public:
}
std::unique_ptr<real_number[]> getMyParticles() final {
- assert(my_particles_positions != nullptr);
+ assert(my_particles_positions != nullptr || nb_particles_for_me == 0);
return std::move(my_particles_positions);
}
--
GitLab
From c0d2aa4b520d619fa5543230312c27160a1423b8 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 2 May 2018 14:30:44 +0200
Subject: [PATCH 154/342] use less particles, so that allocation bugs are
always triggered
---
bfps/TEST.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bfps/TEST.py b/bfps/TEST.py
index d9b3dedd..4983fc61 100644
--- a/bfps/TEST.py
+++ b/bfps/TEST.py
@@ -130,7 +130,7 @@ class TEST(_code):
dns_type = None):
pars = {}
if dns_type == 'test_interpolation':
- pars['nparticles'] = 10
+ pars['nparticles'] = 3
pars['tracers0_integration_steps'] = int(4)
pars['tracers0_neighbours'] = int(1)
pars['tracers0_smoothness'] = int(1)
--
GitLab
From f1b8265696ca457dff982e608af78a3f0de2a258 Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Wed, 2 May 2018 16:20:58 +0200
Subject: [PATCH 155/342] Update another assert when nb particles is zero
---
bfps/cpp/particles/particles_input_hdf5.hpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bfps/cpp/particles/particles_input_hdf5.hpp b/bfps/cpp/particles/particles_input_hdf5.hpp
index c093cd85..20239f06 100644
--- a/bfps/cpp/particles/particles_input_hdf5.hpp
+++ b/bfps/cpp/particles/particles_input_hdf5.hpp
@@ -317,7 +317,7 @@ public:
}
std::unique_ptr<partsize_t[]> getMyParticlesIndexes() final {
- assert(my_particles_indexes != nullptr);
+ assert(my_particles_indexes != nullptr || nb_particles_for_me == 0);
return std::move(my_particles_indexes);
}
};
--
GitLab
From 481a716ae63b69a491a8b665f5dc9965fe6e0d6a Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Thu, 3 May 2018 14:30:58 +0200
Subject: [PATCH 156/342] Ensure not to allocate if size is zero
---
bfps/cpp/particles/abstract_particles_output.hpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bfps/cpp/particles/abstract_particles_output.hpp b/bfps/cpp/particles/abstract_particles_output.hpp
index 7c25502c..6c4c811b 100644
--- a/bfps/cpp/particles/abstract_particles_output.hpp
+++ b/bfps/cpp/particles/abstract_particles_output.hpp
@@ -176,7 +176,7 @@ public:
buffer_particles_rhs_send[idx_rhs].reset(new real_number[size_buffers_send*buffers_size_particle_rhs_send]);
}
}
- else if(buffers_size_particle_rhs_send < size_particle_rhs && size_particle_rhs > 0){
+ else if(buffers_size_particle_rhs_send < size_particle_rhs && size_particle_rhs > 0 && size_buffers_send > 0){
buffers_size_particle_rhs_send = size_particle_rhs;
for(int idx_rhs = 0 ; idx_rhs < nb_rhs ; ++idx_rhs){
buffer_particles_rhs_send[idx_rhs].reset(new real_number[size_buffers_send*buffers_size_particle_rhs_send]);
--
GitLab
From 92dadd3ba3038db1a151e21231d3c9ca9549832f Mon Sep 17 00:00:00 2001
From: Berenger Bramas <berenger.bramas@mpcdf.mpg.de>
Date: Thu, 3 May 2018 15:50:52 +0200
Subject: [PATCH 157/342] Update to ensure no allocation with 0 value in
abstract output
---
bfps/cpp/full_code/test_interpolation.cpp | 20 ++++++--
.../particles/abstract_particles_output.hpp | 47 ++++++++++---------
2 files changed, 40 insertions(+), 27 deletions(-)
diff --git a/bfps/cpp/full_code/test_interpolation.cpp b/bfps/cpp/full_code/test_interpolation.cpp
index e702e708..b194d372 100644
--- a/bfps/cpp/full_code/test_interpolation.cpp
+++ b/bfps/cpp/full_code/test_interpolation.cpp
@@ -131,8 +131,12 @@ int test_interpolation<rnumber>::do_work()
this->nabla_u->rval(741, 1, 2) - this->nabla_u->rval(741, 2, 1));
// allocate interpolation arrays
- std::unique_ptr<double[]> p3data(new double[3*this->ps->getLocalNbParticles()]);
- std::unique_ptr<double[]> p9data(new double[9*this->ps->getLocalNbParticles()]);
+ std::unique_ptr<double[]> p3data;
+ std::unique_ptr<double[]> p9data;
+ if(this->ps->getLocalNbParticles()){
+ p3data.reset(new double[3*this->ps->getLocalNbParticles()]);
+ p9data.reset(new double[9*this->ps->getLocalNbParticles()]);
+ }
/// sample position
std::copy(this->ps->getParticlesState(),
@@ -150,7 +154,9 @@ int test_interpolation<rnumber>::do_work()
/// sample velocity at particles' position
std::fill_n(p3data.get(), 3*this->ps->getLocalNbParticles(), 0);
this->ps->sample_compute_field(*this->velocity, p3data.get());
- DEBUG_MSG("first vel value is %g\n", p3data.get()[0]);
+ if(p3data){
+ DEBUG_MSG("first vel value is %g\n", p3data.get()[0]);
+ }
this->particles_sample_writer_mpi->template save_dataset<3>(
"tracers0",
"velocity",
@@ -162,7 +168,9 @@ int test_interpolation<rnumber>::do_work()
/// sample vorticity at particles' position
std::fill_n(p3data.get(), 3*this->ps->getLocalNbParticles(), 0);
this->ps->sample_compute_field(*this->vorticity, p3data.get());
- DEBUG_MSG("first vort value is %g\n", p3data.get()[0]);
+ if(p3data){
+ DEBUG_MSG("first vort value is %g\n", p3data.get()[0]);
+ }
this->particles_sample_writer_mpi->template save_dataset<3>(
"tracers0",
"vorticity",
@@ -174,7 +182,9 @@ int test_interpolation<rnumber>::do_work()
/// sample velocity gradient at particles' position
std::fill_n(p9data.get(), 9*this->ps->getLocalNbParticles(), 0);
this->ps->sample_compute_field(*this->nabla_u, p9data.get());
- DEBUG_MSG("first vel gradient value is %g\n", p9data.get()[0]);
+ if(p9data){
+ DEBUG_MSG("first vel gradient value is %g\n", p9data.get()[0]);
+ }
this->particles_sample_writer_mpi->template save_dataset<9>(
"tracers0",
"velocity_gradient",
diff --git a/bfps/cpp/particles/abstract_particles_output.hpp b/bfps/cpp/particles/abstract_particles_output.hpp
index 6c4c811b..4fc344d3 100644
--- a/bfps/cpp/particles/abstract_particles_output.hpp
+++ b/bfps/cpp/particles/abstract_particles_output.hpp
@@ -67,10 +67,10 @@ public:
abstract_particles_output(MPI_Comm in_mpi_com, const partsize_t inTotalNbParticles, const int in_nb_rhs) throw()
: mpi_com(in_mpi_com), my_rank(-1), nb_processes(-1),
total_nb_particles(inTotalNbParticles), nb_rhs(in_nb_rhs),
- buffer_particles_rhs_send(in_nb_rhs), size_buffers_send(-1),
- buffers_size_particle_rhs_send(-1),
- buffer_particles_rhs_recv(in_nb_rhs), size_buffers_recv(-1),
- buffers_size_particle_rhs_recv(-1),
+ buffer_particles_rhs_send(in_nb_rhs), size_buffers_send(0),
+ buffers_size_particle_rhs_send(0),
+ buffer_particles_rhs_recv(in_nb_rhs), size_buffers_recv(0),
+ buffers_size_particle_rhs_recv(0),
nb_processes_involved(0), current_is_involved(true), particles_chunk_per_process(0),
particles_chunk_current_size(0), particles_chunk_current_offset(0) {
@@ -139,16 +139,16 @@ public:
void releaseMemory(){
buffer_indexes_send.release();
buffer_particles_positions_send.release();
- size_buffers_send = -1;
+ size_buffers_send = 0;
buffer_indexes_recv.release();
buffer_particles_positions_recv.release();
- size_buffers_recv = -1;
+ size_buffers_recv = 0;
for(int idx_rhs = 0 ; idx_rhs < nb_rhs ; ++idx_rhs){
buffer_particles_rhs_send[idx_rhs].release();
buffer_particles_rhs_recv[idx_rhs].release();
}
- buffers_size_particle_rhs_send = -1;
- buffers_size_particle_rhs_recv = -1;
+ buffers_size_particle_rhs_send = 0;
+ buffers_size_particle_rhs_recv = 0;
}
template <int size_particle_rhs>
@@ -164,7 +164,7 @@ public:
{
TIMEZONE("sort-to-distribute");
- if(size_buffers_send < nb_particles && nb_particles){
+ if(size_buffers_send < nb_particles){
size_buffers_send = nb_particles;
buffer_indexes_send.reset(new std::pair<partsize_t,partsize_t>[size_buffers_send]);
buffer_particles_positions_send.reset(new real_number[size_buffers_send*size_particle_positions]);
@@ -176,10 +176,12 @@ public:
buffer_particles_rhs_send[idx_rhs].reset(new real_number[size_buffers_send*buffers_size_particle_rhs_send]);
}
}
- else if(buffers_size_particle_rhs_send < size_particle_rhs && size_particle_rhs > 0 && size_buffers_send > 0){
+ else if(buffers_size_particle_rhs_send < size_particle_rhs){
buffers_size_particle_rhs_send = size_particle_rhs;
- for(int idx_rhs = 0 ; idx_rhs < nb_rhs ; ++idx_rhs){
- buffer_particles_rhs_send[idx_rhs].reset(new real_number[size_buffers_send*buffers_size_particle_rhs_send]);
+ if(size_buffers_send > 0){
+ for(int idx_rhs = 0 ; idx_rhs < nb_rhs ; ++idx_rhs){
+ buffer_particles_rhs_send[idx_rhs].reset(new real_number[size_buffers_send*buffers_size_particle_rhs_send]);
+ }
}
}
@@ -225,23 +227,23 @@ public:
const int nb_to_receive = exchanger.getTotalToRecv();
assert(nb_to_receive == particles_chunk_current_size);
- if(size_buffers_recv < nb_to_receive && nb_to_receive){
+ if(size_buffers_recv < nb_to_receive){
size_buffers_recv = nb_to_receive;
buffer_indexes_recv.reset(new partsize_t[size_buffers_recv]);
buffer_particles_positions_recv.reset(new real_number[size_buffers_recv*size_particle_positions]);
-
- if(buffers_size_particle_rhs_recv < size_particle_rhs){
- buffers_size_particle_rhs_recv = size_particle_rhs;
- }
+
+ buffers_size_particle_rhs_recv = size_particle_rhs;
for(int idx_rhs = 0 ; idx_rhs < nb_rhs ; ++idx_rhs){
buffer_particles_rhs_recv[idx_rhs].reset(new real_number[size_buffers_recv*buffers_size_particle_rhs_recv]);
}
}
- else if(buffers_size_particle_rhs_recv < size_particle_rhs && size_buffers_recv > 0){
+ else if(buffers_size_particle_rhs_recv < size_particle_rhs){
buffers_size_particle_rhs_recv = size_particle_rhs;
- for(int idx_rhs = 0 ; idx_rhs < nb_rhs ; ++idx_rhs){
- buffer_particles_rhs_recv[idx_rhs].reset(new real_number[size_buffers_recv*buffers_size_particle_rhs_recv]);
- }
+ if(size_buffers_recv > 0){
+ for(int idx_rhs = 0 ; idx_rhs < nb_rhs ; ++idx_rhs){
+ buffer_particles_rhs_recv[idx_rhs].reset(new real_number[size_buffers_recv*buffers_size_particle_rhs_recv]);
+ }
+ }
}
{
@@ -260,10 +262,11 @@ public:
return;
}
- if(size_buffers_send < nb_to_receive && nb_to_receive){
+ if(size_buffers_send < nb_to_receive){
size_buffers_send = nb_to_receive;
buffer_indexes_send.reset(new std::pair<partsize_t,partsize_t>[size_buffers_send]);
buffer_particles_positions_send.reset(new real_number[size_buffers_send*size_particle_positions]);
+ buffers_size_particle_rhs_send = size_particle_rhs;
for(int idx_rhs = 0 ; idx_rhs < nb_rhs ; ++idx_rhs){
buffer_particles_rhs_send[idx_rhs].reset(new real_number[size_buffers_send*buffers_size_particle_rhs_send]);
}
--
GitLab
From 6741a3fdea8703d4afb4e8c754f70cc4faa59e42 Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Sun, 6 May 2018 21:31:46 +0200
Subject: [PATCH 158/342] try to fix initialization for noncubic grids
---
bfps/tools.py | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/bfps/tools.py b/bfps/tools.py
index 9d39e845..ee1e4775 100644
--- a/bfps/tools.py
+++ b/bfps/tools.py
@@ -203,10 +203,11 @@ def padd_with_zeros(
"""
if (type(odtype) == type(None)):
odtype = a.dtype
- assert(a.shape[0] <= n0 and
- a.shape[1] <= n1 and
+ print(a.shape, n0, n1, n2//2+1)
+ assert(a.shape[0] <= n1 and
+ a.shape[1] <= n0 and
a.shape[2] <= n2//2+1)
- b = np.zeros((n0, n1, n2//2 + 1) + a.shape[3:], dtype = odtype)
+ b = np.zeros((n1, n0, n2//2 + 1) + a.shape[3:], dtype = odtype)
m0 = a.shape[1]
m1 = a.shape[0]
m2 = a.shape[2]
--
GitLab
From fca444e1b0c34db35e100478bf97c63e716254c3 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 15 May 2018 17:06:24 +0200
Subject: [PATCH 159/342] clean up old cache differently
---
bfps/NavierStokes.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/bfps/NavierStokes.py b/bfps/NavierStokes.py
index cbd881a7..98169ab0 100644
--- a/bfps/NavierStokes.py
+++ b/bfps/NavierStokes.py
@@ -612,9 +612,12 @@ class NavierStokes(_fluid_particle_base):
for k in ['t', 'vel_max(t)', 'renergy(t)',
'energy(t)', 'enstrophy(t)',
'energy(k)', 'enstrophy(k)',
+ 'energy(t, k)',
+ 'enstrophy(t, k)',
'R_ij(t)',
'ii0', 'ii1', 'iter0', 'iter1']:
- del pp_file[k]
+ if k in pp_file.keys():
+ del pp_file[k]
if computation_needed:
pp_file['iter0'] = iter0
pp_file['iter1'] = iter1
--
GitLab
From b496dd5ead921804f1dd22bf0a31e174e42f2e83 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 24 May 2018 10:33:00 +0200
Subject: [PATCH 160/342] bugfix: generate correct shape when copying initial
condition
---
bfps/DNS.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 6a815080..2e18b51c 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -879,8 +879,8 @@ class DNS(_code):
# the case of increasing/decreasing by the same factor in all directions.
# in principle we could write something more generic, but i'm not sure
# how complicated that would be
- dst_shape = (self.parameters['nz'],
- self.parameters['ny'],
+ dst_shape = (self.parameters['ny'],
+ self.parameters['nz'],
(self.parameters['nx']+2) // 2,
3)
src_file = h5py.File(src_file_name, 'r')
--
GitLab
From 2f1e6d62da0544d430dcf07952109f0dc7b93ed0 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 24 May 2018 17:30:40 +0200
Subject: [PATCH 161/342] add L2norm methods
---
bfps/cpp/field.cpp | 64 ++++++++++++++++++++++++++++++++++++++++++
bfps/cpp/field.hpp | 3 ++
bfps/cpp/kspace.cpp | 68 +++++++++++++++++++++++++++++++++++++++++++++
bfps/cpp/kspace.hpp | 6 ++++
4 files changed, 141 insertions(+)
diff --git a/bfps/cpp/field.cpp b/bfps/cpp/field.cpp
index a9135a89..e4e08f54 100644
--- a/bfps/cpp/field.cpp
+++ b/bfps/cpp/field.cpp
@@ -1152,6 +1152,42 @@ void field<rnumber, be, fc>::compute_stats(
}
}
+template <typename rnumber,
+ field_backend be,
+ field_components fc>
+template <kspace_dealias_type dt>
+double field<rnumber, be, fc>::L2norm(
+ kspace<be, dt> *kk)
+{
+ TIMEZONE("field::L2norm");
+ if (!this->real_space_representation)
+ return kk->template L2norm<rnumber, fc>(this->get_cdata());
+ else
+ {
+ shared_array<double> local_m2_threaded(1, [&](double* local_moment){
+ std::fill_n(local_moment, 1, 0);});
+
+ this->RLOOP(
+ [&](ptrdiff_t rindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex){
+ double *local_m2 = local_m2_threaded.getMine();
+ for (unsigned int i=0; i<ncomp(fc); i++)
+ local_m2[0] += this->data[rindex*ncomp(fc)+i]*this->data[rindex*ncomp(fc)+i];
+ });
+
+ local_m2_threaded.mergeParallel();
+ double m2;
+ MPI_Allreduce(
+ (void*)local_m2_threaded.getMasterData(),
+ &m2,
+ 1,
+ MPI_DOUBLE, MPI_SUM, this->comm);
+ return m2 / this->npoints;
+ }
+}
+
template <typename rnumber,
field_backend be,
field_components fc1,
@@ -1655,6 +1691,34 @@ template void field<double, FFTW, THREExTHREE>::compute_stats<SMOOTH>(
kspace<FFTW, SMOOTH> *,
const hid_t, const std::string, const hsize_t, const double);
+template double field<float, FFTW, ONE>::L2norm<TWO_THIRDS>(
+ kspace<FFTW, TWO_THIRDS> *);
+template double field<float, FFTW, THREE>::L2norm<TWO_THIRDS>(
+ kspace<FFTW, TWO_THIRDS> *);
+template double field<float, FFTW, THREExTHREE>::L2norm<TWO_THIRDS>(
+ kspace<FFTW, TWO_THIRDS> *);
+
+template double field<double, FFTW, ONE>::L2norm<TWO_THIRDS>(
+ kspace<FFTW, TWO_THIRDS> *);
+template double field<double, FFTW, THREE>::L2norm<TWO_THIRDS>(
+ kspace<FFTW, TWO_THIRDS> *);
+template double field<double, FFTW, THREExTHREE>::L2norm<TWO_THIRDS>(
+ kspace<FFTW, TWO_THIRDS> *);
+
+template double field<float, FFTW, ONE>::L2norm<SMOOTH>(
+ kspace<FFTW, SMOOTH> *);
+template double field<float, FFTW, THREE>::L2norm<SMOOTH>(
+ kspace<FFTW, SMOOTH> *);
+template double field<float, FFTW, THREExTHREE>::L2norm<SMOOTH>(
+ kspace<FFTW, SMOOTH> *);
+
+template double field<double, FFTW, ONE>::L2norm<SMOOTH>(
+ kspace<FFTW, SMOOTH> *);
+template double field<double, FFTW, THREE>::L2norm<SMOOTH>(
+ kspace<FFTW, SMOOTH> *);
+template double field<double, FFTW, THREExTHREE>::L2norm<SMOOTH>(
+ kspace<FFTW, SMOOTH> *);
+
template int compute_gradient<float, FFTW, THREE, THREExTHREE, SMOOTH>(
kspace<FFTW, SMOOTH> *,
field<float, FFTW, THREE> *,
diff --git a/bfps/cpp/field.hpp b/bfps/cpp/field.hpp
index da03fc32..a52d2a56 100644
--- a/bfps/cpp/field.hpp
+++ b/bfps/cpp/field.hpp
@@ -249,6 +249,9 @@ class field
const std::string dset_name,
const hsize_t toffset,
const double max_estimate);
+ template <kspace_dealias_type dt>
+ double L2norm(
+ kspace<be, dt> *kk);
inline void impose_zero_mode()
{
if (this->clayout->myrank == this->clayout->rank[0][0] &&
diff --git a/bfps/cpp/kspace.cpp b/bfps/cpp/kspace.cpp
index 01accf4d..e3dabfe9 100644
--- a/bfps/cpp/kspace.cpp
+++ b/bfps/cpp/kspace.cpp
@@ -597,6 +597,48 @@ void kspace<be, dt>::cospectrum(
}
}
+template <field_backend be,
+ kspace_dealias_type dt>
+template <typename rnumber,
+ field_components fc>
+double kspace<be, dt>::L2norm(
+ const rnumber(* __restrict a)[2])
+{
+ TIMEZONE("field::L2norm");
+ shared_array<double> L2_local_thread(1,[&](double* spec_local){
+ std::fill_n(spec_local, 1, 0);
+ });
+
+ this->CLOOP_K2_NXMODES(
+ [&](ptrdiff_t cindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex,
+ double k2,
+ int nxmodes){
+ if (k2 <= this->kM2)
+ {
+ double* L2_local = L2_local_thread.getMine();
+ for (hsize_t i=0; i<ncomp(fc); i++)
+ for (hsize_t j=0; j<ncomp(fc); j++){
+ L2_local[0] += nxmodes * (
+ (a[ncomp(fc)*cindex + i][0] * a[ncomp(fc)*cindex + j][0]) +
+ (a[ncomp(fc)*cindex + i][1] * a[ncomp(fc)*cindex + j][1]));
+ }
+ }
+ });
+
+ L2_local_thread.mergeParallel();
+
+ double L2;
+ MPI_Allreduce(
+ L2_local_thread.getMasterData(),
+ &L2,
+ 1,
+ MPI_DOUBLE, MPI_SUM, this->layout->comm);
+ return L2 * this->dkx * this->dky * this->dkz;
+}
+
template class kspace<FFTW, TWO_THIRDS>;
template class kspace<FFTW, SMOOTH>;
@@ -801,6 +843,32 @@ template void kspace<FFTW, SMOOTH>::cospectrum<double, THREExTHREE>(
const std::string dset_name,
const hsize_t toffset);
+template double kspace<FFTW, TWO_THIRDS>::L2norm<float, ONE>(
+ const typename fftw_interface<float>::complex *__restrict__ a);
+template double kspace<FFTW, TWO_THIRDS>::L2norm<float, THREE>(
+ const typename fftw_interface<float>::complex *__restrict__ a);
+template double kspace<FFTW, TWO_THIRDS>::L2norm<float, THREExTHREE>(
+ const typename fftw_interface<float>::complex *__restrict__ a);
+template double kspace<FFTW, TWO_THIRDS>::L2norm<double, ONE>(
+ const typename fftw_interface<double>::complex *__restrict__ a);
+template double kspace<FFTW, TWO_THIRDS>::L2norm<double, THREE>(
+ const typename fftw_interface<double>::complex *__restrict__ a);
+template double kspace<FFTW, TWO_THIRDS>::L2norm<double, THREExTHREE>(
+ const typename fftw_interface<double>::complex *__restrict__ a);
+
+template double kspace<FFTW, SMOOTH>::L2norm<float, ONE>(
+ const typename fftw_interface<float>::complex *__restrict__ a);
+template double kspace<FFTW, SMOOTH>::L2norm<float, THREE>(
+ const typename fftw_interface<float>::complex *__restrict__ a);
+template double kspace<FFTW, SMOOTH>::L2norm<float, THREExTHREE>(
+ const typename fftw_interface<float>::complex *__restrict__ a);
+template double kspace<FFTW, SMOOTH>::L2norm<double, ONE>(
+ const typename fftw_interface<double>::complex *__restrict__ a);
+template double kspace<FFTW, SMOOTH>::L2norm<double, THREE>(
+ const typename fftw_interface<double>::complex *__restrict__ a);
+template double kspace<FFTW, SMOOTH>::L2norm<double, THREExTHREE>(
+ const typename fftw_interface<double>::complex *__restrict__ a);
+
template void kspace<FFTW, SMOOTH>::force_divfree<float>(
typename fftw_interface<float>::complex *__restrict__ a);
template void kspace<FFTW, SMOOTH>::force_divfree<double>(
diff --git a/bfps/cpp/kspace.hpp b/bfps/cpp/kspace.hpp
index d8bc008d..507ecea3 100644
--- a/bfps/cpp/kspace.hpp
+++ b/bfps/cpp/kspace.hpp
@@ -114,6 +114,12 @@ class kspace
const hid_t group,
const std::string dset_name,
const hsize_t toffset);
+
+ template <typename rnumber,
+ field_components fc>
+ double L2norm(
+ const rnumber(* __restrict__ a)[2]);
+
template <class func_type>
void CLOOP(func_type expression)
{
--
GitLab
From 8922bada894fe322fbae84db888579956f66d74b Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Fri, 25 May 2018 06:40:47 +0200
Subject: [PATCH 162/342] add sanity check to field test
---
bfps/cpp/field.cpp | 10 +++++-----
bfps/cpp/full_code/field_test.cpp | 14 ++++++++++++++
bfps/cpp/kspace.cpp | 10 ++++------
3 files changed, 23 insertions(+), 11 deletions(-)
diff --git a/bfps/cpp/field.cpp b/bfps/cpp/field.cpp
index e4e08f54..489903b7 100644
--- a/bfps/cpp/field.cpp
+++ b/bfps/cpp/field.cpp
@@ -1077,17 +1077,17 @@ void field<rnumber, be, fc>::symmetrize()
fftw_interface<rnumber>::free(buffer);
delete mpistatus;
/* put asymmetric data to 0 */
- /*if (this->clayout->myrank == this->clayout->rank[0][this->clayout->sizes[0]/2])
+ if (this->clayout->myrank == this->clayout->rank[0][this->clayout->sizes[0]/2])
{
- tindex = ncomp(fc)*(this->clayout->sizes[0]/2 - this->clayout->starts[0])*this->clayout->sizes[1]*this->clayout->sizes[2];
+ ptrdiff_t tindex = ncomp(fc)*(this->clayout->sizes[0]/2 - this->clayout->starts[0])*this->clayout->sizes[1]*this->clayout->sizes[2];
for (ii = 0; ii < this->clayout->sizes[1]; ii++)
{
std::fill_n((rnumber*)(data + tindex), ncomp(fc)*2*this->clayout->sizes[2], 0.0);
tindex += ncomp(fc)*this->clayout->sizes[2];
}
}
- tindex = ncomp(fc)*();
- std::fill_n((rnumber*)(data + tindex), ncomp(fc)*2, 0.0);*/
+ //tindex = ncomp(fc)*();
+ //std::fill_n((rnumber*)(data + tindex), ncomp(fc)*2, 0.0);
}
template <typename rnumber,
@@ -1184,7 +1184,7 @@ double field<rnumber, be, fc>::L2norm(
&m2,
1,
MPI_DOUBLE, MPI_SUM, this->comm);
- return m2 / this->npoints;
+ return sqrt(m2 / this->npoints);
}
}
diff --git a/bfps/cpp/full_code/field_test.cpp b/bfps/cpp/full_code/field_test.cpp
index b07f9b39..b6c65a4a 100644
--- a/bfps/cpp/full_code/field_test.cpp
+++ b/bfps/cpp/full_code/field_test.cpp
@@ -76,9 +76,13 @@ int field_test<rnumber>::do_work(void)
});
*scal_field_alt = scal_field->get_rdata();
+ double L2r = scal_field->L2norm(kk);
scal_field->dft();
+ double L2c = scal_field->L2norm(kk);
scal_field->ift();
scal_field->normalize();
+ DEBUG_MSG("L2r = %g, L2c = %g\n",
+ L2r, L2c / scal_field->npoints);
double max_error = 0;
scal_field->RLOOP(
@@ -93,6 +97,16 @@ int field_test<rnumber>::do_work(void)
DEBUG_MSG("maximum error is %g\n", max_error);
+ scal_field->dft();
+ kk->template dealias<rnumber, ONE>(scal_field->get_cdata());
+ scal_field->symmetrize();
+ scal_field->normalize();
+ L2c = scal_field->L2norm(kk);
+ scal_field->ift();
+ L2r = scal_field->L2norm(kk);
+ DEBUG_MSG("L2r = %g, L2c = %g\n",
+ L2r, L2c);
+
// deallocate
delete kk;
delete scal_field;
diff --git a/bfps/cpp/kspace.cpp b/bfps/cpp/kspace.cpp
index e3dabfe9..72f8d70d 100644
--- a/bfps/cpp/kspace.cpp
+++ b/bfps/cpp/kspace.cpp
@@ -616,14 +616,12 @@ double kspace<be, dt>::L2norm(
ptrdiff_t zindex,
double k2,
int nxmodes){
- if (k2 <= this->kM2)
{
double* L2_local = L2_local_thread.getMine();
- for (hsize_t i=0; i<ncomp(fc); i++)
- for (hsize_t j=0; j<ncomp(fc); j++){
+ for (hsize_t i=0; i<ncomp(fc); i++){
L2_local[0] += nxmodes * (
- (a[ncomp(fc)*cindex + i][0] * a[ncomp(fc)*cindex + j][0]) +
- (a[ncomp(fc)*cindex + i][1] * a[ncomp(fc)*cindex + j][1]));
+ (a[ncomp(fc)*cindex + i][0] * a[ncomp(fc)*cindex + i][0]) +
+ (a[ncomp(fc)*cindex + i][1] * a[ncomp(fc)*cindex + i][1]));
}
}
});
@@ -636,7 +634,7 @@ double kspace<be, dt>::L2norm(
&L2,
1,
MPI_DOUBLE, MPI_SUM, this->layout->comm);
- return L2 * this->dkx * this->dky * this->dkz;
+ return sqrt(L2 * this->dkx * this->dky * this->dkz);
}
--
GitLab
From 1b794b9e06784ec8d803ff779801e5befbd37439 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 25 May 2018 10:32:45 +0200
Subject: [PATCH 163/342] fix 2 warnings
---
bfps/cpp/field.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/bfps/cpp/field.cpp b/bfps/cpp/field.cpp
index 489903b7..ccc3fa86 100644
--- a/bfps/cpp/field.cpp
+++ b/bfps/cpp/field.cpp
@@ -944,7 +944,7 @@ void field<rnumber, be, fc>::compute_rspace_zaverage(
MPI_DOUBLE, MPI_SUM, this->comm);
}
// divide by total number of slices
- for (ptrdiff_t n=0; n < slice_size; n++)
+ for (ptrdiff_t n=0; n < ptrdiff_t(slice_size); n++)
zaverage[n] /= this->rlayout->sizes[0];
if (this->myrank == 0)
@@ -1080,7 +1080,7 @@ void field<rnumber, be, fc>::symmetrize()
if (this->clayout->myrank == this->clayout->rank[0][this->clayout->sizes[0]/2])
{
ptrdiff_t tindex = ncomp(fc)*(this->clayout->sizes[0]/2 - this->clayout->starts[0])*this->clayout->sizes[1]*this->clayout->sizes[2];
- for (ii = 0; ii < this->clayout->sizes[1]; ii++)
+ for (ii = 0; ii < ptrdiff_t(this->clayout->sizes[1]); ii++)
{
std::fill_n((rnumber*)(data + tindex), ncomp(fc)*2*this->clayout->sizes[2], 0.0);
tindex += ncomp(fc)*this->clayout->sizes[2];
--
GitLab
From 68ed228066ec5eb3ae347baf800415685c4a85f6 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 25 May 2018 19:56:58 +0200
Subject: [PATCH 164/342] still debugging
---
bfps/cpp/field.cpp | 162 +++++++++++++++++++++---------
bfps/cpp/full_code/field_test.cpp | 2 +-
2 files changed, 118 insertions(+), 46 deletions(-)
diff --git a/bfps/cpp/field.cpp b/bfps/cpp/field.cpp
index ccc3fa86..2b2852a5 100644
--- a/bfps/cpp/field.cpp
+++ b/bfps/cpp/field.cpp
@@ -73,11 +73,10 @@ field<rnumber, be, fc>::field(
nfftw[0] = nz;
nfftw[1] = ny;
nfftw[2] = nx;
- //ptrdiff_t tmp_local_size;
+ ptrdiff_t tmp_local_size;
ptrdiff_t local_n0, local_0_start;
ptrdiff_t local_n1, local_1_start;
- //tmp_local_size = fftw_mpi_local_size_many_transposed(
- fftw_interface<rnumber>::mpi_local_size_many_transposed(
+ tmp_local_size = fftw_interface<rnumber>::mpi_local_size_many_transposed(
3, nfftw, ncomp(fc),
FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK, this->comm,
&local_n0, &local_0_start,
@@ -86,6 +85,8 @@ field<rnumber, be, fc>::field(
sizes[0] = nz; sizes[1] = ny; sizes[2] = nx;
subsizes[0] = local_n0; subsizes[1] = ny; subsizes[2] = nx;
starts[0] = local_0_start; starts[1] = 0; starts[2] = 0;
+ DEBUG_MSG("local_0_start = %ld, local_1_start = %ld\n",
+ local_0_start, local_1_start);
this->rlayout = new field_layout<fc>(
sizes, subsizes, starts, this->comm);
this->npoints = this->rlayout->full_size / ncomp(fc);
@@ -99,6 +100,11 @@ field<rnumber, be, fc>::field(
starts[0] = local_1_start; starts[1] = 0; starts[2] = 0;
this->clayout = new field_layout<fc>(
sizes, subsizes, starts, this->comm);
+ DEBUG_MSG("local_size = %ld, rlayout->local_size = %ld, rmemlayout->local_size = %ld, clayout->local_size = %ld\n",
+ tmp_local_size,
+ this->rlayout->local_size,
+ this->rmemlayout->local_size,
+ this->clayout->local_size);
this->data = fftw_interface<rnumber>::alloc_real(
this->rmemlayout->local_size);
memset(this->data, 0, sizeof(rnumber)*this->rmemlayout->local_size);
@@ -1017,77 +1023,143 @@ void field<rnumber, be, fc>::symmetrize()
{
TIMEZONE("field::symmetrize");
assert(!this->real_space_representation);
- ptrdiff_t ii, cc;
- typename fftw_interface<rnumber>::complex *data = this->get_cdata();
+ //this->ift();
+ //this->dft();
+ //this->normalize();
+ typename fftw_interface<rnumber>::complex *cdata = this->get_cdata();
+ // symmetrize kx = 0 plane, line by line, for ky != 0
MPI_Status *mpistatus = new MPI_Status;
- if (this->myrank == this->clayout->rank[0][0])
- {
- for (cc = 0; cc < ncomp(fc); cc++)
- data[cc][1] = 0.0;
- for (ii = 1; ii < ptrdiff_t(this->clayout->sizes[1]/2); ii++)
- for (cc = 0; cc < ncomp(fc); cc++) {
- ( *(data + cc + ncomp(fc)*(this->clayout->sizes[1] - ii)*this->clayout->sizes[2]))[0] =
- (*(data + cc + ncomp(fc)*( ii)*this->clayout->sizes[2]))[0];
- ( *(data + cc + ncomp(fc)*(this->clayout->sizes[1] - ii)*this->clayout->sizes[2]))[1] =
- -(*(data + cc + ncomp(fc)*( ii)*this->clayout->sizes[2]))[1];
- }
- }
typename fftw_interface<rnumber>::complex *buffer;
buffer = fftw_interface<rnumber>::alloc_complex(ncomp(fc)*this->clayout->sizes[1]);
- ptrdiff_t yy;
/*ptrdiff_t tindex;*/
int ranksrc, rankdst;
- for (yy = 1; yy < ptrdiff_t(this->clayout->sizes[0]/2); yy++) {
- ranksrc = this->clayout->rank[0][yy];
- rankdst = this->clayout->rank[0][this->clayout->sizes[0] - yy];
+ for (ptrdiff_t iy = 1; iy < ptrdiff_t(this->clayout->sizes[0]/2); iy++)
+ {
+ ranksrc = this->clayout->rank[0][iy];
+ rankdst = this->clayout->rank[0][this->clayout->sizes[0] - iy];
if (this->clayout->myrank == ranksrc)
- for (ii = 0; ii < ptrdiff_t(this->clayout->sizes[1]); ii++)
- for (cc = 0; cc < ncomp(fc); cc++)
+ {
+ ptrdiff_t iyy = iy - this->clayout->starts[0];
+ for (ptrdiff_t iz = 0; iz < ptrdiff_t(this->clayout->sizes[1]); iz++)
+ {
+ ptrdiff_t cindex = this->get_cindex(0, iyy, iz);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++)
for (int imag_comp=0; imag_comp<2; imag_comp++)
- (*(buffer + ncomp(fc)*ii+cc))[imag_comp] =
- (*(data + ncomp(fc)*((yy - this->clayout->starts[0])*this->clayout->sizes[1] + ii)*this->clayout->sizes[2] + cc))[imag_comp];
+ (*(buffer + ncomp(fc)*iz+cc))[imag_comp] =
+ (*(cdata + ncomp(fc)*cindex + cc))[imag_comp];
+ }
+ }
if (ranksrc != rankdst)
{
if (this->clayout->myrank == ranksrc)
MPI_Send((void*)buffer,
- ncomp(fc)*this->clayout->sizes[1], mpi_real_type<rnumber>::complex(), rankdst, yy,
- this->clayout->comm);
+ ncomp(fc)*this->clayout->sizes[1],
+ mpi_real_type<rnumber>::complex(),
+ rankdst, iy,
+ this->clayout->comm);
if (this->clayout->myrank == rankdst)
MPI_Recv((void*)buffer,
- ncomp(fc)*this->clayout->sizes[1], mpi_real_type<rnumber>::complex(), ranksrc, yy,
- this->clayout->comm, mpistatus);
+ ncomp(fc)*this->clayout->sizes[1],
+ mpi_real_type<rnumber>::complex(),
+ ranksrc, iy,
+ this->clayout->comm,
+ mpistatus);
}
if (this->clayout->myrank == rankdst)
{
- for (ii = 1; ii < ptrdiff_t(this->clayout->sizes[1]); ii++)
- for (cc = 0; cc < ncomp(fc); cc++)
+ ptrdiff_t iyy = (this->clayout->sizes[0] - iy) - this->clayout->starts[0];
+ for (ptrdiff_t iz = 1; iz < ptrdiff_t(this->clayout->sizes[1]); iz++)
+ {
+ ptrdiff_t izz = (this->clayout->sizes[1] - iz);
+ ptrdiff_t cindex = this->get_cindex(0, iyy, izz);
+ DEBUG_MSG("iy = %ld, iz = %ld\n", iy, iz);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++)
{
- (*(data + ncomp(fc)*((this->clayout->sizes[0] - yy - this->clayout->starts[0])*this->clayout->sizes[1] + ii)*this->clayout->sizes[2] + cc))[0] =
- (*(buffer + ncomp(fc)*(this->clayout->sizes[1]-ii)+cc))[0];
- (*(data + ncomp(fc)*((this->clayout->sizes[0] - yy - this->clayout->starts[0])*this->clayout->sizes[1] + ii)*this->clayout->sizes[2] + cc))[1] =
- -(*(buffer + ncomp(fc)*(this->clayout->sizes[1]-ii)+cc))[1];
+ (*(cdata + ncomp(fc)*cindex + cc))[0] = (*(buffer + ncomp(fc)*iz+cc))[0];
+ (*(cdata + ncomp(fc)*cindex + cc))[1] = -(*(buffer + ncomp(fc)*iz+cc))[1];
}
- for (cc = 0; cc < ncomp(fc); cc++)
+ }
+ ptrdiff_t cindex = this->get_cindex(0, iyy, 0);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++)
{
- (*((data + cc + ncomp(fc)*(this->clayout->sizes[0] - yy - this->clayout->starts[0])*this->clayout->sizes[1]*this->clayout->sizes[2])))[0] = (*(buffer + cc))[0];
- (*((data + cc + ncomp(fc)*(this->clayout->sizes[0] - yy - this->clayout->starts[0])*this->clayout->sizes[1]*this->clayout->sizes[2])))[1] = -(*(buffer + cc))[1];
+ (*(cdata + cc + ncomp(fc)*cindex))[0] = (*(buffer + cc))[0];
+ (*(cdata + cc + ncomp(fc)*cindex))[1] = -(*(buffer + cc))[1];
}
}
}
fftw_interface<rnumber>::free(buffer);
delete mpistatus;
- /* put asymmetric data to 0 */
+ // symmetrize kx = 0, ky = 0 line
+ if (this->clayout->myrank == this->clayout->rank[0][0])
+ {
+ for (ptrdiff_t iz = 1; iz < ptrdiff_t(this->clayout->sizes[1]/2); iz++)
+ {
+ ptrdiff_t cindex0 = this->get_cindex(0, 0, iz);
+ ptrdiff_t cindex1 = this->get_cindex(0, 0, this->clayout->sizes[1] - iz);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++)
+ {
+ rnumber rtmp = ((*(cdata + cc + ncomp(fc)*cindex0))[0] +
+ (*(cdata + cc + ncomp(fc)*cindex1))[0])/2;
+ rnumber ctmp = ((*(cdata + cc + ncomp(fc)*cindex0))[1] -
+ (*(cdata + cc + ncomp(fc)*cindex1))[1])/2;
+ (*(cdata + cc + ncomp(fc)*cindex0))[0] = rtmp;
+ (*(cdata + cc + ncomp(fc)*cindex1))[0] = rtmp;
+ (*(cdata + cc + ncomp(fc)*cindex0))[1] = ctmp;
+ (*(cdata + cc + ncomp(fc)*cindex1))[1] = -ctmp;
+ }
+ }
+ }
+ // make 0 mode real
+ if (this->myrank == this->clayout->rank[0][0])
+ {
+ for (ptrdiff_t cc = 0; cc < ncomp(fc); cc++)
+ cdata[cc][1] = 0.0;
+ }
+ // put kx = nx/2 modes to 0
+ for (ptrdiff_t iy = 0; iy < ptrdiff_t(this->clayout->subsizes[0]); iy++)
+ for (ptrdiff_t iz = 0; iz < ptrdiff_t(this->clayout->subsizes[1]); iz++)
+ {
+ ptrdiff_t cindex = this->get_cindex(this->clayout->sizes[2]-1, iy, iz);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++) {
+ (*(cdata + cc + ncomp(fc)*cindex))[0] = 0.0;
+ (*(cdata + cc + ncomp(fc)*cindex))[1] = 0.0;
+ }
+ }
+ // put ky = ny/2 modes to 0
if (this->clayout->myrank == this->clayout->rank[0][this->clayout->sizes[0]/2])
{
- ptrdiff_t tindex = ncomp(fc)*(this->clayout->sizes[0]/2 - this->clayout->starts[0])*this->clayout->sizes[1]*this->clayout->sizes[2];
- for (ii = 0; ii < ptrdiff_t(this->clayout->sizes[1]); ii++)
+ for (ptrdiff_t iz = 0; iz < ptrdiff_t(this->clayout->subsizes[1]); iz++)
+ for (ptrdiff_t ix = 0; ix < ptrdiff_t(this->clayout->subsizes[2]); ix++)
{
- std::fill_n((rnumber*)(data + tindex), ncomp(fc)*2*this->clayout->sizes[2], 0.0);
- tindex += ncomp(fc)*this->clayout->sizes[2];
+ ptrdiff_t cindex = this->get_cindex(ix, this->clayout->sizes[0]/2-this->clayout->starts[0], iz);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++) {
+ (*(cdata + cc + ncomp(fc)*cindex))[0] = 0.0;
+ (*(cdata + cc + ncomp(fc)*cindex))[1] = 0.0;
+ }
+ }
+ }
+ // put kz = nz/2 modes to 0
+ for (ptrdiff_t iy = 0; iy < ptrdiff_t(this->clayout->subsizes[0]); iy++)
+ for (ptrdiff_t ix = 0; ix < ptrdiff_t(this->clayout->subsizes[2]); ix++)
+ {
+ ptrdiff_t cindex = this->get_cindex(ix, iy, this->clayout->sizes[1]/2);
+ for (int cc = 0; cc < int(ncomp(fc)); cc++) {
+ (*(cdata + cc + ncomp(fc)*cindex))[0] = 0.0;
+ (*(cdata + cc + ncomp(fc)*cindex))[1] = 0.0;
}
}
- //tindex = ncomp(fc)*();
- //std::fill_n((rnumber*)(data + tindex), ncomp(fc)*2, 0.0);
+ ///* put asymmetric data to 0 */
+ //if (this->clayout->myrank == this->clayout->rank[0][this->clayout->sizes[0]/2])
+ //{
+ // ptrdiff_t tindex = ncomp(fc)*(this->clayout->sizes[0]/2 - this->clayout->starts[0])*this->clayout->sizes[1]*this->clayout->sizes[2];
+ // for (ii = 0; ii < ptrdiff_t(this->clayout->sizes[1]); ii++)
+ // {
+ // std::fill_n((rnumber*)(data + tindex), ncomp(fc)*2*this->clayout->sizes[2], 0.0);
+ // tindex += ncomp(fc)*this->clayout->sizes[2];
+ // }
+ //}
+ ////tindex = ncomp(fc)*();
+ ////std::fill_n((rnumber*)(data + tindex), ncomp(fc)*2, 0.0);
}
template <typename rnumber,
diff --git a/bfps/cpp/full_code/field_test.cpp b/bfps/cpp/full_code/field_test.cpp
index b6c65a4a..5c323e54 100644
--- a/bfps/cpp/full_code/field_test.cpp
+++ b/bfps/cpp/full_code/field_test.cpp
@@ -50,7 +50,7 @@ int field_test<rnumber>::do_work(void)
DEFAULT_FFTW_FLAG);
std::default_random_engine rgen;
std::normal_distribution<rnumber> rdist;
- rgen.seed(1);
+ rgen.seed(2);
//auto gaussian = std::bind(rgen, rdist);
kspace<FFTW,SMOOTH> *kk = new kspace<FFTW, SMOOTH>(
scal_field->clayout, this->dkx, this->dky, this->dkz);
--
GitLab
From 662bff21778096a31d04238ff656caee287ee1fd Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Mon, 28 May 2018 09:59:38 +0200
Subject: [PATCH 165/342] still debugging
---
bfps/DNS.py | 1 +
bfps/cpp/field.cpp | 11 +++++++----
2 files changed, 8 insertions(+), 4 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 2e18b51c..7c5fdfa7 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -57,6 +57,7 @@ class DNS(_code):
'mail_address': '',
'mail_events' : None}
self.generate_default_parameters()
+ self.statistics = {}
return None
def set_precision(
self,
diff --git a/bfps/cpp/field.cpp b/bfps/cpp/field.cpp
index 2b2852a5..e3c35a51 100644
--- a/bfps/cpp/field.cpp
+++ b/bfps/cpp/field.cpp
@@ -1026,11 +1026,13 @@ void field<rnumber, be, fc>::symmetrize()
//this->ift();
//this->dft();
//this->normalize();
+ //return;
typename fftw_interface<rnumber>::complex *cdata = this->get_cdata();
// symmetrize kx = 0 plane, line by line, for ky != 0
MPI_Status *mpistatus = new MPI_Status;
- typename fftw_interface<rnumber>::complex *buffer;
- buffer = fftw_interface<rnumber>::alloc_complex(ncomp(fc)*this->clayout->sizes[1]);
+ typename fftw_interface<rnumber>::complex *buffer = new typename fftw_interface<rnumber>::complex[ncomp(fc)*this->clayout->sizes[1]];
+ //typename fftw_interface<rnumber>::complex *buffer;
+ //buffer = fftw_interface<rnumber>::alloc_complex(ncomp(fc)*this->clayout->sizes[1]);
/*ptrdiff_t tindex;*/
int ranksrc, rankdst;
for (ptrdiff_t iy = 1; iy < ptrdiff_t(this->clayout->sizes[0]/2); iy++)
@@ -1072,7 +1074,7 @@ void field<rnumber, be, fc>::symmetrize()
{
ptrdiff_t izz = (this->clayout->sizes[1] - iz);
ptrdiff_t cindex = this->get_cindex(0, iyy, izz);
- DEBUG_MSG("iy = %ld, iz = %ld\n", iy, iz);
+ //DEBUG_MSG("iy = %ld, iz = %ld\n", iy, iz);
for (int cc = 0; cc < int(ncomp(fc)); cc++)
{
(*(cdata + ncomp(fc)*cindex + cc))[0] = (*(buffer + ncomp(fc)*iz+cc))[0];
@@ -1087,7 +1089,8 @@ void field<rnumber, be, fc>::symmetrize()
}
}
}
- fftw_interface<rnumber>::free(buffer);
+ //fftw_interface<rnumber>::free(buffer);
+ delete[] buffer;
delete mpistatus;
// symmetrize kx = 0, ky = 0 line
if (this->clayout->myrank == this->clayout->rank[0][0])
--
GitLab
From 5e05c2ecb17ba981aa5752b39620d786a516a48c Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Mon, 28 May 2018 09:59:58 +0200
Subject: [PATCH 166/342] add Parseval test
---
bfps/test/test_Parseval.py | 33 +++++++++++++++++++++++++++++++++
1 file changed, 33 insertions(+)
create mode 100644 bfps/test/test_Parseval.py
diff --git a/bfps/test/test_Parseval.py b/bfps/test/test_Parseval.py
new file mode 100644
index 00000000..00945185
--- /dev/null
+++ b/bfps/test/test_Parseval.py
@@ -0,0 +1,33 @@
+#! /usr/bin/env python
+
+import os
+import numpy as np
+import h5py
+import sys
+
+import bfps
+from bfps import DNS
+
+def main():
+ niterations = 10
+ c = DNS()
+ c.launch(
+ ['NSVE',
+ '--nx', '32',
+ '--ny', '32',
+ '--nz', '32',
+ '--forcing_type', 'linear',
+ '--np', '4',
+ '--ntpp', '1',
+ '--niter_todo', '{0}'.format(niterations),
+ '--niter_out', '{0}'.format(niterations),
+ '--niter_stat', '1',
+ '--wd', './'] +
+ sys.argv[1:])
+ c.compute_statistics()
+ print(c.statistics['energy(t)'] - c.statistics['renergy(t)'])
+ return None
+
+if __name__ == '__main__':
+ main()
+
--
GitLab
From a010badde210e02153fdb1b27e59cebaad414d06 Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Mon, 28 May 2018 10:00:47 +0200
Subject: [PATCH 167/342] add pure fftw symmetrize test
---
tests/misc/makefile | 14 ++
tests/misc/run.sh | 2 +
tests/misc/test_fftw.c | 335 +++++++++++++++++++++++++++++++++++++++++
3 files changed, 351 insertions(+)
create mode 100644 tests/misc/makefile
create mode 100644 tests/misc/run.sh
create mode 100644 tests/misc/test_fftw.c
diff --git a/tests/misc/makefile b/tests/misc/makefile
new file mode 100644
index 00000000..c8f5f788
--- /dev/null
+++ b/tests/misc/makefile
@@ -0,0 +1,14 @@
+test_fftw: test_fftw.c
+ mpicc \
+ -I/stuff/ext_installs/include \
+ -fopenmp \
+ test_fftw.c \
+ -o test_fftw \
+ -L/stuff/ext_installs/lib \
+ -lfftw3_mpi \
+ -lfftw3 \
+ -lfftw3f_mpi \
+ -lfftw3f \
+ -lfftw3_threads \
+ -lfftw3f_threads \
+ -lm
diff --git a/tests/misc/run.sh b/tests/misc/run.sh
new file mode 100644
index 00000000..48120912
--- /dev/null
+++ b/tests/misc/run.sh
@@ -0,0 +1,2 @@
+make
+mpirun -np 4 -x OMP_NUM_THREADS=1 test_fftw
diff --git a/tests/misc/test_fftw.c b/tests/misc/test_fftw.c
new file mode 100644
index 00000000..bed5e695
--- /dev/null
+++ b/tests/misc/test_fftw.c
@@ -0,0 +1,335 @@
+#include <fftw3-mpi.h>
+#include <omp.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <math.h>
+
+//#define NO_FFTWOMP
+
+#define NX 24
+#define NY 24
+#define NZ 24
+
+const int nx = NX;
+const int ny = NY;
+const int nz = NZ;
+const int npoints = NX*NY*NZ;
+
+const double dkx = 1.0;
+const double dky = 1.0;
+const double dkz = 1.0;
+
+int myrank, nprocs;
+
+int main(
+ int argc,
+ char *argv[])
+{
+ ////////////////////////////////////
+ /* initialize MPI environment */
+#ifdef NO_FFTWOMP
+ MPI_Init(&argc, &argv);
+ MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
+ MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
+ fftw_mpi_init();
+ fftwf_mpi_init();
+ printf("There are %d processes\n", nprocs);
+#else
+ int mpiprovided;
+ MPI_Init_thread(&argc, &argv, MPI_THREAD_FUNNELED, &mpiprovided);
+ assert(mpiprovided >= MPI_THREAD_FUNNELED);
+ MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
+ MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
+ const int nThreads = omp_get_max_threads();
+ printf("Number of threads for the FFTW = %d\n",
+ nThreads);
+ if (nThreads > 1){
+ fftw_init_threads();
+ fftwf_init_threads();
+ }
+ fftw_mpi_init();
+ fftwf_mpi_init();
+ printf("There are %d processes and %d threads\n",
+ nprocs,
+ nThreads);
+ if (nThreads > 1){
+ fftw_plan_with_nthreads(nThreads);
+ fftwf_plan_with_nthreads(nThreads);
+ }
+#endif
+
+ ////////////////////////////////////
+ /* do useful work */
+
+ // declarations
+ ptrdiff_t nfftw[3];
+ ptrdiff_t tmp_local_size;
+ ptrdiff_t local_n0, local_0_start;
+ ptrdiff_t local_n1, local_1_start;
+ ptrdiff_t local_size;
+ ptrdiff_t ix, iy, iz;
+ ptrdiff_t jx, jy, jz;
+ ptrdiff_t rindex, cindex;
+ int cc;
+ float *data0, *data;
+ fftwf_complex *cdata;
+ double L2norm0, L2norm1, L2norm2, L2normk;
+ double local_L2norm0, local_L2norm1;
+ fftwf_plan c2r_plan, r2c_plan;
+ double *kx, *ky, *kz;
+
+ // get sizes
+ nfftw[0] = nz;
+ nfftw[1] = ny;
+ nfftw[2] = nx;
+ tmp_local_size = fftwf_mpi_local_size_many_transposed(
+ 3, nfftw, 3,
+ FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK, MPI_COMM_WORLD,
+ &local_n0, &local_0_start,
+ &local_n1, &local_1_start);
+
+ local_size = local_n1 * nz * nx * 3 * 2;
+
+ // allocate field
+ data = fftwf_alloc_real(
+ local_size);
+ data0 = fftwf_alloc_real(
+ local_size);
+ cdata = (fftwf_complex*)(data);
+
+ c2r_plan = fftwf_mpi_plan_many_dft_c2r(
+ 3, nfftw, 3,
+ FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
+ cdata,
+ data,
+ MPI_COMM_WORLD,
+ FFTW_ESTIMATE | FFTW_MPI_TRANSPOSED_IN);
+
+ r2c_plan = fftwf_mpi_plan_many_dft_r2c(
+ 3, nfftw, 3,
+ FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
+ data,
+ cdata,
+ MPI_COMM_WORLD,
+ FFTW_ESTIMATE | FFTW_MPI_TRANSPOSED_OUT);
+
+ kx = (double*)malloc(sizeof(double)*(nx/2+1));
+ ky = (double*)malloc(sizeof(double)*local_n1);
+ kz = (double*)malloc(sizeof(double)*nz);
+
+ // generate wavenumbers
+ for (jy = 0; jy < local_n1; jy++)
+ {
+ if (jy + local_1_start <= ny/2)
+ ky[jy] = dky*(jy + local_1_start);
+ else
+ ky[jy] = dky*((jy + local_1_start) - ny);
+ }
+ for (jz = 0; jz < nz; jz++)
+ {
+ if (jz <= nz/2)
+ kz[jz] = dkz*jz;
+ else
+ kz[jz] = dkz*(jz - nz);
+ }
+ for (jx = 0; jx < nx/2+1; jx++)
+ {
+ kx[jx] = dkx*jx;
+ }
+
+ // fill field with random numbers
+ // I'm generating cindex the stupid way, but we can also use
+ // cindex = (jy*nz + jz)*(nx/2+1) + jx
+ cindex = 0;
+ for (jy = 0; jy < local_n1; jy++)
+ for (jz = 0; jz < nz; jz++)
+ {
+ for (jx = 0; jx < nx/2+1; jx++)
+ {
+ double k2 = (kx[jx]*kx[jx] +
+ ky[jy]*ky[jy] +
+ kz[jz]*kz[jz]);
+ if (jx == 0 && (jy + local_1_start) == 0 && jz == 0)
+ k2 = dkx*dkx + dky*dky + dkz*dkz;
+ for (cc = 0; cc<3; cc++)
+ {
+ cdata[cindex*3+cc][0] = (drand48()-0.5) / sqrt(k2);
+ cdata[cindex*3+cc][1] = (drand48()-0.5) / sqrt(k2);
+ }
+ cindex++;
+ }
+ }
+
+ // go back and forth so that the
+ // Fourier space representation is properly symmetrized
+ fftwf_execute(c2r_plan);
+ fftwf_execute(r2c_plan);
+ // normalize, compute Fourier space L2 norm
+ cindex = 0;
+ local_L2norm0 = 0;
+ for (jy = 0; jy < local_n1; jy++)
+ for (jz = 0; jz < nz; jz++)
+ {
+ for (cc = 0; cc<3; cc++)
+ {
+ cdata[cindex*3+cc][0] /= npoints;
+ cdata[cindex*3+cc][1] /= npoints;
+ local_L2norm0 += (cdata[cindex*3+cc][0]*cdata[cindex*3+cc][0] +
+ cdata[cindex*3+cc][1]*cdata[cindex*3+cc][1]);
+ }
+ cindex++;
+ for (jx = 1; jx < nx/2+1; jx++)
+ {
+ for (cc = 0; cc<3; cc++)
+ {
+ cdata[cindex*3+cc][0] /= npoints;
+ cdata[cindex*3+cc][1] /= npoints;
+ local_L2norm0 += 2*(cdata[cindex*3+cc][0]*cdata[cindex*3+cc][0] +
+ cdata[cindex*3+cc][1]*cdata[cindex*3+cc][1]);
+ }
+ cindex++;
+ }
+ }
+ MPI_Allreduce(
+ &local_L2norm0,
+ &L2normk,
+ 1,
+ MPI_DOUBLE,
+ MPI_SUM,
+ MPI_COMM_WORLD);
+ L2normk = sqrt(L2normk);
+
+ // go to real space
+ fftwf_execute(c2r_plan);
+
+ // rindex = (iz*ny + iy)*(nx+2) + ix
+ rindex = 0;
+ local_L2norm0 = 0;
+ for (iz = 0; iz < local_n0; iz++)
+ for (iy = 0; iy < ny; iy++)
+ {
+ for (ix = 0; ix < nx; ix++)
+ {
+ for (cc = 0; cc<3; cc++)
+ {
+ local_L2norm0 += data[rindex*3+cc]*data[rindex*3+cc];
+ }
+ rindex++;
+ }
+ for (ix = nx; ix < nx+2; ix++)
+ {
+ rindex++;
+ }
+ }
+ MPI_Allreduce(
+ &local_L2norm0,
+ &L2norm1,
+ 1,
+ MPI_DOUBLE,
+ MPI_SUM,
+ MPI_COMM_WORLD);
+ L2norm1 = sqrt(L2norm1 / npoints);
+
+ //fftwf_execute(r2c_plan);
+
+ //cindex = 0;
+ //local_L2norm0 = 0;
+ //for (jy = 0; jy < local_n1; jy++)
+ // for (jz = 0; jz < nz; jz++)
+ // {
+ // for (cc = 0; cc<3; cc++)
+ // {
+ // local_L2norm0 += (cdata[cindex*3+cc][0]*cdata[cindex*3+cc][0] +
+ // cdata[cindex*3+cc][1]*cdata[cindex*3+cc][1]);
+ // }
+ // cindex++;
+ // // I am not adding the energy from mode nx/2 as a matter of principle.
+ // for (jx = 1; jx < nx/2+1; jx++)
+ // {
+ // for (cc = 0; cc<3; cc++)
+ // {
+ // local_L2norm0 += 2*(cdata[cindex*3+cc][0]*cdata[cindex*3+cc][0] +
+ // cdata[cindex*3+cc][1]*cdata[cindex*3+cc][1]);
+ // }
+ // cindex++;
+ // }
+ // }
+ //MPI_Allreduce(
+ // &local_L2norm0,
+ // &L2normk,
+ // 1,
+ // MPI_DOUBLE,
+ // MPI_SUM,
+ // MPI_COMM_WORLD);
+ //L2normk = sqrt(L2normk) / (nx*ny*nz);
+ //fftwf_execute(c2r_plan);
+
+ //// normalize
+ //rindex = 0;
+ //local_L2norm0 = 0;
+ //local_L2norm1 = 0;
+ //for (iz = 0; iz < local_n0; iz++)
+ // for (iy = 0; iy < ny; iy++)
+ // {
+ // for (ix = 0; ix < nx; ix++)
+ // {
+ // for (cc = 0; cc<3; cc++)
+ // {
+ // data[rindex*3+cc] /= (nx*ny*nz);
+ // local_L2norm0 += data[rindex*3+cc]*data[rindex*3+cc];
+ // local_L2norm1 += ((data0[rindex*3+cc] - data[rindex*3+cc])*
+ // (data0[rindex*3+cc] - data[rindex*3+cc]));
+ // }
+ // rindex++;
+ // }
+ // for (ix = nx; ix < nx+2; ix++)
+ // {
+ // rindex++;
+ // }
+ // }
+ //MPI_Allreduce(
+ // &local_L2norm0,
+ // &L2norm1,
+ // 1,
+ // MPI_DOUBLE,
+ // MPI_SUM,
+ // MPI_COMM_WORLD);
+ //MPI_Allreduce(
+ // &local_L2norm1,
+ // &L2norm2,
+ // 1,
+ // MPI_DOUBLE,
+ // MPI_SUM,
+ // MPI_COMM_WORLD);
+ //L2norm1 = sqrt(L2norm1 / (nx*ny*nz));
+ //L2norm2 = sqrt(L2norm2 / (nx*ny*nz));
+
+ printf("L2normk = %g, L2norm1 = %g, normed_diff = %g\n",
+ L2normk, L2norm1, fabs(L2normk - L2norm1)/npoints);
+
+ // deallocate
+ fftwf_destroy_plan(r2c_plan);
+ fftwf_destroy_plan(c2r_plan);
+ fftwf_free(data);
+ fftwf_free(data0);
+ free(kx);
+ free(ky);
+ free(kz);
+
+ ////////////////////////////////////
+ /* clean up */
+ fftwf_mpi_cleanup();
+ fftw_mpi_cleanup();
+
+#ifndef NO_FFTWOMP
+ if (nThreads > 1){
+ fftw_cleanup_threads();
+ fftwf_cleanup_threads();
+ }
+#endif
+
+ MPI_Finalize();
+ return EXIT_SUCCESS;
+}
+
--
GitLab
From 34fd4e42633e824c7018f41418b8784909428a16 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 28 May 2018 23:19:17 +0200
Subject: [PATCH 168/342] still debugging
---
bfps/TEST.py | 5 +++++
bfps/cpp/vorticity_equation.hpp | 1 +
bfps/test/test_Parseval.py | 2 +-
setup.py | 1 +
tests/misc/test_fftw.c | 6 +++---
5 files changed, 11 insertions(+), 4 deletions(-)
diff --git a/bfps/TEST.py b/bfps/TEST.py
index 4983fc61..0de39dd0 100644
--- a/bfps/TEST.py
+++ b/bfps/TEST.py
@@ -124,6 +124,7 @@ class TEST(_code):
self.parameters['dky'] = float(1.0)
self.parameters['dkz'] = float(1.0)
self.parameters['filter_length'] = float(1.0)
+ self.parameters['random_seed'] = int(1)
return None
def generate_extra_parameters(
self,
@@ -268,6 +269,9 @@ class TEST(_code):
parser_field_test = subparsers.add_parser(
'field_test',
help = 'plain field test')
+ parser_symmetrize_test = subparsers.add_parser(
+ 'symmetrize_test',
+ help = 'plain symmetrize test')
parser_field_output_test = subparsers.add_parser(
'field_output_test',
help = 'plain field output test')
@@ -276,6 +280,7 @@ class TEST(_code):
help = 'test velocity gradient interpolation')
for parser in ['parser_filter_test',
'parser_field_test',
+ 'parser_symmetrize_test',
'parser_field_output_test',
'parser_test_interpolation']:
eval('self.simulation_parser_arguments(' + parser + ')')
diff --git a/bfps/cpp/vorticity_equation.hpp b/bfps/cpp/vorticity_equation.hpp
index 81f0cb66..230ec37a 100644
--- a/bfps/cpp/vorticity_equation.hpp
+++ b/bfps/cpp/vorticity_equation.hpp
@@ -154,6 +154,7 @@ class vorticity_equation
this->kk->template low_pass<rnumber, THREE>(this->cvorticity->get_cdata(), this->kk->kM);
this->kk->template force_divfree<rnumber>(this->cvorticity->get_cdata());
#endif
+ this->cvorticity->symmetrize();
}
}
diff --git a/bfps/test/test_Parseval.py b/bfps/test/test_Parseval.py
index 00945185..6bc19574 100644
--- a/bfps/test/test_Parseval.py
+++ b/bfps/test/test_Parseval.py
@@ -25,7 +25,7 @@ def main():
'--wd', './'] +
sys.argv[1:])
c.compute_statistics()
- print(c.statistics['energy(t)'] - c.statistics['renergy(t)'])
+ print((c.statistics['energy(t)'] - c.statistics['renergy(t)']) / c.statistics['renergy(t)'])
return None
if __name__ == '__main__':
diff --git a/setup.py b/setup.py
index 350bccf2..7640e417 100644
--- a/setup.py
+++ b/setup.py
@@ -93,6 +93,7 @@ src_file_list = ['full_code/NSVEcomplex_particles',
'full_code/test',
'full_code/filter_test',
'full_code/field_test',
+ 'full_code/symmetrize_test',
'full_code/field_output_test',
'full_code/test_interpolation',
'hdf5_tools',
diff --git a/tests/misc/test_fftw.c b/tests/misc/test_fftw.c
index bed5e695..df4dc2eb 100644
--- a/tests/misc/test_fftw.c
+++ b/tests/misc/test_fftw.c
@@ -7,9 +7,9 @@
//#define NO_FFTWOMP
-#define NX 24
-#define NY 24
-#define NZ 24
+#define NX 36
+#define NY 36
+#define NZ 12
const int nx = NX;
const int ny = NY;
--
GitLab
From ff2840b955956732c9c8562b17a3ab4ad61239fd Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 28 May 2018 23:19:37 +0200
Subject: [PATCH 169/342] add symmetrize test files
---
bfps/cpp/full_code/symmetrize_test.cpp | 177 +++++++++++++++++++++++++
bfps/cpp/full_code/symmetrize_test.hpp | 62 +++++++++
2 files changed, 239 insertions(+)
create mode 100644 bfps/cpp/full_code/symmetrize_test.cpp
create mode 100644 bfps/cpp/full_code/symmetrize_test.hpp
diff --git a/bfps/cpp/full_code/symmetrize_test.cpp b/bfps/cpp/full_code/symmetrize_test.cpp
new file mode 100644
index 00000000..4bc98cb8
--- /dev/null
+++ b/bfps/cpp/full_code/symmetrize_test.cpp
@@ -0,0 +1,177 @@
+#include <string>
+#include <cmath>
+#include <random>
+#include "symmetrize_test.hpp"
+#include "scope_timer.hpp"
+
+
+template <typename rnumber>
+int symmetrize_test<rnumber>::initialize(void)
+{
+ this->read_parameters();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int symmetrize_test<rnumber>::finalize(void)
+{
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int symmetrize_test<rnumber>::read_parameters()
+{
+ this->test::read_parameters();
+ hid_t parameter_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ this->random_seed = hdf5_tools::read_value<int>(
+ parameter_file, "/parameters/random_seed");
+ H5Fclose(parameter_file);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int symmetrize_test<rnumber>::do_work(void)
+{
+ // allocate
+ field<rnumber, FFTW, THREE> *scal_field0 = new field<rnumber, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ DEFAULT_FFTW_FLAG);
+ field<rnumber, FFTW, THREE> *scal_field1 = new field<rnumber, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ DEFAULT_FFTW_FLAG);
+ std::default_random_engine rgen;
+ std::normal_distribution<rnumber> rdist;
+ rgen.seed(1);
+ kspace<FFTW,SMOOTH> *kk = new kspace<FFTW, SMOOTH>(
+ scal_field0->clayout, this->dkx, this->dky, this->dkz);
+
+ if (this->myrank == 0)
+ {
+ hid_t stat_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDWR,
+ H5P_DEFAULT);
+ kk->store(stat_file);
+ H5Fclose(stat_file);
+ }
+
+ // fill up scal_field0
+ *scal_field0 = 0.0;
+ scal_field0->real_space_representation = false;
+ kk->CLOOP_K2(
+ [&](ptrdiff_t cindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex,
+ double k2){
+ //if (k2 < kk->kM2)
+ {
+ scal_field0->cval(cindex, 0, 0) = rdist(rgen);
+ scal_field0->cval(cindex, 0, 1) = rdist(rgen);
+ scal_field0->cval(cindex, 1, 0) = rdist(rgen);
+ scal_field0->cval(cindex, 1, 1) = rdist(rgen);
+ scal_field0->cval(cindex, 2, 0) = rdist(rgen);
+ scal_field0->cval(cindex, 2, 1) = rdist(rgen);
+ }
+ if (k2 > 0)
+ {
+ scal_field0->cval(cindex, 0, 0) /= sqrt(k2);
+ scal_field0->cval(cindex, 0, 1) /= sqrt(k2);
+ scal_field0->cval(cindex, 1, 0) /= sqrt(k2);
+ scal_field0->cval(cindex, 1, 1) /= sqrt(k2);
+ scal_field0->cval(cindex, 2, 0) /= sqrt(k2);
+ scal_field0->cval(cindex, 2, 1) /= sqrt(k2);
+ }
+ else
+ {
+ scal_field0->cval(cindex, 0, 0) = 0;
+ scal_field0->cval(cindex, 0, 1) = 0;
+ scal_field0->cval(cindex, 1, 0) = 0;
+ scal_field0->cval(cindex, 1, 1) = 0;
+ scal_field0->cval(cindex, 2, 0) = 0;
+ scal_field0->cval(cindex, 2, 1) = 0;
+ }
+ });
+ // dealias (?!)
+ //kk->template low_pass<rnumber, THREE>(scal_field0->get_cdata(), kk->kM);
+ kk->template dealias<rnumber, THREE>(scal_field0->get_cdata());
+ // make the field divergence free
+ kk->template force_divfree<rnumber>(scal_field0->get_cdata());
+ // apply symmetrize to scal_field0
+ scal_field0->symmetrize();
+
+
+ // make copy in scal_field1
+ // this MUST be made after symmetrizing scal_field0
+ // (alternatively, we may symmetrize scal_field1 as well before the ift-dft cycle
+ scal_field1->real_space_representation = false;
+ *scal_field1 = scal_field0->get_cdata();
+
+ // go back and forth with scal_field1, to enforce symmetry
+ scal_field1->ift();
+ scal_field1->dft();
+ scal_field1->normalize();
+
+ // now compare the two fields
+ double max_diff = 0;
+ ptrdiff_t ix, iy, iz;
+ double k_at_max_diff = 0;
+ double a0, a1;
+
+ kk->CLOOP_K2(
+ [&](ptrdiff_t cindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex,
+ double k2){
+ double diff_re0 = scal_field0->cval(cindex, 0, 0) - scal_field1->cval(cindex, 0, 0);
+ double diff_re1 = scal_field0->cval(cindex, 1, 0) - scal_field1->cval(cindex, 1, 0);
+ double diff_re2 = scal_field0->cval(cindex, 2, 0) - scal_field1->cval(cindex, 2, 0);
+ double diff_im0 = scal_field0->cval(cindex, 0, 1) - scal_field1->cval(cindex, 0, 1);
+ double diff_im1 = scal_field0->cval(cindex, 1, 1) - scal_field1->cval(cindex, 1, 1);
+ double diff_im2 = scal_field0->cval(cindex, 2, 1) - scal_field1->cval(cindex, 2, 1);
+ double diff = sqrt(diff_re0*diff_re0 + diff_re1*diff_re1 + diff_re2*diff_re2 +
+ diff_im0*diff_im0 + diff_im1*diff_im1 + diff_im2*diff_im2);
+ double amplitude0 = (scal_field0->cval(cindex, 0, 0)*scal_field0->cval(cindex, 0, 0) +
+ scal_field0->cval(cindex, 1, 0)*scal_field0->cval(cindex, 1, 0) +
+ scal_field0->cval(cindex, 2, 0)*scal_field0->cval(cindex, 2, 0) +
+ scal_field0->cval(cindex, 0, 1)*scal_field0->cval(cindex, 0, 1) +
+ scal_field0->cval(cindex, 1, 1)*scal_field0->cval(cindex, 1, 1) +
+ scal_field0->cval(cindex, 2, 1)*scal_field0->cval(cindex, 2, 1));
+ double amplitude1 = (scal_field1->cval(cindex, 0, 0)*scal_field1->cval(cindex, 0, 0) +
+ scal_field1->cval(cindex, 1, 0)*scal_field1->cval(cindex, 1, 0) +
+ scal_field1->cval(cindex, 2, 0)*scal_field1->cval(cindex, 2, 0) +
+ scal_field1->cval(cindex, 0, 1)*scal_field1->cval(cindex, 0, 1) +
+ scal_field1->cval(cindex, 1, 1)*scal_field1->cval(cindex, 1, 1) +
+ scal_field1->cval(cindex, 2, 1)*scal_field1->cval(cindex, 2, 1));
+ double amplitude = sqrt((amplitude0 + amplitude1)/2);
+ if (amplitude > 0)
+ if (diff/amplitude > max_diff)
+ {
+ max_diff = diff / amplitude;
+ ix = xindex;
+ iy = yindex + scal_field0->clayout->starts[0];
+ iz = zindex;
+ k_at_max_diff = sqrt(k2);
+ a0 = sqrt(amplitude0);
+ a1 = sqrt(amplitude1);
+ }
+ });
+ DEBUG_MSG("found maximum relative difference %g at ix = %ld, iy = %ld, iz = %ld, wavenumber = %g, amplitudes %g %g\n",
+ max_diff, ix, iy, iz, k_at_max_diff, a0, a1);
+
+ // deallocate
+ delete kk;
+ delete scal_field1;
+ delete scal_field0;
+ return EXIT_SUCCESS;
+}
+
+template class symmetrize_test<float>;
+template class symmetrize_test<double>;
+
diff --git a/bfps/cpp/full_code/symmetrize_test.hpp b/bfps/cpp/full_code/symmetrize_test.hpp
new file mode 100644
index 00000000..d3fbbaeb
--- /dev/null
+++ b/bfps/cpp/full_code/symmetrize_test.hpp
@@ -0,0 +1,62 @@
+/**********************************************************************
+* *
+* Copyright 2018 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef SYMMETRIZE_TEST_HPP
+#define SYMMETRIZE_TEST_HPP
+
+
+
+#include <cstdlib>
+#include "base.hpp"
+#include "kspace.hpp"
+#include "field.hpp"
+#include "full_code/test.hpp"
+
+/** \brief A class for testing basic field class functionality.
+ */
+
+template <typename rnumber>
+class symmetrize_test: public test
+{
+ public:
+ int random_seed;
+
+ symmetrize_test(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ test(
+ COMMUNICATOR,
+ simulation_name){}
+ ~symmetrize_test(){}
+
+ int initialize(void);
+ int do_work(void);
+ int finalize(void);
+ int read_parameters(void);
+};
+
+#endif//SYMMETRIZE_TEST_HPP
+
--
GitLab
From eaf6dbd74d4470d99831fc974a458a84e93d0f0e Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Tue, 29 May 2018 09:46:22 +0200
Subject: [PATCH 170/342] add spectrum plot
---
bfps/test/test_Parseval.py | 17 ++++++++++++++++-
1 file changed, 16 insertions(+), 1 deletion(-)
diff --git a/bfps/test/test_Parseval.py b/bfps/test/test_Parseval.py
index 6bc19574..52552628 100644
--- a/bfps/test/test_Parseval.py
+++ b/bfps/test/test_Parseval.py
@@ -8,6 +8,8 @@ import sys
import bfps
from bfps import DNS
+import matplotlib.pyplot as plt
+
def main():
niterations = 10
c = DNS()
@@ -25,7 +27,20 @@ def main():
'--wd', './'] +
sys.argv[1:])
c.compute_statistics()
- print((c.statistics['energy(t)'] - c.statistics['renergy(t)']) / c.statistics['renergy(t)'])
+ print((c.statistics['energy(t)']*3 - c.statistics['renergy(t)']) / c.statistics['renergy(t)'])
+ print(list(c.statistics.keys()))
+ energyk = np.mean(c.statistics['energy(t, k)'], axis = 0)
+ nshell = c.get_data_file()['kspace/nshell'].value
+ renergy = np.mean(c.statistics['renergy(t)'])
+ print(renergy, np.trapz(energyk[:-2], c.statistics['kshell'][:-2]))
+
+ f = plt.figure()
+ a = f.add_subplot(111)
+ a.plot(c.statistics['kshell'], energyk)
+ a.plot(c.statistics['kshell'], (energyk / nshell)*(4*np.pi*c.statistics['kshell']**2))
+ a.set_yscale('log')
+ a.set_xscale('log')
+ f.savefig('spectrum.pdf')
return None
if __name__ == '__main__':
--
GitLab
From dcc3b1c19107ac37765856b69c261b4167b40f7f Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 29 May 2018 12:32:51 +0200
Subject: [PATCH 171/342] fix energy spectrum computation for nontrivial box
size
---
bfps/DNS.py | 108 ++++++++++++++++++++++++++++---------------
bfps/NavierStokes.py | 9 ++--
2 files changed, 76 insertions(+), 41 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 7c5fdfa7..c407a955 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -208,8 +208,12 @@ class DNS(_code):
return os.path.join(self.work_dir, self.simname + '_particles.h5')
def get_particle_file(self):
return h5py.File(self.get_particle_file_name(), 'r')
+ def get_cache_file_name(self):
+ return os.path.join(self.work_dir, self.simname + '_cache.h5')
+ def get_cache_file(self):
+ return h5py.File(self.get_cache_file_name(), 'r')
def get_postprocess_file_name(self):
- return os.path.join(self.work_dir, self.simname + '_postprocess.h5')
+ return self.get_cache_file_name()
def get_postprocess_file(self):
return h5py.File(self.get_postprocess_file_name(), 'r')
def compute_statistics(self, iter0 = 0, iter1 = None):
@@ -225,7 +229,7 @@ class DNS(_code):
tensors, and the enstrophy spectrum is also used to
compute the dissipation :math:`\\varepsilon(t)`.
These basic quantities are stored in a newly created HDF5 file,
- ``simname_postprocess.h5``.
+ ``simname_cache.h5``.
"""
if len(list(self.statistics.keys())) > 0:
return None
@@ -254,8 +258,15 @@ class DNS(_code):
computation_needed = not (ii0 == pp_file['ii0'].value and
ii1 == pp_file['ii1'].value)
if computation_needed:
- for k in pp_file.keys():
- del pp_file[k]
+ for k in ['t', 'vel_max(t)', 'renergy(t)',
+ 'energy(t)', 'enstrophy(t)',
+ 'energy(k)', 'enstrophy(k)',
+ 'energy(t, k)',
+ 'enstrophy(t, k)',
+ 'R_ij(t)',
+ 'ii0', 'ii1', 'iter0', 'iter1']:
+ if k in pp_file.keys():
+ del pp_file[k]
if computation_needed:
pp_file['iter0'] = iter0
pp_file['iter1'] = iter1
@@ -264,39 +275,63 @@ class DNS(_code):
pp_file['t'] = (self.parameters['dt']*
self.parameters['niter_stat']*
(np.arange(ii0, ii1+1).astype(np.float)))
- pp_file['energy(t, k)'] = (
- data_file['statistics/spectra/velocity_velocity'][ii0:ii1+1, :, 0, 0] +
- data_file['statistics/spectra/velocity_velocity'][ii0:ii1+1, :, 1, 1] +
- data_file['statistics/spectra/velocity_velocity'][ii0:ii1+1, :, 2, 2])/2
- pp_file['enstrophy(t, k)'] = (
+ phi_ij = data_file['statistics/spectra/velocity_velocity'][ii0:ii1+1]
+ discrete_Fourier_prefactor = 1. / (self.parameters['dkx']*
+ self.parameters['dky']*
+ self.parameters['dkz'])
+ pp_file['R_ij(t)'] = self.statistics['dk']*np.sum(phi_ij, axis = 1)*discrete_Fourier_prefactor
+ energy_tk = discrete_Fourier_prefactor*(
+ phi_ij[:, :, 0, 0] +
+ phi_ij[:, :, 1, 1] +
+ phi_ij[:, :, 2, 2])/2
+ pp_file['energy(t)'] = (self.statistics['dk'] *
+ np.sum(energy_tk, axis = 1))
+ pp_file['energy(k)'] = np.mean(energy_tk, axis = 0)
+ enstrophy_tk = discrete_Fourier_prefactor*(
data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 0, 0] +
data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 1, 1] +
data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 2, 2])/2
- pp_file['vel_max(t)'] = data_file['statistics/moments/velocity'] [ii0:ii1+1, 9, 3]
+ pp_file['enstrophy(t)'] = (self.statistics['dk'] *
+ np.sum(enstrophy_tk, axis = 1))
+ pp_file['enstrophy(k)'] = np.mean(enstrophy_tk, axis = 0)
+ pp_file['vel_max(t)'] = data_file['statistics/moments/velocity'][ii0:ii1+1, 9, 3]
pp_file['renergy(t)'] = data_file['statistics/moments/velocity'][ii0:ii1+1, 2, 3]/2
for k in ['t',
- 'energy(t, k)',
- 'enstrophy(t, k)',
+ 'energy(t)',
+ 'energy(k)',
+ 'enstrophy(t)',
+ 'enstrophy(k)',
+ 'R_ij(t)',
'vel_max(t)',
'renergy(t)']:
if k in pp_file.keys():
self.statistics[k] = pp_file[k].value
self.compute_time_averages()
return None
+ def compute_Reynolds_stress_invariants(
+ self):
+ Rij = self.statistics['R_ij(t)']
+ Rij /= (2*self.statistics['energy(t)'][:, None, None])
+ Rij[:, 0, 0] -= 1./3
+ Rij[:, 1, 1] -= 1./3
+ Rij[:, 2, 2] -= 1./3
+ self.statistics['I2(t)'] = np.sqrt(np.einsum('...ij,...ij', Rij, Rij, optimize = True) / 6)
+ self.statistics['I3(t)'] = np.cbrt(np.einsum('...ij,...jk,...ki', Rij, Rij, Rij, optimize = True) / 6)
+ return None
def compute_time_averages(self):
"""Compute easy stats.
Further computation of statistics based on the contents of
- ``simname_postprocess.h5``.
+ ``simname_cache.h5``.
Standard quantities are as follows
(consistent with [Ishihara]_):
.. math::
U_{\\textrm{int}}(t) = \\sqrt{\\frac{2E(t)}{3}}, \\hskip .5cm
- L_{\\textrm{int}}(t) = \\frac{\pi}{2U_{int}^2(t)} \\int \\frac{dk}{k} E(t, k), \\hskip .5cm
- T_{\\textrm{int}}(t) =
- \\frac{L_{\\textrm{int}}(t)}{U_{\\textrm{int}}(t)}
+ L_{\\textrm{int}} = \\frac{\pi}{2U_{int}^2} \\int \\frac{dk}{k} E(k), \\hskip .5cm
+ T_{\\textrm{int}} =
+ \\frac{L_{\\textrm{int}}}{U_{\\textrm{int}}}
\\eta_K = \\left(\\frac{\\nu^3}{\\varepsilon}\\right)^{1/4}, \\hskip .5cm
\\tau_K = \\left(\\frac{\\nu}{\\varepsilon}\\right)^{1/2}, \\hskip .5cm
@@ -313,21 +348,14 @@ class DNS(_code):
J. Fluid Mech.,
**592**, 335-366, 2007
"""
- for key in ['energy', 'enstrophy']:
- self.statistics[key + '(t)'] = (self.statistics['dk'] *
- np.sum(self.statistics[key + '(t, k)'], axis = 1))
self.statistics['Uint(t)'] = np.sqrt(2*self.statistics['energy(t)'] / 3)
- self.statistics['Lint(t)'] = ((self.statistics['dk']*np.pi /
- (2*self.statistics['Uint(t)']**2)) *
- np.nansum(self.statistics['energy(t, k)'] /
- self.statistics['kshell'][None, :], axis = 1))
for key in ['energy',
'enstrophy',
- 'vel_max',
- 'Uint',
- 'Lint']:
+ 'mean_trS2',
+ 'Uint']:
if key + '(t)' in self.statistics.keys():
self.statistics[key] = np.average(self.statistics[key + '(t)'], axis = 0)
+ self.statistics['vel_max'] = np.max(self.statistics['vel_max(t)'])
for suffix in ['', '(t)']:
self.statistics['diss' + suffix] = (self.parameters['nu'] *
self.statistics['enstrophy' + suffix]*2)
@@ -335,9 +363,6 @@ class DNS(_code):
self.statistics['diss' + suffix])**.25
self.statistics['tauK' + suffix] = (self.parameters['nu'] /
self.statistics['diss' + suffix])**.5
- self.statistics['Re' + suffix] = (self.statistics['Uint' + suffix] *
- self.statistics['Lint' + suffix] /
- self.parameters['nu'])
self.statistics['lambda' + suffix] = (15 * self.parameters['nu'] *
self.statistics['Uint' + suffix]**2 /
self.statistics['diss' + suffix])**.5
@@ -348,6 +373,13 @@ class DNS(_code):
self.statistics['etaK' + suffix])
if self.parameters['dealias_type'] == 1:
self.statistics['kMeta' + suffix] *= 0.8
+ self.statistics['Lint'] = ((self.statistics['dk']*np.pi /
+ (2*self.statistics['Uint']**2)) *
+ np.nansum(self.statistics['energy(k)'] /
+ self.statistics['kshell']))
+ self.statistics['Re'] = (self.statistics['Uint'] *
+ self.statistics['Lint'] /
+ self.parameters['nu'])
self.statistics['Tint'] = self.statistics['Lint'] / self.statistics['Uint']
self.statistics['Taylor_microscale'] = self.statistics['lambda']
return None
@@ -836,21 +868,21 @@ class DNS(_code):
"""
np.random.seed(rseed)
Kdata00 = scalar_generator(
- self.parameters['nz']//2,
- self.parameters['ny']//2,
- self.parameters['nx']//2,
+ self.parameters['nz'],
+ self.parameters['ny'],
+ self.parameters['nx'],
p = spectra_slope,
amplitude = amplitude).astype(self.ctype)
Kdata01 = scalar_generator(
- self.parameters['nz']//2,
- self.parameters['ny']//2,
- self.parameters['nx']//2,
+ self.parameters['nz'],
+ self.parameters['ny'],
+ self.parameters['nx'],
p = spectra_slope,
amplitude = amplitude).astype(self.ctype)
Kdata02 = scalar_generator(
- self.parameters['nz']//2,
- self.parameters['ny']//2,
- self.parameters['nx']//2,
+ self.parameters['nz'],
+ self.parameters['ny'],
+ self.parameters['nx'],
p = spectra_slope,
amplitude = amplitude).astype(self.ctype)
Kdata0 = np.zeros(
diff --git a/bfps/NavierStokes.py b/bfps/NavierStokes.py
index 98169ab0..62a2263e 100644
--- a/bfps/NavierStokes.py
+++ b/bfps/NavierStokes.py
@@ -627,15 +627,18 @@ class NavierStokes(_fluid_particle_base):
self.parameters['niter_stat']*
(np.arange(ii0, ii1+1).astype(np.float)))
phi_ij = data_file['statistics/spectra/velocity_velocity'][ii0:ii1+1]
- pp_file['R_ij(t)'] = self.statistics['dk']*np.sum(phi_ij, axis = 1)
- energy_tk = (
+ discrete_Fourier_prefactor = 1. / (self.parameters['dkx']*
+ self.parameters['dky']*
+ self.parameters['dkz'])
+ pp_file['R_ij(t)'] = self.statistics['dk']*np.sum(phi_ij, axis = 1)*discrete_Fourier_prefactor
+ energy_tk = discrete_Fourier_prefactor*(
phi_ij[:, :, 0, 0] +
phi_ij[:, :, 1, 1] +
phi_ij[:, :, 2, 2])/2
pp_file['energy(t)'] = (self.statistics['dk'] *
np.sum(energy_tk, axis = 1))
pp_file['energy(k)'] = np.mean(energy_tk, axis = 0)
- enstrophy_tk = (
+ enstrophy_tk = discrete_Fourier_prefactor*(
data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 0, 0] +
data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 1, 1] +
data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 2, 2])/2
--
GitLab
From 34d6c547eb520c3d70be0d3d5240cc20995b230f Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 29 May 2018 12:35:41 +0200
Subject: [PATCH 172/342] update to use new DNS stats
---
bfps/test/test_Parseval.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/bfps/test/test_Parseval.py b/bfps/test/test_Parseval.py
index 52552628..903ad75b 100644
--- a/bfps/test/test_Parseval.py
+++ b/bfps/test/test_Parseval.py
@@ -27,9 +27,9 @@ def main():
'--wd', './'] +
sys.argv[1:])
c.compute_statistics()
- print((c.statistics['energy(t)']*3 - c.statistics['renergy(t)']) / c.statistics['renergy(t)'])
+ print((c.statistics['energy(t)'] - c.statistics['renergy(t)']) / c.statistics['renergy(t)'])
print(list(c.statistics.keys()))
- energyk = np.mean(c.statistics['energy(t, k)'], axis = 0)
+ energyk = c.statistics['energy(k)']
nshell = c.get_data_file()['kspace/nshell'].value
renergy = np.mean(c.statistics['renergy(t)'])
print(renergy, np.trapz(energyk[:-2], c.statistics['kshell'][:-2]))
--
GitLab
From 511066ef9eb98c525a82775cb729ff28d789dfa8 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 29 May 2018 12:36:34 +0200
Subject: [PATCH 173/342] output more sensible error
---
tests/misc/run.sh | 2 +-
tests/misc/test_fftw.c | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/tests/misc/run.sh b/tests/misc/run.sh
index 48120912..ada649ca 100644
--- a/tests/misc/run.sh
+++ b/tests/misc/run.sh
@@ -1,2 +1,2 @@
make
-mpirun -np 4 -x OMP_NUM_THREADS=1 test_fftw
+mpirun -np 2 -x OMP_NUM_THREADS=1 test_fftw
diff --git a/tests/misc/test_fftw.c b/tests/misc/test_fftw.c
index df4dc2eb..6da0099c 100644
--- a/tests/misc/test_fftw.c
+++ b/tests/misc/test_fftw.c
@@ -305,8 +305,8 @@ int main(
//L2norm1 = sqrt(L2norm1 / (nx*ny*nz));
//L2norm2 = sqrt(L2norm2 / (nx*ny*nz));
- printf("L2normk = %g, L2norm1 = %g, normed_diff = %g\n",
- L2normk, L2norm1, fabs(L2normk - L2norm1)/npoints);
+ printf("L2normk = %g, L2norm1 = %g, relative error = %g\n",
+ L2normk, L2norm1, fabs(L2normk - L2norm1) / (L2normk));
// deallocate
fftwf_destroy_plan(r2c_plan);
--
GitLab
From a6247072c26af7c761157e8f6548c69c954575a6 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 29 May 2018 14:53:22 +0200
Subject: [PATCH 174/342] fix wavenumber computation
---
bfps/cpp/kspace.cpp | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git a/bfps/cpp/kspace.cpp b/bfps/cpp/kspace.cpp
index 72f8d70d..21d37379 100644
--- a/bfps/cpp/kspace.cpp
+++ b/bfps/cpp/kspace.cpp
@@ -66,17 +66,17 @@ kspace<be, dt>::kspace(
for (i = 0; i<int(this->layout->subsizes[0]); i++)
{
ii = i + this->layout->starts[0];
- if (ii <= int(this->layout->sizes[1]/2))
+ if (ii <= int(this->layout->sizes[0]/2))
this->ky[i] = this->dky*ii;
else
this->ky[i] = this->dky*(ii - int(this->layout->sizes[1]));
}
for (i = 0; i<int(this->layout->sizes[1]); i++)
{
- if (i <= int(this->layout->sizes[0]/2))
+ if (i <= int(this->layout->sizes[1]/2))
this->kz[i] = this->dkz*i;
else
- this->kz[i] = this->dkz*(i - int(this->layout->sizes[0]));
+ this->kz[i] = this->dkz*(i - int(this->layout->sizes[1]));
}
switch(dt)
{
@@ -476,7 +476,11 @@ void kspace<be, dt>::dealias(typename fftw_interface<rnumber>::complex *__restri
ptrdiff_t yindex,
ptrdiff_t zindex,
double k2){
- double tval = this->dealias_filter[int(round(k2 / this->dk2))];
+ //double tval = this->dealias_filter[int(round(k2 / this->dk2))];
+ double kk2 = (pow(this->kx[xindex]/this->kMx, 2) +
+ pow(this->ky[yindex]/this->kMy, 2) +
+ pow(this->kz[zindex]/this->kMz, 2));
+ double tval = exp(-36.0 * (pow(kk2, 18.)));
for (unsigned int tcounter=0; tcounter<2*ncomp(fc); tcounter++)
((rnumber*)a)[2*ncomp(fc)*cindex + tcounter] *= tval;
});
--
GitLab
From cab0e2886eb96552f8e8dd08a9b11f662a9e00a4 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 29 May 2018 15:02:16 +0200
Subject: [PATCH 175/342] use own symmetrize, which is fine
---
bfps/cpp/field.cpp | 13 +------------
1 file changed, 1 insertion(+), 12 deletions(-)
diff --git a/bfps/cpp/field.cpp b/bfps/cpp/field.cpp
index e3c35a51..6535c07a 100644
--- a/bfps/cpp/field.cpp
+++ b/bfps/cpp/field.cpp
@@ -1023,6 +1023,7 @@ void field<rnumber, be, fc>::symmetrize()
{
TIMEZONE("field::symmetrize");
assert(!this->real_space_representation);
+ // for debugging, just use FFTW
//this->ift();
//this->dft();
//this->normalize();
@@ -1151,18 +1152,6 @@ void field<rnumber, be, fc>::symmetrize()
(*(cdata + cc + ncomp(fc)*cindex))[1] = 0.0;
}
}
- ///* put asymmetric data to 0 */
- //if (this->clayout->myrank == this->clayout->rank[0][this->clayout->sizes[0]/2])
- //{
- // ptrdiff_t tindex = ncomp(fc)*(this->clayout->sizes[0]/2 - this->clayout->starts[0])*this->clayout->sizes[1]*this->clayout->sizes[2];
- // for (ii = 0; ii < ptrdiff_t(this->clayout->sizes[1]); ii++)
- // {
- // std::fill_n((rnumber*)(data + tindex), ncomp(fc)*2*this->clayout->sizes[2], 0.0);
- // tindex += ncomp(fc)*this->clayout->sizes[2];
- // }
- //}
- ////tindex = ncomp(fc)*();
- ////std::fill_n((rnumber*)(data + tindex), ncomp(fc)*2, 0.0);
}
template <typename rnumber,
--
GitLab
From 10b0fdbda9b138e5ed7486ea782089defdf0d331 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 29 May 2018 15:03:58 +0200
Subject: [PATCH 176/342] rename fields, add output
---
bfps/cpp/full_code/symmetrize_test.cpp | 132 ++++++++++++++-----------
1 file changed, 72 insertions(+), 60 deletions(-)
diff --git a/bfps/cpp/full_code/symmetrize_test.cpp b/bfps/cpp/full_code/symmetrize_test.cpp
index 4bc98cb8..471e234c 100644
--- a/bfps/cpp/full_code/symmetrize_test.cpp
+++ b/bfps/cpp/full_code/symmetrize_test.cpp
@@ -36,11 +36,11 @@ template <typename rnumber>
int symmetrize_test<rnumber>::do_work(void)
{
// allocate
- field<rnumber, FFTW, THREE> *scal_field0 = new field<rnumber, FFTW, THREE>(
+ field<rnumber, FFTW, THREE> *test_field0 = new field<rnumber, FFTW, THREE>(
this->nx, this->ny, this->nz,
this->comm,
DEFAULT_FFTW_FLAG);
- field<rnumber, FFTW, THREE> *scal_field1 = new field<rnumber, FFTW, THREE>(
+ field<rnumber, FFTW, THREE> *test_field1 = new field<rnumber, FFTW, THREE>(
this->nx, this->ny, this->nz,
this->comm,
DEFAULT_FFTW_FLAG);
@@ -48,7 +48,7 @@ int symmetrize_test<rnumber>::do_work(void)
std::normal_distribution<rnumber> rdist;
rgen.seed(1);
kspace<FFTW,SMOOTH> *kk = new kspace<FFTW, SMOOTH>(
- scal_field0->clayout, this->dkx, this->dky, this->dkz);
+ test_field0->clayout, this->dkx, this->dky, this->dkz);
if (this->myrank == 0)
{
@@ -60,62 +60,62 @@ int symmetrize_test<rnumber>::do_work(void)
H5Fclose(stat_file);
}
- // fill up scal_field0
- *scal_field0 = 0.0;
- scal_field0->real_space_representation = false;
+ // fill up test_field0
+ *test_field0 = 0.0;
+ test_field0->real_space_representation = false;
kk->CLOOP_K2(
[&](ptrdiff_t cindex,
ptrdiff_t xindex,
ptrdiff_t yindex,
ptrdiff_t zindex,
double k2){
- //if (k2 < kk->kM2)
+ if (k2 < kk->kM2)
{
- scal_field0->cval(cindex, 0, 0) = rdist(rgen);
- scal_field0->cval(cindex, 0, 1) = rdist(rgen);
- scal_field0->cval(cindex, 1, 0) = rdist(rgen);
- scal_field0->cval(cindex, 1, 1) = rdist(rgen);
- scal_field0->cval(cindex, 2, 0) = rdist(rgen);
- scal_field0->cval(cindex, 2, 1) = rdist(rgen);
+ test_field0->cval(cindex, 0, 0) = rdist(rgen);
+ test_field0->cval(cindex, 0, 1) = rdist(rgen);
+ test_field0->cval(cindex, 1, 0) = rdist(rgen);
+ test_field0->cval(cindex, 1, 1) = rdist(rgen);
+ test_field0->cval(cindex, 2, 0) = rdist(rgen);
+ test_field0->cval(cindex, 2, 1) = rdist(rgen);
}
if (k2 > 0)
{
- scal_field0->cval(cindex, 0, 0) /= sqrt(k2);
- scal_field0->cval(cindex, 0, 1) /= sqrt(k2);
- scal_field0->cval(cindex, 1, 0) /= sqrt(k2);
- scal_field0->cval(cindex, 1, 1) /= sqrt(k2);
- scal_field0->cval(cindex, 2, 0) /= sqrt(k2);
- scal_field0->cval(cindex, 2, 1) /= sqrt(k2);
+ test_field0->cval(cindex, 0, 0) /= sqrt(k2);
+ test_field0->cval(cindex, 0, 1) /= sqrt(k2);
+ test_field0->cval(cindex, 1, 0) /= sqrt(k2);
+ test_field0->cval(cindex, 1, 1) /= sqrt(k2);
+ test_field0->cval(cindex, 2, 0) /= sqrt(k2);
+ test_field0->cval(cindex, 2, 1) /= sqrt(k2);
}
else
{
- scal_field0->cval(cindex, 0, 0) = 0;
- scal_field0->cval(cindex, 0, 1) = 0;
- scal_field0->cval(cindex, 1, 0) = 0;
- scal_field0->cval(cindex, 1, 1) = 0;
- scal_field0->cval(cindex, 2, 0) = 0;
- scal_field0->cval(cindex, 2, 1) = 0;
+ test_field0->cval(cindex, 0, 0) = 0;
+ test_field0->cval(cindex, 0, 1) = 0;
+ test_field0->cval(cindex, 1, 0) = 0;
+ test_field0->cval(cindex, 1, 1) = 0;
+ test_field0->cval(cindex, 2, 0) = 0;
+ test_field0->cval(cindex, 2, 1) = 0;
}
});
// dealias (?!)
- //kk->template low_pass<rnumber, THREE>(scal_field0->get_cdata(), kk->kM);
- kk->template dealias<rnumber, THREE>(scal_field0->get_cdata());
+ //kk->template low_pass<rnumber, THREE>(test_field0->get_cdata(), kk->kM);
+ kk->template dealias<rnumber, THREE>(test_field0->get_cdata());
// make the field divergence free
- kk->template force_divfree<rnumber>(scal_field0->get_cdata());
- // apply symmetrize to scal_field0
- scal_field0->symmetrize();
+ kk->template force_divfree<rnumber>(test_field0->get_cdata());
+ // apply symmetrize to test_field0
+ //test_field0->symmetrize();
- // make copy in scal_field1
- // this MUST be made after symmetrizing scal_field0
- // (alternatively, we may symmetrize scal_field1 as well before the ift-dft cycle
- scal_field1->real_space_representation = false;
- *scal_field1 = scal_field0->get_cdata();
+ // make copy in test_field1
+ // this MUST be made after symmetrizing test_field0
+ // (alternatively, we may symmetrize test_field1 as well before the ift-dft cycle
+ test_field1->real_space_representation = false;
+ *test_field1 = test_field0->get_cdata();
- // go back and forth with scal_field1, to enforce symmetry
- scal_field1->ift();
- scal_field1->dft();
- scal_field1->normalize();
+ // go back and forth with test_field1, to enforce symmetry
+ test_field1->ift();
+ test_field1->dft();
+ test_field1->normalize();
// now compare the two fields
double max_diff = 0;
@@ -129,33 +129,33 @@ int symmetrize_test<rnumber>::do_work(void)
ptrdiff_t yindex,
ptrdiff_t zindex,
double k2){
- double diff_re0 = scal_field0->cval(cindex, 0, 0) - scal_field1->cval(cindex, 0, 0);
- double diff_re1 = scal_field0->cval(cindex, 1, 0) - scal_field1->cval(cindex, 1, 0);
- double diff_re2 = scal_field0->cval(cindex, 2, 0) - scal_field1->cval(cindex, 2, 0);
- double diff_im0 = scal_field0->cval(cindex, 0, 1) - scal_field1->cval(cindex, 0, 1);
- double diff_im1 = scal_field0->cval(cindex, 1, 1) - scal_field1->cval(cindex, 1, 1);
- double diff_im2 = scal_field0->cval(cindex, 2, 1) - scal_field1->cval(cindex, 2, 1);
+ double diff_re0 = test_field0->cval(cindex, 0, 0) - test_field1->cval(cindex, 0, 0);
+ double diff_re1 = test_field0->cval(cindex, 1, 0) - test_field1->cval(cindex, 1, 0);
+ double diff_re2 = test_field0->cval(cindex, 2, 0) - test_field1->cval(cindex, 2, 0);
+ double diff_im0 = test_field0->cval(cindex, 0, 1) - test_field1->cval(cindex, 0, 1);
+ double diff_im1 = test_field0->cval(cindex, 1, 1) - test_field1->cval(cindex, 1, 1);
+ double diff_im2 = test_field0->cval(cindex, 2, 1) - test_field1->cval(cindex, 2, 1);
double diff = sqrt(diff_re0*diff_re0 + diff_re1*diff_re1 + diff_re2*diff_re2 +
diff_im0*diff_im0 + diff_im1*diff_im1 + diff_im2*diff_im2);
- double amplitude0 = (scal_field0->cval(cindex, 0, 0)*scal_field0->cval(cindex, 0, 0) +
- scal_field0->cval(cindex, 1, 0)*scal_field0->cval(cindex, 1, 0) +
- scal_field0->cval(cindex, 2, 0)*scal_field0->cval(cindex, 2, 0) +
- scal_field0->cval(cindex, 0, 1)*scal_field0->cval(cindex, 0, 1) +
- scal_field0->cval(cindex, 1, 1)*scal_field0->cval(cindex, 1, 1) +
- scal_field0->cval(cindex, 2, 1)*scal_field0->cval(cindex, 2, 1));
- double amplitude1 = (scal_field1->cval(cindex, 0, 0)*scal_field1->cval(cindex, 0, 0) +
- scal_field1->cval(cindex, 1, 0)*scal_field1->cval(cindex, 1, 0) +
- scal_field1->cval(cindex, 2, 0)*scal_field1->cval(cindex, 2, 0) +
- scal_field1->cval(cindex, 0, 1)*scal_field1->cval(cindex, 0, 1) +
- scal_field1->cval(cindex, 1, 1)*scal_field1->cval(cindex, 1, 1) +
- scal_field1->cval(cindex, 2, 1)*scal_field1->cval(cindex, 2, 1));
+ double amplitude0 = (test_field0->cval(cindex, 0, 0)*test_field0->cval(cindex, 0, 0) +
+ test_field0->cval(cindex, 1, 0)*test_field0->cval(cindex, 1, 0) +
+ test_field0->cval(cindex, 2, 0)*test_field0->cval(cindex, 2, 0) +
+ test_field0->cval(cindex, 0, 1)*test_field0->cval(cindex, 0, 1) +
+ test_field0->cval(cindex, 1, 1)*test_field0->cval(cindex, 1, 1) +
+ test_field0->cval(cindex, 2, 1)*test_field0->cval(cindex, 2, 1));
+ double amplitude1 = (test_field1->cval(cindex, 0, 0)*test_field1->cval(cindex, 0, 0) +
+ test_field1->cval(cindex, 1, 0)*test_field1->cval(cindex, 1, 0) +
+ test_field1->cval(cindex, 2, 0)*test_field1->cval(cindex, 2, 0) +
+ test_field1->cval(cindex, 0, 1)*test_field1->cval(cindex, 0, 1) +
+ test_field1->cval(cindex, 1, 1)*test_field1->cval(cindex, 1, 1) +
+ test_field1->cval(cindex, 2, 1)*test_field1->cval(cindex, 2, 1));
double amplitude = sqrt((amplitude0 + amplitude1)/2);
if (amplitude > 0)
if (diff/amplitude > max_diff)
{
max_diff = diff / amplitude;
ix = xindex;
- iy = yindex + scal_field0->clayout->starts[0];
+ iy = yindex + test_field0->clayout->starts[0];
iz = zindex;
k_at_max_diff = sqrt(k2);
a0 = sqrt(amplitude0);
@@ -165,10 +165,22 @@ int symmetrize_test<rnumber>::do_work(void)
DEBUG_MSG("found maximum relative difference %g at ix = %ld, iy = %ld, iz = %ld, wavenumber = %g, amplitudes %g %g\n",
max_diff, ix, iy, iz, k_at_max_diff, a0, a1);
+ test_field1->io(
+ this->simname + "_fields.h5",
+ "field1",
+ 0,
+ false);
+ test_field1->ift();
+ test_field1->io(
+ this->simname + "_fields.h5",
+ "field1",
+ 0,
+ false);
+
// deallocate
delete kk;
- delete scal_field1;
- delete scal_field0;
+ delete test_field1;
+ delete test_field0;
return EXIT_SUCCESS;
}
--
GitLab
From 4e01d930cb0f2f74644dd541c8c81d265657e302 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 29 May 2018 15:04:33 +0200
Subject: [PATCH 177/342] don't print list of statistics keys
---
bfps/test/test_Parseval.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/bfps/test/test_Parseval.py b/bfps/test/test_Parseval.py
index 903ad75b..aa680426 100644
--- a/bfps/test/test_Parseval.py
+++ b/bfps/test/test_Parseval.py
@@ -28,7 +28,6 @@ def main():
sys.argv[1:])
c.compute_statistics()
print((c.statistics['energy(t)'] - c.statistics['renergy(t)']) / c.statistics['renergy(t)'])
- print(list(c.statistics.keys()))
energyk = c.statistics['energy(k)']
nshell = c.get_data_file()['kspace/nshell'].value
renergy = np.mean(c.statistics['renergy(t)'])
--
GitLab
From b8bd79ead7cf7507801ced56440153fb6efea92b Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 29 May 2018 15:31:49 +0200
Subject: [PATCH 178/342] compute dealias factor on the fly
this approach implicitly solves the anisotropic grid problem
---
bfps/cpp/kspace.cpp | 25 ++++++-------------------
bfps/cpp/kspace.hpp | 6 ++----
2 files changed, 8 insertions(+), 23 deletions(-)
diff --git a/bfps/cpp/kspace.cpp b/bfps/cpp/kspace.cpp
index 21d37379..3fb25000 100644
--- a/bfps/cpp/kspace.cpp
+++ b/bfps/cpp/kspace.cpp
@@ -69,7 +69,7 @@ kspace<be, dt>::kspace(
if (ii <= int(this->layout->sizes[0]/2))
this->ky[i] = this->dky*ii;
else
- this->ky[i] = this->dky*(ii - int(this->layout->sizes[1]));
+ this->ky[i] = this->dky*(ii - int(this->layout->sizes[0]));
}
for (i = 0; i<int(this->layout->sizes[1]); i++)
{
@@ -116,8 +116,6 @@ kspace<be, dt>::kspace(
std::fill_n(nshell_local, this->nshells, 0);
});
- std::vector<std::unordered_map<int, double>> dealias_filter_threaded(omp_get_max_threads());
-
this->CLOOP_K2_NXMODES(
[&](ptrdiff_t cindex,
ptrdiff_t xindex,
@@ -131,9 +129,6 @@ kspace<be, dt>::kspace(
kshell_local_thread.getMine()[int(knorm/this->dk)] += nxmodes*knorm;
nshell_local_thread.getMine()[int(knorm/this->dk)] += nxmodes;
}
- if (dt == SMOOTH){
- dealias_filter_threaded[omp_get_thread_num()][int(round(k2 / this->dk2))] = exp(-36.0 * pow(k2/this->kM2, 18.));
- }
});
// Merge results
@@ -141,14 +136,6 @@ kspace<be, dt>::kspace(
kshell_local_thread.mergeParallel();
nshell_local_thread.mergeParallel();
- if (dt == SMOOTH){
- for(int idxMerge = 0 ; idxMerge < int(dealias_filter_threaded.size()) ; ++idxMerge){
- for(const auto kv : dealias_filter_threaded[idxMerge]){
- this->dealias_filter[kv.first] = kv.second;
- }
- }
- }
-
MPI_Allreduce(
nshell_local_thread.getMasterData(),
&this->nshell.front(),
@@ -437,6 +424,7 @@ int kspace<be, dt>::filter_calibrated_ell(
const double ell,
std::string filter_type)
{
+ TIMEZONE("kspace::filter_calibrated_ell");
if (filter_type == std::string("sharp_Fourier_sphere"))
{
this->template low_pass<rnumber, fc>(
@@ -464,23 +452,22 @@ template <typename rnumber,
field_components fc>
void kspace<be, dt>::dealias(typename fftw_interface<rnumber>::complex *__restrict__ a)
{
+ TIMEZONE("kspace::dealias");
switch(dt)
{
case TWO_THIRDS:
this->low_pass<rnumber, fc>(a, this->kM);
break;
case SMOOTH:
- this->CLOOP_K2(
+ this->CLOOP(
[&](ptrdiff_t cindex,
ptrdiff_t xindex,
ptrdiff_t yindex,
- ptrdiff_t zindex,
- double k2){
- //double tval = this->dealias_filter[int(round(k2 / this->dk2))];
+ ptrdiff_t zindex){
double kk2 = (pow(this->kx[xindex]/this->kMx, 2) +
pow(this->ky[yindex]/this->kMy, 2) +
pow(this->kz[zindex]/this->kMz, 2));
- double tval = exp(-36.0 * (pow(kk2, 18.)));
+ double tval = exp(-36.0 * (pow(kk2, 18)));
for (unsigned int tcounter=0; tcounter<2*ncomp(fc); tcounter++)
((rnumber*)a)[2*ncomp(fc)*cindex + tcounter] *= tval;
});
diff --git a/bfps/cpp/kspace.hpp b/bfps/cpp/kspace.hpp
index 507ecea3..c0bf2583 100644
--- a/bfps/cpp/kspace.hpp
+++ b/bfps/cpp/kspace.hpp
@@ -54,7 +54,6 @@ class kspace
/* mode and dealiasing information */
double kMx, kMy, kMz, kM, kM2;
std::vector<double> kx, ky, kz;
- std::unordered_map<int, double> dealias_filter;
std::vector<double> kshell;
std::vector<int64_t> nshell;
int nshells;
@@ -156,8 +155,8 @@ class kspace
for (hsize_t xindex = 0; xindex < this->layout->subsizes[2]; xindex++)
{
double k2 = (this->kx[xindex]*this->kx[xindex] +
- this->ky[yindex]*this->ky[yindex] +
- this->kz[zindex]*this->kz[zindex]);
+ this->ky[yindex]*this->ky[yindex] +
+ this->kz[zindex]*this->kz[zindex]);
expression(cindex, xindex, yindex, zindex, k2);
cindex++;
}
@@ -179,7 +178,6 @@ class kspace
+ zindex*this->layout->subsizes[2];
hsize_t xindex = 0;
double k2 = (
- this->kx[xindex]*this->kx[xindex] +
this->ky[yindex]*this->ky[yindex] +
this->kz[zindex]*this->kz[zindex]);
expression(cindex, xindex, yindex, zindex, k2, 1);
--
GitLab
From 149020bbe90ca76563ff0ba2bc4cbb78b7ec98ad Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 29 May 2018 16:10:59 +0200
Subject: [PATCH 179/342] fix dkz/Lz options
---
bfps/DNS.py | 2 +-
bfps/test/test_Parseval.py | 11 +++++++----
2 files changed, 8 insertions(+), 5 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index c407a955..28291be5 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -718,7 +718,7 @@ class DNS(_code):
opt.dkx = 2. / opt.Lx
if type(opt.dky) == type(None):
opt.dky = 2. / opt.Ly
- if type(opt.dkx) == type(None):
+ if type(opt.dkz) == type(None):
opt.dkz = 2. / opt.Lz
if type(opt.nx) == type(None):
opt.nx = opt.n
diff --git a/bfps/test/test_Parseval.py b/bfps/test/test_Parseval.py
index aa680426..b2afdbee 100644
--- a/bfps/test/test_Parseval.py
+++ b/bfps/test/test_Parseval.py
@@ -31,14 +31,17 @@ def main():
energyk = c.statistics['energy(k)']
nshell = c.get_data_file()['kspace/nshell'].value
renergy = np.mean(c.statistics['renergy(t)'])
- print(renergy, np.trapz(energyk[:-2], c.statistics['kshell'][:-2]))
+ print(renergy, np.sum(energyk[:-2]) * (c.parameters['dkx']*c.parameters['dky']*c.parameters['dkz']))
+ print(c.parameters['dkx'], c.parameters['dky'], c.parameters['dkz'])
f = plt.figure()
a = f.add_subplot(111)
- a.plot(c.statistics['kshell'], energyk)
- a.plot(c.statistics['kshell'], (energyk / nshell)*(4*np.pi*c.statistics['kshell']**2))
- a.set_yscale('log')
+ a.plot(c.statistics['kshell'], energyk, label = 'unnormalized')
+ a.plot(c.statistics['kshell'], (energyk / nshell)*(4*np.pi*c.statistics['kshell']**2), label = 'normalized')
+ #a.set_yscale('log')
a.set_xscale('log')
+ a.legend(loc = 'best')
+ f.tight_layout()
f.savefig('spectrum.pdf')
return None
--
GitLab
From 424a01ec77cabdabc6cd17ae76619dbd0e8e2906 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 29 May 2018 21:41:14 +0200
Subject: [PATCH 180/342] tweak spectrum computation
---
bfps/DNS.py | 17 ++++++-----------
bfps/NavierStokes.py | 17 ++++++-----------
bfps/test/test_Parseval.py | 6 ++++--
3 files changed, 16 insertions(+), 24 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 28291be5..538285e5 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -276,23 +276,18 @@ class DNS(_code):
self.parameters['niter_stat']*
(np.arange(ii0, ii1+1).astype(np.float)))
phi_ij = data_file['statistics/spectra/velocity_velocity'][ii0:ii1+1]
- discrete_Fourier_prefactor = 1. / (self.parameters['dkx']*
- self.parameters['dky']*
- self.parameters['dkz'])
- pp_file['R_ij(t)'] = self.statistics['dk']*np.sum(phi_ij, axis = 1)*discrete_Fourier_prefactor
- energy_tk = discrete_Fourier_prefactor*(
+ pp_file['R_ij(t)'] = np.sum(phi_ij, axis = 1)
+ energy_tk = (
phi_ij[:, :, 0, 0] +
phi_ij[:, :, 1, 1] +
phi_ij[:, :, 2, 2])/2
- pp_file['energy(t)'] = (self.statistics['dk'] *
- np.sum(energy_tk, axis = 1))
+ pp_file['energy(t)'] = np.sum(energy_tk, axis = 1)
pp_file['energy(k)'] = np.mean(energy_tk, axis = 0)
- enstrophy_tk = discrete_Fourier_prefactor*(
+ enstrophy_tk = (
data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 0, 0] +
data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 1, 1] +
data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 2, 2])/2
- pp_file['enstrophy(t)'] = (self.statistics['dk'] *
- np.sum(enstrophy_tk, axis = 1))
+ pp_file['enstrophy(t)'] = np.sum(enstrophy_tk, axis = 1)
pp_file['enstrophy(k)'] = np.mean(enstrophy_tk, axis = 0)
pp_file['vel_max(t)'] = data_file['statistics/moments/velocity'][ii0:ii1+1, 9, 3]
pp_file['renergy(t)'] = data_file['statistics/moments/velocity'][ii0:ii1+1, 2, 3]/2
@@ -373,7 +368,7 @@ class DNS(_code):
self.statistics['etaK' + suffix])
if self.parameters['dealias_type'] == 1:
self.statistics['kMeta' + suffix] *= 0.8
- self.statistics['Lint'] = ((self.statistics['dk']*np.pi /
+ self.statistics['Lint'] = ((np.pi /
(2*self.statistics['Uint']**2)) *
np.nansum(self.statistics['energy(k)'] /
self.statistics['kshell']))
diff --git a/bfps/NavierStokes.py b/bfps/NavierStokes.py
index 62a2263e..d158f5b2 100644
--- a/bfps/NavierStokes.py
+++ b/bfps/NavierStokes.py
@@ -627,23 +627,18 @@ class NavierStokes(_fluid_particle_base):
self.parameters['niter_stat']*
(np.arange(ii0, ii1+1).astype(np.float)))
phi_ij = data_file['statistics/spectra/velocity_velocity'][ii0:ii1+1]
- discrete_Fourier_prefactor = 1. / (self.parameters['dkx']*
- self.parameters['dky']*
- self.parameters['dkz'])
- pp_file['R_ij(t)'] = self.statistics['dk']*np.sum(phi_ij, axis = 1)*discrete_Fourier_prefactor
- energy_tk = discrete_Fourier_prefactor*(
+ pp_file['R_ij(t)'] = np.sum(phi_ij, axis = 1)
+ energy_tk = (
phi_ij[:, :, 0, 0] +
phi_ij[:, :, 1, 1] +
phi_ij[:, :, 2, 2])/2
- pp_file['energy(t)'] = (self.statistics['dk'] *
- np.sum(energy_tk, axis = 1))
+ pp_file['energy(t)'] = np.sum(energy_tk, axis = 1)
pp_file['energy(k)'] = np.mean(energy_tk, axis = 0)
- enstrophy_tk = discrete_Fourier_prefactor*(
+ enstrophy_tk = (
data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 0, 0] +
data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 1, 1] +
data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 2, 2])/2
- pp_file['enstrophy(t)'] = (self.statistics['dk'] *
- np.sum(enstrophy_tk, axis = 1))
+ pp_file['enstrophy(t)'] = np.sum(enstrophy_tk, axis = 1)
pp_file['enstrophy(k)'] = np.mean(enstrophy_tk, axis = 0)
pp_file['vel_max(t)'] = data_file['statistics/moments/velocity'][ii0:ii1+1, 9, 3]
pp_file['renergy(t)'] = data_file['statistics/moments/velocity'][ii0:ii1+1, 2, 3]/2
@@ -727,7 +722,7 @@ class NavierStokes(_fluid_particle_base):
self.statistics['etaK' + suffix])
if self.parameters['dealias_type'] == 1:
self.statistics['kMeta' + suffix] *= 0.8
- self.statistics['Lint'] = ((self.statistics['dk']*np.pi /
+ self.statistics['Lint'] = ((np.pi /
(2*self.statistics['Uint']**2)) *
np.nansum(self.statistics['energy(k)'] /
self.statistics['kshell']))
diff --git a/bfps/test/test_Parseval.py b/bfps/test/test_Parseval.py
index b2afdbee..80a16d6a 100644
--- a/bfps/test/test_Parseval.py
+++ b/bfps/test/test_Parseval.py
@@ -31,13 +31,15 @@ def main():
energyk = c.statistics['energy(k)']
nshell = c.get_data_file()['kspace/nshell'].value
renergy = np.mean(c.statistics['renergy(t)'])
- print(renergy, np.sum(energyk[:-2]) * (c.parameters['dkx']*c.parameters['dky']*c.parameters['dkz']))
+ print(renergy, np.sum(energyk[:-2]))
print(c.parameters['dkx'], c.parameters['dky'], c.parameters['dkz'])
+ energyk_alt = (energyk / nshell)*(4*np.pi*c.statistics['kshell']**2)
+ print(renergy, np.sum(energyk_alt[:-2])*c.statistics['dk'] / (c.parameters['dkx']*c.parameters['dky']*c.parameters['dkz']))
f = plt.figure()
a = f.add_subplot(111)
a.plot(c.statistics['kshell'], energyk, label = 'unnormalized')
- a.plot(c.statistics['kshell'], (energyk / nshell)*(4*np.pi*c.statistics['kshell']**2), label = 'normalized')
+ a.plot(c.statistics['kshell'], energyk_alt, label = 'normalized')
#a.set_yscale('log')
a.set_xscale('log')
a.legend(loc = 'best')
--
GitLab
From 107b66ac3526df9787098bcc463f75e7d443dad9 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 29 May 2018 21:43:45 +0200
Subject: [PATCH 181/342] add fftw --- numpy.fft comparison
---
bfps/test/test_fftw.py | 56 ++++++++++++++++++++++++++++++++++++++++++
1 file changed, 56 insertions(+)
create mode 100644 bfps/test/test_fftw.py
diff --git a/bfps/test/test_fftw.py b/bfps/test/test_fftw.py
new file mode 100644
index 00000000..3de54502
--- /dev/null
+++ b/bfps/test/test_fftw.py
@@ -0,0 +1,56 @@
+#! /usr/bin/env python
+
+import os
+import numpy as np
+#import cupy as np
+import h5py
+import sys
+
+import bfps
+from bfps import TEST
+
+import matplotlib.pyplot as plt
+
+def main():
+ niterations = 10
+ c = TEST()
+ c.launch(
+ ['symmetrize_test',
+ '--nx', '32',
+ '--ny', '32',
+ '--nz', '32',
+ '--np', '4',
+ '--ntpp', '1',
+ '--wd', './'] +
+ sys.argv[1:])
+
+ df = h5py.File(c.simname + '.h5', 'r')
+ print(df['kspace/kx'].value)
+ print(df['kspace/ky'].value)
+ print(df['kspace/kz'].value)
+ df = h5py.File(c.simname + '_fields.h5', 'r')
+ field1_complex = df['field1/complex/0'].value
+ field1_real = df['field1/real/0'].value
+
+ np_field1_real = np.fft.irfftn(field1_complex, axes = (0, 1, 2)).transpose(1, 0, 2, 3)
+ print(np.mean(field1_real**2))
+ print(np.mean(np_field1_real**2)*(np_field1_real.size/3)**2)
+ print(np.max(np.abs(field1_real - np_field1_real*(np_field1_real.size/3))))
+
+ np_field1_complex = np.fft.rfftn(field1_real.transpose(1, 0, 2, 3), axes = (0, 1, 2))
+
+ print(np.sum(np.abs(field1_complex)**2))
+ print(np.sum(np.abs(np_field1_complex)**2))
+ print(np.max(np.abs(np_field1_complex - field1_complex)))
+
+ f = plt.figure()
+ a = f.add_subplot(121)
+ a.imshow(np.log(np.abs(np_field1_complex[:, :, 0, 0])), interpolation = 'nearest')
+ a = f.add_subplot(122)
+ a.imshow(np.log(np.abs(field1_complex[:, :, 0, 0])), interpolation = 'nearest')
+ f.savefig('symmetrize_test.pdf')
+ return None
+
+if __name__ == '__main__':
+ main()
+
--
GitLab
From a93dfedfbd831aafb8e8dc9fe247083a483f213a Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Tue, 29 May 2018 22:35:13 +0200
Subject: [PATCH 182/342] remove some debug messages
---
bfps/PP.py | 1 -
bfps/TEST.py | 1 -
bfps/test/test_Parseval.py | 3 ++-
bfps/tools.py | 1 -
4 files changed, 2 insertions(+), 4 deletions(-)
diff --git a/bfps/PP.py b/bfps/PP.py
index 95c5f253..4d19095a 100644
--- a/bfps/PP.py
+++ b/bfps/PP.py
@@ -640,7 +640,6 @@ class PP(_code):
for kz in range(src_file[src_dset_name].shape[0]):
dst_file[dst_dset_name][kz] = src_file[src_dset_name][kz]
else:
- print('aloha')
min_shape = (min(dst_shape[0], src_file[src_dset_name].shape[0]),
min(dst_shape[1], src_file[src_dset_name].shape[1]),
min(dst_shape[2], src_file[src_dset_name].shape[2]),
diff --git a/bfps/TEST.py b/bfps/TEST.py
index 0de39dd0..cd4d3e4a 100644
--- a/bfps/TEST.py
+++ b/bfps/TEST.py
@@ -288,7 +288,6 @@ class TEST(_code):
eval('self.parameters_to_parser_arguments(' + parser + ')')
eval('self.parameters_to_parser_arguments(' + parser + ',' +
'parameters = self.generate_extra_parameters(dns_type = \'' + parser + '\'))')
- print(self.parameters.keys())
return None
def prepare_launch(
self,
diff --git a/bfps/test/test_Parseval.py b/bfps/test/test_Parseval.py
index 80a16d6a..cca9d3d1 100644
--- a/bfps/test/test_Parseval.py
+++ b/bfps/test/test_Parseval.py
@@ -27,7 +27,8 @@ def main():
'--wd', './'] +
sys.argv[1:])
c.compute_statistics()
- print((c.statistics['energy(t)'] - c.statistics['renergy(t)']) / c.statistics['renergy(t)'])
+ Parseval_error = np.abs((c.statistics['energy(t)'] - c.statistics['renergy(t)']) / c.statistics['renergy(t)'])
+ assert(np.max(Parseval_error) < 1e-6)
energyk = c.statistics['energy(k)']
nshell = c.get_data_file()['kspace/nshell'].value
renergy = np.mean(c.statistics['renergy(t)'])
diff --git a/bfps/tools.py b/bfps/tools.py
index ee1e4775..a3fbc9d9 100644
--- a/bfps/tools.py
+++ b/bfps/tools.py
@@ -203,7 +203,6 @@ def padd_with_zeros(
"""
if (type(odtype) == type(None)):
odtype = a.dtype
- print(a.shape, n0, n1, n2//2+1)
assert(a.shape[0] <= n1 and
a.shape[1] <= n0 and
a.shape[2] <= n2//2+1)
--
GitLab
From 1cb82685b7889ce90e27f5ab73c8214a384a1813 Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Tue, 29 May 2018 22:50:39 +0200
Subject: [PATCH 183/342] update Parseval test
---
bfps/test/test_Parseval.py | 57 +++++++++++++++-----------------------
1 file changed, 23 insertions(+), 34 deletions(-)
diff --git a/bfps/test/test_Parseval.py b/bfps/test/test_Parseval.py
index cca9d3d1..491ad29c 100644
--- a/bfps/test/test_Parseval.py
+++ b/bfps/test/test_Parseval.py
@@ -12,40 +12,29 @@ import matplotlib.pyplot as plt
def main():
niterations = 10
- c = DNS()
- c.launch(
- ['NSVE',
- '--nx', '32',
- '--ny', '32',
- '--nz', '32',
- '--forcing_type', 'linear',
- '--np', '4',
- '--ntpp', '1',
- '--niter_todo', '{0}'.format(niterations),
- '--niter_out', '{0}'.format(niterations),
- '--niter_stat', '1',
- '--wd', './'] +
- sys.argv[1:])
- c.compute_statistics()
- Parseval_error = np.abs((c.statistics['energy(t)'] - c.statistics['renergy(t)']) / c.statistics['renergy(t)'])
- assert(np.max(Parseval_error) < 1e-6)
- energyk = c.statistics['energy(k)']
- nshell = c.get_data_file()['kspace/nshell'].value
- renergy = np.mean(c.statistics['renergy(t)'])
- print(renergy, np.sum(energyk[:-2]))
- print(c.parameters['dkx'], c.parameters['dky'], c.parameters['dkz'])
- energyk_alt = (energyk / nshell)*(4*np.pi*c.statistics['kshell']**2)
- print(renergy, np.sum(energyk_alt[:-2])*c.statistics['dk'] / (c.parameters['dkx']*c.parameters['dky']*c.parameters['dkz']))
-
- f = plt.figure()
- a = f.add_subplot(111)
- a.plot(c.statistics['kshell'], energyk, label = 'unnormalized')
- a.plot(c.statistics['kshell'], energyk_alt, label = 'normalized')
- #a.set_yscale('log')
- a.set_xscale('log')
- a.legend(loc = 'best')
- f.tight_layout()
- f.savefig('spectrum.pdf')
+ nlist = [16, 32, 48, 24, 64, 12]
+ for ii in range(len(nlist)):
+ c = DNS()
+ c.launch(
+ ['NSVE',
+ '--nx', str(nlist[ii]),
+ '--ny', str(nlist[(ii+1)%(len(nlist))]),
+ '--nz', str(nlist[(ii+2)%(len(nlist))]),
+ '--Lx', str(2+np.random.random()),
+ '--Ly', str(2+np.random.random()),
+ '--Lz', str(2+np.random.random()),
+ '--simname', 'test_Parseval_{0}'.format(ii),
+ '--np', '4',
+ '--ntpp', '1',
+ '--niter_todo', '{0}'.format(niterations),
+ '--niter_out', '{0}'.format(niterations),
+ '--niter_stat', '1',
+ '--wd', './'] +
+ sys.argv[1:])
+ c.compute_statistics()
+ Parseval_error = np.abs((c.statistics['energy(t)'] - c.statistics['renergy(t)']) / c.statistics['renergy(t)'])
+ assert(np.max(Parseval_error) < 1e-6)
+ print('SUCCESS!!! Parseval test passed for unequal nx, ny, nz and random Lx, Ly, Lz')
return None
if __name__ == '__main__':
--
GitLab
From 4b00ebb2f6538721169bcb73927bb34afee9067c Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Tue, 29 May 2018 23:16:36 +0200
Subject: [PATCH 184/342] revert to old symmetrize
I know the arithmetic mean thing is more polite, but I'm going to use
the old version, so that the test still passes as is.
And everyone does it this way anyway...
---
bfps/cpp/field.cpp | 10 ++--------
bfps/test/test_bfps_NSVEparticles.py | 9 ++++++---
2 files changed, 8 insertions(+), 11 deletions(-)
diff --git a/bfps/cpp/field.cpp b/bfps/cpp/field.cpp
index 6535c07a..2558552d 100644
--- a/bfps/cpp/field.cpp
+++ b/bfps/cpp/field.cpp
@@ -1102,14 +1102,8 @@ void field<rnumber, be, fc>::symmetrize()
ptrdiff_t cindex1 = this->get_cindex(0, 0, this->clayout->sizes[1] - iz);
for (int cc = 0; cc < int(ncomp(fc)); cc++)
{
- rnumber rtmp = ((*(cdata + cc + ncomp(fc)*cindex0))[0] +
- (*(cdata + cc + ncomp(fc)*cindex1))[0])/2;
- rnumber ctmp = ((*(cdata + cc + ncomp(fc)*cindex0))[1] -
- (*(cdata + cc + ncomp(fc)*cindex1))[1])/2;
- (*(cdata + cc + ncomp(fc)*cindex0))[0] = rtmp;
- (*(cdata + cc + ncomp(fc)*cindex1))[0] = rtmp;
- (*(cdata + cc + ncomp(fc)*cindex0))[1] = ctmp;
- (*(cdata + cc + ncomp(fc)*cindex1))[1] = -ctmp;
+ (*(cdata + cc + ncomp(fc)*cindex1))[0] = (*(cdata + cc + ncomp(fc)*cindex0))[0];
+ (*(cdata + cc + ncomp(fc)*cindex1))[1] = -(*(cdata + cc + ncomp(fc)*cindex0))[1];
}
}
}
diff --git a/bfps/test/test_bfps_NSVEparticles.py b/bfps/test/test_bfps_NSVEparticles.py
index 33212e76..7ceab8e5 100644
--- a/bfps/test/test_bfps_NSVEparticles.py
+++ b/bfps/test/test_bfps_NSVEparticles.py
@@ -42,13 +42,16 @@ def main():
for iteration in [0, 32, 64]:
field0 = f0['vorticity/complex/{0}'.format(iteration)].value
field1 = f1['vorticity/complex/{0}'.format(iteration)].value
- assert(np.max(np.abs(field0 - field1)) < 1e-5)
+ field_error = np.max(np.abs(field0 - field1))
x0 = f0['tracers0/state/{0}'.format(iteration)].value
x1 = f1['tracers0/state/{0}'.format(iteration)].value
- assert(np.max(np.abs(x0 - x1)) < 1e-5)
+ traj_error = np.max(np.abs(x0 - x1))
y0 = f0['tracers0/rhs/{0}'.format(iteration)].value
y1 = f1['tracers0/rhs/{0}'.format(iteration)].value
- assert(np.max(np.abs(y0 - y1)) < 1e-5)
+ rhs_error = np.max(np.abs(y0 - y1))
+ assert(field_error < 1e-5)
+ assert(traj_error < 1e-5)
+ assert(rhs_error < 1e-5)
print('SUCCESS! Basic test passed.')
return None
--
GitLab
From a23ec27a649556ebe5dc5067a9b9b243121666ce Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Tue, 29 May 2018 23:20:22 +0200
Subject: [PATCH 185/342] add new command line test
---
setup.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/setup.py b/setup.py
index 7640e417..1cb8ad93 100644
--- a/setup.py
+++ b/setup.py
@@ -290,7 +290,8 @@ setup(
'bfps = bfps.__main__:main',
'bfps1 = bfps.__main__:main',
'bfps.test_NSVEparticles = bfps.test.test_bfps_NSVEparticles:main',
- 'bfps.test_particles = bfps.test.test_particles:main'],
+ 'bfps.test_particles = bfps.test.test_particles:main',
+ 'bfps.test_Parseval = bfps.test.test_Parseval:main'],
},
version = VERSION,
########################################################################
--
GitLab
From f4fb131935c7e6a864d3c947b217fdda8383d523 Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Tue, 29 May 2018 23:22:38 +0200
Subject: [PATCH 186/342] add Parseval test to CI script
---
tests/ci-scripts/test.sh | 2 ++
1 file changed, 2 insertions(+)
diff --git a/tests/ci-scripts/test.sh b/tests/ci-scripts/test.sh
index ddde2489..a2178e44 100644
--- a/tests/ci-scripts/test.sh
+++ b/tests/ci-scripts/test.sh
@@ -38,6 +38,8 @@ ls $destdir/bin/
$pythonbin $destdir/bin/bfps.test_NSVEparticles
+$pythonbin $destdir/bin/bfps.test_Parseval
+
# Clean
if [[ -d $destdir ]] ; then
rm -rf $destdir ;
--
GitLab
From b50335d10ce5e54ed15278b0236666439e51f3f5 Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Wed, 30 May 2018 00:01:54 +0200
Subject: [PATCH 187/342] strip down test_Parseval file
---
bfps/test/test_Parseval.py | 4 ----
1 file changed, 4 deletions(-)
diff --git a/bfps/test/test_Parseval.py b/bfps/test/test_Parseval.py
index 491ad29c..00a88d24 100644
--- a/bfps/test/test_Parseval.py
+++ b/bfps/test/test_Parseval.py
@@ -1,15 +1,11 @@
#! /usr/bin/env python
-import os
import numpy as np
-import h5py
import sys
import bfps
from bfps import DNS
-import matplotlib.pyplot as plt
-
def main():
niterations = 10
nlist = [16, 32, 48, 24, 64, 12]
--
GitLab
From fcf9ac62d90f01a6e07f9156cef93de5beb34fab Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Wed, 30 May 2018 09:44:25 +0200
Subject: [PATCH 188/342] tweak fftw test
---
bfps/test/test_fftw.py | 19 +++++++++----------
1 file changed, 9 insertions(+), 10 deletions(-)
diff --git a/bfps/test/test_fftw.py b/bfps/test/test_fftw.py
index 3de54502..38f82e62 100644
--- a/bfps/test/test_fftw.py
+++ b/bfps/test/test_fftw.py
@@ -25,23 +25,22 @@ def main():
sys.argv[1:])
df = h5py.File(c.simname + '.h5', 'r')
- print(df['kspace/kx'].value)
- print(df['kspace/ky'].value)
- print(df['kspace/kz'].value)
df = h5py.File(c.simname + '_fields.h5', 'r')
field1_complex = df['field1/complex/0'].value
field1_real = df['field1/real/0'].value
np_field1_real = np.fft.irfftn(field1_complex, axes = (0, 1, 2)).transpose(1, 0, 2, 3)
- print(np.mean(field1_real**2))
- print(np.mean(np_field1_real**2)*(np_field1_real.size/3)**2)
- print(np.max(np.abs(field1_real - np_field1_real*(np_field1_real.size/3))))
+ L2normr = np.sqrt(np.mean(field1_real**2))
+ err = np.max(np.abs(field1_real - np_field1_real*(np_field1_real.size/3)))
+ assert(err < 1e-5)
- np_field1_complex = np.fft.rfftn(field1_real.transpose(1, 0, 2, 3), axes = (0, 1, 2))
+ np_field1_complex = np.fft.rfftn(field1_real.transpose(1, 0, 2, 3), axes = (0, 1, 2)) / (np_field1_real.size/3)
- print(np.sum(np.abs(field1_complex)**2))
- print(np.sum(np.abs(np_field1_complex)**2))
- print(np.max(np.abs(np_field1_complex - field1_complex)))
+ L2norm0 = np.sqrt(np.sum(np.abs(field1_complex[:, :, 0])**2) + 2*np.sum(np.abs(field1_complex[:, :, 1:])**2))
+ L2norm1 = np.sqrt(np.sum(np.abs(np_field1_complex[:, :, 0])**2) + 2*np.sum(np.abs(np_field1_complex[:, :, 1:])**2))
+ err = np.max(np.abs(np_field1_complex - field1_complex))
+ assert(err < 1e-5)
+ print(L2normr, L2norm0, L2norm1)
f = plt.figure()
a = f.add_subplot(121)
--
GitLab
From c1b51c1ddbc6a60c331fc21f06aad6937ba9c116 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 30 May 2018 10:24:44 +0200
Subject: [PATCH 189/342] update fftw test, put in CI script
---
bfps/cpp/full_code/symmetrize_test.cpp | 18 +++---
bfps/test/test_fftw.py | 87 +++++++++++++++-----------
setup.py | 3 +-
tests/ci-scripts/test.sh | 4 +-
4 files changed, 61 insertions(+), 51 deletions(-)
diff --git a/bfps/cpp/full_code/symmetrize_test.cpp b/bfps/cpp/full_code/symmetrize_test.cpp
index 471e234c..13d48d60 100644
--- a/bfps/cpp/full_code/symmetrize_test.cpp
+++ b/bfps/cpp/full_code/symmetrize_test.cpp
@@ -69,15 +69,12 @@ int symmetrize_test<rnumber>::do_work(void)
ptrdiff_t yindex,
ptrdiff_t zindex,
double k2){
- if (k2 < kk->kM2)
- {
- test_field0->cval(cindex, 0, 0) = rdist(rgen);
- test_field0->cval(cindex, 0, 1) = rdist(rgen);
- test_field0->cval(cindex, 1, 0) = rdist(rgen);
- test_field0->cval(cindex, 1, 1) = rdist(rgen);
- test_field0->cval(cindex, 2, 0) = rdist(rgen);
- test_field0->cval(cindex, 2, 1) = rdist(rgen);
- }
+ test_field0->cval(cindex, 0, 0) = rdist(rgen);
+ test_field0->cval(cindex, 0, 1) = rdist(rgen);
+ test_field0->cval(cindex, 1, 0) = rdist(rgen);
+ test_field0->cval(cindex, 1, 1) = rdist(rgen);
+ test_field0->cval(cindex, 2, 0) = rdist(rgen);
+ test_field0->cval(cindex, 2, 1) = rdist(rgen);
if (k2 > 0)
{
test_field0->cval(cindex, 0, 0) /= sqrt(k2);
@@ -98,12 +95,11 @@ int symmetrize_test<rnumber>::do_work(void)
}
});
// dealias (?!)
- //kk->template low_pass<rnumber, THREE>(test_field0->get_cdata(), kk->kM);
kk->template dealias<rnumber, THREE>(test_field0->get_cdata());
// make the field divergence free
kk->template force_divfree<rnumber>(test_field0->get_cdata());
// apply symmetrize to test_field0
- //test_field0->symmetrize();
+ test_field0->symmetrize();
// make copy in test_field1
diff --git a/bfps/test/test_fftw.py b/bfps/test/test_fftw.py
index 38f82e62..3de2d97d 100644
--- a/bfps/test/test_fftw.py
+++ b/bfps/test/test_fftw.py
@@ -1,53 +1,64 @@
#! /usr/bin/env python
-import os
import numpy as np
-#import cupy as np
import h5py
import sys
import bfps
from bfps import TEST
-import matplotlib.pyplot as plt
+try:
+ import matplotlib.pyplot as plt
+except:
+ plt = None
def main():
niterations = 10
- c = TEST()
- c.launch(
- ['symmetrize_test',
- '--nx', '32',
- '--ny', '32',
- '--nz', '32',
- '--np', '4',
- '--ntpp', '1',
- '--wd', './'] +
- sys.argv[1:])
-
- df = h5py.File(c.simname + '.h5', 'r')
- df = h5py.File(c.simname + '_fields.h5', 'r')
- field1_complex = df['field1/complex/0'].value
- field1_real = df['field1/real/0'].value
-
- np_field1_real = np.fft.irfftn(field1_complex, axes = (0, 1, 2)).transpose(1, 0, 2, 3)
- L2normr = np.sqrt(np.mean(field1_real**2))
- err = np.max(np.abs(field1_real - np_field1_real*(np_field1_real.size/3)))
- assert(err < 1e-5)
-
- np_field1_complex = np.fft.rfftn(field1_real.transpose(1, 0, 2, 3), axes = (0, 1, 2)) / (np_field1_real.size/3)
-
- L2norm0 = np.sqrt(np.sum(np.abs(field1_complex[:, :, 0])**2) + 2*np.sum(np.abs(field1_complex[:, :, 1:])**2))
- L2norm1 = np.sqrt(np.sum(np.abs(np_field1_complex[:, :, 0])**2) + 2*np.sum(np.abs(np_field1_complex[:, :, 1:])**2))
- err = np.max(np.abs(np_field1_complex - field1_complex))
- assert(err < 1e-5)
- print(L2normr, L2norm0, L2norm1)
-
- f = plt.figure()
- a = f.add_subplot(121)
- a.imshow(np.log(np.abs(np_field1_complex[:, :, 0, 0])), interpolation = 'nearest')
- a = f.add_subplot(122)
- a.imshow(np.log(np.abs(field1_complex[:, :, 0, 0])), interpolation = 'nearest')
- f.savefig('symmetrize_test.pdf')
+ nlist = [16, 32, 48, 24, 64, 12]
+ for ii in range(len(nlist)):
+ c = TEST()
+ c.launch(
+ ['symmetrize_test',
+ '--nx', str(nlist[ii]),
+ '--ny', str(nlist[(ii+1)%(len(nlist))]),
+ '--nz', str(nlist[(ii+2)%(len(nlist))]),
+ '--Lx', str(2+np.random.random()),
+ '--Ly', str(2+np.random.random()),
+ '--Lz', str(2+np.random.random()),
+ '--simname', 'fftw_vs_numpy_{0}'.format(ii),
+ '--np', '4',
+ '--ntpp', '1',
+ '--wd', './'] +
+ sys.argv[1:])
+ df = h5py.File(c.simname + '.h5', 'r')
+ df = h5py.File(c.simname + '_fields.h5', 'r')
+ field1_complex = df['field1/complex/0'].value
+ field1_real = df['field1/real/0'].value
+ npoints = field1_real.size//3
+
+ np_field1_real = np.fft.irfftn(field1_complex, axes = (0, 1, 2)).transpose(1, 0, 2, 3)
+ L2normr = np.sqrt(np.mean(np.sum(field1_real**2, axis = 3)))
+ np_L2normr = np.sqrt(np.mean(np.sum(np_field1_real**2, axis = 3)))
+ err = np.max(np.abs(field1_real - np_field1_real*npoints)) / L2normr
+ assert(err < 1e-5)
+
+ np_field1_complex = np.fft.rfftn(field1_real.transpose(1, 0, 2, 3), axes = (0, 1, 2)) / npoints
+
+ L2norm0 = np.sqrt(np.sum(np.abs(field1_complex[:, :, 0])**2) + 2*np.sum(np.abs(field1_complex[:, :, 1:])**2))
+ L2norm1 = np.sqrt(np.sum(np.abs(np_field1_complex[:, :, 0])**2) + 2*np.sum(np.abs(np_field1_complex[:, :, 1:])**2))
+ err = np.max(np.abs(np_field1_complex - field1_complex)) / L2norm0
+ assert(err < 1e-5)
+
+ err = abs(L2normr - L2norm0) / L2norm0
+ assert(err < 1e-5)
+
+ if not type(plt) == type(None):
+ f = plt.figure()
+ a = f.add_subplot(121)
+ a.imshow(np.log(np.abs(np_field1_complex[:, :, 0, 0])), interpolation = 'nearest')
+ a = f.add_subplot(122)
+ a.imshow(np.log(np.abs(field1_complex[:, :, 0, 0])), interpolation = 'nearest')
+ f.savefig(c.simname + '_complex_slice_kx0.pdf')
return None
if __name__ == '__main__':
diff --git a/setup.py b/setup.py
index 1cb8ad93..9671a3c6 100644
--- a/setup.py
+++ b/setup.py
@@ -291,7 +291,8 @@ setup(
'bfps1 = bfps.__main__:main',
'bfps.test_NSVEparticles = bfps.test.test_bfps_NSVEparticles:main',
'bfps.test_particles = bfps.test.test_particles:main',
- 'bfps.test_Parseval = bfps.test.test_Parseval:main'],
+ 'bfps.test_Parseval = bfps.test.test_Parseval:main',
+ 'bfps.test_fftw = bfps.test.test_fftw:main'],
},
version = VERSION,
########################################################################
diff --git a/tests/ci-scripts/test.sh b/tests/ci-scripts/test.sh
index a2178e44..e2adc661 100644
--- a/tests/ci-scripts/test.sh
+++ b/tests/ci-scripts/test.sh
@@ -36,10 +36,12 @@ $pythonbin setup.py install --prefix=$destdir
ls $destdir
ls $destdir/bin/
-$pythonbin $destdir/bin/bfps.test_NSVEparticles
+$pythonbin $destdir/bin/bfps.test_fftw
$pythonbin $destdir/bin/bfps.test_Parseval
+$pythonbin $destdir/bin/bfps.test_NSVEparticles
+
# Clean
if [[ -d $destdir ]] ; then
rm -rf $destdir ;
--
GitLab
From d265e8779b27adabc85941fb9ec47c553cd90f0c Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 30 May 2018 13:08:31 +0200
Subject: [PATCH 190/342] use "normalized" spectrum
I'm not sure how much sense this makes, but I am now explicitly dividing
the computed spectrum by dk, such that the integral of E(k)dk is the
total energy.
---
bfps/DNS.py | 5 +++--
bfps/NavierStokes.py | 5 +++--
2 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 538285e5..90a42c79 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -247,6 +247,7 @@ class DNS(_code):
ii0 = iter0 // self.parameters['niter_stat']
ii1 = iter1 // self.parameters['niter_stat']
self.statistics['kshell'] = data_file['kspace/kshell'].value
+ self.statistics['nshell'] = data_file['kspace/nshell'].value
for kk in [-1, -2]:
if (self.statistics['kshell'][kk] == 0):
self.statistics['kshell'][kk] = np.nan
@@ -282,13 +283,13 @@ class DNS(_code):
phi_ij[:, :, 1, 1] +
phi_ij[:, :, 2, 2])/2
pp_file['energy(t)'] = np.sum(energy_tk, axis = 1)
- pp_file['energy(k)'] = np.mean(energy_tk, axis = 0)
+ pp_file['energy(k)'] = np.mean(energy_tk, axis = 0)*(4*np.pi*self.statistics['kshell']**2) / (self.statistics['dk']*self.statistics['nshell'])
enstrophy_tk = (
data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 0, 0] +
data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 1, 1] +
data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 2, 2])/2
pp_file['enstrophy(t)'] = np.sum(enstrophy_tk, axis = 1)
- pp_file['enstrophy(k)'] = np.mean(enstrophy_tk, axis = 0)
+ pp_file['enstrophy(k)'] = np.mean(enstrophy_tk, axis = 0)*(4*np.pi*self.statistics['kshell']**2) / self.statistics['nshell']
pp_file['vel_max(t)'] = data_file['statistics/moments/velocity'][ii0:ii1+1, 9, 3]
pp_file['renergy(t)'] = data_file['statistics/moments/velocity'][ii0:ii1+1, 2, 3]/2
for k in ['t',
diff --git a/bfps/NavierStokes.py b/bfps/NavierStokes.py
index d158f5b2..9c12ca20 100644
--- a/bfps/NavierStokes.py
+++ b/bfps/NavierStokes.py
@@ -598,6 +598,7 @@ class NavierStokes(_fluid_particle_base):
ii0 = iter0 // self.parameters['niter_stat']
ii1 = iter1 // self.parameters['niter_stat']
self.statistics['kshell'] = data_file['kspace/kshell'].value
+ self.statistics['nshell'] = data_file['kspace/nshell'].value
for kk in [-1, -2]:
if (self.statistics['kshell'][kk] == 0):
self.statistics['kshell'][kk] = np.nan
@@ -633,13 +634,13 @@ class NavierStokes(_fluid_particle_base):
phi_ij[:, :, 1, 1] +
phi_ij[:, :, 2, 2])/2
pp_file['energy(t)'] = np.sum(energy_tk, axis = 1)
- pp_file['energy(k)'] = np.mean(energy_tk, axis = 0)
+ pp_file['energy(k)'] = np.mean(energy_tk, axis = 0)*(4*np.pi*self.statistics['kshell']**2) / (self.statistics['dk']*self.statistics['nshell'])
enstrophy_tk = (
data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 0, 0] +
data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 1, 1] +
data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 2, 2])/2
pp_file['enstrophy(t)'] = np.sum(enstrophy_tk, axis = 1)
- pp_file['enstrophy(k)'] = np.mean(enstrophy_tk, axis = 0)
+ pp_file['enstrophy(k)'] = np.mean(enstrophy_tk, axis = 0)*(4*np.pi*self.statistics['kshell']**2) / (self.statistics['dk']*self.statistics['nshell'])
pp_file['vel_max(t)'] = data_file['statistics/moments/velocity'][ii0:ii1+1, 9, 3]
pp_file['renergy(t)'] = data_file['statistics/moments/velocity'][ii0:ii1+1, 2, 3]/2
if 'trS2_Q_R' in data_file['statistics/moments'].keys():
--
GitLab
From 39a161c2d3fcf8d64c73c4602b01961a0b5645b9 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 30 May 2018 17:14:55 +0200
Subject: [PATCH 191/342] remove debug messages
---
bfps/cpp/field.cpp | 7 -------
1 file changed, 7 deletions(-)
diff --git a/bfps/cpp/field.cpp b/bfps/cpp/field.cpp
index 2558552d..25d71641 100644
--- a/bfps/cpp/field.cpp
+++ b/bfps/cpp/field.cpp
@@ -85,8 +85,6 @@ field<rnumber, be, fc>::field(
sizes[0] = nz; sizes[1] = ny; sizes[2] = nx;
subsizes[0] = local_n0; subsizes[1] = ny; subsizes[2] = nx;
starts[0] = local_0_start; starts[1] = 0; starts[2] = 0;
- DEBUG_MSG("local_0_start = %ld, local_1_start = %ld\n",
- local_0_start, local_1_start);
this->rlayout = new field_layout<fc>(
sizes, subsizes, starts, this->comm);
this->npoints = this->rlayout->full_size / ncomp(fc);
@@ -100,11 +98,6 @@ field<rnumber, be, fc>::field(
starts[0] = local_1_start; starts[1] = 0; starts[2] = 0;
this->clayout = new field_layout<fc>(
sizes, subsizes, starts, this->comm);
- DEBUG_MSG("local_size = %ld, rlayout->local_size = %ld, rmemlayout->local_size = %ld, clayout->local_size = %ld\n",
- tmp_local_size,
- this->rlayout->local_size,
- this->rmemlayout->local_size,
- this->clayout->local_size);
this->data = fftw_interface<rnumber>::alloc_real(
this->rmemlayout->local_size);
memset(this->data, 0, sizeof(rnumber)*this->rmemlayout->local_size);
--
GitLab
From 3c3a23a0d853f3207dc64f31cc77853a0aa39f13 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 31 May 2018 16:32:23 +0200
Subject: [PATCH 192/342] tweak statistics cache
basic stats can now be read even in the absence of the main simname.h5
file, using just the cache file.
---
bfps/DNS.py | 162 +++++++++++++++++++++++------------------
bfps/NavierStokes.py | 168 +++++++++++++++++++++++++------------------
bfps/_base.py | 6 +-
3 files changed, 195 insertions(+), 141 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 90a42c79..495278bf 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -233,76 +233,102 @@ class DNS(_code):
"""
if len(list(self.statistics.keys())) > 0:
return None
- self.read_parameters()
- with self.get_data_file() as data_file:
- if 'moments' not in data_file['statistics'].keys():
- return None
- iter0 = min((data_file['statistics/moments/velocity'].shape[0] *
- self.parameters['niter_stat']-1),
- iter0)
- if type(iter1) == type(None):
- iter1 = data_file['iteration'].value
- else:
- iter1 = min(data_file['iteration'].value, iter1)
- ii0 = iter0 // self.parameters['niter_stat']
- ii1 = iter1 // self.parameters['niter_stat']
- self.statistics['kshell'] = data_file['kspace/kshell'].value
- self.statistics['nshell'] = data_file['kspace/nshell'].value
- for kk in [-1, -2]:
- if (self.statistics['kshell'][kk] == 0):
- self.statistics['kshell'][kk] = np.nan
- self.statistics['kM'] = data_file['kspace/kM'].value
- self.statistics['dk'] = data_file['kspace/dk'].value
- computation_needed = True
- pp_file = h5py.File(self.get_postprocess_file_name(), 'a')
- if 'ii0' in pp_file.keys():
- computation_needed = not (ii0 == pp_file['ii0'].value and
- ii1 == pp_file['ii1'].value)
- if computation_needed:
- for k in ['t', 'vel_max(t)', 'renergy(t)',
- 'energy(t)', 'enstrophy(t)',
- 'energy(k)', 'enstrophy(k)',
- 'energy(t, k)',
- 'enstrophy(t, k)',
+ if not os.path.exists(self.get_data_file_name()):
+ if os.path.exists(self.get_cache_file_name()):
+ self.read_parameters(fname = self.get_cache_file_name())
+ with self.get_cache_file() as pp_file:
+ for k in ['t',
+ 'energy(t)',
+ 'energy(k)',
+ 'enstrophy(t)',
+ 'enstrophy(k)',
'R_ij(t)',
- 'ii0', 'ii1', 'iter0', 'iter1']:
+ 'vel_max(t)',
+ 'renergy(t)']:
if k in pp_file.keys():
- del pp_file[k]
- if computation_needed:
- pp_file['iter0'] = iter0
- pp_file['iter1'] = iter1
- pp_file['ii0'] = ii0
- pp_file['ii1'] = ii1
- pp_file['t'] = (self.parameters['dt']*
- self.parameters['niter_stat']*
- (np.arange(ii0, ii1+1).astype(np.float)))
- phi_ij = data_file['statistics/spectra/velocity_velocity'][ii0:ii1+1]
- pp_file['R_ij(t)'] = np.sum(phi_ij, axis = 1)
- energy_tk = (
- phi_ij[:, :, 0, 0] +
- phi_ij[:, :, 1, 1] +
- phi_ij[:, :, 2, 2])/2
- pp_file['energy(t)'] = np.sum(energy_tk, axis = 1)
- pp_file['energy(k)'] = np.mean(energy_tk, axis = 0)*(4*np.pi*self.statistics['kshell']**2) / (self.statistics['dk']*self.statistics['nshell'])
- enstrophy_tk = (
- data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 0, 0] +
- data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 1, 1] +
- data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 2, 2])/2
- pp_file['enstrophy(t)'] = np.sum(enstrophy_tk, axis = 1)
- pp_file['enstrophy(k)'] = np.mean(enstrophy_tk, axis = 0)*(4*np.pi*self.statistics['kshell']**2) / self.statistics['nshell']
- pp_file['vel_max(t)'] = data_file['statistics/moments/velocity'][ii0:ii1+1, 9, 3]
- pp_file['renergy(t)'] = data_file['statistics/moments/velocity'][ii0:ii1+1, 2, 3]/2
- for k in ['t',
- 'energy(t)',
- 'energy(k)',
- 'enstrophy(t)',
- 'enstrophy(k)',
- 'R_ij(t)',
- 'vel_max(t)',
- 'renergy(t)']:
- if k in pp_file.keys():
- self.statistics[k] = pp_file[k].value
- self.compute_time_averages()
+ self.statistics[k] = pp_file[k].value
+ self.statistics['kM'] = pp_file['kspace/kM'].value
+ self.statistics['dk'] = pp_file['kspace/dk'].value
+ self.statistics['kshell'] = pp_file['kspace/kshell'].value
+ self.statistics['nshell'] = pp_file['kspace/nshell'].value
+ else:
+ self.read_parameters()
+ with self.get_data_file() as data_file:
+ if 'moments' not in data_file['statistics'].keys():
+ return None
+ iter0 = min((data_file['statistics/moments/velocity'].shape[0] *
+ self.parameters['niter_stat']-1),
+ iter0)
+ if type(iter1) == type(None):
+ iter1 = data_file['iteration'].value
+ else:
+ iter1 = min(data_file['iteration'].value, iter1)
+ ii0 = iter0 // self.parameters['niter_stat']
+ ii1 = iter1 // self.parameters['niter_stat']
+ self.statistics['kshell'] = data_file['kspace/kshell'].value
+ self.statistics['nshell'] = data_file['kspace/nshell'].value
+ for kk in [-1, -2]:
+ if (self.statistics['kshell'][kk] == 0):
+ self.statistics['kshell'][kk] = np.nan
+ self.statistics['kM'] = data_file['kspace/kM'].value
+ self.statistics['dk'] = data_file['kspace/dk'].value
+ computation_needed = True
+ pp_file = h5py.File(self.get_postprocess_file_name(), 'a')
+ if not ('parameters' in pp_file.keys()):
+ data_file.copy('parameters', pp_file)
+ data_file.copy('kspace', pp_file)
+ if 'ii0' in pp_file.keys():
+ computation_needed = not (ii0 == pp_file['ii0'].value and
+ ii1 == pp_file['ii1'].value)
+ if computation_needed:
+ for k in ['t', 'vel_max(t)', 'renergy(t)',
+ 'energy(t)', 'enstrophy(t)',
+ 'energy(k)', 'enstrophy(k)',
+ 'energy(t, k)',
+ 'enstrophy(t, k)',
+ 'R_ij(t)',
+ 'ii0', 'ii1', 'iter0', 'iter1']:
+ if k in pp_file.keys():
+ del pp_file[k]
+ if computation_needed:
+ pp_file['iter0'] = iter0
+ pp_file['iter1'] = iter1
+ pp_file['ii0'] = ii0
+ pp_file['ii1'] = ii1
+ pp_file['t'] = (self.parameters['dt']*
+ self.parameters['niter_stat']*
+ (np.arange(ii0, ii1+1).astype(np.float)))
+ phi_ij = data_file['statistics/spectra/velocity_velocity'][ii0:ii1+1]
+ pp_file['R_ij(t)'] = np.sum(phi_ij, axis = 1)
+ energy_tk = (
+ phi_ij[:, :, 0, 0] +
+ phi_ij[:, :, 1, 1] +
+ phi_ij[:, :, 2, 2])/2
+ pp_file['energy(t)'] = np.sum(energy_tk, axis = 1)
+ pp_file['energy(k)'] = np.mean(energy_tk, axis = 0)*(4*np.pi*self.statistics['kshell']**2) / (self.statistics['dk']*self.statistics['nshell'])
+ enstrophy_tk = (
+ data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 0, 0] +
+ data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 1, 1] +
+ data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 2, 2])/2
+ pp_file['enstrophy(t)'] = np.sum(enstrophy_tk, axis = 1)
+ pp_file['enstrophy(k)'] = np.mean(enstrophy_tk, axis = 0)*(4*np.pi*self.statistics['kshell']**2) / (self.statistics['dk']*self.statistics['nshell'])
+ pp_file['vel_max(t)'] = data_file['statistics/moments/velocity'][ii0:ii1+1, 9, 3]
+ pp_file['renergy(t)'] = data_file['statistics/moments/velocity'][ii0:ii1+1, 2, 3]/2
+ for k in ['t',
+ 'energy(t)',
+ 'energy(k)',
+ 'enstrophy(t)',
+ 'enstrophy(k)',
+ 'R_ij(t)',
+ 'vel_max(t)',
+ 'renergy(t)']:
+ if k in pp_file.keys():
+ self.statistics[k] = pp_file[k].value
+ # sanity check --- Parseval theorem check
+ assert(np.max(np.abs(
+ self.statistics['renergy(t)'] -
+ self.statistics['energy(t)']) / self.statistics['energy(t)']) < 1e-5)
+ self.compute_time_averages()
return None
def compute_Reynolds_stress_invariants(
self):
diff --git a/bfps/NavierStokes.py b/bfps/NavierStokes.py
index 9c12ca20..c30adbe2 100644
--- a/bfps/NavierStokes.py
+++ b/bfps/NavierStokes.py
@@ -584,79 +584,105 @@ class NavierStokes(_fluid_particle_base):
"""
if len(list(self.statistics.keys())) > 0:
return None
- self.read_parameters()
- with self.get_data_file() as data_file:
- if 'moments' not in data_file['statistics'].keys():
- return None
- iter0 = min((data_file['statistics/moments/velocity'].shape[0] *
- self.parameters['niter_stat']-1),
- iter0)
- if type(iter1) == type(None):
- iter1 = data_file['iteration'].value
- else:
- iter1 = min(data_file['iteration'].value, iter1)
- ii0 = iter0 // self.parameters['niter_stat']
- ii1 = iter1 // self.parameters['niter_stat']
- self.statistics['kshell'] = data_file['kspace/kshell'].value
- self.statistics['nshell'] = data_file['kspace/nshell'].value
- for kk in [-1, -2]:
- if (self.statistics['kshell'][kk] == 0):
- self.statistics['kshell'][kk] = np.nan
- self.statistics['kM'] = data_file['kspace/kM'].value
- self.statistics['dk'] = data_file['kspace/dk'].value
- computation_needed = True
- pp_file = h5py.File(self.get_postprocess_file_name(), 'a')
- if 'ii0' in pp_file.keys():
- computation_needed = not (ii0 == pp_file['ii0'].value and
- ii1 == pp_file['ii1'].value)
- if computation_needed:
- for k in ['t', 'vel_max(t)', 'renergy(t)',
- 'energy(t)', 'enstrophy(t)',
- 'energy(k)', 'enstrophy(k)',
- 'energy(t, k)',
- 'enstrophy(t, k)',
+ if not os.path.exists(self.get_data_file_name()):
+ if os.path.exists(self.get_cache_file_name()):
+ self.read_parameters(fname = self.get_cache_file_name())
+ with self.get_cache_file() as pp_file:
+ for k in ['t',
+ 'energy(t)',
+ 'energy(k)',
+ 'enstrophy(t)',
+ 'enstrophy(k)',
'R_ij(t)',
- 'ii0', 'ii1', 'iter0', 'iter1']:
+ 'vel_max(t)',
+ 'renergy(t)']:
if k in pp_file.keys():
- del pp_file[k]
- if computation_needed:
- pp_file['iter0'] = iter0
- pp_file['iter1'] = iter1
- pp_file['ii0'] = ii0
- pp_file['ii1'] = ii1
- pp_file['t'] = (self.parameters['dt']*
- self.parameters['niter_stat']*
- (np.arange(ii0, ii1+1).astype(np.float)))
- phi_ij = data_file['statistics/spectra/velocity_velocity'][ii0:ii1+1]
- pp_file['R_ij(t)'] = np.sum(phi_ij, axis = 1)
- energy_tk = (
- phi_ij[:, :, 0, 0] +
- phi_ij[:, :, 1, 1] +
- phi_ij[:, :, 2, 2])/2
- pp_file['energy(t)'] = np.sum(energy_tk, axis = 1)
- pp_file['energy(k)'] = np.mean(energy_tk, axis = 0)*(4*np.pi*self.statistics['kshell']**2) / (self.statistics['dk']*self.statistics['nshell'])
- enstrophy_tk = (
- data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 0, 0] +
- data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 1, 1] +
- data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 2, 2])/2
- pp_file['enstrophy(t)'] = np.sum(enstrophy_tk, axis = 1)
- pp_file['enstrophy(k)'] = np.mean(enstrophy_tk, axis = 0)*(4*np.pi*self.statistics['kshell']**2) / (self.statistics['dk']*self.statistics['nshell'])
- pp_file['vel_max(t)'] = data_file['statistics/moments/velocity'][ii0:ii1+1, 9, 3]
- pp_file['renergy(t)'] = data_file['statistics/moments/velocity'][ii0:ii1+1, 2, 3]/2
- if 'trS2_Q_R' in data_file['statistics/moments'].keys():
- pp_file['mean_trS2(t)'] = data_file['statistics/moments/trS2_Q_R'][:, 1, 0]
- for k in ['t',
- 'energy(t)',
- 'energy(k)',
- 'enstrophy(t)',
- 'enstrophy(k)',
- 'R_ij(t)',
- 'vel_max(t)',
- 'renergy(t)',
- 'mean_trS2(t)']:
- if k in pp_file.keys():
- self.statistics[k] = pp_file[k].value
- self.compute_time_averages()
+ self.statistics[k] = pp_file[k].value
+ self.statistics['kM'] = pp_file['kspace/kM'].value
+ self.statistics['dk'] = pp_file['kspace/dk'].value
+ self.statistics['kshell'] = pp_file['kspace/kshell'].value
+ self.statistics['nshell'] = pp_file['kspace/nshell'].value
+ else:
+ self.read_parameters()
+ with self.get_data_file() as data_file:
+ if 'moments' not in data_file['statistics'].keys():
+ return None
+ iter0 = min((data_file['statistics/moments/velocity'].shape[0] *
+ self.parameters['niter_stat']-1),
+ iter0)
+ if type(iter1) == type(None):
+ iter1 = data_file['iteration'].value
+ else:
+ iter1 = min(data_file['iteration'].value, iter1)
+ ii0 = iter0 // self.parameters['niter_stat']
+ ii1 = iter1 // self.parameters['niter_stat']
+ self.statistics['kshell'] = data_file['kspace/kshell'].value
+ self.statistics['nshell'] = data_file['kspace/nshell'].value
+ for kk in [-1, -2]:
+ if (self.statistics['kshell'][kk] == 0):
+ self.statistics['kshell'][kk] = np.nan
+ self.statistics['kM'] = data_file['kspace/kM'].value
+ self.statistics['dk'] = data_file['kspace/dk'].value
+ computation_needed = True
+ pp_file = h5py.File(self.get_postprocess_file_name(), 'a')
+ if not ('parameters' in pp_file.keys()):
+ data_file.copy('parameters', pp_file)
+ data_file.copy('kspace', pp_file)
+ if 'ii0' in pp_file.keys():
+ computation_needed = not (ii0 == pp_file['ii0'].value and
+ ii1 == pp_file['ii1'].value)
+ if computation_needed:
+ for k in ['t', 'vel_max(t)', 'renergy(t)',
+ 'energy(t)', 'enstrophy(t)',
+ 'energy(k)', 'enstrophy(k)',
+ 'energy(t, k)',
+ 'enstrophy(t, k)',
+ 'R_ij(t)',
+ 'ii0', 'ii1', 'iter0', 'iter1']:
+ if k in pp_file.keys():
+ del pp_file[k]
+ if computation_needed:
+ pp_file['iter0'] = iter0
+ pp_file['iter1'] = iter1
+ pp_file['ii0'] = ii0
+ pp_file['ii1'] = ii1
+ pp_file['t'] = (self.parameters['dt']*
+ self.parameters['niter_stat']*
+ (np.arange(ii0, ii1+1).astype(np.float)))
+ phi_ij = data_file['statistics/spectra/velocity_velocity'][ii0:ii1+1]
+ pp_file['R_ij(t)'] = np.sum(phi_ij, axis = 1)
+ energy_tk = (
+ phi_ij[:, :, 0, 0] +
+ phi_ij[:, :, 1, 1] +
+ phi_ij[:, :, 2, 2])/2
+ pp_file['energy(t)'] = np.sum(energy_tk, axis = 1)
+ pp_file['energy(k)'] = np.mean(energy_tk, axis = 0)*(4*np.pi*self.statistics['kshell']**2) / (self.statistics['dk']*self.statistics['nshell'])
+ enstrophy_tk = (
+ data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 0, 0] +
+ data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 1, 1] +
+ data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 2, 2])/2
+ pp_file['enstrophy(t)'] = np.sum(enstrophy_tk, axis = 1)
+ pp_file['enstrophy(k)'] = np.mean(enstrophy_tk, axis = 0)*(4*np.pi*self.statistics['kshell']**2) / (self.statistics['dk']*self.statistics['nshell'])
+ pp_file['vel_max(t)'] = data_file['statistics/moments/velocity'][ii0:ii1+1, 9, 3]
+ pp_file['renergy(t)'] = data_file['statistics/moments/velocity'][ii0:ii1+1, 2, 3]/2
+ if 'trS2_Q_R' in data_file['statistics/moments'].keys():
+ pp_file['mean_trS2(t)'] = data_file['statistics/moments/trS2_Q_R'][:, 1, 0]
+ for k in ['t',
+ 'energy(t)',
+ 'energy(k)',
+ 'enstrophy(t)',
+ 'enstrophy(k)',
+ 'R_ij(t)',
+ 'vel_max(t)',
+ 'renergy(t)',
+ 'mean_trS2(t)']:
+ if k in pp_file.keys():
+ self.statistics[k] = pp_file[k].value
+ # sanity check --- Parseval theorem check
+ assert(np.max(np.abs(
+ self.statistics['renergy(t)'] -
+ self.statistics['energy(t)']) / self.statistics['energy(t)']) < 1e-5)
+ self.compute_time_averages()
return None
def compute_Reynolds_stress_invariants(
self):
diff --git a/bfps/_base.py b/bfps/_base.py
index 2ea8fbbb..57506727 100644
--- a/bfps/_base.py
+++ b/bfps/_base.py
@@ -224,8 +224,10 @@ class _base(object):
ofile[group + '/' + k][...] = parameters[k]
ofile.close()
return None
- def read_parameters(self):
- with h5py.File(os.path.join(self.work_dir, self.simname + '.h5'), 'r') as data_file:
+ def read_parameters(self, fname = None):
+ if type(fname) == type(None):
+ fname = os.path.join(self.work_dir, self.simname + '.h5')
+ with h5py.File(fname, 'r') as data_file:
for k in data_file['parameters'].keys():
if k in self.parameters.keys():
if type(self.parameters[k]) in [int, str, float]:
--
GitLab
From 0e6891a2407ca0157088d3ad4aba611dba3be3d6 Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Sun, 10 Jun 2018 20:50:01 +0200
Subject: [PATCH 193/342] addinitial version of pressure Hessian sampler
---
bfps/DNS.py | 21 +++-
bfps/cpp/full_code/NSVEp_extra_sampling.cpp | 126 ++++++++++++++++++++
bfps/cpp/full_code/NSVEp_extra_sampling.hpp | 47 ++++++++
setup.py | 3 +-
4 files changed, 190 insertions(+), 7 deletions(-)
create mode 100644 bfps/cpp/full_code/NSVEp_extra_sampling.cpp
create mode 100644 bfps/cpp/full_code/NSVEp_extra_sampling.hpp
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 495278bf..c7d77956 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -128,11 +128,11 @@ class DNS(_code):
template_class = '{0}<{1}>::'.format(self.dns_type, rnumber),
template_prefix = 'template '.format(rnumber),
just_declaration = True) + '\n\n')
- if self.dns_type in ['NSVEparticles', 'NSVE_no_output', 'NSVEparticles_no_output', 'NSVEcomplex_particles']:
+ if self.dns_type in ['NSVEparticles', 'NSVE_no_output', 'NSVEparticles_no_output', 'NSVEcomplex_particles', 'NSVEp_extra_sampling']:
outfile.write('template <typename rnumber> int NSVE<rnumber>::read_parameters(){return EXIT_SUCCESS;}\n')
outfile.write('template int NSVE<float>::read_parameters();\n')
outfile.write('template int NSVE<double>::read_parameters();\n\n')
- if self.dns_type in ['NSVEparticles_no_output']:
+ if self.dns_type in ['NSVEparticles_no_output', 'NSVEp_extra_sampling']:
outfile.write('template <typename rnumber> int NSVEparticles<rnumber>::read_parameters(){return EXIT_SUCCESS;}\n')
outfile.write('template int NSVEparticles<float>::read_parameters();\n')
outfile.write('template int NSVEparticles<double>::read_parameters();\n\n')
@@ -682,7 +682,11 @@ class DNS(_code):
'NSVEcomplex_particles',
help = 'plain Navier-Stokes vorticity formulation, with oriented active particles')
- for parser in ['NSVEparticles_no_output', 'NSVEp2', 'NSVEp2p']:
+ parser_NSVEp_extra = subparsers.add_parser(
+ 'NSVEp_extra_sampling',
+ help = 'plain Navier-Stokes vorticity formulation, with basic fluid tracers, that sample velocity gradient, as well as pressure and its derivatives.')
+
+ for parser in ['NSVEparticles_no_output', 'NSVEp2', 'NSVEp2p', 'NSVEp_extra']:
eval('self.simulation_parser_arguments({0})'.format('parser_' + parser))
eval('self.job_parser_arguments({0})'.format('parser_' + parser))
eval('self.particle_parser_arguments({0})'.format('parser_' + parser))
@@ -725,7 +729,7 @@ class DNS(_code):
self.dns_type = opt.DNS_class
self.name = self.dns_type + '-' + self.fluid_precision + '-v' + bfps.__version__
# merge parameters if needed
- if self.dns_type in ['NSVEparticles', 'NSVEcomplex_particles', 'NSVEparticles_no_output']:
+ if self.dns_type in ['NSVEparticles', 'NSVEcomplex_particles', 'NSVEparticles_no_output', 'NSVEp_extra_sampling']:
for k in self.NSVEp_extra_parameters.keys():
self.parameters[k] = self.NSVEp_extra_parameters[k]
if type(extra_parameters) != type(None):
@@ -790,7 +794,7 @@ class DNS(_code):
# hardcoded FFTW complex representation size
field_size = 3*(opt.nx+2)*opt.ny*opt.nz*self.fluid_dtype.itemsize
checkpoint_size = field_size
- if self.dns_type in ['NSVEparticles', 'NSVEcomplex_particles', 'NSVEparticles_no_output']:
+ if self.dns_type in ['NSVEparticles', 'NSVEcomplex_particles', 'NSVEparticles_no_output', 'NSVEp_extra_sampling']:
rhs_size = self.parameters['tracers0_integration_steps']
if type(opt.tracers0_integration_steps) != type(None):
rhs_size = opt.tracers0_integration_steps
@@ -997,6 +1001,11 @@ class DNS(_code):
if self.dns_type in ['NSVEcomplex_particles']:
particle_file.create_group('tracers0/orientation')
particle_file.create_group('tracers0/velocity_gradient')
+ if self.dns_type in ['NSVEp_extra_sampling']:
+ particle_file.create_group('tracers0/velocity_gradient')
+ particle_file.create_group('tracers0/pressure')
+ particle_file.create_group('tracers0/pressure_gradient')
+ particle_file.create_group('tracers0/pressure_Hessian')
return None
def launch_jobs(
self,
@@ -1056,7 +1065,7 @@ class DNS(_code):
# particle_initial_condition[..., 2] += onedarray[None, :, None, None]
self.write_par(
particle_ic = None)
- if self.dns_type in ['NSVEparticles', 'NSVEcomplex_particles', 'NSVEparticles_no_output']:
+ if self.dns_type in ['NSVEparticles', 'NSVEcomplex_particles', 'NSVEparticles_no_output', 'NSVEp_extra_sampling']:
self.generate_particle_data(opt = opt)
self.run(
nb_processes = opt.nb_processes,
diff --git a/bfps/cpp/full_code/NSVEp_extra_sampling.cpp b/bfps/cpp/full_code/NSVEp_extra_sampling.cpp
new file mode 100644
index 00000000..3d002d34
--- /dev/null
+++ b/bfps/cpp/full_code/NSVEp_extra_sampling.cpp
@@ -0,0 +1,126 @@
+#include "full_code/NSVEp_extra_sampling.hpp"
+
+
+
+template <typename rnumber>
+int NSVEp_extra_sampling<rnumber>::initialize(void)
+{
+ this->NSVEparticles<rnumber>::initialize();
+
+ /// allocate grad vel field
+ this->nabla_u = new field<rnumber, FFTW, THREExTHREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ this->fs->cvorticity->fftw_plan_rigor);
+ this->pressure = new field<rnumber, FFTW, ONE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ this->fs->cvorticity->fftw_plan_rigor);
+ this->nabla_p = new field<rnumber, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ this->fs->cvorticity->fftw_plan_rigor);
+ this->Hessian_p = new field<rnumber, FFTW, THREExTHREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ this->fs->cvorticity->fftw_plan_rigor);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVEp_extra_sampling<rnumber>::finalize(void)
+{
+ delete this->nabla_u;
+ delete this->pressure;
+ delete this->nabla_p;
+ delete this->Hessian_p;
+ this->NSVEparticles<rnumber>::finalize();
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVEp_extra_sampling<rnumber>::do_stats()
+{
+ this->NSVEparticles<rnumber>::do_stats();
+ if (!(this->iteration % this->niter_part == 0))
+ return EXIT_SUCCESS;
+
+ /// fs->cvelocity should contain the velocity in Fourier space
+ this->fs->compute_velocity(this->fs->cvorticity);
+ compute_gradient(
+ this->fs->kk,
+ this->fs->cvelocity,
+ this->nabla_u);
+ this->nabla_u->ift();
+
+ this->fs->compute_pressure(this->pressure);
+
+ compute_gradient(
+ this->fs->kk,
+ this->pressure,
+ this->nabla_p);
+
+ compute_gradient(
+ this->fs->kk,
+ this->nabla_p,
+ this->Hessian_p);
+
+ this->pressure->ift();
+ this->nabla_p->ift();
+ this->Hessian_p->ift();
+
+ // sample velocity gradient
+ std::unique_ptr<double[]> pdata(new double[9*this->ps->getLocalNbParticles()]);
+ std::fill_n(pdata.get(), 9*this->ps->getLocalNbParticles(), 0);
+ this->ps->sample_compute_field(*this->nabla_u, pdata.get());
+
+ this->particles_sample_writer_mpi->template save_dataset<9>(
+ "tracers0",
+ "velocity_gradient",
+ this->ps->getParticlesState(),
+ &pdata,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+
+ // sample pressure
+ std::fill_n(pdata.get(), this->ps->getLocalNbParticles(), 0);
+ this->ps->sample_compute_field(*this->pressure, pdata.get());
+ this->particles_sample_writer_mpi->template save_dataset<1>(
+ "tracers0",
+ "pressure",
+ this->ps->getParticlesState(),
+ &pdata,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+
+ // sample pressure gradient
+ std::fill_n(pdata.get(), 3*this->ps->getLocalNbParticles(), 0);
+ this->ps->sample_compute_field(*this->nabla_p, pdata.get());
+ this->particles_sample_writer_mpi->template save_dataset<3>(
+ "tracers0",
+ "pressure_gradient",
+ this->ps->getParticlesState(),
+ &pdata,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+
+ // sample pressure gradient
+ std::fill_n(pdata.get(), 9*this->ps->getLocalNbParticles(), 0);
+ this->ps->sample_compute_field(*this->Hessian_p, pdata.get());
+ this->particles_sample_writer_mpi->template save_dataset<9>(
+ "tracers0",
+ "pressure_Hessian",
+ this->ps->getParticlesState(),
+ &pdata,
+ this->ps->getParticlesIndexes(),
+ this->ps->getLocalNbParticles(),
+ this->ps->get_step_idx()-1);
+ return EXIT_SUCCESS;
+}
+
+template class NSVEp_extra_sampling<float>;
+template class NSVEp_extra_sampling<double>;
+
diff --git a/bfps/cpp/full_code/NSVEp_extra_sampling.hpp b/bfps/cpp/full_code/NSVEp_extra_sampling.hpp
new file mode 100644
index 00000000..0857cde7
--- /dev/null
+++ b/bfps/cpp/full_code/NSVEp_extra_sampling.hpp
@@ -0,0 +1,47 @@
+#ifndef NSVEP_EXTRA_SAMPLING_HPP
+#define NSVEP_EXTRA_SAMPLING_HPP
+
+
+
+#include <cstdlib>
+#include "base.hpp"
+#include "vorticity_equation.hpp"
+#include "full_code/NSVEparticles.hpp"
+#include "particles/particles_system_builder.hpp"
+#include "particles/particles_output_hdf5.hpp"
+#include "particles/particles_sampling.hpp"
+
+/** \brief Navier-Stokes solver with tracers that sample velocity gradient
+ * and pressure Hessian.
+ *
+ */
+
+template <typename rnumber>
+class NSVEp_extra_sampling: public NSVEparticles<rnumber>
+{
+ public:
+
+ /* other stuff */
+ field<rnumber, FFTW, ONE> *pressure;
+ field<rnumber, FFTW, THREE> *nabla_p;
+ field<rnumber, FFTW, THREExTHREE> *nabla_u;
+ field<rnumber, FFTW, THREExTHREE> *Hessian_p;
+
+ NSVEp_extra_sampling(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ NSVEparticles<rnumber>(
+ COMMUNICATOR,
+ simulation_name){}
+ ~NSVEp_extra_sampling(){}
+
+ int initialize(void);
+ int finalize(void);
+
+ int read_parameters(void);
+ int do_stats(void);
+};
+
+#endif//NSVEP_EXTRA_SAMPLING_HPP
+
+
diff --git a/setup.py b/setup.py
index 9671a3c6..8152e5fe 100644
--- a/setup.py
+++ b/setup.py
@@ -133,7 +133,8 @@ src_file_list = ['full_code/NSVEcomplex_particles',
'spline_n10',
'Lagrange_polys',
'scope_timer',
- 'full_code/NSVEparticles']
+ 'full_code/NSVEparticles',
+ 'full_code/NSVEp_extra_sampling']
particle_headers = [
'cpp/particles/abstract_particles_input.hpp',
--
GitLab
From 658b72f09b890bfaf1eba4fa54b07a6e49ab0143 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 19 Jun 2018 15:57:40 +0200
Subject: [PATCH 194/342] add a test of divergence
---
bfps/test/test_interpolation.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/bfps/test/test_interpolation.py b/bfps/test/test_interpolation.py
index d5cda159..eeb40248 100644
--- a/bfps/test/test_interpolation.py
+++ b/bfps/test/test_interpolation.py
@@ -42,6 +42,10 @@ def main():
vort1[:, 1] = vel_gradient[:, 6] - vel_gradient[:, 2]
vort1[:, 2] = vel_gradient[:, 1] - vel_gradient[:, 3]
assert(np.max(np.abs(vort0-vort1) / np.abs(vort0)) <= 1e-5)
+ divergence = vel_gradient[:, 0] + vel_gradient[:, 4] + vel_gradient[:, 8]
+ divergence_error = np.abs(divergence) / (vel_gradient[:, 0]**2 + vel_gradient[:, 1]**2 + vel_gradient[:, 2]**2)**.5
+ print('mean divergence error is ', np.mean(divergence_error))
+ print('maximum divergence error is ', np.max(divergence_error))
print('SUCCESS! Interpolated vorticity agrees with vorticity from interpolated velocity gradient.')
return None
--
GitLab
From 5214b9e47b2b6cfe9e8d359df008198331ebf130 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 19 Jun 2018 16:31:11 +0200
Subject: [PATCH 195/342] clean up arrays before sampling vel gradient etc
---
bfps/cpp/full_code/NSVEcomplex_particles.cpp | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/bfps/cpp/full_code/NSVEcomplex_particles.cpp b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
index 93d0edc1..950d2c15 100644
--- a/bfps/cpp/full_code/NSVEcomplex_particles.cpp
+++ b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
@@ -118,6 +118,7 @@ int NSVEcomplex_particles<rnumber>::do_stats()
return EXIT_SUCCESS;
/// allocate temporary data array
+ /// initialize pdata0 with the positions, and pdata1 with the orientations
std::unique_ptr<double[]> pdata0 = this->ps->extractParticlesState(0, 3);
std::unique_ptr<double[]> pdata1 = this->ps->extractParticlesState(3, 6);
std::unique_ptr<double[]> pdata2(new double[9*this->ps->getLocalNbParticles()]);
@@ -143,6 +144,8 @@ int NSVEcomplex_particles<rnumber>::do_stats()
this->ps->get_step_idx()-1);
/// sample velocity
+ /// from now on, we need to clean up data arrays before interpolation
+ std::fill_n(pdata1.get(), 3*this->ps->getLocalNbParticles(), 0);
this->ps->sample_compute_field(*this->tmp_vec_field, pdata1.get());
this->particles_sample_writer_mpi->template save_dataset<3>(
"tracers0",
@@ -161,6 +164,7 @@ int NSVEcomplex_particles<rnumber>::do_stats()
this->fs->cvelocity,
this->nabla_u);
this->nabla_u->ift();
+ std::fill_n(pdata2.get(), 9*this->ps->getLocalNbParticles(), 0);
this->ps->sample_compute_field(*this->nabla_u, pdata2.get());
this->particles_sample_writer_mpi->template save_dataset<9>(
"tracers0",
@@ -174,6 +178,7 @@ int NSVEcomplex_particles<rnumber>::do_stats()
/// compute acceleration and sample it
this->fs->compute_Lagrangian_acceleration(this->tmp_vec_field);
this->tmp_vec_field->ift();
+ std::fill_n(pdata1.get(), 3*this->ps->getLocalNbParticles(), 0);
this->ps->sample_compute_field(*this->tmp_vec_field, pdata1.get());
this->particles_sample_writer_mpi->template save_dataset<3>(
"tracers0",
--
GitLab
From 0db01970d7214bb4f0bcf0bb2a74c9b069c6c8e8 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 20 Jun 2018 13:51:06 +0200
Subject: [PATCH 196/342] clean up temp vorticity array before interpolating
---
bfps/cpp/particles/abstract_particles_system.hpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/bfps/cpp/particles/abstract_particles_system.hpp b/bfps/cpp/particles/abstract_particles_system.hpp
index 871f0459..ffa89892 100644
--- a/bfps/cpp/particles/abstract_particles_system.hpp
+++ b/bfps/cpp/particles/abstract_particles_system.hpp
@@ -75,6 +75,7 @@ public:
const field<rnumber, be, fc>& in_field) {
static_assert(fc == THREE, "only THREE is supported for now");
std::unique_ptr<real_number[]> extra_rhs(new real_number[getLocalNbParticles()*3]());
+ std::fill_n(extra_rhs.get(), 3*getLocalNbParticles(), 0);
sample_compute_field(in_field, extra_rhs.get());
completeLoopWithVorticity(dt, extra_rhs.get());
}
--
GitLab
From 6f474a6ed85f27a1939bbf485c8ef21703481630 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 21 Jun 2018 15:08:33 +0200
Subject: [PATCH 197/342] hardcode type of particles to simple spheres that can
rotate.
---
bfps/cpp/full_code/NSVEcomplex_particles.cpp | 2 ++
bfps/cpp/particles/particles_inner_computer.hpp | 12 ++++++------
2 files changed, 8 insertions(+), 6 deletions(-)
diff --git a/bfps/cpp/full_code/NSVEcomplex_particles.cpp b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
index 950d2c15..da7015fd 100644
--- a/bfps/cpp/full_code/NSVEcomplex_particles.cpp
+++ b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
@@ -14,10 +14,12 @@ int NSVEcomplex_particles<rnumber>::initialize(void)
p2p_computer<double, long long int> current_p2p_computer;
// TODO: particle interactions are switched off manually for testing purposes.
// this needs to be fixed once particle interactions can be properly resolved.
+ this->enable_p2p = false;
current_p2p_computer.setEnable(enable_p2p);
//current_p2p_computer.setEnable(false);
particles_inner_computer<double, long long int> current_particles_inner_computer(inner_v0);
+ this->enable_inner = false;
current_particles_inner_computer.setEnable(enable_inner);
this->cutoff = 1.0;
diff --git a/bfps/cpp/particles/particles_inner_computer.hpp b/bfps/cpp/particles/particles_inner_computer.hpp
index 5c855cf5..f1fe322a 100644
--- a/bfps/cpp/particles/particles_inner_computer.hpp
+++ b/bfps/cpp/particles/particles_inner_computer.hpp
@@ -98,12 +98,12 @@ public:
#pragma omp parallel for
for(partsize_t idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
// Cross product vorticity/orientation
- rhs_part[idx_part*size_particle_rhs + 3+IDX_X] += (rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Y]*pos_part[idx_part*size_particle_positions + 3+IDX_Z] -
- rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Z]*pos_part[idx_part*size_particle_positions + 3+IDX_Y]);
- rhs_part[idx_part*size_particle_rhs + 3+IDX_Y] += (rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Z]*pos_part[idx_part*size_particle_positions + 3+IDX_X] -
- rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_X]*pos_part[idx_part*size_particle_positions + 3+IDX_Z]);
- rhs_part[idx_part*size_particle_rhs + 3+IDX_Z] += (rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_X]*pos_part[idx_part*size_particle_positions + 3+IDX_Y] -
- rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Y]*pos_part[idx_part*size_particle_positions + 3+IDX_X]);
+ rhs_part[idx_part*size_particle_rhs + 3+IDX_X] += 0.5*(rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Y]*pos_part[idx_part*size_particle_positions + 3+IDX_Z] -
+ rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Z]*pos_part[idx_part*size_particle_positions + 3+IDX_Y]);
+ rhs_part[idx_part*size_particle_rhs + 3+IDX_Y] += 0.5*(rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Z]*pos_part[idx_part*size_particle_positions + 3+IDX_X] -
+ rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_X]*pos_part[idx_part*size_particle_positions + 3+IDX_Z]);
+ rhs_part[idx_part*size_particle_rhs + 3+IDX_Z] += 0.5*(rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_X]*pos_part[idx_part*size_particle_positions + 3+IDX_Y] -
+ rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Y]*pos_part[idx_part*size_particle_positions + 3+IDX_X]);
}
}
--
GitLab
From fcb557e94153cb316167db35231c554a53ff671c Mon Sep 17 00:00:00 2001
From: Debarghya Banerjee <debarghya.banerjee@ds.mpg.de>
Date: Mon, 20 Aug 2018 10:48:42 +0200
Subject: [PATCH 198/342] 3 scale PDFs
modified: field.cpp
modified: field.hpp
---
bfps/cpp/field.cpp | 153 +++++++++++++++++++++++++++++++++++++++++++++
bfps/cpp/field.hpp | 13 ++++
2 files changed, 166 insertions(+)
diff --git a/bfps/cpp/field.cpp b/bfps/cpp/field.cpp
index 25d71641..5fe146b9 100644
--- a/bfps/cpp/field.cpp
+++ b/bfps/cpp/field.cpp
@@ -1559,6 +1559,138 @@ int joint_rspace_PDF(
return EXIT_SUCCESS;
}
+// Debarghya edit for 3 scale PDFs //
+
+template <typename rnumber,
+ field_backend be>
+int joint_rspace_3PDF(
+ field<rnumber, be, ONE> *f1,
+ field<rnumber, be, ONE> *f2,
+ field<rnumber, be, ONE> *f3,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const std::vector<double> max_f1_estimate,
+ const std::vector<double> max_f2_estimate,
+ const std::vector<double> max_f3_estimate)
+{
+ TIMEZONE("joint_rspace_3PDF");
+ assert(f1->real_space_representation);
+ assert(f2->real_space_representation);
+ assert(f3->real_space_representation);
+
+ assert(max_f1_estimate.size() == 1);
+ assert(max_f2_estimate.size() == 1);
+ assert(max_f3_estimate.size() == 1);
+
+ int nbins;
+ std::string dsetc, dsetm;
+ dsetc = "histograms/" + dset_name + "_components";
+ dsetm = "histograms/" + dset_name;
+ if (f1->myrank == 0)
+ {
+ hid_t dset, wspace;
+ hsize_t dims[5];
+ int ndims;
+ dset = H5Dopen(
+ group,
+ dsetm.c_str(),
+ H5P_DEFAULT);
+ wspace = H5Dget_space(dset);
+ ndims = H5Sget_simple_extent_dims(wspace, dims, NULL);
+ assert(ndims == 4);
+ H5Sclose(wspace);
+ H5Dclose(dset);
+ nbins = dims[1];
+ }
+ {
+ TIMEZONE("MPI_Bcast");
+ MPI_Bcast(&nbins, 1, MPI_INT, 0, f1->comm);
+ }
+
+
+ /// histogram magnitudes
+ shared_array<ptrdiff_t> local_histm_threaded(
+ nbins*nbins*nbins,
+ [&](ptrdiff_t* local_hist){
+ std::fill_n(local_hist, nbins*nbins*nbins, 0);
+ });
+
+ /// set up bin sizes
+ std::vector<double> bin1size, bin2size, bin3size;
+ bin1size.resize(1);
+ bin2size.resize(1);
+ bin3size.resize(1);
+
+ bin1size[0] = 2*max_f1_estimate[0] / nbins;
+ bin2size[0] = 2*max_f2_estimate[0] / nbins;
+ bin3size[0] = 2*max_f3_estimate[0] / nbins;
+
+
+ {
+ TIMEZONE("field::RLOOP");
+ f1->RLOOP(
+ [&](ptrdiff_t rindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex){
+ ptrdiff_t *local_histm = local_histm_threaded.getMine();
+ int bin1 = 0;
+ int bin2 = 0;
+ int bin3 = 0;
+
+ bin1 = int(floor((f1->rval(rindex) + max_f1_estimate[0])/bin1size[0]));
+ bin2 = int(floor((f2->rval(rindex) + max_f2_estimate[0])/bin2size[0]));
+ bin3 = int(floor((f3->rval(rindex) + max_f3_estimate[0])/bin3size[0]));
+ if ((bin1 >= 0 && bin1 < nbins) &&
+ (bin2 >= 0 && bin2 < nbins) &&
+ (bin3 >= 0 && bin3 < nbins))
+ local_histm[bin1*nbins*nbins + bin2*nbins + bin3]++;
+ });
+ }
+ local_histm_threaded.mergeParallel();
+ ptrdiff_t *histm = new ptrdiff_t[nbins*nbins*nbins];
+ ptrdiff_t *histc = NULL;
+ {
+ MPI_Allreduce(
+ (void*)local_histm_threaded.getMasterData(),
+ (void*)histm,
+ nbins*nbins*nbins,
+ MPI_INT64_T, MPI_SUM, f1->comm);
+ }
+
+ if (f1->myrank == 0)
+ {
+ TIMEZONE("root-work");
+ hid_t dset, wspace, mspace;
+ hsize_t count[5], offset[5];
+
+ dset = H5Dopen(group, dsetm.c_str(), H5P_DEFAULT);
+ assert(dset > 0);
+ offset[0] = toffset;
+ offset[1] = 0;
+ offset[2] = 0;
+ offset[3] = 0;
+ count[0] = 1;
+ count[1] = nbins;
+ count[2] = nbins;
+ count[3] = nbins;
+ mspace = H5Screate_simple(4, count, NULL);
+ wspace = H5Dget_space(dset);
+ H5Sselect_hyperslab(wspace, H5S_SELECT_SET, offset, NULL, count, NULL);
+ H5Dwrite(dset, H5T_NATIVE_INT64, mspace, wspace, H5P_DEFAULT, histm);
+ H5Sclose(wspace);
+ H5Sclose(mspace);
+ H5Dclose(dset);
+ }
+
+ delete[] histm;
+
+ return EXIT_SUCCESS;
+}
+
+
+
template <typename rnumber,
field_backend be,
field_components fc>
@@ -1831,3 +1963,24 @@ template int joint_rspace_PDF<double, FFTW, ONE>(
const std::vector<double>,
const std::vector<double>);
+template int joint_rspace_3PDF<float, FFTW>(
+ field<float, FFTW, ONE> *,
+ field<float, FFTW, ONE> *,
+ field<float, FFTW, ONE> *,
+ const hid_t,
+ const std::string,
+ const hsize_t,
+ const std::vector<double>,
+ const std::vector<double>,
+ const std::vector<double>);
+template int joint_rspace_3PDF<double, FFTW>(
+ field<double, FFTW, ONE> *,
+ field<double, FFTW, ONE> *,
+ field<double, FFTW, ONE> *,
+ const hid_t,
+ const std::string,
+ const hsize_t,
+ const std::vector<double>,
+ const std::vector<double>,
+ const std::vector<double>);
+
diff --git a/bfps/cpp/field.hpp b/bfps/cpp/field.hpp
index a52d2a56..d48722db 100644
--- a/bfps/cpp/field.hpp
+++ b/bfps/cpp/field.hpp
@@ -347,5 +347,18 @@ int joint_rspace_PDF(
const std::vector<double> max_f1_estimate,
const std::vector<double> max_f2_estimate);
+template <typename rnumber,
+ field_backend be>
+int joint_rspace_3PDF(
+ field<rnumber, be, ONE> *f1,
+ field<rnumber, be, ONE> *f2,
+ field<rnumber, be, ONE> *f3,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset,
+ const std::vector<double> max_f1_estimate,
+ const std::vector<double> max_f2_estimate,
+ const std::vector<double> max_f3_estimate);
+
#endif//FIELD_HPP
--
GitLab
From 2d42f2555dbf3baa825769105a16a425713276af Mon Sep 17 00:00:00 2001
From: Cristian Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 31 Aug 2018 11:18:30 +0200
Subject: [PATCH 199/342] use snprintf instead of sprintf
---
bfps/_base.py | 4 ++--
bfps/_code.py | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/bfps/_base.py b/bfps/_base.py
index 57506727..3d7c747b 100644
--- a/bfps/_base.py
+++ b/bfps/_base.py
@@ -97,7 +97,7 @@ class _base(object):
'char fname[256];\n' +
'hsize_t dims[1];\n' +
'char *string_data;\n' +
- 'sprintf(fname, "%s.h5", {0});\n'.format(simname_variable) +
+ 'snprintf(fname, 255, "%s.h5", {0});\n'.format(simname_variable) +
'parameter_file = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT);\n')
key_prefix = ''
if prepend_this:
@@ -118,7 +118,7 @@ class _base(object):
'memtype = H5Dget_type(dset);\n' +
'string_data = (char*)malloc(256);\n' +
'H5Dread(dset, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &string_data);\n' +
- 'sprintf({0}, "%s", string_data);\n'.format(key_prefix + key[i]) +
+ 'snprintf({0}, 255, "%s", string_data);\n'.format(key_prefix + key[i]) +
'free(string_data);\n'
'H5Sclose(space);\n' +
'H5Tclose(memtype);\n' +
diff --git a/bfps/_code.py b/bfps/_code.py
index 78302109..73af7661 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -116,7 +116,7 @@ class _code(_base):
}
#endif
strcpy(simname, argv[1]);
- sprintf(fname, "%s.h5", simname);
+ snprintf(fname, 255, "%s.h5", simname);
parameter_file = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT);
Cdset = H5Dopen(parameter_file, "iteration", H5P_DEFAULT);
H5Dread(
--
GitLab
From 5545631e50950d8b7e1fe9355c97956993678b28 Mon Sep 17 00:00:00 2001
From: Cristian Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 31 Aug 2018 13:02:27 +0200
Subject: [PATCH 200/342] minor update of test_io
---
tests/base.py | 3 +++
tests/test_io_03_run.py | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
diff --git a/tests/base.py b/tests/base.py
index 1c06974e..6f110716 100644
--- a/tests/base.py
+++ b/tests/base.py
@@ -50,6 +50,9 @@ def get_parser(base_class = bfps.NavierStokes,
parser.add_argument('-n',
type = int, dest = 'n',
default = n)
+ parser.add_argument('--np',
+ type = int, dest = 'np',
+ default = ncpu)
parser.add_argument('--ncpu',
type = int, dest = 'ncpu',
default = ncpu)
diff --git a/tests/test_io_03_run.py b/tests/test_io_03_run.py
index a789ac66..5b4905ba 100644
--- a/tests/test_io_03_run.py
+++ b/tests/test_io_03_run.py
@@ -35,5 +35,5 @@ if __name__ == '__main__':
c.write_src()
c.write_par()
c.set_host_info(bfps.host_info)
- c.run()
+ c.run(opt.ncpu, 1)
--
GitLab
From ca42fb6676831a9d86d78b6d02db961a7bece7f1 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 31 Aug 2018 13:20:38 +0200
Subject: [PATCH 201/342] update sge script for openmp usage
---
bfps/_code.py | 28 +++++++++++++++-----------
bfps/cpp/full_code/symmetrize_test.cpp | 4 ++++
2 files changed, 20 insertions(+), 12 deletions(-)
diff --git a/bfps/_code.py b/bfps/_code.py
index 73af7661..d8b32d59 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -62,7 +62,7 @@ class _code(_base):
#include <string>
#include <cstring>
#include <fftw3-mpi.h>
- #include <omp.h>
+ #include <omp.h>
#include <fenv.h>
#include <cstdlib>
//endcpp
@@ -277,7 +277,8 @@ class _code(_base):
hours = hours,
minutes = minutes,
out_file = out_file + '_' + suffix,
- err_file = err_file + '_' + suffix)
+ err_file = err_file + '_' + suffix,
+ nb_threads_per_process = nb_threads_per_process)
os.chdir(self.work_dir)
qsub_atoms = ['qsub']
if len(job_name_list) >= 1:
@@ -299,7 +300,7 @@ class _code(_base):
out_file = out_file + '_' + suffix,
err_file = err_file + '_' + suffix,
nb_mpi_processes = nb_processes,
- nb_threads_per_process = nb_threads_per_process)
+ nb_threads_per_process = nb_threads_per_process)
os.chdir(self.work_dir)
qsub_atoms = ['sbatch']
@@ -326,7 +327,7 @@ class _code(_base):
out_file = out_file + '_' + suffix,
err_file = err_file + '_' + suffix,
nb_mpi_processes = nb_processes,
- nb_threads_per_process = nb_threads_per_process)
+ nb_threads_per_process = nb_threads_per_process)
else:
self.write_IBMLoadLeveler_file_many_job(
file_name = os.path.join(self.work_dir, job_script_name),
@@ -338,7 +339,7 @@ class _code(_base):
err_file = err_file + '_' + suffix,
njobs = njobs,
nb_mpi_processes = nb_processes,
- nb_threads_per_process = nb_threads_per_process)
+ nb_threads_per_process = nb_threads_per_process)
submit_atoms = ['llsubmit']
if not no_submit:
@@ -367,8 +368,8 @@ class _code(_base):
minutes = None,
out_file = None,
err_file = None,
- nb_mpi_processes = None,
- nb_threads_per_process = None):
+ nb_mpi_processes = None,
+ nb_threads_per_process = None):
script_file = open(file_name, 'w')
script_file.write('# @ shell=/bin/bash\n')
@@ -465,8 +466,8 @@ class _code(_base):
out_file = None,
err_file = None,
njobs = 2,
- nb_mpi_processes = None,
- nb_threads_per_process = None):
+ nb_mpi_processes = None,
+ nb_threads_per_process = None):
assert(type(self.host_info['environment']) != type(None))
script_file = open(file_name, 'w')
script_file.write('# @ shell=/bin/bash\n')
@@ -563,7 +564,8 @@ class _code(_base):
hours = None,
minutes = None,
out_file = None,
- err_file = None):
+ err_file = None,
+ nb_threads_per_process = 1):
script_file = open(file_name, 'w')
script_file.write('#!/bin/bash\n')
# export all environment variables
@@ -586,6 +588,8 @@ class _code(_base):
script_file.write('echo "got $NSLOTS slots."\n')
script_file.write('echo "Start time is `date`"\n')
script_file.write('mpiexec -machinefile $TMPDIR/machines ' +
+ '-genv OMP_NUM_THREADS={0} '.format(nb_threads_per_process)
+ '-genv OMP_PLACES=cores '
'-genv LD_LIBRARY_PATH ' +
'"' +
':'.join([bfps.lib_dir] + bfps.install_info['library_dirs']) +
@@ -604,8 +608,8 @@ class _code(_base):
minutes = None,
out_file = None,
err_file = None,
- nb_mpi_processes = None,
- nb_threads_per_process = None):
+ nb_mpi_processes = None,
+ nb_threads_per_process = None):
script_file = open(file_name, 'w')
script_file.write('#!/bin/bash -l\n')
# job name
diff --git a/bfps/cpp/full_code/symmetrize_test.cpp b/bfps/cpp/full_code/symmetrize_test.cpp
index 13d48d60..3b4cd5a5 100644
--- a/bfps/cpp/full_code/symmetrize_test.cpp
+++ b/bfps/cpp/full_code/symmetrize_test.cpp
@@ -36,14 +36,18 @@ template <typename rnumber>
int symmetrize_test<rnumber>::do_work(void)
{
// allocate
+ DEBUG_MSG("about to allocate field0\n");
field<rnumber, FFTW, THREE> *test_field0 = new field<rnumber, FFTW, THREE>(
this->nx, this->ny, this->nz,
this->comm,
DEFAULT_FFTW_FLAG);
+ DEBUG_MSG("finished allocating field0\n");
+ DEBUG_MSG("about to allocate field1\n");
field<rnumber, FFTW, THREE> *test_field1 = new field<rnumber, FFTW, THREE>(
this->nx, this->ny, this->nz,
this->comm,
DEFAULT_FFTW_FLAG);
+ DEBUG_MSG("finished allocating field1\n");
std::default_random_engine rgen;
std::normal_distribution<rnumber> rdist;
rgen.seed(1);
--
GitLab
From fdb252ee168e6996a0271cb32109c0f95881744a Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 31 Aug 2018 13:37:46 +0200
Subject: [PATCH 202/342] fix typo
---
bfps/_code.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/bfps/_code.py b/bfps/_code.py
index d8b32d59..fae02beb 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -588,8 +588,8 @@ class _code(_base):
script_file.write('echo "got $NSLOTS slots."\n')
script_file.write('echo "Start time is `date`"\n')
script_file.write('mpiexec -machinefile $TMPDIR/machines ' +
- '-genv OMP_NUM_THREADS={0} '.format(nb_threads_per_process)
- '-genv OMP_PLACES=cores '
+ '-genv OMP_NUM_THREADS={0} '.format(nb_threads_per_process) +
+ '-genv OMP_PLACES=cores ' +
'-genv LD_LIBRARY_PATH ' +
'"' +
':'.join([bfps.lib_dir] + bfps.install_info['library_dirs']) +
--
GitLab
From d31003f9ad8be7522a0a6c71ed07a70406893d6b Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 5 Sep 2018 21:50:03 +0200
Subject: [PATCH 203/342] add code for single-double conversion
---
bfps/PP.py | 6 ++
bfps/cpp/full_code/field_single_to_double.cpp | 85 +++++++++++++++++++
bfps/cpp/full_code/field_single_to_double.hpp | 62 ++++++++++++++
setup.py | 1 +
4 files changed, 154 insertions(+)
create mode 100644 bfps/cpp/full_code/field_single_to_double.cpp
create mode 100644 bfps/cpp/full_code/field_single_to_double.hpp
diff --git a/bfps/PP.py b/bfps/PP.py
index 4d19095a..27a35928 100644
--- a/bfps/PP.py
+++ b/bfps/PP.py
@@ -434,6 +434,12 @@ class PP(_code):
self.simulation_parser_arguments(parser_native_binary_to_hdf5)
self.job_parser_arguments(parser_native_binary_to_hdf5)
self.parameters_to_parser_arguments(parser_native_binary_to_hdf5)
+ parser_field_single_to_double = subparsers.add_parser(
+ 'field_single_to_double',
+ help = 'convert complex vorticity from single to double')
+ self.simulation_parser_arguments(parser_field_single_to_double)
+ self.job_parser_arguments(parser_field_single_to_double)
+ self.parameters_to_parser_arguments(parser_field_single_to_double)
parser_get_rfields = subparsers.add_parser(
'get_rfields',
help = 'get real space velocity field')
diff --git a/bfps/cpp/full_code/field_single_to_double.cpp b/bfps/cpp/full_code/field_single_to_double.cpp
new file mode 100644
index 00000000..bbc7e292
--- /dev/null
+++ b/bfps/cpp/full_code/field_single_to_double.cpp
@@ -0,0 +1,85 @@
+#include <string>
+#include <cmath>
+#include "field_single_to_double.hpp"
+#include "scope_timer.hpp"
+
+
+template <typename rnumber>
+int field_single_to_double<rnumber>::initialize(void)
+{
+ this->NSVE_field_stats<rnumber>::initialize();
+ DEBUG_MSG("after NSVE_field_stats::initialize\n");
+ this->vec_field_double = new field<double, FFTW, THREE>(
+ this->nx, this->ny, this->nz,
+ this->comm,
+ this->vorticity->fftw_plan_rigor);
+ hid_t parameter_file = H5Fopen(
+ (this->simname + std::string(".h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ hid_t dset = H5Dopen(parameter_file, "/parameters/niter_out", H5P_DEFAULT);
+ H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->niter_out);
+ H5Dclose(dset);
+ if (H5Lexists(parameter_file, "/parameters/checkpoints_per_file", H5P_DEFAULT))
+ {
+ dset = H5Dopen(parameter_file, "/parameters/checkpoints_per_file", H5P_DEFAULT);
+ H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->checkpoints_per_file);
+ H5Dclose(dset);
+ }
+ else
+ this->checkpoints_per_file = 1;
+ H5Fclose(parameter_file);
+ parameter_file = H5Fopen(
+ (this->simname + std::string("_post.h5")).c_str(),
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT);
+ DEBUG_MSG("before read_vector\n");
+ this->iteration_list = hdf5_tools::read_vector<int>(
+ parameter_file,
+ "/get_rfields/parameters/iteration_list");
+ H5Fclose(parameter_file);
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int field_single_to_double<rnumber>::work_on_current_iteration(void)
+{
+ DEBUG_MSG("entered field_single_to_double::work_on_current_iteration\n");
+ this->read_current_cvorticity();
+
+ this->vorticity->RLOOP(
+ [&](ptrdiff_t rindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex){
+ {
+ this->vec_field_double->rval(rindex, 0) = this->vorticity->rval(rindex, 0);
+ this->vec_field_double->rval(rindex, 1) = this->vorticity->rval(rindex, 1);
+ this->vec_field_double->rval(rindex, 2) = this->vorticity->rval(rindex, 2);
+ }
+ }
+ );
+
+ std::string fname = (
+ this->simname +
+ std::string("_checkpoint_double_") +
+ std::to_string(this->iteration / (this->niter_out*this->checkpoints_per_file)) +
+ std::string(".h5"));
+ this->vec_field_double->io(
+ fname,
+ "vorticity",
+ this->iteration,
+ false);
+
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int field_single_to_double<rnumber>::finalize(void)
+{
+ delete this->vec_field_double;
+ return EXIT_SUCCESS;
+}
+
+template class field_single_to_double<float>;
+
diff --git a/bfps/cpp/full_code/field_single_to_double.hpp b/bfps/cpp/full_code/field_single_to_double.hpp
new file mode 100644
index 00000000..c4afc439
--- /dev/null
+++ b/bfps/cpp/full_code/field_single_to_double.hpp
@@ -0,0 +1,62 @@
+/**********************************************************************
+* *
+* Copyright 2017 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#ifndef FIELD_SINGLE_TO_DOUBLE_HPP
+#define FIELD_SINGLE_TO_DOUBLE_HPP
+
+#include <cstdlib>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <vector>
+#include "base.hpp"
+#include "field.hpp"
+#include "field_binary_IO.hpp"
+#include "full_code/NSVE_field_stats.hpp"
+
+template <typename rnumber>
+class field_single_to_double: public NSVE_field_stats<rnumber>
+{
+ public:
+ int checkpoints_per_file;
+ int niter_out;
+
+ field<double, FFTW, THREE> *vec_field_double;
+
+ field_single_to_double(
+ const MPI_Comm COMMUNICATOR,
+ const std::string &simulation_name):
+ NSVE_field_stats<rnumber>(
+ COMMUNICATOR,
+ simulation_name){}
+ virtual ~field_single_to_double(){}
+
+ int initialize(void);
+ int work_on_current_iteration(void);
+ int finalize(void);
+};
+
+#endif//FIELD_SINGLE_TO_DOUBLE_HPP
+
diff --git a/setup.py b/setup.py
index 8152e5fe..7889c927 100644
--- a/setup.py
+++ b/setup.py
@@ -98,6 +98,7 @@ src_file_list = ['full_code/NSVEcomplex_particles',
'full_code/test_interpolation',
'hdf5_tools',
'full_code/get_rfields',
+ 'full_code/field_single_to_double',
'full_code/resize',
'full_code/NSVE_field_stats',
'full_code/native_binary_to_hdf5',
--
GitLab
From 89752a41ae0c08b2a9fa35bb00461b55b0a01c4e Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Wed, 5 Sep 2018 22:04:42 +0200
Subject: [PATCH 204/342] fix parameter location
---
bfps/cpp/full_code/field_single_to_double.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bfps/cpp/full_code/field_single_to_double.cpp b/bfps/cpp/full_code/field_single_to_double.cpp
index bbc7e292..3300573b 100644
--- a/bfps/cpp/full_code/field_single_to_double.cpp
+++ b/bfps/cpp/full_code/field_single_to_double.cpp
@@ -36,7 +36,7 @@ int field_single_to_double<rnumber>::initialize(void)
DEBUG_MSG("before read_vector\n");
this->iteration_list = hdf5_tools::read_vector<int>(
parameter_file,
- "/get_rfields/parameters/iteration_list");
+ "/field_single_to_double/parameters/iteration_list");
H5Fclose(parameter_file);
return EXIT_SUCCESS;
}
--
GitLab
From 27dcccfc87972712000d92f5897fa88c26328515 Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Wed, 5 Sep 2018 23:00:27 +0200
Subject: [PATCH 205/342] add kspace, use std::copy
---
bfps/cpp/full_code/field_single_to_double.cpp | 15 ++++++++++-----
bfps/cpp/full_code/field_single_to_double.hpp | 1 +
2 files changed, 11 insertions(+), 5 deletions(-)
diff --git a/bfps/cpp/full_code/field_single_to_double.cpp b/bfps/cpp/full_code/field_single_to_double.cpp
index 3300573b..51a4e667 100644
--- a/bfps/cpp/full_code/field_single_to_double.cpp
+++ b/bfps/cpp/full_code/field_single_to_double.cpp
@@ -9,10 +9,13 @@ int field_single_to_double<rnumber>::initialize(void)
{
this->NSVE_field_stats<rnumber>::initialize();
DEBUG_MSG("after NSVE_field_stats::initialize\n");
+ this->kk = new kspace<FFTW, SMOOTH>(
+ this->vorticity->clayout, this->dkx, this->dky, this->dkz);
this->vec_field_double = new field<double, FFTW, THREE>(
this->nx, this->ny, this->nz,
this->comm,
this->vorticity->fftw_plan_rigor);
+ this->vec_field_double->real_space_representation = false;
hid_t parameter_file = H5Fopen(
(this->simname + std::string(".h5")).c_str(),
H5F_ACC_RDONLY,
@@ -47,15 +50,16 @@ int field_single_to_double<rnumber>::work_on_current_iteration(void)
DEBUG_MSG("entered field_single_to_double::work_on_current_iteration\n");
this->read_current_cvorticity();
- this->vorticity->RLOOP(
- [&](ptrdiff_t rindex,
+ this->kk->CLOOP(
+ [&](ptrdiff_t cindex,
ptrdiff_t xindex,
ptrdiff_t yindex,
ptrdiff_t zindex){
{
- this->vec_field_double->rval(rindex, 0) = this->vorticity->rval(rindex, 0);
- this->vec_field_double->rval(rindex, 1) = this->vorticity->rval(rindex, 1);
- this->vec_field_double->rval(rindex, 2) = this->vorticity->rval(rindex, 2);
+ std::copy(
+ (rnumber*)(this->vorticity->get_cdata() + cindex),
+ (rnumber*)(this->vorticity->get_cdata() + cindex) + 6,
+ (double*)(this->vec_field_double->get_cdata() + cindex));
}
}
);
@@ -78,6 +82,7 @@ template <typename rnumber>
int field_single_to_double<rnumber>::finalize(void)
{
delete this->vec_field_double;
+ delete this->kk;
return EXIT_SUCCESS;
}
diff --git a/bfps/cpp/full_code/field_single_to_double.hpp b/bfps/cpp/full_code/field_single_to_double.hpp
index c4afc439..0a7550e4 100644
--- a/bfps/cpp/full_code/field_single_to_double.hpp
+++ b/bfps/cpp/full_code/field_single_to_double.hpp
@@ -42,6 +42,7 @@ class field_single_to_double: public NSVE_field_stats<rnumber>
public:
int checkpoints_per_file;
int niter_out;
+ kspace<FFTW, SMOOTH> *kk;
field<double, FFTW, THREE> *vec_field_double;
--
GitLab
From 843daf044064d2db48546acc75dd891360d66346 Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Thu, 6 Sep 2018 07:38:30 +0200
Subject: [PATCH 206/342] fix memory access
---
bfps/cpp/full_code/field_single_to_double.cpp | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/bfps/cpp/full_code/field_single_to_double.cpp b/bfps/cpp/full_code/field_single_to_double.cpp
index 51a4e667..bb34abd2 100644
--- a/bfps/cpp/full_code/field_single_to_double.cpp
+++ b/bfps/cpp/full_code/field_single_to_double.cpp
@@ -50,6 +50,8 @@ int field_single_to_double<rnumber>::work_on_current_iteration(void)
DEBUG_MSG("entered field_single_to_double::work_on_current_iteration\n");
this->read_current_cvorticity();
+ // using CLOOP as opposed to a global std::copy because CLOOP
+ // is openmp parallelized.
this->kk->CLOOP(
[&](ptrdiff_t cindex,
ptrdiff_t xindex,
@@ -57,9 +59,9 @@ int field_single_to_double<rnumber>::work_on_current_iteration(void)
ptrdiff_t zindex){
{
std::copy(
- (rnumber*)(this->vorticity->get_cdata() + cindex),
- (rnumber*)(this->vorticity->get_cdata() + cindex) + 6,
- (double*)(this->vec_field_double->get_cdata() + cindex));
+ (rnumber*)(this->vorticity->get_cdata() + cindex*3),
+ (rnumber*)(this->vorticity->get_cdata() + cindex*3) + 6,
+ (double*)(this->vec_field_double->get_cdata() + cindex*3));
}
}
);
--
GitLab
From f2ec407249c664a90de72974971417cc5f1dd5b1 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 6 Sep 2018 09:39:26 +0200
Subject: [PATCH 207/342] add assertion for local_size
---
bfps/cpp/field.cpp | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/bfps/cpp/field.cpp b/bfps/cpp/field.cpp
index 25d71641..d5696d56 100644
--- a/bfps/cpp/field.cpp
+++ b/bfps/cpp/field.cpp
@@ -73,10 +73,10 @@ field<rnumber, be, fc>::field(
nfftw[0] = nz;
nfftw[1] = ny;
nfftw[2] = nx;
- ptrdiff_t tmp_local_size;
+ hsize_t tmp_local_size;
ptrdiff_t local_n0, local_0_start;
ptrdiff_t local_n1, local_1_start;
- tmp_local_size = fftw_interface<rnumber>::mpi_local_size_many_transposed(
+ tmp_local_size = fftw_interface<rnumber>::mpi_local_size_many_transposed(
3, nfftw, ncomp(fc),
FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK, this->comm,
&local_n0, &local_0_start,
@@ -87,6 +87,7 @@ field<rnumber, be, fc>::field(
starts[0] = local_0_start; starts[1] = 0; starts[2] = 0;
this->rlayout = new field_layout<fc>(
sizes, subsizes, starts, this->comm);
+ assert(tmp_local_size == this->rlayout->local_size);
this->npoints = this->rlayout->full_size / ncomp(fc);
sizes[0] = nz; sizes[1] = ny; sizes[2] = nx+2;
subsizes[0] = local_n0; subsizes[1] = ny; subsizes[2] = nx+2;
--
GitLab
From 5a58dff60e7ddfc51ea26afafe5557503191e2cb Mon Sep 17 00:00:00 2001
From: Cristian Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 6 Sep 2018 13:19:12 +0200
Subject: [PATCH 208/342] tweak e-mail options for IBMLoadLeveler
---
bfps/_code.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/bfps/_code.py b/bfps/_code.py
index fae02beb..61b79fea 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -389,7 +389,7 @@ class _code(_base):
script_file.write('# @ node_usage = not_shared\n')
script_file.write('# @ notification = complete\n')
- script_file.write('# @ notify_user = $(user)@rzg.mpg.de\n')
+ script_file.write('# @ notify_user = {0}\n'.format(self.host_info['mail_address'])
nb_cpus_per_node = self.host_info['deltanprocs']
assert isinstance(nb_cpus_per_node, int) and \
@@ -484,6 +484,9 @@ class _code(_base):
assert(type(self.host_info['environment']) != type(None))
script_file.write('# @ class = {0}\n'.format(self.host_info['environment']))
script_file.write('# @ node_usage = not_shared\n')
+
+ script_file.write('# @ notification = error\n')
+ script_file.write('# @ notify_user = {0}\n'.format(self.host_info['mail_address'])
script_file.write('#\n')
nb_cpus_per_node = self.host_info['deltanprocs']
--
GitLab
From 366c6ada92989e45ddf0b5dcac1595176893fda5 Mon Sep 17 00:00:00 2001
From: Cristian Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 6 Sep 2018 13:49:31 +0200
Subject: [PATCH 209/342] fix typo
---
bfps/_code.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/bfps/_code.py b/bfps/_code.py
index 61b79fea..fe7c35ab 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -389,7 +389,7 @@ class _code(_base):
script_file.write('# @ node_usage = not_shared\n')
script_file.write('# @ notification = complete\n')
- script_file.write('# @ notify_user = {0}\n'.format(self.host_info['mail_address'])
+ script_file.write('# @ notify_user = {0}\n'.format(self.host_info['mail_address']))
nb_cpus_per_node = self.host_info['deltanprocs']
assert isinstance(nb_cpus_per_node, int) and \
@@ -486,7 +486,7 @@ class _code(_base):
script_file.write('# @ node_usage = not_shared\n')
script_file.write('# @ notification = error\n')
- script_file.write('# @ notify_user = {0}\n'.format(self.host_info['mail_address'])
+ script_file.write('# @ notify_user = {0}\n'.format(self.host_info['mail_address']))
script_file.write('#\n')
nb_cpus_per_node = self.host_info['deltanprocs']
--
GitLab
From 20b0a4894905930f8493f3c704d617eda68fd76d Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Fri, 7 Sep 2018 22:02:15 +0200
Subject: [PATCH 210/342] optimize Lagrangian acceleration computation
by default a new field object (pressure) is generated every time the
acceleraation is computed in vorticity_equation. I now allow for an
externally allocated pressure field to be used, thus avoiding the
creation of a new field object every niter_part.
---
bfps/cpp/full_code/NSVEparticles.cpp | 9 ++++++++-
bfps/cpp/full_code/NSVEparticles.hpp | 1 +
bfps/cpp/vorticity_equation.cpp | 13 ++++++++++---
bfps/cpp/vorticity_equation.hpp | 4 +++-
4 files changed, 22 insertions(+), 5 deletions(-)
diff --git a/bfps/cpp/full_code/NSVEparticles.cpp b/bfps/cpp/full_code/NSVEparticles.cpp
index c4db9ee7..bbdd96c3 100644
--- a/bfps/cpp/full_code/NSVEparticles.cpp
+++ b/bfps/cpp/full_code/NSVEparticles.cpp
@@ -7,6 +7,12 @@ template <typename rnumber>
int NSVEparticles<rnumber>::initialize(void)
{
this->NSVE<rnumber>::initialize();
+ this->pressure = new field<rnumber, FFTW, ONE>(
+ this->fs->cvelocity->rlayout->sizes[2],
+ this->fs->cvelocity->rlayout->sizes[1],
+ this->fs->cvelocity->rlayout->sizes[0],
+ this->fs->cvelocity->rlayout->comm,
+ this->fs->cvelocity->fftw_plan_rigor);
this->ps = particles_system_builder(
this->fs->cvelocity, // (field object)
@@ -64,6 +70,7 @@ int NSVEparticles<rnumber>::write_checkpoint(void)
template <typename rnumber>
int NSVEparticles<rnumber>::finalize(void)
{
+ delete this->pressure;
this->ps.release();
delete this->particles_output_writer_mpi;
delete this->particles_sample_writer_mpi;
@@ -115,7 +122,7 @@ int NSVEparticles<rnumber>::do_stats()
this->ps->get_step_idx()-1);
/// compute acceleration and sample it
- this->fs->compute_Lagrangian_acceleration(this->tmp_vec_field);
+ this->fs->compute_Lagrangian_acceleration(this->tmp_vec_field, this->pressure);
this->tmp_vec_field->ift();
std::fill_n(pdata.get(), 3*this->ps->getLocalNbParticles(), 0);
this->ps->sample_compute_field(*this->tmp_vec_field, pdata.get());
diff --git a/bfps/cpp/full_code/NSVEparticles.hpp b/bfps/cpp/full_code/NSVEparticles.hpp
index 1a2a6535..1d73e129 100644
--- a/bfps/cpp/full_code/NSVEparticles.hpp
+++ b/bfps/cpp/full_code/NSVEparticles.hpp
@@ -58,6 +58,7 @@ class NSVEparticles: public NSVE<rnumber>
/* other stuff */
std::unique_ptr<abstract_particles_system<long long int, double>> ps;
+ field<rnumber, FFTW, ONE> *pressure;
particles_output_hdf5<long long int, double,3> *particles_output_writer_mpi;
particles_output_sampling_hdf5<long long int, double, 3> *particles_sample_writer_mpi;
diff --git a/bfps/cpp/vorticity_equation.cpp b/bfps/cpp/vorticity_equation.cpp
index cfffc26c..ead9345a 100644
--- a/bfps/cpp/vorticity_equation.cpp
+++ b/bfps/cpp/vorticity_equation.cpp
@@ -679,14 +679,20 @@ void vorticity_equation<rnumber, be>::compute_pressure(field<rnumber, be, ONE> *
template <class rnumber,
field_backend be>
void vorticity_equation<rnumber, be>::compute_Lagrangian_acceleration(
- field<rnumber, be, THREE> *acceleration)
+ field<rnumber, be, THREE> *acceleration,
+ field<rnumber, be, ONE> *pressure)
{
- field<rnumber, be, ONE> *pressure = new field<rnumber, be, ONE>(
+ bool own_pressure = false;
+ if (pressure == NULL)
+ {
+ pressure = new field<rnumber, be, ONE>(
this->cvelocity->rlayout->sizes[2],
this->cvelocity->rlayout->sizes[1],
this->cvelocity->rlayout->sizes[0],
this->cvelocity->rlayout->comm,
this->cvelocity->fftw_plan_rigor);
+ own_pressure = true;
+ }
this->compute_velocity(this->cvorticity);
this->cvelocity->ift();
this->compute_pressure(pressure);
@@ -724,7 +730,8 @@ void vorticity_equation<rnumber, be>::compute_Lagrangian_acceleration(
acceleration->get_cdata()[tindex+2][1] -= this->kk->kz[zindex]*pressure->get_cdata()[cindex][0];
}
});
- delete pressure;
+ if (own_pressure)
+ delete pressure;
}
template <class rnumber,
diff --git a/bfps/cpp/vorticity_equation.hpp b/bfps/cpp/vorticity_equation.hpp
index 230ec37a..67bd9891 100644
--- a/bfps/cpp/vorticity_equation.hpp
+++ b/bfps/cpp/vorticity_equation.hpp
@@ -161,7 +161,9 @@ class vorticity_equation
/* statistics and general postprocessing */
void compute_pressure(field<rnumber, be, ONE> *pressure);
void compute_Eulerian_acceleration(field<rnumber, be, THREE> *acceleration);
- void compute_Lagrangian_acceleration(field<rnumber, be, THREE> *acceleration);
+ void compute_Lagrangian_acceleration(
+ field<rnumber, be, THREE> *acceleration,
+ field<rnumber, be, ONE> *pressure = NULL);
};
#endif//VORTICITY_EQUATION
--
GitLab
From 89a5e91521b38b64cb51ea8acf7b8fb41da74137 Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Sat, 8 Sep 2018 21:52:03 +0200
Subject: [PATCH 211/342] partial move read_parameters to cpp files
NSVEparticles test works.
---
bfps/DNS.py | 20 -------------
bfps/cpp/full_code/NSVE.cpp | 22 ++++++++++++++
bfps/cpp/full_code/NSVEparticles.cpp | 14 +++++++++
bfps/cpp/full_code/code_base.cpp | 14 +++++++++
bfps/cpp/full_code/code_base.hpp | 1 +
.../full_code/direct_numerical_simulation.cpp | 11 +++++++
.../full_code/direct_numerical_simulation.hpp | 1 +
setup.py | 29 +++++++------------
8 files changed, 74 insertions(+), 38 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index c7d77956..e6ef006a 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -116,26 +116,6 @@ class DNS(_code):
with open(self.name + '.cpp', 'w') as outfile:
outfile.write(self.version_message + '\n\n')
outfile.write(self.includes + '\n\n')
- outfile.write(
- self.cread_pars(
- template_class = '{0}<rnumber>::'.format(self.dns_type),
- template_prefix = 'template <typename rnumber> ',
- simname_variable = 'this->simname.c_str()',
- prepend_this = True) +
- '\n\n')
- for rnumber in ['float', 'double']:
- outfile.write(self.cread_pars(
- template_class = '{0}<{1}>::'.format(self.dns_type, rnumber),
- template_prefix = 'template '.format(rnumber),
- just_declaration = True) + '\n\n')
- if self.dns_type in ['NSVEparticles', 'NSVE_no_output', 'NSVEparticles_no_output', 'NSVEcomplex_particles', 'NSVEp_extra_sampling']:
- outfile.write('template <typename rnumber> int NSVE<rnumber>::read_parameters(){return EXIT_SUCCESS;}\n')
- outfile.write('template int NSVE<float>::read_parameters();\n')
- outfile.write('template int NSVE<double>::read_parameters();\n\n')
- if self.dns_type in ['NSVEparticles_no_output', 'NSVEp_extra_sampling']:
- outfile.write('template <typename rnumber> int NSVEparticles<rnumber>::read_parameters(){return EXIT_SUCCESS;}\n')
- outfile.write('template int NSVEparticles<float>::read_parameters();\n')
- outfile.write('template int NSVEparticles<double>::read_parameters();\n\n')
outfile.write(self.main + '\n')
return None
def generate_default_parameters(self):
diff --git a/bfps/cpp/full_code/NSVE.cpp b/bfps/cpp/full_code/NSVE.cpp
index e8bf9fd2..ce0a9544 100644
--- a/bfps/cpp/full_code/NSVE.cpp
+++ b/bfps/cpp/full_code/NSVE.cpp
@@ -137,6 +137,28 @@ int NSVE<rnumber>::do_stats()
return EXIT_SUCCESS;
}
+template <typename rnumber>
+int NSVE<rnumber>::read_parameters(void)
+{
+ this->direct_numerical_simulation::read_parameters();
+ hid_t parameter_file = H5Fopen((this->simname + ".h5").c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
+ this->nu = hdf5_tools::read_value<double>(parameter_file, "parameters/nu");
+ this->dt = hdf5_tools::read_value<double>(parameter_file, "parameters/dt");
+ this->fmode = hdf5_tools::read_value<int>(parameter_file, "parameters/fmode");
+ this->famplitude = hdf5_tools::read_value<double>(parameter_file, "parameters/famplitude");
+ this->friction_coefficient = hdf5_tools::read_value<double>(parameter_file, "parameters/friction_coefficient");
+ this->fk0 = hdf5_tools::read_value<double>(parameter_file, "parameters/fk0");
+ this->fk1 = hdf5_tools::read_value<double>(parameter_file, "parameters/fk1");
+ this->energy = hdf5_tools::read_value<double>(parameter_file, "parameters/energy");
+ this->histogram_bins = hdf5_tools::read_value<int>(parameter_file, "parameters/histogram_bins");
+ this->max_velocity_estimate = hdf5_tools::read_value<double>(parameter_file, "parameters/max_velocity_estimate");
+ this->max_vorticity_estimate = hdf5_tools::read_value<double>(parameter_file, "parameters/max_vorticity_estimate");
+ std::string tmp = hdf5_tools::read_string(parameter_file, "parameters/forcing_type");
+ snprintf(this->forcing_type, 511, "%s", tmp.c_str());
+ H5Fclose(parameter_file);
+ return EXIT_SUCCESS;
+}
+
template class NSVE<float>;
template class NSVE<double>;
diff --git a/bfps/cpp/full_code/NSVEparticles.cpp b/bfps/cpp/full_code/NSVEparticles.cpp
index bbdd96c3..78f3d6a0 100644
--- a/bfps/cpp/full_code/NSVEparticles.cpp
+++ b/bfps/cpp/full_code/NSVEparticles.cpp
@@ -142,6 +142,20 @@ int NSVEparticles<rnumber>::do_stats()
return EXIT_SUCCESS;
}
+template <typename rnumber>
+int NSVEparticles<rnumber>::read_parameters(void)
+{
+ this->NSVE<rnumber>::read_parameters();
+ hid_t parameter_file = H5Fopen((this->simname + ".h5").c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
+ this->niter_part = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_part");
+ this->nparticles = hdf5_tools::read_value<int>(parameter_file, "parameters/nparticles");
+ this->tracers0_integration_steps = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_integration_steps");
+ this->tracers0_neighbours = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_neighbours");
+ this->tracers0_smoothness = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_smoothness");
+ H5Fclose(parameter_file);
+ return EXIT_SUCCESS;
+}
+
template class NSVEparticles<float>;
template class NSVEparticles<double>;
diff --git a/bfps/cpp/full_code/code_base.cpp b/bfps/cpp/full_code/code_base.cpp
index 1b06fe8e..32fe9c1c 100644
--- a/bfps/cpp/full_code/code_base.cpp
+++ b/bfps/cpp/full_code/code_base.cpp
@@ -34,3 +34,17 @@ int code_base::check_stopping_condition(void)
return EXIT_SUCCESS;
}
+int code_base::read_parameters(void)
+{
+ hid_t parameter_file = H5Fopen((this->simname + ".h5").c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
+ this->dkx = hdf5_tools::read_value<double>(parameter_file, "parameters/dkx");
+ this->dky = hdf5_tools::read_value<double>(parameter_file, "parameters/dky");
+ this->dkz = hdf5_tools::read_value<double>(parameter_file, "parameters/dkz");
+ this->nx = hdf5_tools::read_value<int>(parameter_file, "parameters/nx");
+ this->ny = hdf5_tools::read_value<int>(parameter_file, "parameters/ny");
+ this->nz = hdf5_tools::read_value<int>(parameter_file, "parameters/nz");
+ this->dealias_type = hdf5_tools::read_value<int>(parameter_file, "parameters/dealias_type");
+ H5Fclose(parameter_file);
+ return EXIT_SUCCESS;
+}
+
diff --git a/bfps/cpp/full_code/code_base.hpp b/bfps/cpp/full_code/code_base.hpp
index cf0521e2..5ec4260d 100644
--- a/bfps/cpp/full_code/code_base.hpp
+++ b/bfps/cpp/full_code/code_base.hpp
@@ -108,6 +108,7 @@ class code_base
return EXIT_SUCCESS;
}
+ virtual int read_parameters(void);
virtual int initialize(void) = 0;
virtual int main_loop(void) = 0;
virtual int finalize(void) = 0;
diff --git a/bfps/cpp/full_code/direct_numerical_simulation.cpp b/bfps/cpp/full_code/direct_numerical_simulation.cpp
index edc2f994..f763cde9 100644
--- a/bfps/cpp/full_code/direct_numerical_simulation.cpp
+++ b/bfps/cpp/full_code/direct_numerical_simulation.cpp
@@ -117,3 +117,14 @@ int direct_numerical_simulation::main_loop(void)
return EXIT_SUCCESS;
}
+int direct_numerical_simulation::read_parameters(void)
+{
+ this->code_base::read_parameters();
+ hid_t parameter_file = H5Fopen((this->simname + ".h5").c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
+ this->checkpoints_per_file = hdf5_tools::read_value<int>(parameter_file, "parameters/checkpoints_per_file");
+ this->niter_out = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_out");
+ this->niter_stat = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_stat");
+ this->niter_todo = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_todo");
+ H5Fclose(parameter_file);
+ return EXIT_SUCCESS;
+}
diff --git a/bfps/cpp/full_code/direct_numerical_simulation.hpp b/bfps/cpp/full_code/direct_numerical_simulation.hpp
index 8050bb04..15ab698a 100644
--- a/bfps/cpp/full_code/direct_numerical_simulation.hpp
+++ b/bfps/cpp/full_code/direct_numerical_simulation.hpp
@@ -51,6 +51,7 @@ class direct_numerical_simulation: public code_base
simulation_name){}
virtual ~direct_numerical_simulation(){}
+ virtual int read_parameters(void);
virtual int write_checkpoint(void) = 0;
virtual int initialize(void) = 0;
virtual int step(void) = 0;
diff --git a/setup.py b/setup.py
index 7889c927..b427ebe7 100644
--- a/setup.py
+++ b/setup.py
@@ -88,40 +88,29 @@ print('This is bfps version ' + VERSION)
### lists of files and MANIFEST.in
-src_file_list = ['full_code/NSVEcomplex_particles',
+src_file_list = [
+ 'full_code/code_base',
+ 'full_code/direct_numerical_simulation',
+ 'full_code/NSVE',
'full_code/joint_acc_vel_stats',
'full_code/test',
'full_code/filter_test',
'full_code/field_test',
'full_code/symmetrize_test',
'full_code/field_output_test',
- 'full_code/test_interpolation',
- 'hdf5_tools',
'full_code/get_rfields',
'full_code/field_single_to_double',
'full_code/resize',
'full_code/NSVE_field_stats',
'full_code/native_binary_to_hdf5',
'full_code/postprocess',
- 'full_code/code_base',
- 'full_code/direct_numerical_simulation',
- 'full_code/NSVE',
- 'field_binary_IO',
- 'vorticity_equation',
'field',
'kspace',
'field_layout',
- 'field_descriptor',
- 'rFFTW_distributed_particles',
- 'distributed_particles',
- 'particles',
- 'particles_base',
- 'rFFTW_interpolator',
- 'interpolator',
- 'interpolator_base',
- 'fluid_solver',
- 'fluid_solver_base',
+ 'hdf5_tools',
'fftw_tools',
+ 'vorticity_equation',
+ 'field_binary_IO',
'spline_n1',
'spline_n2',
'spline_n3',
@@ -134,7 +123,11 @@ src_file_list = ['full_code/NSVEcomplex_particles',
'spline_n10',
'Lagrange_polys',
'scope_timer',
+ 'interpolator',
+ 'interpolator_base',
+ 'full_code/test_interpolation',
'full_code/NSVEparticles',
+ 'full_code/NSVEcomplex_particles',
'full_code/NSVEp_extra_sampling']
particle_headers = [
--
GitLab
From d4772c2c63c1d8fff3c6bdac483cb4458048a21e Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Sun, 9 Sep 2018 00:00:59 +0200
Subject: [PATCH 212/342] make complex_particles use new read_parameters
---
bfps/DNS.py | 5 ++
bfps/cpp/full_code/NSVE_no_output.hpp | 1 -
bfps/cpp/full_code/NSVEcomplex_particles.cpp | 23 +++++++++
.../cpp/full_code/NSVEparticles_no_output.hpp | 1 -
bfps/cpp/full_code/postprocess.cpp | 48 +++++--------------
bfps/cpp/full_code/postprocess.hpp | 1 +
bfps/cpp/full_code/test.cpp | 33 -------------
bfps/cpp/full_code/test.hpp | 1 -
8 files changed, 42 insertions(+), 71 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index e6ef006a..e6ace758 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -148,6 +148,11 @@ class DNS(_code):
self.NSVEp_extra_parameters['tracers0_integration_steps'] = int(4)
self.NSVEp_extra_parameters['tracers0_neighbours'] = int(1)
self.NSVEp_extra_parameters['tracers0_smoothness'] = int(1)
+ self.NSVEp_extra_parameters['tracers0_enable_p2p'] = int(0)
+ self.NSVEp_extra_parameters['tracers0_enable_inner'] = int(0)
+ self.NSVEp_extra_parameters['tracers0_enable_vorticity_omega'] = int(0)
+ self.NSVEp_extra_parameters['tracers0_cutoff'] = float(1)
+ self.NSVEp_extra_parameters['tracers0_inner_v0'] = float(1)
#self.extra_parameters = {}
#for key in ['NSVE', 'NSVE_no_output', 'NSVEparticles', 'NSVEparticles_no_output', 'NSVEcomplex_particles']:
# self.extra_parameters[key] = {}
diff --git a/bfps/cpp/full_code/NSVE_no_output.hpp b/bfps/cpp/full_code/NSVE_no_output.hpp
index 0047a45a..b98f89f6 100644
--- a/bfps/cpp/full_code/NSVE_no_output.hpp
+++ b/bfps/cpp/full_code/NSVE_no_output.hpp
@@ -18,7 +18,6 @@ class NSVE_no_output: public NSVE<rnumber>
{
return 0;
}
- int read_parameters(void);
};
#endif//NSVE_NO_OUTPUT_HPP
diff --git a/bfps/cpp/full_code/NSVEcomplex_particles.cpp b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
index 93d0edc1..1e138711 100644
--- a/bfps/cpp/full_code/NSVEcomplex_particles.cpp
+++ b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
@@ -184,6 +184,29 @@ int NSVEcomplex_particles<rnumber>::do_stats()
this->ps->getLocalNbParticles(),
this->ps->get_step_idx()-1);
+ // deallocate temporary data array
+ // TODO: is it required/safe to call the release method here?
+ //pdata.release();
+
+ return EXIT_SUCCESS;
+}
+
+template <typename rnumber>
+int NSVEcomplex_particles<rnumber>::read_parameters(void)
+{
+ this->NSVE<rnumber>::read_parameters();
+ hid_t parameter_file = H5Fopen((this->simname + ".h5").c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
+ this->niter_part = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_part");
+ this->nparticles = hdf5_tools::read_value<int>(parameter_file, "parameters/nparticles");
+ this->tracers0_integration_steps = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_integration_steps");
+ this->tracers0_neighbours = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_neighbours");
+ this->tracers0_smoothness = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_smoothness");
+ this->enable_p2p = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_enable_p2p");
+ this->enable_inner = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_enable_inner");
+ this->enable_vorticity_omega = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_enable_vorticity_omega");
+ this->cutoff = hdf5_tools::read_value<double>(parameter_file, "parameters/tracers0_cutoff");
+ this->inner_v0 = hdf5_tools::read_value<double>(parameter_file, "parameters/tracers0_inner_v0");
+ H5Fclose(parameter_file);
return EXIT_SUCCESS;
}
diff --git a/bfps/cpp/full_code/NSVEparticles_no_output.hpp b/bfps/cpp/full_code/NSVEparticles_no_output.hpp
index 264fd75a..d3d40139 100644
--- a/bfps/cpp/full_code/NSVEparticles_no_output.hpp
+++ b/bfps/cpp/full_code/NSVEparticles_no_output.hpp
@@ -18,7 +18,6 @@ class NSVEparticles_no_output: public NSVEparticles<rnumber>
{
return 0;
}
- int read_parameters(void);
};
#endif//NSVEPARTICLES_NO_OUTPUT_HPP
diff --git a/bfps/cpp/full_code/postprocess.cpp b/bfps/cpp/full_code/postprocess.cpp
index cb04a030..430d87a5 100644
--- a/bfps/cpp/full_code/postprocess.cpp
+++ b/bfps/cpp/full_code/postprocess.cpp
@@ -33,41 +33,19 @@ int postprocess::main_loop(void)
int postprocess::read_parameters()
{
- hid_t parameter_file;
- char fname[256];
- sprintf(fname, "%s.h5", this->simname.c_str());
- parameter_file = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT);
- this->dealias_type = hdf5_tools::read_value<int>(
- parameter_file, "/parameters/dealias_type");
- this->dkx = hdf5_tools::read_value<double>(
- parameter_file, "/parameters/dkx");
- this->dky = hdf5_tools::read_value<double>(
- parameter_file, "/parameters/dky");
- this->dkz = hdf5_tools::read_value<double>(
- parameter_file, "/parameters/dkz");
- this->dt = hdf5_tools::read_value<double>(
- parameter_file, "/parameters/dt");
- this->famplitude = hdf5_tools::read_value<double>(
- parameter_file, "/parameters/famplitude");
- this->friction_coefficient = hdf5_tools::read_value<double>(
- parameter_file, "/parameters/friction_coefficient");
- this->fk0 = hdf5_tools::read_value<double>(
- parameter_file, "/parameters/fk0");
- this->fk1 = hdf5_tools::read_value<double>(
- parameter_file, "/parameters/fk1");
- this->fmode = hdf5_tools::read_value<int>(
- parameter_file, "/parameters/fmode");
- sprintf(this->forcing_type, "%s",
- hdf5_tools::read_string(parameter_file, "/parameters/forcing_type").c_str());
- this->nu = hdf5_tools::read_value<double>(
- parameter_file, "/parameters/nu");
- this->nx = hdf5_tools::read_value<int>(
- parameter_file, "/parameters/nx");
- this->ny = hdf5_tools::read_value<int>(
- parameter_file, "/parameters/ny");
- this->nz = hdf5_tools::read_value<int>(
- parameter_file, "/parameters/nz");
+ this->code_base::read_parameters();
+ hid_t parameter_file = H5Fopen((this->simname + ".h5").c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
+ this->nu = hdf5_tools::read_value<double>(parameter_file, "parameters/nu");
+ this->dt = hdf5_tools::read_value<double>(parameter_file, "parameters/dt");
+ this->fmode = hdf5_tools::read_value<int>(parameter_file, "parameters/fmode");
+ this->famplitude = hdf5_tools::read_value<double>(parameter_file, "parameters/famplitude");
+ this->friction_coefficient = hdf5_tools::read_value<double>(parameter_file, "parameters/friction_coefficient");
+ this->fk0 = hdf5_tools::read_value<double>(parameter_file, "parameters/fk0");
+ this->fk1 = hdf5_tools::read_value<double>(parameter_file, "parameters/fk1");
+ this->energy = hdf5_tools::read_value<double>(parameter_file, "parameters/energy");
+ std::string tmp = hdf5_tools::read_string(parameter_file, "parameters/forcing_type");
+ snprintf(this->forcing_type, 511, "%s", tmp.c_str());
H5Fclose(parameter_file);
- return 0;
+ return EXIT_SUCCESS;
}
diff --git a/bfps/cpp/full_code/postprocess.hpp b/bfps/cpp/full_code/postprocess.hpp
index 660e561a..65e6eadd 100644
--- a/bfps/cpp/full_code/postprocess.hpp
+++ b/bfps/cpp/full_code/postprocess.hpp
@@ -46,6 +46,7 @@ class postprocess: public code_base
double friction_coefficient;
double fk0;
double fk1;
+ double energy;
int fmode;
char forcing_type[512];
double nu;
diff --git a/bfps/cpp/full_code/test.cpp b/bfps/cpp/full_code/test.cpp
index 9c2e4e67..e382efb4 100644
--- a/bfps/cpp/full_code/test.cpp
+++ b/bfps/cpp/full_code/test.cpp
@@ -18,36 +18,3 @@ int test::main_loop(void)
return EXIT_SUCCESS;
}
-
-int test::read_parameters()
-{
- hid_t parameter_file;
- hid_t dset;
- char fname[256];
- sprintf(fname, "%s.h5", this->simname.c_str());
- parameter_file = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT);
- dset = H5Dopen(parameter_file, "/parameters/dealias_type", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->dealias_type);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/dkx", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->dkx);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/dky", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->dky);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/dkz", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->dkz);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/nx", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->nx);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/ny", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->ny);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/parameters/nz", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->nz);
- H5Dclose(dset);
- H5Fclose(parameter_file);
- return 0;
-}
-
diff --git a/bfps/cpp/full_code/test.hpp b/bfps/cpp/full_code/test.hpp
index 134a0151..96ddaf81 100644
--- a/bfps/cpp/full_code/test.hpp
+++ b/bfps/cpp/full_code/test.hpp
@@ -56,7 +56,6 @@ class test: public code_base
virtual int finalize(void) = 0;
int main_loop(void);
- virtual int read_parameters(void);
};
#endif//TEST_HPP
--
GitLab
From c53c68b4331499481fbdddc6f01472c334b8f5f2 Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Sun, 9 Sep 2018 12:09:01 +0200
Subject: [PATCH 213/342] update timer to output debug message
---
bfps/cpp/full_code/NSVE.cpp | 6 ++++
bfps/cpp/full_code/NSVE_field_stats.cpp | 4 +++
bfps/cpp/full_code/NSVE_no_output.hpp | 29 ++++++++++++++++-
bfps/cpp/full_code/NSVEcomplex_particles.cpp | 32 +++++++++++++++++++
bfps/cpp/full_code/NSVEp_extra_sampling.cpp | 3 ++
bfps/cpp/full_code/NSVEparticles.cpp | 9 ++++++
.../cpp/full_code/NSVEparticles_no_output.hpp | 3 +-
bfps/cpp/full_code/code_base.cpp | 32 +++++++++++++++++++
.../full_code/direct_numerical_simulation.cpp | 5 +++
bfps/cpp/full_code/field_output_test.cpp | 4 +++
bfps/cpp/full_code/field_single_to_double.cpp | 4 ++-
bfps/cpp/full_code/field_test.cpp | 13 ++++----
bfps/cpp/full_code/filter_test.cpp | 13 ++++----
bfps/cpp/full_code/get_rfields.cpp | 4 ++-
bfps/cpp/full_code/joint_acc_vel_stats.cpp | 4 ++-
bfps/cpp/full_code/native_binary_to_hdf5.cpp | 4 +++
bfps/cpp/full_code/postprocess.cpp | 2 ++
bfps/cpp/full_code/resize.cpp | 4 ++-
bfps/cpp/full_code/symmetrize_test.cpp | 4 +++
bfps/cpp/full_code/test.cpp | 4 +--
bfps/cpp/full_code/test_interpolation.cpp | 4 +++
bfps/cpp/scope_timer.hpp | 3 +-
22 files changed, 168 insertions(+), 22 deletions(-)
diff --git a/bfps/cpp/full_code/NSVE.cpp b/bfps/cpp/full_code/NSVE.cpp
index ce0a9544..d9cb72a2 100644
--- a/bfps/cpp/full_code/NSVE.cpp
+++ b/bfps/cpp/full_code/NSVE.cpp
@@ -7,6 +7,7 @@
template <typename rnumber>
int NSVE<rnumber>::initialize(void)
{
+ TIMEZONE("NSVE::initialize");
this->read_iteration();
this->read_parameters();
if (this->myrank == 0)
@@ -67,6 +68,7 @@ int NSVE<rnumber>::initialize(void)
template <typename rnumber>
int NSVE<rnumber>::step(void)
{
+ TIMEZONE("NSVE::step");
this->fs->step(this->dt);
this->iteration = this->fs->iteration;
return EXIT_SUCCESS;
@@ -75,6 +77,7 @@ int NSVE<rnumber>::step(void)
template <typename rnumber>
int NSVE<rnumber>::write_checkpoint(void)
{
+ TIMEZONE("NSVE::write_checkpoint");
this->fs->io_checkpoint(false);
this->checkpoint = this->fs->checkpoint;
this->write_iteration();
@@ -84,6 +87,7 @@ int NSVE<rnumber>::write_checkpoint(void)
template <typename rnumber>
int NSVE<rnumber>::finalize(void)
{
+ TIMEZONE("NSVE::finalize");
if (this->myrank == 0)
H5Fclose(this->stat_file);
delete this->fs;
@@ -104,6 +108,7 @@ int NSVE<rnumber>::finalize(void)
template <typename rnumber>
int NSVE<rnumber>::do_stats()
{
+ TIMEZONE("NSVE::do_stats");
if (!(this->iteration % this->niter_stat == 0))
return EXIT_SUCCESS;
hid_t stat_group;
@@ -140,6 +145,7 @@ int NSVE<rnumber>::do_stats()
template <typename rnumber>
int NSVE<rnumber>::read_parameters(void)
{
+ TIMEZONE("NSVE::read_parameters");
this->direct_numerical_simulation::read_parameters();
hid_t parameter_file = H5Fopen((this->simname + ".h5").c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
this->nu = hdf5_tools::read_value<double>(parameter_file, "parameters/nu");
diff --git a/bfps/cpp/full_code/NSVE_field_stats.cpp b/bfps/cpp/full_code/NSVE_field_stats.cpp
index 7e33acf9..15980a20 100644
--- a/bfps/cpp/full_code/NSVE_field_stats.cpp
+++ b/bfps/cpp/full_code/NSVE_field_stats.cpp
@@ -7,6 +7,7 @@
template <typename rnumber>
int NSVE_field_stats<rnumber>::initialize(void)
{
+ TIMEZONE("NSVE_field_stats::initialize");
this->postprocess::read_parameters();
this->vorticity = new field<rnumber, FFTW, THREE>(
nx, ny, nz,
@@ -49,6 +50,7 @@ int NSVE_field_stats<rnumber>::initialize(void)
template <typename rnumber>
int NSVE_field_stats<rnumber>::read_current_cvorticity(void)
{
+ TIMEZONE("NSVE_field_stats::read_current_cvorticity");
this->vorticity->real_space_representation = false;
if (this->bin_IO != NULL)
{
@@ -76,6 +78,7 @@ int NSVE_field_stats<rnumber>::read_current_cvorticity(void)
template <typename rnumber>
int NSVE_field_stats<rnumber>::finalize(void)
{
+ TIMEZONE("NSVE_field_stats::finalize");
if (this->bin_IO != NULL)
delete this->bin_IO;
delete this->vorticity;
@@ -85,6 +88,7 @@ int NSVE_field_stats<rnumber>::finalize(void)
template <typename rnumber>
int NSVE_field_stats<rnumber>::work_on_current_iteration(void)
{
+ TIMEZONE("NSVE_field_stats::work_on_current_iteration");
return EXIT_SUCCESS;
}
diff --git a/bfps/cpp/full_code/NSVE_no_output.hpp b/bfps/cpp/full_code/NSVE_no_output.hpp
index b98f89f6..045db08e 100644
--- a/bfps/cpp/full_code/NSVE_no_output.hpp
+++ b/bfps/cpp/full_code/NSVE_no_output.hpp
@@ -1,3 +1,29 @@
+/**********************************************************************
+* *
+* Copyright 2017 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
#ifndef NSVE_NO_OUTPUT_HPP
#define NSVE_NO_OUTPUT_HPP
@@ -16,7 +42,8 @@ class NSVE_no_output: public NSVE<rnumber>
~NSVE_no_output(){}
int write_checkpoint(void)
{
- return 0;
+ TIMEZONE("NSVE_no_output::write_checkpoint");
+ return EXIT_SUCCESS;
}
};
diff --git a/bfps/cpp/full_code/NSVEcomplex_particles.cpp b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
index 1e138711..81c6cd5f 100644
--- a/bfps/cpp/full_code/NSVEcomplex_particles.cpp
+++ b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
@@ -1,3 +1,29 @@
+/**********************************************************************
+* *
+* Copyright 2017 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
#include <string>
#include <cmath>
#include "NSVEcomplex_particles.hpp"
@@ -9,6 +35,7 @@
template <typename rnumber>
int NSVEcomplex_particles<rnumber>::initialize(void)
{
+ TIMEZONE("NSVEcomplex_particles::initialize");
this->NSVE<rnumber>::initialize();
p2p_computer<double, long long int> current_p2p_computer;
@@ -63,6 +90,7 @@ int NSVEcomplex_particles<rnumber>::initialize(void)
template <typename rnumber>
int NSVEcomplex_particles<rnumber>::step(void)
{
+ TIMEZONE("NSVEcomplex_particles::step");
this->fs->compute_velocity(this->fs->cvorticity);
this->fs->cvelocity->ift();
if(enable_vorticity_omega){
@@ -80,6 +108,7 @@ int NSVEcomplex_particles<rnumber>::step(void)
template <typename rnumber>
int NSVEcomplex_particles<rnumber>::write_checkpoint(void)
{
+ TIMEZONE("NSVEcomplex_particles::write_checkpoint");
this->NSVE<rnumber>::write_checkpoint();
this->particles_output_writer_mpi->open_file(this->fs->get_current_fname());
// TODO P2P write particle data too
@@ -96,6 +125,7 @@ int NSVEcomplex_particles<rnumber>::write_checkpoint(void)
template <typename rnumber>
int NSVEcomplex_particles<rnumber>::finalize(void)
{
+ TIMEZONE("NSVEcomplex_particles::finalize");
delete this->nabla_u;
delete this->particles_output_writer_mpi;
delete this->particles_sample_writer_mpi;
@@ -109,6 +139,7 @@ int NSVEcomplex_particles<rnumber>::finalize(void)
template <typename rnumber>
int NSVEcomplex_particles<rnumber>::do_stats()
{
+ TIMEZONE("NSVEcomplex_particles::do_stats");
/// perform fluid stats
this->NSVE<rnumber>::do_stats();
@@ -194,6 +225,7 @@ int NSVEcomplex_particles<rnumber>::do_stats()
template <typename rnumber>
int NSVEcomplex_particles<rnumber>::read_parameters(void)
{
+ TIMEZONE("NSVEcomplex_particles::read_parameters");
this->NSVE<rnumber>::read_parameters();
hid_t parameter_file = H5Fopen((this->simname + ".h5").c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
this->niter_part = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_part");
diff --git a/bfps/cpp/full_code/NSVEp_extra_sampling.cpp b/bfps/cpp/full_code/NSVEp_extra_sampling.cpp
index 3d002d34..22357510 100644
--- a/bfps/cpp/full_code/NSVEp_extra_sampling.cpp
+++ b/bfps/cpp/full_code/NSVEp_extra_sampling.cpp
@@ -5,6 +5,7 @@
template <typename rnumber>
int NSVEp_extra_sampling<rnumber>::initialize(void)
{
+ TIMEZONE("NSVEp_extra_sampling::initialize");
this->NSVEparticles<rnumber>::initialize();
/// allocate grad vel field
@@ -30,6 +31,7 @@ int NSVEp_extra_sampling<rnumber>::initialize(void)
template <typename rnumber>
int NSVEp_extra_sampling<rnumber>::finalize(void)
{
+ TIMEZONE("NSVEp_extra_sampling::finalize");
delete this->nabla_u;
delete this->pressure;
delete this->nabla_p;
@@ -41,6 +43,7 @@ int NSVEp_extra_sampling<rnumber>::finalize(void)
template <typename rnumber>
int NSVEp_extra_sampling<rnumber>::do_stats()
{
+ TIMEZONE("NSVEp_extra_sampling::do_stats");
this->NSVEparticles<rnumber>::do_stats();
if (!(this->iteration % this->niter_part == 0))
return EXIT_SUCCESS;
diff --git a/bfps/cpp/full_code/NSVEparticles.cpp b/bfps/cpp/full_code/NSVEparticles.cpp
index 78f3d6a0..b09e3280 100644
--- a/bfps/cpp/full_code/NSVEparticles.cpp
+++ b/bfps/cpp/full_code/NSVEparticles.cpp
@@ -1,3 +1,6 @@
+
+
+
#include <string>
#include <cmath>
#include "NSVEparticles.hpp"
@@ -6,6 +9,7 @@
template <typename rnumber>
int NSVEparticles<rnumber>::initialize(void)
{
+ TIMEZONE("NSVEparticles::intialize");
this->NSVE<rnumber>::initialize();
this->pressure = new field<rnumber, FFTW, ONE>(
this->fs->cvelocity->rlayout->sizes[2],
@@ -45,6 +49,7 @@ int NSVEparticles<rnumber>::initialize(void)
template <typename rnumber>
int NSVEparticles<rnumber>::step(void)
{
+ TIMEZONE("NSVEparticles::step");
this->fs->compute_velocity(this->fs->cvorticity);
this->fs->cvelocity->ift();
this->ps->completeLoop(this->dt);
@@ -55,6 +60,7 @@ int NSVEparticles<rnumber>::step(void)
template <typename rnumber>
int NSVEparticles<rnumber>::write_checkpoint(void)
{
+ TIMEZONE("NSVEparticles::write_checkpoint");
this->NSVE<rnumber>::write_checkpoint();
this->particles_output_writer_mpi->open_file(this->fs->get_current_fname());
this->particles_output_writer_mpi->template save<3>(
@@ -70,6 +76,7 @@ int NSVEparticles<rnumber>::write_checkpoint(void)
template <typename rnumber>
int NSVEparticles<rnumber>::finalize(void)
{
+ TIMEZONE("NSVEparticles::finalize");
delete this->pressure;
this->ps.release();
delete this->particles_output_writer_mpi;
@@ -84,6 +91,7 @@ int NSVEparticles<rnumber>::finalize(void)
template <typename rnumber>
int NSVEparticles<rnumber>::do_stats()
{
+ TIMEZONE("NSVEparticles::do_stats");
/// fluid stats go here
this->NSVE<rnumber>::do_stats();
@@ -145,6 +153,7 @@ int NSVEparticles<rnumber>::do_stats()
template <typename rnumber>
int NSVEparticles<rnumber>::read_parameters(void)
{
+ TIMEZONE("NSVEparticles::read_parameters");
this->NSVE<rnumber>::read_parameters();
hid_t parameter_file = H5Fopen((this->simname + ".h5").c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
this->niter_part = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_part");
diff --git a/bfps/cpp/full_code/NSVEparticles_no_output.hpp b/bfps/cpp/full_code/NSVEparticles_no_output.hpp
index d3d40139..5b9d5e15 100644
--- a/bfps/cpp/full_code/NSVEparticles_no_output.hpp
+++ b/bfps/cpp/full_code/NSVEparticles_no_output.hpp
@@ -16,7 +16,8 @@ class NSVEparticles_no_output: public NSVEparticles<rnumber>
~NSVEparticles_no_output(){}
int write_checkpoint(void)
{
- return 0;
+ TIMEZONE("NSVEparticles_no_output::write_checkpoint");
+ return EXIT_SUCCESS;
}
};
diff --git a/bfps/cpp/full_code/code_base.cpp b/bfps/cpp/full_code/code_base.cpp
index 32fe9c1c..a6487c72 100644
--- a/bfps/cpp/full_code/code_base.cpp
+++ b/bfps/cpp/full_code/code_base.cpp
@@ -1,12 +1,42 @@
+/**********************************************************************
+* *
+* Copyright 2017 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#define NDEBUG
+
#include "code_base.hpp"
#include "scope_timer.hpp"
+
code_base::code_base(
const MPI_Comm COMMUNICATOR,
const std::string &simulation_name):
comm(COMMUNICATOR),
simname(simulation_name)
{
+ TIMEZONE("code_base::code_base");
MPI_Comm_rank(this->comm, &this->myrank);
MPI_Comm_size(this->comm, &this->nprocs);
this->stop_code_now = false;
@@ -14,6 +44,7 @@ code_base::code_base(
int code_base::check_stopping_condition(void)
{
+ TIMEZONE("code_base::check_stopping_condition");
if (myrank == 0)
{
std::string fname = (
@@ -36,6 +67,7 @@ int code_base::check_stopping_condition(void)
int code_base::read_parameters(void)
{
+ TIMEZONE("code_base::read_parameters");
hid_t parameter_file = H5Fopen((this->simname + ".h5").c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
this->dkx = hdf5_tools::read_value<double>(parameter_file, "parameters/dkx");
this->dky = hdf5_tools::read_value<double>(parameter_file, "parameters/dky");
diff --git a/bfps/cpp/full_code/direct_numerical_simulation.cpp b/bfps/cpp/full_code/direct_numerical_simulation.cpp
index f763cde9..c0b0441e 100644
--- a/bfps/cpp/full_code/direct_numerical_simulation.cpp
+++ b/bfps/cpp/full_code/direct_numerical_simulation.cpp
@@ -8,6 +8,7 @@
int direct_numerical_simulation::grow_file_datasets()
{
+ TIMEZONE("direct_numerical_simulation::grow_file_datasets");
return hdf5_tools::grow_file_datasets(
this->stat_file,
"statistics",
@@ -16,6 +17,7 @@ int direct_numerical_simulation::grow_file_datasets()
int direct_numerical_simulation::read_iteration(void)
{
+ TIMEZONE("direct_numerical_simulation::read_iteration");
/* read iteration */
hid_t dset;
hid_t iteration_file = H5Fopen(
@@ -56,6 +58,7 @@ int direct_numerical_simulation::read_iteration(void)
int direct_numerical_simulation::write_iteration(void)
{
+ TIMEZONE("direct_numerical_simulation::write_iteration");
if (this->myrank == 0)
{
hid_t dset = H5Dopen(
@@ -88,6 +91,7 @@ int direct_numerical_simulation::write_iteration(void)
int direct_numerical_simulation::main_loop(void)
{
+ TIMEZONE("direct_numerical_simulation::main_loop");
this->start_simple_timer();
int max_iter = (this->iteration + this->niter_todo -
(this->iteration % this->niter_todo));
@@ -119,6 +123,7 @@ int direct_numerical_simulation::main_loop(void)
int direct_numerical_simulation::read_parameters(void)
{
+ TIMEZONE("direct_numerical_simulation::read_parameters");
this->code_base::read_parameters();
hid_t parameter_file = H5Fopen((this->simname + ".h5").c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
this->checkpoints_per_file = hdf5_tools::read_value<int>(parameter_file, "parameters/checkpoints_per_file");
diff --git a/bfps/cpp/full_code/field_output_test.cpp b/bfps/cpp/full_code/field_output_test.cpp
index f5440064..30df4e75 100644
--- a/bfps/cpp/full_code/field_output_test.cpp
+++ b/bfps/cpp/full_code/field_output_test.cpp
@@ -8,6 +8,7 @@
template <typename rnumber>
int field_output_test<rnumber>::initialize(void)
{
+ TIMEZONE("field_output_test::initialize");
this->read_parameters();
return EXIT_SUCCESS;
}
@@ -15,12 +16,14 @@ int field_output_test<rnumber>::initialize(void)
template <typename rnumber>
int field_output_test<rnumber>::finalize(void)
{
+ TIMEZONE("field_output_test::finalize");
return EXIT_SUCCESS;
}
template <typename rnumber>
int field_output_test<rnumber>::read_parameters()
{
+ TIMEZONE("field_output_test::read_parameters");
this->test::read_parameters();
return EXIT_SUCCESS;
}
@@ -28,6 +31,7 @@ int field_output_test<rnumber>::read_parameters()
template <typename rnumber>
int field_output_test<rnumber>::do_work(void)
{
+ TIMEZONE("field_output_test::do_work");
// allocate
field<rnumber, FFTW, ONE> *scal_field = new field<rnumber, FFTW, ONE>(
this->nx, this->ny, this->nz,
diff --git a/bfps/cpp/full_code/field_single_to_double.cpp b/bfps/cpp/full_code/field_single_to_double.cpp
index bb34abd2..92976ecf 100644
--- a/bfps/cpp/full_code/field_single_to_double.cpp
+++ b/bfps/cpp/full_code/field_single_to_double.cpp
@@ -7,6 +7,7 @@
template <typename rnumber>
int field_single_to_double<rnumber>::initialize(void)
{
+ TIMEZONE("field_single_to_double::intialize");
this->NSVE_field_stats<rnumber>::initialize();
DEBUG_MSG("after NSVE_field_stats::initialize\n");
this->kk = new kspace<FFTW, SMOOTH>(
@@ -47,7 +48,7 @@ int field_single_to_double<rnumber>::initialize(void)
template <typename rnumber>
int field_single_to_double<rnumber>::work_on_current_iteration(void)
{
- DEBUG_MSG("entered field_single_to_double::work_on_current_iteration\n");
+ TIMEZONE("field_single_to_double::work_on_current_iteration");
this->read_current_cvorticity();
// using CLOOP as opposed to a global std::copy because CLOOP
@@ -83,6 +84,7 @@ int field_single_to_double<rnumber>::work_on_current_iteration(void)
template <typename rnumber>
int field_single_to_double<rnumber>::finalize(void)
{
+ TIMEZONE("field_single_to_double::finalize");
delete this->vec_field_double;
delete this->kk;
return EXIT_SUCCESS;
diff --git a/bfps/cpp/full_code/field_test.cpp b/bfps/cpp/full_code/field_test.cpp
index 5c323e54..1627bc40 100644
--- a/bfps/cpp/full_code/field_test.cpp
+++ b/bfps/cpp/full_code/field_test.cpp
@@ -8,6 +8,7 @@
template <typename rnumber>
int field_test<rnumber>::initialize(void)
{
+ TIMEZONE("field_test::initialize");
this->read_parameters();
return EXIT_SUCCESS;
}
@@ -15,23 +16,22 @@ int field_test<rnumber>::initialize(void)
template <typename rnumber>
int field_test<rnumber>::finalize(void)
{
+ TIMEZONE("field_test::finalize");
+ this->read_parameters();
return EXIT_SUCCESS;
}
template <typename rnumber>
int field_test<rnumber>::read_parameters()
{
+ TIMEZONE("field_test::read_parameters");
this->test::read_parameters();
// in case any parameters are needed, this is where they should be read
- hid_t parameter_file;
- hid_t dset;
- parameter_file = H5Fopen(
+ hid_t parameter_file = H5Fopen(
(this->simname + std::string(".h5")).c_str(),
H5F_ACC_RDONLY,
H5P_DEFAULT);
- dset = H5Dopen(parameter_file, "/parameters/filter_length", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->filter_length);
- H5Dclose(dset);
+ this->filter_length = hdf5_tools::read_value<double>(parameter_file, "/parameters/filter_length");
H5Fclose(parameter_file);
return EXIT_SUCCESS;
}
@@ -39,6 +39,7 @@ int field_test<rnumber>::read_parameters()
template <typename rnumber>
int field_test<rnumber>::do_work(void)
{
+ TIMEZONE("field_test::do_work");
// allocate
field<rnumber, FFTW, ONE> *scal_field = new field<rnumber, FFTW, ONE>(
this->nx, this->ny, this->nz,
diff --git a/bfps/cpp/full_code/filter_test.cpp b/bfps/cpp/full_code/filter_test.cpp
index 80c4f83d..4db13843 100644
--- a/bfps/cpp/full_code/filter_test.cpp
+++ b/bfps/cpp/full_code/filter_test.cpp
@@ -7,6 +7,7 @@
template <typename rnumber>
int filter_test<rnumber>::initialize(void)
{
+ TIMEZONE("filter_test::initialize");
this->read_parameters();
this->scal_field = new field<rnumber, FFTW, ONE>(
nx, ny, nz,
@@ -30,6 +31,7 @@ int filter_test<rnumber>::initialize(void)
template <typename rnumber>
int filter_test<rnumber>::finalize(void)
{
+ TIMEZONE("filter_test::finalize");
delete this->scal_field;
delete this->kk;
return EXIT_SUCCESS;
@@ -38,16 +40,13 @@ int filter_test<rnumber>::finalize(void)
template <typename rnumber>
int filter_test<rnumber>::read_parameters()
{
+ TIMEZONE("filter_test::read_parameters");
this->test::read_parameters();
- hid_t parameter_file;
- hid_t dset;
- parameter_file = H5Fopen(
+ hid_t parameter_file = H5Fopen(
(this->simname + std::string(".h5")).c_str(),
H5F_ACC_RDONLY,
H5P_DEFAULT);
- dset = H5Dopen(parameter_file, "/parameters/filter_length", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->filter_length);
- H5Dclose(dset);
+ this->filter_length = hdf5_tools::read_value<double>(parameter_file, "/parameters/filter_length");
H5Fclose(parameter_file);
return EXIT_SUCCESS;
}
@@ -56,6 +55,7 @@ template <typename rnumber>
int filter_test<rnumber>::reset_field(
int dimension)
{
+ TIMEZONE("filter_test::reset_field");
this->scal_field->real_space_representation = true;
*this->scal_field = 0.0;
if (this->scal_field->rlayout->starts[0] == 0)
@@ -95,6 +95,7 @@ int filter_test<rnumber>::reset_field(
template <typename rnumber>
int filter_test<rnumber>::do_work(void)
{
+ TIMEZONE("filter_test::do_work");
std::string filename = this->simname + std::string("_fields.h5");
for (int dimension = 0; dimension < 3; dimension++)
{
diff --git a/bfps/cpp/full_code/get_rfields.cpp b/bfps/cpp/full_code/get_rfields.cpp
index 3986b5f9..5d872b72 100644
--- a/bfps/cpp/full_code/get_rfields.cpp
+++ b/bfps/cpp/full_code/get_rfields.cpp
@@ -7,6 +7,7 @@
template <typename rnumber>
int get_rfields<rnumber>::initialize(void)
{
+ TIMEZONE("get_rfields::initialize");
this->NSVE_field_stats<rnumber>::initialize();
DEBUG_MSG("after NSVE_field_stats::initialize\n");
this->kk = new kspace<FFTW, SMOOTH>(
@@ -42,7 +43,7 @@ int get_rfields<rnumber>::initialize(void)
template <typename rnumber>
int get_rfields<rnumber>::work_on_current_iteration(void)
{
- DEBUG_MSG("entered get_rfields::work_on_current_iteration\n");
+ TIMEZONE("get_rfields::work_on_current_iteration");
this->read_current_cvorticity();
field<rnumber, FFTW, THREE> *vel = new field<rnumber, FFTW, THREE>(
this->nx, this->ny, this->nz,
@@ -97,6 +98,7 @@ int get_rfields<rnumber>::work_on_current_iteration(void)
template <typename rnumber>
int get_rfields<rnumber>::finalize(void)
{
+ TIMEZONE("get_rfields::finalize");
delete this->kk;
this->NSVE_field_stats<rnumber>::finalize();
return EXIT_SUCCESS;
diff --git a/bfps/cpp/full_code/joint_acc_vel_stats.cpp b/bfps/cpp/full_code/joint_acc_vel_stats.cpp
index e4f4d5d4..1c28527e 100644
--- a/bfps/cpp/full_code/joint_acc_vel_stats.cpp
+++ b/bfps/cpp/full_code/joint_acc_vel_stats.cpp
@@ -7,6 +7,7 @@
template <typename rnumber>
int joint_acc_vel_stats<rnumber>::initialize(void)
{
+ TIMEZONE("joint_acc_vel_stats::initialize");
this->NSVE_field_stats<rnumber>::initialize();
this->kk = new kspace<FFTW, SMOOTH>(
this->vorticity->clayout, this->dkx, this->dky, this->dkz);
@@ -85,7 +86,7 @@ int joint_acc_vel_stats<rnumber>::initialize(void)
template <typename rnumber>
int joint_acc_vel_stats<rnumber>::work_on_current_iteration(void)
{
- DEBUG_MSG("entered joint_acc_vel_stats::work_on_current_iteration\n");
+ TIMEZONE("joint_acc_vel_stats::work_on_current_iteration");
/// read current vorticity, place it in this->ve->cvorticity
this->read_current_cvorticity();
*this->ve->cvorticity = this->vorticity->get_cdata();
@@ -156,6 +157,7 @@ int joint_acc_vel_stats<rnumber>::work_on_current_iteration(void)
template <typename rnumber>
int joint_acc_vel_stats<rnumber>::finalize(void)
{
+ DEBUG_MSG("entered joint_acc_vel_stats::finalize\n");
delete this->ve;
delete this->kk;
if (this->myrank == 0)
diff --git a/bfps/cpp/full_code/native_binary_to_hdf5.cpp b/bfps/cpp/full_code/native_binary_to_hdf5.cpp
index 7774e2de..fb5a39c2 100644
--- a/bfps/cpp/full_code/native_binary_to_hdf5.cpp
+++ b/bfps/cpp/full_code/native_binary_to_hdf5.cpp
@@ -7,6 +7,7 @@
template <typename rnumber>
int native_binary_to_hdf5<rnumber>::initialize(void)
{
+ TIMEZONE("native_binary_to_hdf5::initialize");
this->read_parameters();
this->vec_field = new field<rnumber, FFTW, THREE>(
nx, ny, nz,
@@ -24,6 +25,7 @@ int native_binary_to_hdf5<rnumber>::initialize(void)
template <typename rnumber>
int native_binary_to_hdf5<rnumber>::work_on_current_iteration(void)
{
+ TIMEZONE("native_binary_to_hdf5::work_on_current_iteration");
char itername[16];
sprintf(itername, "i%.5x", this->iteration);
std::string native_binary_fname = (
@@ -45,6 +47,7 @@ int native_binary_to_hdf5<rnumber>::work_on_current_iteration(void)
template <typename rnumber>
int native_binary_to_hdf5<rnumber>::finalize(void)
{
+ TIMEZONE("native_binary_to_hdf5::finalize");
delete this->bin_IO;
delete this->vec_field;
return EXIT_SUCCESS;
@@ -53,6 +56,7 @@ int native_binary_to_hdf5<rnumber>::finalize(void)
template <typename rnumber>
int native_binary_to_hdf5<rnumber>::read_parameters(void)
{
+ TIMEZONE("native_binary_to_hdf5::read_parameters");
this->postprocess::read_parameters();
hid_t parameter_file = H5Fopen(
(this->simname + std::string(".h5")).c_str(),
diff --git a/bfps/cpp/full_code/postprocess.cpp b/bfps/cpp/full_code/postprocess.cpp
index 430d87a5..13bee700 100644
--- a/bfps/cpp/full_code/postprocess.cpp
+++ b/bfps/cpp/full_code/postprocess.cpp
@@ -8,6 +8,7 @@
int postprocess::main_loop(void)
{
+ TIMEZONE("postprocess::main_loop");
this->start_simple_timer();
for (unsigned int iteration_counter = 0;
iteration_counter < iteration_list.size();
@@ -33,6 +34,7 @@ int postprocess::main_loop(void)
int postprocess::read_parameters()
{
+ TIMEZONE("postprocess::read_parameters");
this->code_base::read_parameters();
hid_t parameter_file = H5Fopen((this->simname + ".h5").c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
this->nu = hdf5_tools::read_value<double>(parameter_file, "parameters/nu");
diff --git a/bfps/cpp/full_code/resize.cpp b/bfps/cpp/full_code/resize.cpp
index 41d68ef7..de555a74 100644
--- a/bfps/cpp/full_code/resize.cpp
+++ b/bfps/cpp/full_code/resize.cpp
@@ -7,6 +7,7 @@
template <typename rnumber>
int resize<rnumber>::initialize(void)
{
+ TIMEZONE("resize::initialize");
this->NSVE_field_stats<rnumber>::initialize();
DEBUG_MSG("after NSVE_field_stats::initialize\n");
hid_t parameter_file = H5Fopen(
@@ -46,7 +47,7 @@ int resize<rnumber>::initialize(void)
template <typename rnumber>
int resize<rnumber>::work_on_current_iteration(void)
{
- DEBUG_MSG("entered resize::work_on_current_iteration\n");
+ TIMEZONE("resize::work_on_current_iteration");
this->read_current_cvorticity();
std::string fname = (
@@ -64,6 +65,7 @@ int resize<rnumber>::work_on_current_iteration(void)
template <typename rnumber>
int resize<rnumber>::finalize(void)
{
+ TIMEZONE("resize::finalize");
delete this->new_field;
this->NSVE_field_stats<rnumber>::finalize();
return EXIT_SUCCESS;
diff --git a/bfps/cpp/full_code/symmetrize_test.cpp b/bfps/cpp/full_code/symmetrize_test.cpp
index 3b4cd5a5..821161da 100644
--- a/bfps/cpp/full_code/symmetrize_test.cpp
+++ b/bfps/cpp/full_code/symmetrize_test.cpp
@@ -8,6 +8,7 @@
template <typename rnumber>
int symmetrize_test<rnumber>::initialize(void)
{
+ TIMEZONE("symmetrize_test::initialize");
this->read_parameters();
return EXIT_SUCCESS;
}
@@ -15,12 +16,14 @@ int symmetrize_test<rnumber>::initialize(void)
template <typename rnumber>
int symmetrize_test<rnumber>::finalize(void)
{
+ TIMEZONE("symmetrize_test::finalize");
return EXIT_SUCCESS;
}
template <typename rnumber>
int symmetrize_test<rnumber>::read_parameters()
{
+ TIMEZONE("symmetrize_test::read_parameters");
this->test::read_parameters();
hid_t parameter_file = H5Fopen(
(this->simname + std::string(".h5")).c_str(),
@@ -35,6 +38,7 @@ int symmetrize_test<rnumber>::read_parameters()
template <typename rnumber>
int symmetrize_test<rnumber>::do_work(void)
{
+ TIMEZONE("symmetrize_test::do_work");
// allocate
DEBUG_MSG("about to allocate field0\n");
field<rnumber, FFTW, THREE> *test_field0 = new field<rnumber, FFTW, THREE>(
diff --git a/bfps/cpp/full_code/test.cpp b/bfps/cpp/full_code/test.cpp
index e382efb4..aa909362 100644
--- a/bfps/cpp/full_code/test.cpp
+++ b/bfps/cpp/full_code/test.cpp
@@ -8,9 +8,7 @@
int test::main_loop(void)
{
- #ifdef USE_TIMINGOUTPUT
- TIMEZONE("test::main_loop");
- #endif
+ TIMEZONE("test::main_loop");
this->start_simple_timer();
this->do_work();
this->print_simple_timer(
diff --git a/bfps/cpp/full_code/test_interpolation.cpp b/bfps/cpp/full_code/test_interpolation.cpp
index b194d372..5ef11de4 100644
--- a/bfps/cpp/full_code/test_interpolation.cpp
+++ b/bfps/cpp/full_code/test_interpolation.cpp
@@ -4,6 +4,7 @@
template <typename rnumber>
int test_interpolation<rnumber>::read_parameters(void)
{
+ TIMEZONE("test_interpolation::read_parameters");
this->test::read_parameters();
hid_t parameter_file = H5Fopen(
(this->simname + std::string(".h5")).c_str(),
@@ -24,6 +25,7 @@ int test_interpolation<rnumber>::read_parameters(void)
template <typename rnumber>
int test_interpolation<rnumber>::initialize(void)
{
+ TIMEZONE("test_interpolation::initialize");
this->read_parameters();
this->vorticity = new field<rnumber, FFTW, THREE>(
this->nx, this->ny, this->nz,
@@ -86,6 +88,7 @@ int test_interpolation<rnumber>::initialize(void)
template <typename rnumber>
int test_interpolation<rnumber>::finalize(void)
{
+ TIMEZONE("test_interpolation::finalize");
delete this->nabla_u;
delete this->velocity;
delete this->vorticity;
@@ -99,6 +102,7 @@ int test_interpolation<rnumber>::finalize(void)
template <typename rnumber>
int test_interpolation<rnumber>::do_work()
{
+ TIMEZONE("test_interpolation::do_work");
*this->nabla_u = 0.0;
this->velocity->real_space_representation = false;
this->vorticity->real_space_representation = false;
diff --git a/bfps/cpp/scope_timer.hpp b/bfps/cpp/scope_timer.hpp
index 2c48e2ed..890f522c 100644
--- a/bfps/cpp/scope_timer.hpp
+++ b/bfps/cpp/scope_timer.hpp
@@ -791,7 +791,8 @@ extern EventManager global_timer_manager;
#define TIMEZONE(NAME) \
ScopeEvent TIMEZONE_Core_Pre_Merge(____TIMEZONE_AUTO_ID, __LINE__)( \
- NAME, global_timer_manager, ScopeEventUniqueKey);
+ NAME, global_timer_manager, ScopeEventUniqueKey); \
+ DEBUG_MSG((NAME + std::string("\n")).c_str());
#define TIMEZONE_MULTI_REF(NAME) \
ScopeEvent TIMEZONE_Core_Pre_Merge(____TIMEZONE_AUTO_ID, __LINE__)( \
NAME, global_timer_manager, ScopeEventMultiRefKey);
--
GitLab
From 441bb388a9a7272090c2eccc2de5584a24660ff2 Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Sun, 9 Sep 2018 17:19:06 +0200
Subject: [PATCH 214/342] add script to run all tests
also add preliminary script to generate dependency list.
we should use this for compiling in the future.
---
cpp_build.py | 85 ++++++++++++++++++++++++++++++++++++++++++
tests/run_all_tests.sh | 11 ++++++
2 files changed, 96 insertions(+)
create mode 100644 cpp_build.py
create mode 100644 tests/run_all_tests.sh
diff --git a/cpp_build.py b/cpp_build.py
new file mode 100644
index 00000000..a312191a
--- /dev/null
+++ b/cpp_build.py
@@ -0,0 +1,85 @@
+#######################################################################
+# #
+# Copyright 2015 Max Planck Institute #
+# for Dynamics and Self-Organization #
+# #
+# This file is part of bfps. #
+# #
+# bfps is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# bfps is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with bfps. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+#######################################################################
+
+
+
+import os
+import subprocess
+
+src_file_list = ['hdf5_tools',
+ 'full_code/get_rfields',
+ 'full_code/NSVE_field_stats',
+ 'full_code/native_binary_to_hdf5',
+ 'full_code/postprocess',
+ 'full_code/code_base',
+ 'full_code/direct_numerical_simulation',
+ 'full_code/NSVE',
+ 'full_code/NSVEparticles',
+ 'field_binary_IO',
+ 'vorticity_equation',
+ 'field',
+ 'kspace',
+ 'field_layout',
+ 'field_descriptor',
+ 'rFFTW_distributed_particles',
+ 'distributed_particles',
+ 'particles',
+ 'particles_base',
+ 'rFFTW_interpolator',
+ 'interpolator',
+ 'interpolator_base',
+ 'fluid_solver',
+ 'fluid_solver_base',
+ 'fftw_tools',
+ 'spline_n1',
+ 'spline_n2',
+ 'spline_n3',
+ 'spline_n4',
+ 'spline_n5',
+ 'spline_n6',
+ 'spline_n7',
+ 'spline_n8',
+ 'spline_n9',
+ 'spline_n10',
+ 'Lagrange_polys',
+ 'scope_timer']
+
+def get_dependency_list():
+ ofile = open('dependencies.txt', 'w')
+ for src_file in src_file_list:
+ p = subprocess.Popen(
+ ['g++', '-Ibfps/cpp', '-MM', 'bfps/cpp/' + src_file + '.cpp'],
+ stdout = subprocess.PIPE)
+ out, err = p.communicate()
+ p.terminate()
+ deps = str(out, 'ASCII').replace('\\\n', '')
+ print(deps.split()[0])
+ ofile.write(' '.join(deps.split()[1:]) + '\n')
+ ofile.close()
+ return None
+
+if __name__ == '__main__':
+ #pass
+ get_dependency_list()
+
diff --git a/tests/run_all_tests.sh b/tests/run_all_tests.sh
new file mode 100644
index 00000000..2b2b4087
--- /dev/null
+++ b/tests/run_all_tests.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+bfps.test_fftw
+bfps.test_Parseval
+bfps.test_NSVEparticles
+
+# test postprocessing
+bfps PP field_single_to_double --simname dns_nsveparticles --iter0 32 --iter1 32
+bfps PP get_rfields --simname dns_nsveparticles --iter0 0 --iter1 64
+bfps PP joint_acc_vel_stats --simname dns_nsveparticles --iter0 0 --iter1 64
+bfps PP resize --simname dns_nsveparticles --new_nx 96 --new_ny 96 --new_nz 96 --new_simname dns_nsveparticles_resized
--
GitLab
From 160c37f485a5eb022daac22c281c0dcf786de49f Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Sun, 9 Sep 2018 17:50:48 +0200
Subject: [PATCH 215/342] replace HDF5 calls with hdf5_tools calls
---
bfps/cpp/full_code/get_rfields.cpp | 14 +++-----------
tests/run_all_tests.sh | 2 ++
2 files changed, 5 insertions(+), 11 deletions(-)
diff --git a/bfps/cpp/full_code/get_rfields.cpp b/bfps/cpp/full_code/get_rfields.cpp
index 5d872b72..376a2659 100644
--- a/bfps/cpp/full_code/get_rfields.cpp
+++ b/bfps/cpp/full_code/get_rfields.cpp
@@ -16,23 +16,15 @@ int get_rfields<rnumber>::initialize(void)
(this->simname + std::string(".h5")).c_str(),
H5F_ACC_RDONLY,
H5P_DEFAULT);
- hid_t dset = H5Dopen(parameter_file, "/parameters/niter_out", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->niter_out);
- H5Dclose(dset);
- if (H5Lexists(parameter_file, "/parameters/checkpoints_per_file", H5P_DEFAULT))
- {
- dset = H5Dopen(parameter_file, "/parameters/checkpoints_per_file", H5P_DEFAULT);
- H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &this->checkpoints_per_file);
- H5Dclose(dset);
- }
- else
+ this->niter_out = hdf5_tools::read_value<int>(parameter_file, "/parameters/niter_out");
+ this->checkpoints_per_file = hdf5_tools::read_value<int>(parameter_file, "/parameters/checkpoints_per_file");
+ if (this->checkpoints_per_file == INT_MAX) // value returned if dataset does not exist
this->checkpoints_per_file = 1;
H5Fclose(parameter_file);
parameter_file = H5Fopen(
(this->simname + std::string("_post.h5")).c_str(),
H5F_ACC_RDONLY,
H5P_DEFAULT);
- DEBUG_MSG("before read_vector\n");
this->iteration_list = hdf5_tools::read_vector<int>(
parameter_file,
"/get_rfields/parameters/iteration_list");
diff --git a/tests/run_all_tests.sh b/tests/run_all_tests.sh
index 2b2b4087..7865a8a3 100644
--- a/tests/run_all_tests.sh
+++ b/tests/run_all_tests.sh
@@ -1,5 +1,7 @@
#!/bin/bash
+set -e
+
bfps.test_fftw
bfps.test_Parseval
bfps.test_NSVEparticles
--
GitLab
From bd36aacbed22046d4950f92ceeb07b6e46331e7d Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Mon, 10 Sep 2018 09:21:44 +0200
Subject: [PATCH 216/342] add broken field::write_filtered
---
bfps/cpp/field.cpp | 132 +++++++++++++++++++++++++++++++++++++++++++++
bfps/cpp/field.hpp | 5 ++
2 files changed, 137 insertions(+)
diff --git a/bfps/cpp/field.cpp b/bfps/cpp/field.cpp
index d5696d56..ba7fa5e4 100644
--- a/bfps/cpp/field.cpp
+++ b/bfps/cpp/field.cpp
@@ -612,6 +612,138 @@ int field<rnumber, be, fc>::write_0slice(
return EXIT_SUCCESS;
}
+template <typename rnumber,
+ field_backend be,
+ field_components fc>
+int field<rnumber, be, fc>::write_filtered(
+ const std::string fname,
+ const std::string field_name,
+ const int iteration,
+ int nx)
+{
+ /* file dataset has same dimensions as field */
+ TIMEZONE("field::write_filtered");
+ // only works in Fourier representation
+ assert(!this->real_space_representation);
+ assert(nx <= this->rlayout->sizes[2]);
+ hid_t file_id, dset_id, plist_id;
+ dset_id = H5I_BADID;
+ std::string dset_name = (
+ "/" + field_name +
+ "/complex" +
+ "/" + std::to_string(iteration));
+
+ /* open/create file */
+ plist_id = H5Pcreate(H5P_FILE_ACCESS);
+ H5Pset_fapl_mpio(plist_id, this->comm, MPI_INFO_NULL);
+ bool file_exists = false;
+ struct stat file_buffer;
+ file_exists = (stat(fname.c_str(), &file_buffer) == 0);
+ if (file_exists)
+ file_id = H5Fopen(fname.c_str(), H5F_ACC_RDWR, plist_id);
+ else
+ file_id = H5Fcreate(fname.c_str(), H5F_ACC_EXCL, H5P_DEFAULT, plist_id);
+ assert(file_id >= 0);
+ H5Pclose(plist_id);
+
+ /* generic space initialization */
+ hid_t fspace, mspace;
+ hsize_t count[ndim(fc)], offset[ndim(fc)], dims[ndim(fc)], fdims[ndim(fc)];
+ hsize_t memoffset[ndim(fc)], memshape[ndim(fc)];
+
+ // set up dimensions
+ for (unsigned int i=3; i<ndim(fc); i++)
+ {
+ count [i] = this->clayout->subsizes[i];
+ offset[i] = this->clayout->starts[i];
+ dims [i] = this->clayout->sizes[i];
+ memshape [i] = count[i];
+ memoffset[i] = 0;
+ }
+ // set up smaller dimensions
+ unsigned int ii = 0;
+ count [ii] = this->clayout->subsizes[ii];
+ offset[ii] = this->clayout->starts[ii];
+ dims [ii] = this->clayout->sizes[ii];
+ memshape [ii] = count[ii];
+ memoffset[ii] = 0;
+ ii = 1;
+ count [ii] = this->clayout->subsizes[ii];
+ offset[ii] = this->clayout->starts[ii];
+ dims [ii] = this->clayout->sizes[ii];
+ memshape [ii] = count[ii];
+ memoffset[ii] = 0;
+ ii = 2;
+ count [ii] = nx/2+1;
+ offset[ii] = 0;
+ dims [ii] = nx/2+1;
+ memshape [ii] = this->clayout->subsizes[ii];
+ memoffset[ii] = 0;
+
+ mspace = H5Screate_simple(ndim(fc), memshape, NULL);
+ H5Sselect_hyperslab(mspace, H5S_SELECT_SET, memoffset, NULL, count, NULL);
+
+ /* open/create data set */
+ if (!H5Lexists(file_id, field_name.c_str(), H5P_DEFAULT))
+ {
+ hid_t gid_tmp = H5Gcreate(
+ file_id, field_name.c_str(),
+ H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ H5Gclose(gid_tmp);
+ }
+ if (!H5Lexists(file_id, (field_name + "/complex").c_str(), H5P_DEFAULT))
+ {
+ hid_t gid_tmp = H5Gcreate(
+ file_id, ("/" + field_name + "/complex").c_str(),
+ H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ H5Gclose(gid_tmp);
+ }
+ if (H5Lexists(file_id, dset_name.c_str(), H5P_DEFAULT))
+ {
+ dset_id = H5Dopen(file_id, dset_name.c_str(), H5P_DEFAULT);
+ fspace = H5Dget_space(dset_id);
+ }
+ else
+ {
+ fspace = H5Screate_simple(
+ ndim(fc),
+ dims,
+ NULL);
+ /* chunking needs to go in here */
+ dset_id = H5Dcreate(
+ file_id,
+ dset_name.c_str(),
+ this->cnumber_H5T,
+ fspace,
+ H5P_DEFAULT,
+ H5P_DEFAULT,
+ H5P_DEFAULT);
+ }
+ /* both dset_id and fspace now have sane values */
+
+ /* check file space */
+ int ndims_fspace = H5Sget_simple_extent_dims(fspace, fdims, NULL);
+ assert(((unsigned int)(ndims_fspace)) == ndim(fc));
+
+ for (unsigned int i=0; i<ndim(fc); i++)
+ {
+ offset[i] = this->clayout->starts[i];
+ assert(dims[i] == fdims[i]);
+ }
+ H5Sselect_hyperslab(fspace, H5S_SELECT_SET, offset, NULL, count, NULL);
+ H5Dwrite(dset_id, this->cnumber_H5T, mspace, fspace, H5P_DEFAULT, this->data);
+ H5Sclose(mspace);
+
+
+ /* close file data space */
+ H5Sclose(fspace);
+ /* close data set */
+ H5Dclose(dset_id);
+ /* close file */
+ H5Fclose(file_id);
+ return EXIT_SUCCESS;
+}
+
template <typename rnumber,
field_backend be,
diff --git a/bfps/cpp/field.hpp b/bfps/cpp/field.hpp
index a52d2a56..861246a4 100644
--- a/bfps/cpp/field.hpp
+++ b/bfps/cpp/field.hpp
@@ -103,6 +103,11 @@ class field
const hid_t group,
const std::string field_name,
const int iteration);
+ int write_filtered(
+ const std::string fname,
+ const std::string field_name,
+ const int iteration,
+ const int nx);
int io_binary(
const std::string fname,
--
GitLab
From a21224a477bf015ee90657535bca1249a507a83c Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 10 Sep 2018 19:02:28 +0200
Subject: [PATCH 217/342] filter both in x and z
---
bfps/cpp/field.cpp | 75 +++++++++++++++++++++++++++-------------------
bfps/cpp/field.hpp | 4 ++-
2 files changed, 47 insertions(+), 32 deletions(-)
diff --git a/bfps/cpp/field.cpp b/bfps/cpp/field.cpp
index ba7fa5e4..ecb3d838 100644
--- a/bfps/cpp/field.cpp
+++ b/bfps/cpp/field.cpp
@@ -619,13 +619,17 @@ int field<rnumber, be, fc>::write_filtered(
const std::string fname,
const std::string field_name,
const int iteration,
- int nx)
+ int nx,
+ int ny,
+ int nz)
{
/* file dataset has same dimensions as field */
TIMEZONE("field::write_filtered");
// only works in Fourier representation
assert(!this->real_space_representation);
- assert(nx <= this->rlayout->sizes[2]);
+ assert(hsize_t(nx) <= this->rlayout->sizes[2]);
+ assert(hsize_t(ny) <= this->rlayout->sizes[1]);
+ assert(hsize_t(nz) <= this->rlayout->sizes[0]);
hid_t file_id, dset_id, plist_id;
dset_id = H5I_BADID;
std::string dset_name = (
@@ -660,28 +664,11 @@ int field<rnumber, be, fc>::write_filtered(
memshape [i] = count[i];
memoffset[i] = 0;
}
- // set up smaller dimensions
- unsigned int ii = 0;
- count [ii] = this->clayout->subsizes[ii];
- offset[ii] = this->clayout->starts[ii];
- dims [ii] = this->clayout->sizes[ii];
- memshape [ii] = count[ii];
- memoffset[ii] = 0;
- ii = 1;
- count [ii] = this->clayout->subsizes[ii];
- offset[ii] = this->clayout->starts[ii];
- dims [ii] = this->clayout->sizes[ii];
- memshape [ii] = count[ii];
- memoffset[ii] = 0;
- ii = 2;
- count [ii] = nx/2+1;
- offset[ii] = 0;
- dims [ii] = nx/2+1;
- memshape [ii] = this->clayout->subsizes[ii];
- memoffset[ii] = 0;
-
- mspace = H5Screate_simple(ndim(fc), memshape, NULL);
- H5Sselect_hyperslab(mspace, H5S_SELECT_SET, memoffset, NULL, count, NULL);
+ // these are dimensions of dataset, needed
+ // to create dataset
+ dims[0] = nz;
+ dims[1] = ny;
+ dims[2] = nx/2+1;
/* open/create data set */
if (!H5Lexists(file_id, field_name.c_str(), H5P_DEFAULT))
@@ -719,20 +706,46 @@ int field<rnumber, be, fc>::write_filtered(
H5P_DEFAULT,
H5P_DEFAULT);
}
- /* both dset_id and fspace now have sane values */
-
/* check file space */
int ndims_fspace = H5Sget_simple_extent_dims(fspace, fdims, NULL);
assert(((unsigned int)(ndims_fspace)) == ndim(fc));
-
for (unsigned int i=0; i<ndim(fc); i++)
{
- offset[i] = this->clayout->starts[i];
assert(dims[i] == fdims[i]);
}
- H5Sselect_hyperslab(fspace, H5S_SELECT_SET, offset, NULL, count, NULL);
- H5Dwrite(dset_id, this->cnumber_H5T, mspace, fspace, H5P_DEFAULT, this->data);
- H5Sclose(mspace);
+ /* both dset_id and fspace now have sane values */
+
+ /// set up counts and offsets
+ /// x is easy, since only positive modes are present
+ count [2] = nx/2+1;
+ offset[2] = 0;
+ memshape [2] = this->clayout->subsizes[2];
+ memoffset[2] = 0;
+
+ /// now we need to visit the four corners of the data
+ for (int cy = 0; cy<2; cy++)
+ {
+ for (int cz = 0; cz < 2; cz++)
+ {
+ // for z, we need to take into account that there are
+ // both positive and negative modes
+ count [1] = nz/2;
+ offset[1] = cz*nz/2;
+ memshape [1] = this->clayout->sizes[1];
+ memoffset[1] = cz*(this->clayout->sizes[1] - nz/2);
+ }
+ count [0] = this->clayout->subsizes[0];
+ offset[0] = this->clayout->starts[0];
+ memshape [0] = this->clayout->subsizes[0];
+ memoffset[0] = 0;
+
+ //now write data
+ mspace = H5Screate_simple(ndim(fc), memshape, NULL);
+ H5Sselect_hyperslab(mspace, H5S_SELECT_SET, memoffset, NULL, count, NULL);
+ H5Sselect_hyperslab(fspace, H5S_SELECT_SET, offset, NULL, count, NULL);
+ H5Dwrite(dset_id, this->cnumber_H5T, mspace, fspace, H5P_DEFAULT, this->data);
+ H5Sclose(mspace);
+ }
/* close file data space */
diff --git a/bfps/cpp/field.hpp b/bfps/cpp/field.hpp
index 861246a4..fe6f8975 100644
--- a/bfps/cpp/field.hpp
+++ b/bfps/cpp/field.hpp
@@ -107,7 +107,9 @@ class field
const std::string fname,
const std::string field_name,
const int iteration,
- const int nx);
+ const int nx,
+ const int ny,
+ const int nz);
int io_binary(
const std::string fname,
--
GitLab
From e455dfd4a052d28a9cb933951d17f2b5379e50e1 Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Mon, 10 Sep 2018 23:37:57 +0200
Subject: [PATCH 218/342] write_filtered works for unfiltered
---
bfps/cpp/field.cpp | 55 +++++++++++++++++++++++++++++++++++-----------
1 file changed, 42 insertions(+), 13 deletions(-)
diff --git a/bfps/cpp/field.cpp b/bfps/cpp/field.cpp
index ecb3d838..ea0bd7a4 100644
--- a/bfps/cpp/field.cpp
+++ b/bfps/cpp/field.cpp
@@ -630,6 +630,8 @@ int field<rnumber, be, fc>::write_filtered(
assert(hsize_t(nx) <= this->rlayout->sizes[2]);
assert(hsize_t(ny) <= this->rlayout->sizes[1]);
assert(hsize_t(nz) <= this->rlayout->sizes[0]);
+ // current algorithm only works for more than one process
+ assert(this->nprocs >= 2);
hid_t file_id, dset_id, plist_id;
dset_id = H5I_BADID;
std::string dset_name = (
@@ -666,6 +668,7 @@ int field<rnumber, be, fc>::write_filtered(
}
// these are dimensions of dataset, needed
// to create dataset
+ //dims[0] = nz;
dims[0] = nz;
dims[1] = ny;
dims[2] = nx/2+1;
@@ -722,22 +725,48 @@ int field<rnumber, be, fc>::write_filtered(
memshape [2] = this->clayout->subsizes[2];
memoffset[2] = 0;
- /// now we need to visit the four corners of the data
- for (int cy = 0; cy<2; cy++)
+ /// three options for y:
+ /// this->starts[0] <= ny/2
+ /// ny / 2 < this->starts[0] +this->clayout->subsizes[0] < this->sizes[0] - ny/2
+ /// this->starts[0] >= this->sizes[0] - ny/2
+ /// we don't care about saving the ny/2 mode, because of symmetry
+ offset[0] = this->clayout->starts[0];
+ memshape[0] = this->clayout->subsizes[0];
+ if (this->clayout->starts[0] <= ny/2)
{
- for (int cz = 0; cz < 2; cz++)
+ if (this->clayout->starts[0] + this->clayout->subsizes[0] <= ny/2)
+ count[0] = this->clayout->subsizes[0];
+ else
+ count[0] = (ny/2 - (this->clayout->starts[0] + this->clayout->subsizes[0]));
+ memoffset[0] = 0;
+ }
+ else if (ny/2 < this->clayout->starts[0] &&
+ this->clayout->starts[0]+this->clayout->subsizes[0] < this->clayout->sizes[0] - ny/2)
+ {
+ count[0] = 0;
+ memoffset[0] = 0;
+ }
+ else
+ {
+ if (this->clayout->starts[0] < this->clayout->sizes[0] - ny/2)
{
- // for z, we need to take into account that there are
- // both positive and negative modes
- count [1] = nz/2;
- offset[1] = cz*nz/2;
- memshape [1] = this->clayout->sizes[1];
- memoffset[1] = cz*(this->clayout->sizes[1] - nz/2);
+ count[0] = this->clayout->sizes[0] - ny/2 - this->clayout->starts[0];
+ memoffset[0] = this->clayout->sizes[0] - ny/2 - this->clayout->starts[0];
}
- count [0] = this->clayout->subsizes[0];
- offset[0] = this->clayout->starts[0];
- memshape [0] = this->clayout->subsizes[0];
- memoffset[0] = 0;
+ else
+ {
+ memoffset[0] = 0;
+ count[0] = this->clayout->subsizes[0];
+ }
+ }
+ /// for z, we need to take into account that there are
+ /// both positive and negative modes
+ for (int cz = 0; cz < 2; cz++)
+ {
+ count [1] = nz/2;
+ offset[1] = cz*nz/2;
+ memshape [1] = this->clayout->sizes[1];
+ memoffset[1] = cz*(this->clayout->sizes[1] - nz/2);
//now write data
mspace = H5Screate_simple(ndim(fc), memshape, NULL);
--
GitLab
From c0d757413560c1d08868903da2ceb502184b74f3 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 11 Sep 2018 14:00:59 +0200
Subject: [PATCH 219/342] fix filtered output
---
bfps/cpp/field.cpp | 55 ++++++++++++++++++++++++++++++----------------
1 file changed, 36 insertions(+), 19 deletions(-)
diff --git a/bfps/cpp/field.cpp b/bfps/cpp/field.cpp
index ea0bd7a4..e64a1b95 100644
--- a/bfps/cpp/field.cpp
+++ b/bfps/cpp/field.cpp
@@ -669,8 +669,8 @@ int field<rnumber, be, fc>::write_filtered(
// these are dimensions of dataset, needed
// to create dataset
//dims[0] = nz;
- dims[0] = nz;
- dims[1] = ny;
+ dims[0] = ny;
+ dims[1] = nz;
dims[2] = nx/2+1;
/* open/create data set */
@@ -730,35 +730,50 @@ int field<rnumber, be, fc>::write_filtered(
/// ny / 2 < this->starts[0] +this->clayout->subsizes[0] < this->sizes[0] - ny/2
/// this->starts[0] >= this->sizes[0] - ny/2
/// we don't care about saving the ny/2 mode, because of symmetry
- offset[0] = this->clayout->starts[0];
+ hsize_t y0 = this->clayout->starts[0];
+ hsize_t y1 = this->clayout->starts[0] + this->clayout->subsizes[0];
memshape[0] = this->clayout->subsizes[0];
- if (this->clayout->starts[0] <= ny/2)
+ if (y1 <= ny/2)
{
- if (this->clayout->starts[0] + this->clayout->subsizes[0] <= ny/2)
- count[0] = this->clayout->subsizes[0];
- else
- count[0] = (ny/2 - (this->clayout->starts[0] + this->clayout->subsizes[0]));
- memoffset[0] = 0;
- }
- else if (ny/2 < this->clayout->starts[0] &&
- this->clayout->starts[0]+this->clayout->subsizes[0] < this->clayout->sizes[0] - ny/2)
- {
- count[0] = 0;
+ count[0] = this->clayout->subsizes[0];
+ offset[0] = y0;
memoffset[0] = 0;
}
else
{
- if (this->clayout->starts[0] < this->clayout->sizes[0] - ny/2)
+ if (y0 < ny/2)
{
- count[0] = this->clayout->sizes[0] - ny/2 - this->clayout->starts[0];
- memoffset[0] = this->clayout->sizes[0] - ny/2 - this->clayout->starts[0];
+ count[0] = ny/2 - y0;
+ offset[0] = y0;
+ memoffset[0] = 0;
}
else
{
- memoffset[0] = 0;
- count[0] = this->clayout->subsizes[0];
+ if (y1 <= this->clayout->sizes[0] - ny/2 + 1)
+ { // y0 < y1 therefore y0 <= this->clayout->sizes[0] - ny/2
+ count[0] = 0;
+ offset[0] = ny/2;
+ memoffset[0] = 0;
+ }
+ else
+ {
+ if (y0 <= this->clayout->sizes[0] - ny/2)
+ {
+ count[0] = y1 - (this->clayout->sizes[0] - ny/2);
+ offset[0] = ny/2;
+ memoffset[0] = this->clayout->subsizes[0] - count[0];
+ }
+ else
+ {
+ count[0] = this->clayout->subsizes[0];
+ offset[0] = y0;
+ memoffset[0] = 0;
+ }
+ }
}
}
+ DEBUG_MSG("count[0] = %ld, offset[0] = %ld\n",
+ count[0], offset[0]);
/// for z, we need to take into account that there are
/// both positive and negative modes
for (int cz = 0; cz < 2; cz++)
@@ -767,6 +782,8 @@ int field<rnumber, be, fc>::write_filtered(
offset[1] = cz*nz/2;
memshape [1] = this->clayout->sizes[1];
memoffset[1] = cz*(this->clayout->sizes[1] - nz/2);
+ DEBUG_MSG("cz = %d, count[1] + offset[1] = %ld\n",
+ cz, count[1] + offset[1]);
//now write data
mspace = H5Screate_simple(ndim(fc), memshape, NULL);
--
GitLab
From 99081cbbbe90899ca977b3337473caa59bc0c89e Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 11 Sep 2018 14:17:30 +0200
Subject: [PATCH 220/342] get rid of comparison warnings
---
bfps/cpp/field.cpp | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/bfps/cpp/field.cpp b/bfps/cpp/field.cpp
index e64a1b95..b70825cb 100644
--- a/bfps/cpp/field.cpp
+++ b/bfps/cpp/field.cpp
@@ -733,7 +733,7 @@ int field<rnumber, be, fc>::write_filtered(
hsize_t y0 = this->clayout->starts[0];
hsize_t y1 = this->clayout->starts[0] + this->clayout->subsizes[0];
memshape[0] = this->clayout->subsizes[0];
- if (y1 <= ny/2)
+ if (y1 <= hsize_t(ny/2))
{
count[0] = this->clayout->subsizes[0];
offset[0] = y0;
@@ -741,7 +741,7 @@ int field<rnumber, be, fc>::write_filtered(
}
else
{
- if (y0 < ny/2)
+ if (y0 < hsize_t(ny)/2)
{
count[0] = ny/2 - y0;
offset[0] = y0;
@@ -749,7 +749,7 @@ int field<rnumber, be, fc>::write_filtered(
}
else
{
- if (y1 <= this->clayout->sizes[0] - ny/2 + 1)
+ if (y1 <= hsize_t(this->clayout->sizes[0] - ny/2 + 1))
{ // y0 < y1 therefore y0 <= this->clayout->sizes[0] - ny/2
count[0] = 0;
offset[0] = ny/2;
@@ -757,7 +757,7 @@ int field<rnumber, be, fc>::write_filtered(
}
else
{
- if (y0 <= this->clayout->sizes[0] - ny/2)
+ if (y0 <= hsize_t(this->clayout->sizes[0] - ny/2))
{
count[0] = y1 - (this->clayout->sizes[0] - ny/2);
offset[0] = ny/2;
--
GitLab
From f9090f38879e50a041d8848de9aaf27d5189446b Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 12 Sep 2018 14:18:17 +0200
Subject: [PATCH 221/342] tweak commenting for complex particles
---
bfps/cpp/full_code/NSVEcomplex_particles.cpp | 26 +++++++++-----------
1 file changed, 11 insertions(+), 15 deletions(-)
diff --git a/bfps/cpp/full_code/NSVEcomplex_particles.cpp b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
index 81c6cd5f..4ffa148a 100644
--- a/bfps/cpp/full_code/NSVEcomplex_particles.cpp
+++ b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
@@ -39,25 +39,21 @@ int NSVEcomplex_particles<rnumber>::initialize(void)
this->NSVE<rnumber>::initialize();
p2p_computer<double, long long int> current_p2p_computer;
- // TODO: particle interactions are switched off manually for testing purposes.
- // this needs to be fixed once particle interactions can be properly resolved.
- current_p2p_computer.setEnable(enable_p2p);
- //current_p2p_computer.setEnable(false);
+ current_p2p_computer.setEnable(this->enable_p2p);
particles_inner_computer<double, long long int> current_particles_inner_computer(inner_v0);
current_particles_inner_computer.setEnable(enable_inner);
- this->cutoff = 1.0;
this->ps = particles_system_builder_with_p2p(
- this->fs->cvelocity, // (field object)
- this->fs->kk, // (kspace object, contains dkx, dky, dkz)
- tracers0_integration_steps, // to check coherency between parameters and hdf input file (nb rhs)
- (long long int)nparticles, // to check coherency between parameters and hdf input file
- this->fs->get_current_fname(), // particles input filename
- std::string("/tracers0/state/") + std::to_string(this->fs->iteration), // dataset name for initial input
- std::string("/tracers0/rhs/") + std::to_string(this->fs->iteration), // dataset name for initial input
- tracers0_neighbours, // parameter (interpolation no neighbours)
- tracers0_smoothness, // parameter
+ this->fs->cvelocity, // (field object)
+ this->fs->kk, // (kspace object, contains dkx, dky, dkz)
+ tracers0_integration_steps, // to check coherency between parameters and hdf input file (nb rhs)
+ (long long int)nparticles, // to check coherency between parameters and hdf input file
+ this->fs->get_current_fname(), // particles input filename
+ std::string("/tracers0/state/") + std::to_string(this->fs->iteration), // dataset name for initial input
+ std::string("/tracers0/rhs/") + std::to_string(this->fs->iteration), // dataset name for initial input
+ tracers0_neighbours, // parameter (interpolation no neighbours)
+ tracers0_smoothness, // parameter (how many continuous derivatives)
this->comm,
this->fs->iteration+1,
std::move(current_p2p_computer),
@@ -93,7 +89,7 @@ int NSVEcomplex_particles<rnumber>::step(void)
TIMEZONE("NSVEcomplex_particles::step");
this->fs->compute_velocity(this->fs->cvorticity);
this->fs->cvelocity->ift();
- if(enable_vorticity_omega){
+ if(this->enable_vorticity_omega){
*this->tmp_vec_field = this->fs->cvorticity->get_cdata();
this->tmp_vec_field->ift();
this->ps->completeLoopWithVorticity(this->dt, *this->tmp_vec_field);
--
GitLab
From 97c2fa2678ceb8adfdb175c401bc7a65c5d9536d Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 12 Sep 2018 14:34:54 +0200
Subject: [PATCH 222/342] don't include deprecated field_descriptor header
---
bfps/cpp/vorticity_equation.hpp | 1 -
1 file changed, 1 deletion(-)
diff --git a/bfps/cpp/vorticity_equation.hpp b/bfps/cpp/vorticity_equation.hpp
index 67bd9891..cbff223e 100644
--- a/bfps/cpp/vorticity_equation.hpp
+++ b/bfps/cpp/vorticity_equation.hpp
@@ -28,7 +28,6 @@
#include <iostream>
#include "field.hpp"
-#include "field_descriptor.hpp"
#ifndef VORTICITY_EQUATION
--
GitLab
From 75ff20540c5ab04d1185100ffd4c9f0ef8457f7d Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 12 Sep 2018 14:51:42 +0200
Subject: [PATCH 223/342] update installation instructions
---
README.rst | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 58 insertions(+), 1 deletion(-)
diff --git a/README.rst b/README.rst
index ddb9f244..5418d711 100644
--- a/README.rst
+++ b/README.rst
@@ -60,7 +60,8 @@ Use a console; navigate to the ``bfps`` folder, and type:
If you want to run simulations on the machine where you're installing,
you will need to call `compile_library` before installing.
Your machine needs to have an MPI compiler installed, the HDF5 C library
-and FFTW >= 3.4.
+and FFTW >= 3.4 --- a detailed prerequisite installation list is
+included at the end of this document.
The file `machine_settings_py.py` should be modified
appropriately for your machine (otherwise the `compile_library` command will most
likely fail).
@@ -102,7 +103,63 @@ Comments
* particles: initialization of multistep solvers is done with lower
order methods, so direct convergence tests will fail.
+* Code is only known to work with HDF5 1.8.x.
+
* Code is used mainly with Python 3.4 and 3.5.
In principle it should be easy to maintain compatibility with Python
2.7.x, but as of `bfps 1.8` this is no longer a main concern.
+-------------------------------
+Installation with prerequisites
+-------------------------------
+
+These installation steps assume that you have a working MPI compiler,
+properly configured on your system (i.e. the various configure scripts
+are able to find it).
+If this is not the case, please consult the FFTW and HDF5 compilation
+instructions for detailed instructions (`./configure --help` should be
+enough).
+
+1. Make directory PREFIX on local fast partition.
+
+2. Download, compile, install FFTW (latest version 3.x from http://www.fftw.org/).
+ Execute the following commands in order, feel free to customize
+ optimisation flags for your own computer:
+
+.. code:: bash
+ ./configure --prefix=PREFIX --enable-single --enable-sse --enable-mpi --enable-openmp --enable-threads
+ make
+ make install
+ ./configure --prefix=PREFIX --enable-sse --enable-sse2 --enable-mpi --enable-openmp --enable-threads
+ make
+ make install
+
+3. Download, compile, install HDF5 (version 1.8.x, currently available
+ at https://support.hdfgroup.org/HDF5/release/obtainsrc518.html.
+ We are using parallel I/O, therefore we use the plain C interface of HDF5:
+
+.. code:: bash
+ ./configure --prefix=PREFIX --enable-parallel
+ make
+ make install
+
+3. This step may be ommited.
+ I recommend the creation of a virtual python3 environment (also under PREFIX) that will be used for installing bfps and dependencies.
+ Please see https://docs.python-guide.org/dev/virtualenvs/.
+
+4. Clone bfps repository.
+
+.. code:: bash
+ git clone git@gitlab.mpcdf.mpg.de:clalescu/bfps.git
+
+5. Tweak host_information.py and machine_settings.py for your user and your machine and place under ~/.config/bfps.
+
+6. Activate virtual environment.
+
+7. Go into bfps repository, execute
+
+.. code:: bash
+
+ python setup.py compile_library
+ python setup.py install
+
--
GitLab
From f06df48595d34cf61f49f88d66e61637ac46113b Mon Sep 17 00:00:00 2001
From: Jose Agustin Arguedas Leiva <arguedas@telica.nld.ds.mpg.de>
Date: Wed, 12 Sep 2018 16:29:32 +0200
Subject: [PATCH 224/342] debugging: orientation does not change
---
bfps/cpp/full_code/NSVEcomplex_particles.cpp | 6 +++++-
bfps/cpp/particles/particles_system.hpp | 2 +-
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/bfps/cpp/full_code/NSVEcomplex_particles.cpp b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
index 4f0ccb50..364827cf 100644
--- a/bfps/cpp/full_code/NSVEcomplex_particles.cpp
+++ b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
@@ -45,6 +45,7 @@ int NSVEcomplex_particles<rnumber>::initialize(void)
this->enable_inner = false;
current_particles_inner_computer.setEnable(enable_inner);
+
this->ps = particles_system_builder_with_p2p(
this->fs->cvelocity, // (field object)
this->fs->kk, // (kspace object, contains dkx, dky, dkz)
@@ -237,7 +238,10 @@ int NSVEcomplex_particles<rnumber>::read_parameters(void)
this->tracers0_smoothness = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_smoothness");
this->enable_p2p = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_enable_p2p");
this->enable_inner = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_enable_inner");
- this->enable_vorticity_omega = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_enable_vorticity_omega");
+ int tval = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_enable_vorticity_omega");
+ this->enable_vorticity_omega = tval;
+ DEBUG_MSG("tracers0_enable_vorticity_omega = %d, this->enable_vorticity_omega = %d\n",
+ tval, this->enable_vorticity_omega);
this->cutoff = hdf5_tools::read_value<double>(parameter_file, "parameters/tracers0_cutoff");
this->inner_v0 = hdf5_tools::read_value<double>(parameter_file, "parameters/tracers0_inner_v0");
H5Fclose(parameter_file);
diff --git a/bfps/cpp/particles/particles_system.hpp b/bfps/cpp/particles/particles_system.hpp
index db2f33fa..e451f56a 100644
--- a/bfps/cpp/particles/particles_system.hpp
+++ b/bfps/cpp/particles/particles_system.hpp
@@ -278,7 +278,7 @@ public:
void completeLoopWithVorticity(const real_number dt,
const real_number particle_extra_rhs[]) final {
- TIMEZONE("particles_system::completeLoop");
+ TIMEZONE("particles_system::completeLoopWithVorticity");
compute();
compute_p2p();
compute_particles_inner(particle_extra_rhs);
--
GitLab
From c8310a19165f6fb9ed21c7df470cc343d15cb294 Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Thu, 13 Sep 2018 00:06:57 +0200
Subject: [PATCH 225/342] actually sample vorticity for rotations
also, remove hardcoded lack of activity/rotation.
---
bfps/cpp/full_code/NSVEcomplex_particles.cpp | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/bfps/cpp/full_code/NSVEcomplex_particles.cpp b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
index 364827cf..9b910e9b 100644
--- a/bfps/cpp/full_code/NSVEcomplex_particles.cpp
+++ b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
@@ -42,7 +42,6 @@ int NSVEcomplex_particles<rnumber>::initialize(void)
current_p2p_computer.setEnable(this->enable_p2p);
particles_inner_computer<double, long long int> current_particles_inner_computer(inner_v0);
- this->enable_inner = false;
current_particles_inner_computer.setEnable(enable_inner);
@@ -94,7 +93,11 @@ int NSVEcomplex_particles<rnumber>::step(void)
if(this->enable_vorticity_omega){
*this->tmp_vec_field = this->fs->cvorticity->get_cdata();
this->tmp_vec_field->ift();
- this->ps->completeLoopWithVorticity(this->dt, *this->tmp_vec_field);
+ std::unique_ptr<double[]> sampled_vorticity(new double[3*this->ps->getLocalNbParticles()]);
+ std::fill_n(sampled_vorticity.get(), 3*this->ps->getLocalNbParticles(), 0);
+ this->ps->sample_compute_field(*this->tmp_vec_field, sampled_vorticity.get());
+ DEBUG_MSG("sampled vorticity is %g\n", sampled_vorticity[0]);
+ this->ps->completeLoopWithVorticity(this->dt, sampled_vorticity.get());
}
else{
this->ps->completeLoop(this->dt);
--
GitLab
From ec9261527fed261cc0a17533d3b0264f90ce97f7 Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Fri, 14 Sep 2018 07:13:53 +0200
Subject: [PATCH 226/342] decide to compile based on actual dependency list
---
cpp_build.py | 9 +++++++++
setup.py | 25 ++++++++++++++++++-------
2 files changed, 27 insertions(+), 7 deletions(-)
diff --git a/cpp_build.py b/cpp_build.py
index a312191a..39371214 100644
--- a/cpp_build.py
+++ b/cpp_build.py
@@ -65,6 +65,15 @@ src_file_list = ['hdf5_tools',
'Lagrange_polys',
'scope_timer']
+def get_file_dependency_list(src_file):
+ p = subprocess.Popen(
+ ['g++', '-Ibfps/cpp', '-MM', 'bfps/cpp/' + src_file + '.cpp'],
+ stdout = subprocess.PIPE)
+ out, err = p.communicate()
+ p.terminate()
+ deps = str(out, 'ASCII').replace('\\\n', '')
+ return deps
+
def get_dependency_list():
ofile = open('dependencies.txt', 'w')
for src_file in src_file_list:
diff --git a/setup.py b/setup.py
index b427ebe7..110f18d5 100644
--- a/setup.py
+++ b/setup.py
@@ -215,13 +215,10 @@ class CompileLibCommand(distutils.cmd.Command):
if not os.path.isfile('bfps/libbfps.a'):
need_to_compile = True
else:
+ need_to_compile = False
ofile = 'bfps/libbfps.a'
libtime = datetime.datetime.fromtimestamp(os.path.getctime(ofile))
latest = libtime
- for fname in header_list:
- latest = max(latest,
- datetime.datetime.fromtimestamp(os.path.getctime('bfps/' + fname)))
- need_to_compile = (latest > libtime)
eca = extra_compile_args
eca += ['-fPIC']
if self.timing_output:
@@ -238,9 +235,14 @@ class CompileLibCommand(distutils.cmd.Command):
if not os.path.exists(ofile):
need_to_compile_file = True
else:
- need_to_compile_file = (need_to_compile or
- (datetime.datetime.fromtimestamp(os.path.getctime(ofile)) <
- datetime.datetime.fromtimestamp(os.path.getctime(ifile))))
+ need_to_compile_file = False
+ if not need_to_compile:
+ latest = libtime
+ dependency_list = get_file_dependency_list(fname)
+ for depname in dependency_list.split()[1:]:
+ latest = max(latest,
+ datetime.datetime.fromtimestamp(os.path.getctime(depname)))
+ need_to_compile_file = (latest > libtime)
if need_to_compile_file:
command_strings = [compiler, '-c']
command_strings += ['bfps/cpp/' + fname + '.cpp']
@@ -269,6 +271,15 @@ class CompileLibCommand(distutils.cmd.Command):
protocol = 2)
return None
+def get_file_dependency_list(src_file):
+ p = subprocess.Popen(
+ ['g++', '-Ibfps/cpp', '-MM', 'bfps/cpp/' + src_file + '.cpp'],
+ stdout = subprocess.PIPE)
+ out, err = p.communicate()
+ p.terminate()
+ deps = str(out, 'ASCII').replace('\\\n', '')
+ return deps
+
from setuptools import setup
setup(
--
GitLab
From 0ff9f81cb1b5c3f9dab61b281a3bbabc91a90400 Mon Sep 17 00:00:00 2001
From: Cristian Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 19 Sep 2018 16:25:23 +0200
Subject: [PATCH 227/342] fix job script for SGE
I messed up the nprocesses/nthreads parameter
---
bfps/_code.py | 17 +++++++++--------
1 file changed, 9 insertions(+), 8 deletions(-)
diff --git a/bfps/_code.py b/bfps/_code.py
index fe7c35ab..fed603e3 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -271,7 +271,7 @@ class _code(_base):
qsub_script_name = 'run_' + suffix + '.sh'
self.write_sge_file(
file_name = os.path.join(self.work_dir, qsub_script_name),
- nprocesses = nb_processes*nb_threads_per_process,
+ nprocesses = nb_processes,
name_of_run = suffix,
command_atoms = command_atoms[5:],
hours = hours,
@@ -590,14 +590,15 @@ class _code(_base):
envprocs))
script_file.write('echo "got $NSLOTS slots."\n')
script_file.write('echo "Start time is `date`"\n')
- script_file.write('mpiexec -machinefile $TMPDIR/machines ' +
- '-genv OMP_NUM_THREADS={0} '.format(nb_threads_per_process) +
- '-genv OMP_PLACES=cores ' +
- '-genv LD_LIBRARY_PATH ' +
- '"' +
+ script_file.write('mpiexec \\\n' +
+ '\t-machinefile $TMPDIR/machines \\\n' +
+ '\t-genv OMP_NUM_THREADS={0} \\\n'.format(nb_threads_per_process) +
+ '\t-genv OMP_PLACES=cores \\\n' +
+ '\t-genv LD_LIBRARY_PATH \\\n' +
+ '\t"' +
':'.join([bfps.lib_dir] + bfps.install_info['library_dirs']) +
- '" ' +
- '-n {0} {1}\n'.format(nprocesses, ' '.join(command_atoms)))
+ '" \\\n' +
+ '\t-n {0} \\\n\t{1}\n'.format(nprocesses, ' '.join(command_atoms)))
script_file.write('echo "End time is `date`"\n')
script_file.write('exit 0\n')
script_file.close()
--
GitLab
From c4143bfe021ec3f9823f6f500bb4808f2fc172e0 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 20 Sep 2018 15:31:30 +0200
Subject: [PATCH 228/342] introduce parameter for fftw_plan_rigor
---
bfps/DNS.py | 1 +
bfps/PP.py | 1 +
bfps/TEST.py | 1 +
bfps/cpp/fftw_interface.hpp | 2 ++
bfps/cpp/fftw_tools.cpp | 7 +++++
bfps/cpp/fftw_tools.hpp | 2 ++
bfps/cpp/full_code/NSVE.cpp | 6 ++--
bfps/cpp/full_code/NSVE.hpp | 1 +
bfps/cpp/full_code/NSVE_field_stats.cpp | 4 ++-
bfps/cpp/full_code/NSVE_field_stats.hpp | 2 ++
bfps/cpp/full_code/field_output_test.cpp | 2 +-
bfps/cpp/full_code/field_test.cpp | 4 +--
bfps/cpp/full_code/filter_test.cpp | 2 +-
bfps/cpp/full_code/joint_acc_vel_stats.cpp | 2 +-
bfps/cpp/full_code/native_binary_to_hdf5.cpp | 2 +-
bfps/cpp/full_code/symmetrize_test.cpp | 6 ++--
bfps/cpp/full_code/symmetrize_test.hpp | 1 +
bfps/cpp/full_code/test_interpolation.cpp | 6 ++--
bfps/cpp/hdf5_tools.cpp | 31 +++++++++++++-------
setup.py | 2 +-
20 files changed, 59 insertions(+), 26 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index e6ace758..b22a309a 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -120,6 +120,7 @@ class DNS(_code):
return None
def generate_default_parameters(self):
# these parameters are relevant for all DNS classes
+ self.parameters['fftw_planner_type'] = 'FFTW_ESTIMATE'
self.parameters['dealias_type'] = int(1)
self.parameters['dkx'] = float(1.0)
self.parameters['dky'] = float(1.0)
diff --git a/bfps/PP.py b/bfps/PP.py
index 27a35928..867864e0 100644
--- a/bfps/PP.py
+++ b/bfps/PP.py
@@ -118,6 +118,7 @@ class PP(_code):
return None
def generate_default_parameters(self):
# these parameters are relevant for all PP classes
+ self.parameters['fftw_planner_type'] = 'FFTW_ESTIMATE'
self.parameters['dealias_type'] = int(1)
self.parameters['dkx'] = float(1.0)
self.parameters['dky'] = float(1.0)
diff --git a/bfps/TEST.py b/bfps/TEST.py
index cd4d3e4a..2a8b37ba 100644
--- a/bfps/TEST.py
+++ b/bfps/TEST.py
@@ -119,6 +119,7 @@ class TEST(_code):
return None
def generate_default_parameters(self):
# these parameters are relevant for all TEST classes
+ self.parameters['fftw_planner_type'] = 'FFTW_ESTIMATE'
self.parameters['dealias_type'] = int(1)
self.parameters['dkx'] = float(1.0)
self.parameters['dky'] = float(1.0)
diff --git a/bfps/cpp/fftw_interface.hpp b/bfps/cpp/fftw_interface.hpp
index 7f5e0144..0a840dd5 100644
--- a/bfps/cpp/fftw_interface.hpp
+++ b/bfps/cpp/fftw_interface.hpp
@@ -26,6 +26,8 @@
#define FFTW_INTERFACE_HPP
#include <fftw3-mpi.h>
+#include <map>
+#include <string>
#ifdef USE_FFTWESTIMATE
#define DEFAULT_FFTW_FLAG FFTW_ESTIMATE
diff --git a/bfps/cpp/fftw_tools.cpp b/bfps/cpp/fftw_tools.cpp
index 61e03d29..9f6e9bfd 100644
--- a/bfps/cpp/fftw_tools.cpp
+++ b/bfps/cpp/fftw_tools.cpp
@@ -31,6 +31,13 @@
#define NDEBUG
+std::map<std::string, unsigned> fftw_planner_string_to_flag = {
+ {"FFTW_ESTIMATE", FFTW_ESTIMATE},
+ {"FFTW_MEASURE", FFTW_MEASURE},
+ {"FFTW_PATIENT", FFTW_PATIENT},
+ {"parameter does not exist", DEFAULT_FFTW_FLAG},
+};
+
template <class rnumber>
int clip_zero_padding(
field_descriptor<rnumber> *f,
diff --git a/bfps/cpp/fftw_tools.hpp b/bfps/cpp/fftw_tools.hpp
index d0f3dbf3..f4480714 100644
--- a/bfps/cpp/fftw_tools.hpp
+++ b/bfps/cpp/fftw_tools.hpp
@@ -34,6 +34,8 @@
extern int myrank, nprocs;
+extern std::map<std::string, unsigned> fftw_planner_string_to_flag;
+
/* given two arrays of the same dimension, we do a simple resize in
* Fourier space: either chop off high modes, or pad with zeros.
* the arrays are assumed to use 3D mpi fftw layout.
diff --git a/bfps/cpp/full_code/NSVE.cpp b/bfps/cpp/full_code/NSVE.cpp
index d9cb72a2..74593b37 100644
--- a/bfps/cpp/full_code/NSVE.cpp
+++ b/bfps/cpp/full_code/NSVE.cpp
@@ -2,6 +2,7 @@
#include <cmath>
#include "NSVE.hpp"
#include "scope_timer.hpp"
+#include "fftw_tools.hpp"
template <typename rnumber>
@@ -37,11 +38,11 @@ int NSVE<rnumber>::initialize(void)
simname.c_str(),
nx, ny, nz,
dkx, dky, dkz,
- DEFAULT_FFTW_FLAG);
+ fftw_planner_string_to_flag[this->fftw_plan_type]);
this->tmp_vec_field = new field<rnumber, FFTW, THREE>(
nx, ny, nz,
this->comm,
- DEFAULT_FFTW_FLAG);
+ fftw_planner_string_to_flag[this->fftw_plan_type]);
this->fs->checkpoints_per_file = checkpoints_per_file;
@@ -161,6 +162,7 @@ int NSVE<rnumber>::read_parameters(void)
this->max_vorticity_estimate = hdf5_tools::read_value<double>(parameter_file, "parameters/max_vorticity_estimate");
std::string tmp = hdf5_tools::read_string(parameter_file, "parameters/forcing_type");
snprintf(this->forcing_type, 511, "%s", tmp.c_str());
+ this->fftw_plan_type = hdf5_tools::read_string(parameter_file, "parameters/fftw_plan_type");
H5Fclose(parameter_file);
return EXIT_SUCCESS;
}
diff --git a/bfps/cpp/full_code/NSVE.hpp b/bfps/cpp/full_code/NSVE.hpp
index 062627fd..c3ad7a0a 100644
--- a/bfps/cpp/full_code/NSVE.hpp
+++ b/bfps/cpp/full_code/NSVE.hpp
@@ -53,6 +53,7 @@ class NSVE: public direct_numerical_simulation
double max_velocity_estimate;
double max_vorticity_estimate;
double nu;
+ std::string fftw_plan_type;
/* other stuff */
vorticity_equation<rnumber, FFTW> *fs;
diff --git a/bfps/cpp/full_code/NSVE_field_stats.cpp b/bfps/cpp/full_code/NSVE_field_stats.cpp
index 15980a20..142c51e7 100644
--- a/bfps/cpp/full_code/NSVE_field_stats.cpp
+++ b/bfps/cpp/full_code/NSVE_field_stats.cpp
@@ -1,6 +1,7 @@
#include <string>
#include <cmath>
#include "NSVE_field_stats.hpp"
+#include "fftw_tools.hpp"
#include "scope_timer.hpp"
@@ -12,7 +13,7 @@ int NSVE_field_stats<rnumber>::initialize(void)
this->vorticity = new field<rnumber, FFTW, THREE>(
nx, ny, nz,
this->comm,
- DEFAULT_FFTW_FLAG);
+ fftw_planner_string_to_flag[this->fftw_plan_type]);
this->vorticity->real_space_representation = false;
hid_t parameter_file = H5Fopen(
(this->simname + std::string(".h5")).c_str(),
@@ -43,6 +44,7 @@ int NSVE_field_stats<rnumber>::initialize(void)
this->vorticity->clayout->starts,
this->vorticity->clayout->comm);
}
+ this->fftw_plan_type = hdf5_tools::read_string(parameter_file, "parameters/fftw_plan_type");
H5Fclose(parameter_file);
return EXIT_SUCCESS;
}
diff --git a/bfps/cpp/full_code/NSVE_field_stats.hpp b/bfps/cpp/full_code/NSVE_field_stats.hpp
index d544c0c7..ae519fc7 100644
--- a/bfps/cpp/full_code/NSVE_field_stats.hpp
+++ b/bfps/cpp/full_code/NSVE_field_stats.hpp
@@ -42,6 +42,8 @@ class NSVE_field_stats: public postprocess
private:
field_binary_IO<rnumber, COMPLEX, THREE> *bin_IO;
public:
+ std::string fftw_plan_type;
+
field<rnumber, FFTW, THREE> *vorticity;
NSVE_field_stats(
diff --git a/bfps/cpp/full_code/field_output_test.cpp b/bfps/cpp/full_code/field_output_test.cpp
index 30df4e75..72406099 100644
--- a/bfps/cpp/full_code/field_output_test.cpp
+++ b/bfps/cpp/full_code/field_output_test.cpp
@@ -36,7 +36,7 @@ int field_output_test<rnumber>::do_work(void)
field<rnumber, FFTW, ONE> *scal_field = new field<rnumber, FFTW, ONE>(
this->nx, this->ny, this->nz,
this->comm,
- DEFAULT_FFTW_FLAG);
+ FFTW_ESTIMATE);
std::default_random_engine rgen;
std::normal_distribution<rnumber> rdist;
rgen.seed(1);
diff --git a/bfps/cpp/full_code/field_test.cpp b/bfps/cpp/full_code/field_test.cpp
index 1627bc40..a9d531bc 100644
--- a/bfps/cpp/full_code/field_test.cpp
+++ b/bfps/cpp/full_code/field_test.cpp
@@ -44,11 +44,11 @@ int field_test<rnumber>::do_work(void)
field<rnumber, FFTW, ONE> *scal_field = new field<rnumber, FFTW, ONE>(
this->nx, this->ny, this->nz,
this->comm,
- DEFAULT_FFTW_FLAG);
+ FFTW_ESTIMATE);
field<rnumber, FFTW, ONE> *scal_field_alt = new field<rnumber, FFTW, ONE>(
this->nx, this->ny, this->nz,
this->comm,
- DEFAULT_FFTW_FLAG);
+ FFTW_ESTIMATE);
std::default_random_engine rgen;
std::normal_distribution<rnumber> rdist;
rgen.seed(2);
diff --git a/bfps/cpp/full_code/filter_test.cpp b/bfps/cpp/full_code/filter_test.cpp
index 4db13843..6dbd05a9 100644
--- a/bfps/cpp/full_code/filter_test.cpp
+++ b/bfps/cpp/full_code/filter_test.cpp
@@ -12,7 +12,7 @@ int filter_test<rnumber>::initialize(void)
this->scal_field = new field<rnumber, FFTW, ONE>(
nx, ny, nz,
this->comm,
- DEFAULT_FFTW_FLAG);
+ FFTW_ESTIMATE);
this->kk = new kspace<FFTW, SMOOTH>(
this->scal_field->clayout, this->dkx, this->dky, this->dkz);
diff --git a/bfps/cpp/full_code/joint_acc_vel_stats.cpp b/bfps/cpp/full_code/joint_acc_vel_stats.cpp
index 1c28527e..fff2e2f5 100644
--- a/bfps/cpp/full_code/joint_acc_vel_stats.cpp
+++ b/bfps/cpp/full_code/joint_acc_vel_stats.cpp
@@ -110,7 +110,7 @@ int joint_acc_vel_stats<rnumber>::work_on_current_iteration(void)
vel = new field<rnumber, FFTW, THREE>(
this->nx, this->ny, this->nz,
this->comm,
- DEFAULT_FFTW_FLAG);
+ this->vorticity->fftw_plan_rigor);
invert_curl(kk, this->ve->cvorticity, vel);
vel->ift();
diff --git a/bfps/cpp/full_code/native_binary_to_hdf5.cpp b/bfps/cpp/full_code/native_binary_to_hdf5.cpp
index fb5a39c2..fe8e1c41 100644
--- a/bfps/cpp/full_code/native_binary_to_hdf5.cpp
+++ b/bfps/cpp/full_code/native_binary_to_hdf5.cpp
@@ -12,7 +12,7 @@ int native_binary_to_hdf5<rnumber>::initialize(void)
this->vec_field = new field<rnumber, FFTW, THREE>(
nx, ny, nz,
this->comm,
- DEFAULT_FFTW_FLAG);
+ FFTW_ESTIMATE);
this->vec_field->real_space_representation = false;
this->bin_IO = new field_binary_IO<rnumber, COMPLEX, THREE>(
this->vec_field->clayout->sizes,
diff --git a/bfps/cpp/full_code/symmetrize_test.cpp b/bfps/cpp/full_code/symmetrize_test.cpp
index 821161da..45225fcb 100644
--- a/bfps/cpp/full_code/symmetrize_test.cpp
+++ b/bfps/cpp/full_code/symmetrize_test.cpp
@@ -2,6 +2,7 @@
#include <cmath>
#include <random>
#include "symmetrize_test.hpp"
+#include "fftw_tools.hpp"
#include "scope_timer.hpp"
@@ -31,6 +32,7 @@ int symmetrize_test<rnumber>::read_parameters()
H5P_DEFAULT);
this->random_seed = hdf5_tools::read_value<int>(
parameter_file, "/parameters/random_seed");
+ this->fftw_plan_type = hdf5_tools::read_string(parameter_file, "parameters/fftw_plan_type");
H5Fclose(parameter_file);
return EXIT_SUCCESS;
}
@@ -44,13 +46,13 @@ int symmetrize_test<rnumber>::do_work(void)
field<rnumber, FFTW, THREE> *test_field0 = new field<rnumber, FFTW, THREE>(
this->nx, this->ny, this->nz,
this->comm,
- DEFAULT_FFTW_FLAG);
+ fftw_planner_string_to_flag[this->fftw_plan_type]);
DEBUG_MSG("finished allocating field0\n");
DEBUG_MSG("about to allocate field1\n");
field<rnumber, FFTW, THREE> *test_field1 = new field<rnumber, FFTW, THREE>(
this->nx, this->ny, this->nz,
this->comm,
- DEFAULT_FFTW_FLAG);
+ fftw_planner_string_to_flag[this->fftw_plan_type]);
DEBUG_MSG("finished allocating field1\n");
std::default_random_engine rgen;
std::normal_distribution<rnumber> rdist;
diff --git a/bfps/cpp/full_code/symmetrize_test.hpp b/bfps/cpp/full_code/symmetrize_test.hpp
index d3fbbaeb..f674d365 100644
--- a/bfps/cpp/full_code/symmetrize_test.hpp
+++ b/bfps/cpp/full_code/symmetrize_test.hpp
@@ -42,6 +42,7 @@ template <typename rnumber>
class symmetrize_test: public test
{
public:
+ std::string fftw_plan_type;
int random_seed;
symmetrize_test(
diff --git a/bfps/cpp/full_code/test_interpolation.cpp b/bfps/cpp/full_code/test_interpolation.cpp
index 5ef11de4..2acd3c27 100644
--- a/bfps/cpp/full_code/test_interpolation.cpp
+++ b/bfps/cpp/full_code/test_interpolation.cpp
@@ -30,18 +30,18 @@ int test_interpolation<rnumber>::initialize(void)
this->vorticity = new field<rnumber, FFTW, THREE>(
this->nx, this->ny, this->nz,
this->comm,
- DEFAULT_FFTW_FLAG);
+ FFTW_ESTIMATE);
this->vorticity->real_space_representation = false;
this->velocity = new field<rnumber, FFTW, THREE>(
this->nx, this->ny, this->nz,
this->comm,
- DEFAULT_FFTW_FLAG);
+ FFTW_ESTIMATE);
this->nabla_u = new field<rnumber, FFTW, THREExTHREE>(
this->nx, this->ny, this->nz,
this->comm,
- DEFAULT_FFTW_FLAG);
+ FFTW_ESTIMATE);
this->kk = new kspace<FFTW, SMOOTH>(
this->vorticity->clayout, this->dkx, this->dky, this->dkz);
diff --git a/bfps/cpp/hdf5_tools.cpp b/bfps/cpp/hdf5_tools.cpp
index c2ef6aae..25acaf21 100644
--- a/bfps/cpp/hdf5_tools.cpp
+++ b/bfps/cpp/hdf5_tools.cpp
@@ -208,17 +208,26 @@ std::string hdf5_tools::read_string(
const hid_t group,
const std::string dset_name)
{
- hid_t dset = H5Dopen(group, dset_name.c_str(), H5P_DEFAULT);
- hid_t space = H5Dget_space(dset);
- hid_t memtype = H5Dget_type(dset);
- char *string_data = (char*)malloc(256);
- H5Dread(dset, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &string_data);
- std::string std_string_data = std::string(string_data);
- free(string_data);
- H5Sclose(space);
- H5Tclose(memtype);
- H5Dclose(dset);
- return std_string_data;
+ if (H5Lexists(group, dset_name.c_str(), H5P_DEFAULT))
+ {
+ hid_t dset = H5Dopen(group, dset_name.c_str(), H5P_DEFAULT);
+ hid_t space = H5Dget_space(dset);
+ hid_t memtype = H5Dget_type(dset);
+ char *string_data = (char*)malloc(256);
+ H5Dread(dset, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &string_data);
+ std::string std_string_data = std::string(string_data);
+ free(string_data);
+ H5Sclose(space);
+ H5Tclose(memtype);
+ H5Dclose(dset);
+ return std_string_data;
+ }
+ else
+ {
+ DEBUG_MSG("attempted to read dataset %s which does not exist.\n",
+ dset_name.c_str());
+ return std::string("parameter does not exist");
+ }
}
template
diff --git a/setup.py b/setup.py
index 110f18d5..afb25f70 100644
--- a/setup.py
+++ b/setup.py
@@ -273,7 +273,7 @@ class CompileLibCommand(distutils.cmd.Command):
def get_file_dependency_list(src_file):
p = subprocess.Popen(
- ['g++', '-Ibfps/cpp', '-MM', 'bfps/cpp/' + src_file + '.cpp'],
+ ['g++', '-std=c++11', '-Ibfps/cpp', '-MM', 'bfps/cpp/' + src_file + '.cpp'],
stdout = subprocess.PIPE)
out, err = p.communicate()
p.terminate()
--
GitLab
From 40c8fad157181d3638b2affec45b8a6ba645135c Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 21 Sep 2018 09:52:20 +0200
Subject: [PATCH 229/342] remove old files
---
bfps/cpp/distributed_particles.cpp | 472 ----------
bfps/cpp/distributed_particles.hpp | 105 ---
bfps/cpp/fftw_tools.cpp | 189 ----
bfps/cpp/fftw_tools.hpp | 32 -
bfps/cpp/fluid_solver.cpp | 1057 ----------------------
bfps/cpp/fluid_solver.hpp | 120 ---
bfps/cpp/fluid_solver_base.cpp | 834 -----------------
bfps/cpp/fluid_solver_base.hpp | 272 ------
bfps/cpp/interpolator.cpp | 214 -----
bfps/cpp/interpolator.hpp | 79 --
bfps/cpp/interpolator_base.cpp | 113 ---
bfps/cpp/interpolator_base.hpp | 114 ---
bfps/cpp/rFFTW_distributed_particles.cpp | 804 ----------------
bfps/cpp/rFFTW_distributed_particles.hpp | 144 ---
bfps/cpp/rFFTW_interpolator.cpp | 210 -----
bfps/cpp/rFFTW_interpolator.hpp | 118 ---
bfps/cpp/slab_field_particles.cpp | 799 ----------------
bfps/cpp/slab_field_particles.hpp | 149 ---
bfps/cpp/tracers.cpp | 204 -----
bfps/cpp/tracers.hpp | 63 --
setup.py | 2 -
21 files changed, 6094 deletions(-)
delete mode 100644 bfps/cpp/distributed_particles.cpp
delete mode 100644 bfps/cpp/distributed_particles.hpp
delete mode 100644 bfps/cpp/fluid_solver.cpp
delete mode 100644 bfps/cpp/fluid_solver.hpp
delete mode 100644 bfps/cpp/fluid_solver_base.cpp
delete mode 100644 bfps/cpp/fluid_solver_base.hpp
delete mode 100644 bfps/cpp/interpolator.cpp
delete mode 100644 bfps/cpp/interpolator.hpp
delete mode 100644 bfps/cpp/interpolator_base.cpp
delete mode 100644 bfps/cpp/interpolator_base.hpp
delete mode 100644 bfps/cpp/rFFTW_distributed_particles.cpp
delete mode 100644 bfps/cpp/rFFTW_distributed_particles.hpp
delete mode 100644 bfps/cpp/rFFTW_interpolator.cpp
delete mode 100644 bfps/cpp/rFFTW_interpolator.hpp
delete mode 100644 bfps/cpp/slab_field_particles.cpp
delete mode 100644 bfps/cpp/slab_field_particles.hpp
delete mode 100644 bfps/cpp/tracers.cpp
delete mode 100644 bfps/cpp/tracers.hpp
diff --git a/bfps/cpp/distributed_particles.cpp b/bfps/cpp/distributed_particles.cpp
deleted file mode 100644
index 73fd0275..00000000
--- a/bfps/cpp/distributed_particles.cpp
+++ /dev/null
@@ -1,472 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-//#define NDEBUG
-
-#include <cmath>
-#include <cassert>
-#include <cstring>
-#include <string>
-#include <sstream>
-#include <array>
-
-#include "base.hpp"
-#include "distributed_particles.hpp"
-#include "fftw_tools.hpp"
-#include "scope_timer.hpp"
-
-
-extern int myrank, nprocs;
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-distributed_particles<particle_type, rnumber, interp_neighbours>::distributed_particles(
- const char *NAME,
- const hid_t data_file_id,
- interpolator<rnumber, interp_neighbours> *VEL,
- const int TRAJ_SKIP,
- const int INTEGRATION_STEPS) : particles_io_base<particle_type>(
- NAME,
- TRAJ_SKIP,
- data_file_id,
- VEL->descriptor->comm)
-{
- assert((INTEGRATION_STEPS <= 6) &&
- (INTEGRATION_STEPS >= 1));
- this->vel = VEL;
- this->rhs.resize(INTEGRATION_STEPS);
- this->integration_steps = INTEGRATION_STEPS;
- this->state.reserve(2*this->nparticles / this->nprocs);
- for (unsigned int i=0; i<this->rhs.size(); i++)
- this->rhs[i].reserve(2*this->nparticles / this->nprocs);
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-distributed_particles<particle_type, rnumber, interp_neighbours>::~distributed_particles()
-{
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void distributed_particles<particle_type, rnumber, interp_neighbours>::sample(
- interpolator<rnumber, interp_neighbours> *field,
- const std::unordered_map<int, single_particle_state<particle_type>> &x,
- std::unordered_map<int, single_particle_state<POINT3D>> &y)
-{
- std::array<double, 3> yy;
- y.clear();
- for (auto &pp: x)
- {
- (*field)(pp.second.data, &yy.front());
- y[pp.first] = &yy.front();
- }
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void distributed_particles<particle_type, rnumber, interp_neighbours>::get_rhs(
- const std::unordered_map<int, single_particle_state<particle_type>> &x,
- std::unordered_map<int, single_particle_state<particle_type>> &y)
-{
- std::unordered_map<int, single_particle_state<POINT3D>> yy;
- switch(particle_type)
- {
- case VELOCITY_TRACER:
- this->sample(this->vel, this->state, yy);
- y.clear();
- for (auto &pp: x)
- y[pp.first] = yy[pp.first].data;
- break;
- }
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void distributed_particles<particle_type, rnumber, interp_neighbours>::sample(
- interpolator<rnumber, interp_neighbours> *field,
- const char *dset_name)
-{
- std::unordered_map<int, single_particle_state<POINT3D>> y;
- this->sample(field, this->state, y);
- this->write(dset_name, y);
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void distributed_particles<particle_type, rnumber, interp_neighbours>::roll_rhs()
-{
- for (int i=this->integration_steps-2; i>=0; i--)
- rhs[i+1] = rhs[i];
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void distributed_particles<particle_type, rnumber, interp_neighbours>::redistribute(
- std::unordered_map<int, single_particle_state<particle_type>> &x,
- std::vector<std::unordered_map<int, single_particle_state<particle_type>>> &vals)
-{
- TIMEZONE("distributed_particles::redistribute");
- //DEBUG_MSG("entered redistribute\n");
- /* neighbouring rank offsets */
- int ro[2];
- ro[0] = -1;
- ro[1] = 1;
- /* neighbouring ranks */
- int nr[2];
- nr[0] = MOD(this->myrank+ro[0], this->nprocs);
- nr[1] = MOD(this->myrank+ro[1], this->nprocs);
- /* particles to send, particles to receive */
- std::vector<int> ps[2], pr[2];
- /* number of particles to send, number of particles to receive */
- int nps[2], npr[2];
- int rsrc, rdst;
- /* get list of id-s to send */
- for (auto &pp: x)
- for (unsigned int i=0; i<2; i++)
- if (this->vel->get_rank(pp.second.data[2]) == nr[i])
- ps[i].push_back(pp.first);
- /* prepare data for send recv */
- for (unsigned int i=0; i<2; i++)
- nps[i] = ps[i].size();
- for (rsrc = 0; rsrc<this->nprocs; rsrc++)
- for (unsigned int i=0; i<2; i++)
- {
- rdst = MOD(rsrc+ro[i], this->nprocs);
- if (this->myrank == rsrc)
- MPI_Send(
- nps+i,
- 1,
- MPI_INTEGER,
- rdst,
- 2*(rsrc*this->nprocs + rdst)+i,
- this->comm);
- if (this->myrank == rdst)
- MPI_Recv(
- npr+1-i,
- 1,
- MPI_INTEGER,
- rsrc,
- 2*(rsrc*this->nprocs + rdst)+i,
- this->comm,
- MPI_STATUS_IGNORE);
- }
- //DEBUG_MSG("I have to send %d %d particles\n", nps[0], nps[1]);
- //DEBUG_MSG("I have to recv %d %d particles\n", npr[0], npr[1]);
- for (unsigned int i=0; i<2; i++)
- pr[i].resize(npr[i]);
-
- int buffer_size = (nps[0] > nps[1]) ? nps[0] : nps[1];
- buffer_size = (buffer_size > npr[0])? buffer_size : npr[0];
- buffer_size = (buffer_size > npr[1])? buffer_size : npr[1];
- //DEBUG_MSG("buffer size is %d\n", buffer_size);
- double *buffer = new double[buffer_size*state_dimension(particle_type)*(1+vals.size())];
- for (rsrc = 0; rsrc<this->nprocs; rsrc++)
- for (unsigned int i=0; i<2; i++)
- {
- rdst = MOD(rsrc+ro[i], this->nprocs);
- if (this->myrank == rsrc && nps[i] > 0)
- {
- MPI_Send(
- &ps[i].front(),
- nps[i],
- MPI_INTEGER,
- rdst,
- 2*(rsrc*this->nprocs + rdst),
- this->comm);
- int pcounter = 0;
- for (int p: ps[i])
- {
- std::copy(x[p].data,
- x[p].data + state_dimension(particle_type),
- buffer + pcounter*(1+vals.size())*state_dimension(particle_type));
- x.erase(p);
- for (unsigned int tindex=0; tindex<vals.size(); tindex++)
- {
- std::copy(vals[tindex][p].data,
- vals[tindex][p].data + state_dimension(particle_type),
- buffer + (pcounter*(1+vals.size()) + tindex+1)*state_dimension(particle_type));
- vals[tindex].erase(p);
- }
- pcounter++;
- }
- MPI_Send(
- buffer,
- nps[i]*(1+vals.size())*state_dimension(particle_type),
- MPI_DOUBLE,
- rdst,
- 2*(rsrc*this->nprocs + rdst)+1,
- this->comm);
- }
- if (this->myrank == rdst && npr[1-i] > 0)
- {
- MPI_Recv(
- &pr[1-i].front(),
- npr[1-i],
- MPI_INTEGER,
- rsrc,
- 2*(rsrc*this->nprocs + rdst),
- this->comm,
- MPI_STATUS_IGNORE);
- MPI_Recv(
- buffer,
- npr[1-i]*(1+vals.size())*state_dimension(particle_type),
- MPI_DOUBLE,
- rsrc,
- 2*(rsrc*this->nprocs + rdst)+1,
- this->comm,
- MPI_STATUS_IGNORE);
- unsigned int pcounter = 0;
- for (int p: pr[1-i])
- {
- x[p] = buffer + (pcounter*(1+vals.size()))*state_dimension(particle_type);
- for (unsigned int tindex=0; tindex<vals.size(); tindex++)
- {
- vals[tindex][p] = buffer + (pcounter*(1+vals.size()) + tindex+1)*state_dimension(particle_type);
- }
- pcounter++;
- }
- }
- }
- delete[] buffer;
-
-
-#ifndef NDEBUG
- /* check that all particles at x are local */
- for (auto &pp: x)
- if (this->vel->get_rank(pp.second.data[2]) != this->myrank)
- {
- DEBUG_MSG("found particle %d with rank %d\n",
- pp.first,
- this->vel->get_rank(pp.second.data[2]));
- assert(false);
- }
-#endif
- //DEBUG_MSG("exiting redistribute\n");
-}
-
-
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void distributed_particles<particle_type, rnumber, interp_neighbours>::AdamsBashforth(
- const int nsteps)
-{
- this->get_rhs(this->state, this->rhs[0]);
- for (auto &pp: this->state)
- for (unsigned int i=0; i<state_dimension(particle_type); i++)
- switch(nsteps)
- {
- case 1:
- pp.second[i] += this->dt*this->rhs[0][pp.first][i];
- break;
- case 2:
- pp.second[i] += this->dt*(3*this->rhs[0][pp.first][i]
- - this->rhs[1][pp.first][i])/2;
- break;
- case 3:
- pp.second[i] += this->dt*(23*this->rhs[0][pp.first][i]
- - 16*this->rhs[1][pp.first][i]
- + 5*this->rhs[2][pp.first][i])/12;
- break;
- case 4:
- pp.second[i] += this->dt*(55*this->rhs[0][pp.first][i]
- - 59*this->rhs[1][pp.first][i]
- + 37*this->rhs[2][pp.first][i]
- - 9*this->rhs[3][pp.first][i])/24;
- break;
- case 5:
- pp.second[i] += this->dt*(1901*this->rhs[0][pp.first][i]
- - 2774*this->rhs[1][pp.first][i]
- + 2616*this->rhs[2][pp.first][i]
- - 1274*this->rhs[3][pp.first][i]
- + 251*this->rhs[4][pp.first][i])/720;
- break;
- case 6:
- pp.second[i] += this->dt*(4277*this->rhs[0][pp.first][i]
- - 7923*this->rhs[1][pp.first][i]
- + 9982*this->rhs[2][pp.first][i]
- - 7298*this->rhs[3][pp.first][i]
- + 2877*this->rhs[4][pp.first][i]
- - 475*this->rhs[5][pp.first][i])/1440;
- break;
- }
- this->redistribute(this->state, this->rhs);
- this->roll_rhs();
-}
-
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void distributed_particles<particle_type, rnumber, interp_neighbours>::step()
-{
- TIMEZONE("distributed_particles::step");
- this->AdamsBashforth((this->iteration < this->integration_steps) ?
- this->iteration+1 :
- this->integration_steps);
- this->iteration++;
-}
-
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void distributed_particles<particle_type, rnumber, interp_neighbours>::read()
-{
- double *temp = new double[this->chunk_size*state_dimension(particle_type)];
- for (unsigned int cindex=0; cindex<this->get_number_of_chunks(); cindex++)
- {
- //read state
- if (this->myrank == 0)
- this->read_state_chunk(cindex, temp);
- MPI_Bcast(
- temp,
- this->chunk_size*state_dimension(particle_type),
- MPI_DOUBLE,
- 0,
- this->comm);
- for (unsigned int p=0; p<this->chunk_size; p++)
- {
- if (this->vel->get_rank(temp[state_dimension(particle_type)*p+2]) == this->myrank)
- this->state[p+cindex*this->chunk_size] = temp + state_dimension(particle_type)*p;
- }
- //read rhs
- if (this->iteration > 0)
- for (int i=0; i<this->integration_steps; i++)
- {
- if (this->myrank == 0)
- this->read_rhs_chunk(cindex, i, temp);
- MPI_Bcast(
- temp,
- this->chunk_size*state_dimension(particle_type),
- MPI_DOUBLE,
- 0,
- this->comm);
- for (unsigned int p=0; p<this->chunk_size; p++)
- {
- auto pp = this->state.find(p+cindex*this->chunk_size);
- if (pp != this->state.end())
- this->rhs[i][p+cindex*this->chunk_size] = temp + state_dimension(particle_type)*p;
- }
- }
- }
- DEBUG_MSG("%s->state.size = %ld\n", this->name.c_str(), this->state.size());
- delete[] temp;
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void distributed_particles<particle_type, rnumber, interp_neighbours>::write(
- const char *dset_name,
- std::unordered_map<int, single_particle_state<POINT3D>> &y)
-{
- TIMEZONE("distributed_particles::write");
- double *data = new double[this->nparticles*3];
- double *yy = new double[this->nparticles*3];
- for (unsigned int cindex=0; cindex<this->get_number_of_chunks(); cindex++)
- {
- std::fill_n(yy, this->chunk_size*3, 0);
- for (unsigned int p=0; p<this->chunk_size; p++)
- {
- auto pp = y.find(p+cindex*this->chunk_size);
- if (pp != y.end())
- std::copy(pp->second.data,
- pp->second.data + 3,
- yy + pp->first*3);
- }
- MPI_Allreduce(
- yy,
- data,
- 3*this->nparticles,
- MPI_DOUBLE,
- MPI_SUM,
- this->comm);
- if (this->myrank == 0)
- this->write_point3D_chunk(dset_name, cindex, data);
- }
- delete[] yy;
- delete[] data;
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void distributed_particles<particle_type, rnumber, interp_neighbours>::write(
- const bool write_rhs)
-{
- TIMEZONE("distributed_particles::write2");
- double *temp0 = new double[this->chunk_size*state_dimension(particle_type)];
- double *temp1 = new double[this->chunk_size*state_dimension(particle_type)];
- for (unsigned int cindex=0; cindex<this->get_number_of_chunks(); cindex++)
- {
- //write state
- std::fill_n(temp0, state_dimension(particle_type)*this->chunk_size, 0);
- for (unsigned int p=0; p<this->chunk_size; p++)
- {
- auto pp = this->state.find(p + cindex*this->chunk_size);
- if (pp != this->state.end())
- std::copy(pp->second.data,
- pp->second.data + state_dimension(particle_type),
- temp0 + p*state_dimension(particle_type));
- }
- MPI_Allreduce(
- temp0,
- temp1,
- state_dimension(particle_type)*this->chunk_size,
- MPI_DOUBLE,
- MPI_SUM,
- this->comm);
- if (this->myrank == 0)
- this->write_state_chunk(cindex, temp1);
- //write rhs
- if (write_rhs)
- for (int i=0; i<this->integration_steps; i++)
- {
- std::fill_n(temp0, state_dimension(particle_type)*this->chunk_size, 0);
- for (unsigned int p=0; p<this->chunk_size; p++)
- {
- auto pp = this->rhs[i].find(p + cindex*this->chunk_size);
- if (pp != this->rhs[i].end())
- std::copy(pp->second.data,
- pp->second.data + state_dimension(particle_type),
- temp0 + p*state_dimension(particle_type));
- }
- MPI_Allreduce(
- temp0,
- temp1,
- state_dimension(particle_type)*this->chunk_size,
- MPI_DOUBLE,
- MPI_SUM,
- this->comm);
- if (this->myrank == 0)
- this->write_rhs_chunk(cindex, i, temp1);
- }
- }
- delete[] temp0;
- delete[] temp1;
-}
-
-
-/*****************************************************************************/
-template class distributed_particles<VELOCITY_TRACER, float, 1>;
-template class distributed_particles<VELOCITY_TRACER, float, 2>;
-template class distributed_particles<VELOCITY_TRACER, float, 3>;
-template class distributed_particles<VELOCITY_TRACER, float, 4>;
-template class distributed_particles<VELOCITY_TRACER, float, 5>;
-template class distributed_particles<VELOCITY_TRACER, float, 6>;
-template class distributed_particles<VELOCITY_TRACER, double, 1>;
-template class distributed_particles<VELOCITY_TRACER, double, 2>;
-template class distributed_particles<VELOCITY_TRACER, double, 3>;
-template class distributed_particles<VELOCITY_TRACER, double, 4>;
-template class distributed_particles<VELOCITY_TRACER, double, 5>;
-template class distributed_particles<VELOCITY_TRACER, double, 6>;
-/*****************************************************************************/
diff --git a/bfps/cpp/distributed_particles.hpp b/bfps/cpp/distributed_particles.hpp
deleted file mode 100644
index cf6e124a..00000000
--- a/bfps/cpp/distributed_particles.hpp
+++ /dev/null
@@ -1,105 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <iostream>
-#include <unordered_map>
-#include <vector>
-#include <hdf5.h>
-#include "base.hpp"
-#include "particles_base.hpp"
-#include "fluid_solver_base.hpp"
-#include "interpolator.hpp"
-
-#ifndef DISTRIBUTED_PARTICLES
-
-#define DISTRIBUTED_PARTICLES
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-class distributed_particles: public particles_io_base<particle_type>
-{
- private:
- std::unordered_map<int, single_particle_state<particle_type> > state;
- std::vector<std::unordered_map<int, single_particle_state<particle_type>>> rhs;
-
- public:
- int integration_steps;
- // this class only works with buffered interpolator
- interpolator<rnumber, interp_neighbours> *vel;
-
- /* simulation parameters */
- double dt;
-
- /* methods */
-
- /* constructor and destructor.
- * allocate and deallocate:
- * this->state
- * this->rhs
- * */
- distributed_particles(
- const char *NAME,
- const hid_t data_file_id,
- interpolator<rnumber, interp_neighbours> *FIELD,
- const int TRAJ_SKIP,
- const int INTEGRATION_STEPS = 2);
- ~distributed_particles();
-
- void sample(
- interpolator<rnumber, interp_neighbours> *field,
- const char *dset_name);
- void sample(
- interpolator<rnumber, interp_neighbours> *field,
- const std::unordered_map<int, single_particle_state<particle_type>> &x,
- std::unordered_map<int, single_particle_state<POINT3D>> &y);
- void get_rhs(
- const std::unordered_map<int, single_particle_state<particle_type>> &x,
- std::unordered_map<int, single_particle_state<particle_type>> &y);
-
- void redistribute(
- std::unordered_map<int, single_particle_state<particle_type>> &x,
- std::vector<std::unordered_map<int, single_particle_state<particle_type>>> &vals);
-
-
- /* input/output */
- void read();
- void write(
- const char *dset_name,
- std::unordered_map<int, single_particle_state<POINT3D>> &y);
- void write(
- const char *dset_name,
- std::unordered_map<int, single_particle_state<particle_type>> &y);
- void write(const bool write_rhs = true);
-
- /* solvers */
- void step();
- void roll_rhs();
- void AdamsBashforth(const int nsteps);
-};
-
-#endif//DISTRIBUTED_PARTICLES
-
diff --git a/bfps/cpp/fftw_tools.cpp b/bfps/cpp/fftw_tools.cpp
index 9f6e9bfd..55794b41 100644
--- a/bfps/cpp/fftw_tools.cpp
+++ b/bfps/cpp/fftw_tools.cpp
@@ -38,192 +38,3 @@ std::map<std::string, unsigned> fftw_planner_string_to_flag = {
{"parameter does not exist", DEFAULT_FFTW_FLAG},
};
-template <class rnumber>
-int clip_zero_padding(
- field_descriptor<rnumber> *f,
- rnumber *a,
- int howmany)
-{
- if (f->ndims < 3)
- return EXIT_FAILURE;
- rnumber *b = a;
- ptrdiff_t copy_size = f->sizes[2] * howmany;
- ptrdiff_t skip_size = copy_size + 2*howmany;
- for (int i0 = 0; i0 < f->subsizes[0]; i0++)
- for (int i1 = 0; i1 < f->sizes[1]; i1++)
- {
- std::copy(a, a + copy_size, b);
- a += skip_size;
- b += copy_size;
- }
- return EXIT_SUCCESS;
-}
-
-template
-int clip_zero_padding<float>(
- field_descriptor<float> *f,
- float *a,
- int howmany);
-
-template
-int clip_zero_padding<double>(
- field_descriptor<double> *f,
- double *a,
- int howmany);
-
-
-
-template <class rnumber>
-int copy_complex_array(
- field_descriptor<rnumber> *fi,
- rnumber (*ai)[2],
-field_descriptor<rnumber> *fo,
-rnumber (*ao)[2],
-int howmany)
-{
- DEBUG_MSG("entered copy_complex_array\n");
- typename fftw_interface<rnumber>::complex *buffer;
- buffer = fftw_interface<rnumber>::alloc_complex(fi->slice_size*howmany);
-
- int min_fast_dim;
- min_fast_dim =
- (fi->sizes[2] > fo->sizes[2]) ?
- fo->sizes[2] : fi->sizes[2];
-
- /* clean up destination, in case we're padding with zeros
- (even if only for one dimension) */
- std::fill_n((rnumber*)ao, fo->local_size*2, 0.0);
-
- int64_t ii0, ii1;
- int64_t oi0, oi1;
- int64_t delta1, delta0;
- int irank, orank;
- delta0 = (fo->sizes[0] - fi->sizes[0]);
- delta1 = (fo->sizes[1] - fi->sizes[1]);
- for (ii0=0; ii0 < fi->sizes[0]; ii0++)
- {
- if (ii0 <= fi->sizes[0]/2)
- {
- oi0 = ii0;
- if (oi0 > fo->sizes[0]/2)
- continue;
- }
- else
- {
- oi0 = ii0 + delta0;
- if ((oi0 < 0) || ((fo->sizes[0] - oi0) >= fo->sizes[0]/2))
- continue;
- }
- irank = fi->rank[ii0];
- orank = fo->rank[oi0];
- if ((irank == orank) &&
- (irank == fi->myrank))
- {
- std::copy(
- (rnumber*)(ai + (ii0 - fi->starts[0] )*fi->slice_size),
- (rnumber*)(ai + (ii0 - fi->starts[0] + 1)*fi->slice_size),
- (rnumber*)buffer);
- }
- else
- {
- if (fi->myrank == irank)
- {
- MPI_Send(
- (void*)(ai + (ii0-fi->starts[0])*fi->slice_size),
- fi->slice_size,
- mpi_real_type<rnumber>::complex(),
- orank,
- ii0,
- fi->comm);
- }
- if (fi->myrank == orank)
- {
- MPI_Recv(
- (void*)(buffer),
- fi->slice_size,
- mpi_real_type<rnumber>::complex(),
- irank,
- ii0,
- fi->comm,
- MPI_STATUS_IGNORE);
- }
- }
- if (fi->myrank == orank)
- {
- for (ii1 = 0; ii1 < fi->sizes[1]; ii1++)
- {
- if (ii1 <= fi->sizes[1]/2)
- {
- oi1 = ii1;
- if (oi1 > fo->sizes[1]/2)
- continue;
- }
- else
- {
- oi1 = ii1 + delta1;
- if ((oi1 < 0) || ((fo->sizes[1] - oi1) >= fo->sizes[1]/2))
- continue;
- }
- std::copy(
- (rnumber*)(buffer + (ii1*fi->sizes[2]*howmany)),
- (rnumber*)(buffer + (ii1*fi->sizes[2] + min_fast_dim)*howmany),
- (rnumber*)(ao +
- ((oi0 - fo->starts[0])*fo->sizes[1] +
- oi1)*fo->sizes[2]*howmany));
- }
- }
- }
- fftw_interface<rnumber>::free(buffer);
- MPI_Barrier(fi->comm);
-
- DEBUG_MSG("exiting copy_complex_array\n");
- return EXIT_SUCCESS;
-}
-
-template
-int copy_complex_array<float>(
- field_descriptor<float> *fi,
- float (*ai)[2],
- field_descriptor<float> *fo,
- float (*ao)[2],
- int howmany);
-
-template
-int copy_complex_array<double>(
- field_descriptor<double> *fi,
- double (*ai)[2],
- field_descriptor<double> *fo,
- double (*ao)[2],
- int howmany);
-
-
-template <class rnumber>
-int get_descriptors_3D(
- int n0, int n1, int n2,
- field_descriptor<rnumber> **fr,
- field_descriptor<rnumber> **fc)
-{
- int ntmp[3];
- ntmp[0] = n0;
- ntmp[1] = n1;
- ntmp[2] = n2;
- *fr = new field_descriptor<rnumber>(3, ntmp, mpi_real_type<rnumber>::real(), MPI_COMM_WORLD);
- ntmp[0] = n0;
- ntmp[1] = n1;
- ntmp[2] = n2/2+1;
- *fc = new field_descriptor<rnumber>(3, ntmp, mpi_real_type<rnumber>::complex(), MPI_COMM_WORLD);
- return EXIT_SUCCESS;
-}
-
-template
-int get_descriptors_3D<float>(
- int n0, int n1, int n2,
- field_descriptor<float> **fr,
- field_descriptor<float> **fc);
-
-template
-int get_descriptors_3D<double>(
- int n0, int n1, int n2,
- field_descriptor<double> **fr,
- field_descriptor<double> **fc);
-
diff --git a/bfps/cpp/fftw_tools.hpp b/bfps/cpp/fftw_tools.hpp
index f4480714..e32500fd 100644
--- a/bfps/cpp/fftw_tools.hpp
+++ b/bfps/cpp/fftw_tools.hpp
@@ -36,37 +36,5 @@ extern int myrank, nprocs;
extern std::map<std::string, unsigned> fftw_planner_string_to_flag;
-/* given two arrays of the same dimension, we do a simple resize in
- * Fourier space: either chop off high modes, or pad with zeros.
- * the arrays are assumed to use 3D mpi fftw layout.
- * */
-template <class rnumber>
-int copy_complex_array(
- field_descriptor<rnumber> *fi,
- rnumber (*ai)[2],
- field_descriptor<rnumber> *fo,
- rnumber (*ao)[2],
- int howmany=1);
-
-template <class rnumber>
-int clip_zero_padding(
- field_descriptor<rnumber> *f,
- rnumber *a,
- int howmany=1);
-
-/* function to get pair of descriptors for real and Fourier space
- * arrays used with fftw.
- * the n0, n1, n2 correspond to the real space data WITHOUT the zero
- * padding that FFTW needs.
- * IMPORTANT: the real space array must be allocated with
- * 2*fc->local_size, and then the zeros cleaned up before trying
- * to write data.
- * */
-template <class rnumber>
-int get_descriptors_3D(
- int n0, int n1, int n2,
- field_descriptor<rnumber> **fr,
- field_descriptor<rnumber> **fc);
-
#endif//FFTW_TOOLS
diff --git a/bfps/cpp/fluid_solver.cpp b/bfps/cpp/fluid_solver.cpp
deleted file mode 100644
index 7ec0c978..00000000
--- a/bfps/cpp/fluid_solver.cpp
+++ /dev/null
@@ -1,1057 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-//#define NDEBUG
-
-#include <cassert>
-#include <cmath>
-#include <cstring>
-#include "fluid_solver.hpp"
-#include "fftw_tools.hpp"
-#include "scope_timer.hpp"
-#include "shared_array.hpp"
-
-
-template <class rnumber>
-void fluid_solver<rnumber>::impose_zero_modes()
-{
- if (this->cd->myrank == this->cd->rank[0])
- {
- std::fill_n((rnumber*)(this->cu), 6, 0.0);
- std::fill_n((rnumber*)(this->cv[0]), 6, 0.0);
- std::fill_n((rnumber*)(this->cv[1]), 6, 0.0);
- std::fill_n((rnumber*)(this->cv[2]), 6, 0.0);
- }
-}
-/*****************************************************************************/
-/* macro for specializations to numeric types compatible with FFTW */
-
-template <class rnumber>
-fluid_solver<rnumber>::fluid_solver(
- const char *NAME,
- int nx,
- int ny,
- int nz,
- double DKX,
- double DKY,
- double DKZ,
- int DEALIAS_TYPE,
- unsigned FFTW_PLAN_RIGOR) : fluid_solver_base<rnumber>(
- NAME,
- nx , ny , nz,
- DKX, DKY, DKZ,
- DEALIAS_TYPE,
- FFTW_PLAN_RIGOR)
-{
- TIMEZONE("fluid_solver::fluid_solver");
- this->cvorticity = fftw_interface<rnumber>::alloc_complex(this->cd->local_size);
- this->cvelocity = fftw_interface<rnumber>::alloc_complex(this->cd->local_size);
- this->rvorticity = fftw_interface<rnumber>::alloc_real(this->cd->local_size*2);
- /*this->rvelocity = (rnumber*)(this->cvelocity);*/
- this->rvelocity = fftw_interface<rnumber>::alloc_real(this->cd->local_size*2);
-
- this->ru = this->rvelocity;
- this->cu = this->cvelocity;
-
- this->rv[0] = this->rvorticity;
- this->rv[3] = this->rvorticity;
- this->cv[0] = this->cvorticity;
- this->cv[3] = this->cvorticity;
-
- this->cv[1] = fftw_interface<rnumber>::alloc_complex(this->cd->local_size);
- this->cv[2] = this->cv[1];
- this->rv[1] = fftw_interface<rnumber>::alloc_real(this->cd->local_size*2);
- this->rv[2] = this->rv[1];
-
- this->c2r_vorticity = new typename fftw_interface<rnumber>::many_plan;
- this->r2c_vorticity = new typename fftw_interface<rnumber>::many_plan;
- this->c2r_velocity = new typename fftw_interface<rnumber>::many_plan;
- this->r2c_velocity = new typename fftw_interface<rnumber>::many_plan;
-
- ptrdiff_t sizes[] = {nz,
- ny,
- nx};
-
- *this->c2r_vorticity = fftw_interface<rnumber>::mpi_plan_many_dft_c2r(
- 3, sizes, 3, FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
- this->cvorticity, this->rvorticity,
- MPI_COMM_WORLD, this->fftw_plan_rigor | FFTW_MPI_TRANSPOSED_IN);
-
- *this->r2c_vorticity = fftw_interface<rnumber>::mpi_plan_many_dft_r2c(
- 3, sizes, 3, FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
- this->rvorticity, this->cvorticity,
- MPI_COMM_WORLD, this->fftw_plan_rigor | FFTW_MPI_TRANSPOSED_OUT);
-
- *this->c2r_velocity = fftw_interface<rnumber>::mpi_plan_many_dft_c2r(
- 3, sizes, 3, FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
- this->cvelocity, this->rvelocity,
- MPI_COMM_WORLD, this->fftw_plan_rigor | FFTW_MPI_TRANSPOSED_IN);
-
- *this->r2c_velocity = fftw_interface<rnumber>::mpi_plan_many_dft_r2c(
- 3, sizes, 3, FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
- this->rvelocity, this->cvelocity,
- MPI_COMM_WORLD, this->fftw_plan_rigor | FFTW_MPI_TRANSPOSED_OUT);
-
- this->uc2r = this->c2r_velocity;
- this->ur2c = this->r2c_velocity;
- this->vc2r[0] = this->c2r_vorticity;
- this->vr2c[0] = this->r2c_vorticity;
-
- this->vc2r[1] = new typename fftw_interface<rnumber>::many_plan;
- this->vr2c[1] = new typename fftw_interface<rnumber>::many_plan;
- this->vc2r[2] = new typename fftw_interface<rnumber>::many_plan;
- this->vr2c[2] = new typename fftw_interface<rnumber>::many_plan;
-
- *(this->vc2r[1]) = fftw_interface<rnumber>::mpi_plan_many_dft_c2r(
- 3, sizes, 3, FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
- this->cv[1], this->rv[1],
- MPI_COMM_WORLD, this->fftw_plan_rigor | FFTW_MPI_TRANSPOSED_IN);
-
- *this->vc2r[2] = fftw_interface<rnumber>::mpi_plan_many_dft_c2r(
- 3, sizes, 3, FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
- this->cv[2], this->rv[2],
- MPI_COMM_WORLD, this->fftw_plan_rigor | FFTW_MPI_TRANSPOSED_IN);
-
- *this->vr2c[1] = fftw_interface<rnumber>::mpi_plan_many_dft_r2c(
- 3, sizes, 3, FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
- this->rv[1], this->cv[1],
- MPI_COMM_WORLD, this->fftw_plan_rigor | FFTW_MPI_TRANSPOSED_OUT);
-
- *this->vr2c[2] = fftw_interface<rnumber>::mpi_plan_many_dft_r2c(
- 3, sizes, 3, FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
- this->rv[2], this->cv[2],
- MPI_COMM_WORLD, this->fftw_plan_rigor | FFTW_MPI_TRANSPOSED_OUT);
-
- /* ``physical'' parameters etc, initialized here just in case */
-
- this->nu = 0.1;
- this->fmode = 1;
- this->famplitude = 1.0;
- this->fk0 = 0;
- this->fk1 = 3.0;
- /* initialization of fields must be done AFTER planning */
- std::fill_n((rnumber*)this->cvorticity, this->cd->local_size*2, 0.0);
- std::fill_n((rnumber*)this->cvelocity, this->cd->local_size*2, 0.0);
- std::fill_n(this->rvelocity, this->cd->local_size*2, 0.0);
- std::fill_n(this->rvorticity, this->cd->local_size*2, 0.0);
- std::fill_n((rnumber*)this->cv[1], this->cd->local_size*2, 0.0);
- std::fill_n(this->rv[1], this->cd->local_size*2, 0.0);
- std::fill_n(this->rv[2], this->cd->local_size*2, 0.0);
-}
-
-template <class rnumber>
-fluid_solver<rnumber>::~fluid_solver()
-{
- fftw_interface<rnumber>::destroy_plan(*this->c2r_vorticity);
- fftw_interface<rnumber>::destroy_plan(*this->r2c_vorticity);
- fftw_interface<rnumber>::destroy_plan(*this->c2r_velocity );
- fftw_interface<rnumber>::destroy_plan(*this->r2c_velocity );
- fftw_interface<rnumber>::destroy_plan(*this->vc2r[1]);
- fftw_interface<rnumber>::destroy_plan(*this->vr2c[1]);
- fftw_interface<rnumber>::destroy_plan(*this->vc2r[2]);
- fftw_interface<rnumber>::destroy_plan(*this->vr2c[2]);
-
- delete this->c2r_vorticity;
- delete this->r2c_vorticity;
- delete this->c2r_velocity ;
- delete this->r2c_velocity ;
- delete this->vc2r[1];
- delete this->vr2c[1];
- delete this->vc2r[2];
- delete this->vr2c[2];
-
- fftw_interface<rnumber>::free(this->cv[1]);
- fftw_interface<rnumber>::free(this->rv[1]);
- fftw_interface<rnumber>::free(this->cvorticity);
- fftw_interface<rnumber>::free(this->rvorticity);
- fftw_interface<rnumber>::free(this->cvelocity);
- fftw_interface<rnumber>::free(this->rvelocity);
-}
-
-template <class rnumber>
-void fluid_solver<rnumber>::compute_vorticity()
-{
- TIMEZONE("fluid_solver::compute_vorticity");
- CLOOP_K2(
- this,
- [&](ptrdiff_t cindex, ptrdiff_t xindex, ptrdiff_t yindex, ptrdiff_t zindex, double k2){
- // cindex indexing is thread safe (and tindex too) + it is a write
- ptrdiff_t tindex = 3*cindex;
- if (k2 <= this->kM2)
- {
- this->cvorticity[tindex+0][0] = -(this->ky[yindex]*this->cu[tindex+2][1] - this->kz[zindex]*this->cu[tindex+1][1]);
- this->cvorticity[tindex+1][0] = -(this->kz[zindex]*this->cu[tindex+0][1] - this->kx[xindex]*this->cu[tindex+2][1]);
- this->cvorticity[tindex+2][0] = -(this->kx[xindex]*this->cu[tindex+1][1] - this->ky[yindex]*this->cu[tindex+0][1]);
- this->cvorticity[tindex+0][1] = (this->ky[yindex]*this->cu[tindex+2][0] - this->kz[zindex]*this->cu[tindex+1][0]);
- this->cvorticity[tindex+1][1] = (this->kz[zindex]*this->cu[tindex+0][0] - this->kx[xindex]*this->cu[tindex+2][0]);
- this->cvorticity[tindex+2][1] = (this->kx[xindex]*this->cu[tindex+1][0] - this->ky[yindex]*this->cu[tindex+0][0]);
- }
- else{
- std::fill_n((rnumber*)(this->cvorticity+tindex), 6, 0.0);
- }
- }
- );
- this->symmetrize(this->cvorticity, 3);
-}
-
-template <class rnumber>
-void fluid_solver<rnumber>::compute_velocity(rnumber (*__restrict__ vorticity)[2])
-{
- TIMEZONE("fluid_solver::compute_velocity");
- CLOOP_K2(
- this,
- [&](ptrdiff_t cindex, ptrdiff_t xindex, ptrdiff_t yindex, ptrdiff_t zindex, double k2){
- // cindex indexing is thread safe (and tindex too) + it is a write
- ptrdiff_t tindex = 3*cindex;
- if (k2 <= this->kM2 && k2 > 0)
- {
- this->cu[tindex+0][0] = -(this->ky[yindex]*vorticity[tindex+2][1] - this->kz[zindex]*vorticity[tindex+1][1]) / k2;
- this->cu[tindex+1][0] = -(this->kz[zindex]*vorticity[tindex+0][1] - this->kx[xindex]*vorticity[tindex+2][1]) / k2;
- this->cu[tindex+2][0] = -(this->kx[xindex]*vorticity[tindex+1][1] - this->ky[yindex]*vorticity[tindex+0][1]) / k2;
- this->cu[tindex+0][1] = (this->ky[yindex]*vorticity[tindex+2][0] - this->kz[zindex]*vorticity[tindex+1][0]) / k2;
- this->cu[tindex+1][1] = (this->kz[zindex]*vorticity[tindex+0][0] - this->kx[xindex]*vorticity[tindex+2][0]) / k2;
- this->cu[tindex+2][1] = (this->kx[xindex]*vorticity[tindex+1][0] - this->ky[yindex]*vorticity[tindex+0][0]) / k2;
- }
- else
- std::fill_n((rnumber*)(this->cu+tindex), 6, 0.0);
- }
- );
- /*this->symmetrize(this->cu, 3);*/
-}
-
-template <class rnumber>
-void fluid_solver<rnumber>::ift_velocity()
-{
- TIMEZONE("fluid_solver::ift_velocity");
- fftw_interface<rnumber>::execute(*(this->c2r_velocity ));
-}
-
-template <class rnumber>
-void fluid_solver<rnumber>::ift_vorticity()
-{
- TIMEZONE("fluid_solver::ift_vorticity");
- std::fill_n(this->rvorticity, this->cd->local_size*2, 0.0);
- fftw_interface<rnumber>::execute(*(this->c2r_vorticity ));
-}
-
-template <class rnumber>
-void fluid_solver<rnumber>::dft_velocity()
-{
- TIMEZONE("fluid_solver::dft_velocity");
- fftw_interface<rnumber>::execute(*(this->r2c_velocity ));
-}
-
-template <class rnumber>
-void fluid_solver<rnumber>::dft_vorticity()
-{
- TIMEZONE("fluid_solver::dft_vorticity");
- std::fill_n((rnumber*)this->cvorticity, this->cd->local_size*2, 0.0);
- fftw_interface<rnumber>::execute(*(this->r2c_vorticity ));
-}
-
-template <class rnumber>
-void fluid_solver<rnumber>::add_forcing(
- rnumber (*__restrict__ acc_field)[2], rnumber (*__restrict__ vort_field)[2], rnumber factor)
-{
- TIMEZONE("fluid_solver::add_forcing");
- if (strcmp(this->forcing_type, "none") == 0)
- return;
- if (strcmp(this->forcing_type, "Kolmogorov") == 0)
- {
- ptrdiff_t cindex;
- if (this->cd->myrank == this->cd->rank[this->fmode])
- {
- cindex = ((this->fmode - this->cd->starts[0]) * this->cd->sizes[1])*this->cd->sizes[2]*3;
- acc_field[cindex+2][0] -= this->famplitude*factor/2;
- }
- if (this->cd->myrank == this->cd->rank[this->cd->sizes[0] - this->fmode])
- {
- cindex = ((this->cd->sizes[0] - this->fmode - this->cd->starts[0]) * this->cd->sizes[1])*this->cd->sizes[2]*3;
- acc_field[cindex+2][0] -= this->famplitude*factor/2;
- }
- return;
- }
- if (strcmp(this->forcing_type, "linear") == 0)
- {
- CLOOP(
- this,
- [&](ptrdiff_t cindex, ptrdiff_t xindex, ptrdiff_t yindex, ptrdiff_t zindex){
- // cindex indexing is thread safe (and cindex*3+c too)
- double knorm = sqrt(this->kx[xindex]*this->kx[xindex] +
- this->ky[yindex]*this->ky[yindex] +
- this->kz[zindex]*this->kz[zindex]);
- if ((this->fk0 <= knorm) && (this->fk1 >= knorm))
- for (int c=0; c<3; c++)
- for (int i=0; i<2; i++)
- acc_field[cindex*3+c][i] += this->famplitude*vort_field[cindex*3+c][i]*factor;
- }
- );
- return;
- }
-}
-
-template <class rnumber>
-void fluid_solver<rnumber>::omega_nonlin(
- int src)
-{
- TIMEZONE("fluid_solver::omega_nonlin");
- assert(src >= 0 && src < 3);
- this->compute_velocity(this->cv[src]);
- /* get fields from Fourier space to real space */
- {
- TIMEZONE("fluid_solver::omega_nonlin::fftw");
- fftw_interface<rnumber>::execute(*(this->c2r_velocity ));
- fftw_interface<rnumber>::execute(*(this->vc2r[src]));
- }
- /* compute cross product $u \times \omega$, and normalize */
- {
- TIMEZONE("fluid_solver::omega_nonlin::RLOOP");
- RLOOP (
- this,
- [&](ptrdiff_t rindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/, ptrdiff_t /*zindex*/){
- ptrdiff_t tindex = 3*rindex;
- rnumber tmp[3][2];
- for (int cc=0; cc<3; cc++)
- tmp[cc][0] = (this->ru[tindex+(cc+1)%3]*this->rv[src][tindex+(cc+2)%3] -
- this->ru[tindex+(cc+2)%3]*this->rv[src][tindex+(cc+1)%3]);
- // Access to rindex is thread safe so there is no overlap between threads
- for (int cc=0; cc<3; cc++)
- this->ru[(3*rindex)+cc] = tmp[cc][0] / this->normalization_factor;
- }
- );
- }
- /* go back to Fourier space */
- this->clean_up_real_space(this->ru, 3);
- {
- TIMEZONE("fluid_solver::omega_nonlin::fftw-2");
- fftw_interface<rnumber>::execute(*(this->r2c_velocity ));
- }
- this->dealias(this->cu, 3);
- /* $\imath k \times Fourier(u \times \omega)$ */
- {
- TIMEZONE("fluid_solver::omega_nonlin::CLOOP");
- CLOOP(
- this,
- [&](ptrdiff_t cindex, ptrdiff_t xindex, ptrdiff_t yindex, ptrdiff_t zindex){
- rnumber tmp[3][2];
- ptrdiff_t tindex = 3*cindex;
- {
- tmp[0][0] = -(this->ky[yindex]*this->cu[tindex+2][1] - this->kz[zindex]*this->cu[tindex+1][1]);
- tmp[1][0] = -(this->kz[zindex]*this->cu[tindex+0][1] - this->kx[xindex]*this->cu[tindex+2][1]);
- tmp[2][0] = -(this->kx[xindex]*this->cu[tindex+1][1] - this->ky[yindex]*this->cu[tindex+0][1]);
- tmp[0][1] = (this->ky[yindex]*this->cu[tindex+2][0] - this->kz[zindex]*this->cu[tindex+1][0]);
- tmp[1][1] = (this->kz[zindex]*this->cu[tindex+0][0] - this->kx[xindex]*this->cu[tindex+2][0]);
- tmp[2][1] = (this->kx[xindex]*this->cu[tindex+1][0] - this->ky[yindex]*this->cu[tindex+0][0]);
- }
- // cindex indexing is thread safe so it is 3*cindex so there is no overlap between threads
- for (int cc=0; cc<3; cc++)
- for (int i=0; i<2; i++)
- this->cu[tindex+cc][i] = tmp[cc][i];
- }
- );
- }
- {
- TIMEZONE("fluid_solver::omega_nonlin::add_forcing");
- this->add_forcing(this->cu, this->cv[src], 1.0);
- }
- {
- TIMEZONE("fluid_solver::omega_nonlin::force_divfree");
- this->force_divfree(this->cu);
- }
-}
-
-template <class rnumber>
-void fluid_solver<rnumber>::step(double dt)
-{
- TIMEZONE("fluid_solver::step");
- std::fill_n((rnumber*)this->cv[1], this->cd->local_size*2, 0.0);
- this->omega_nonlin(0);
- CLOOP_K2(
- this,
- [&](ptrdiff_t cindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/, ptrdiff_t /*zindex*/, double k2){
- if (k2 <= this->kM2)
- {
- double factor0 = exp(-this->nu * k2 * dt);
- // cindex indexing is thread safe so there is no overlap between threads
- for (int cc=0; cc<3; cc++) for (int i=0; i<2; i++)
- this->cv[1][3*cindex+cc][i] = (this->cv[0][3*cindex+cc][i] +
- dt*this->cu[3*cindex+cc][i])*factor0;
- }
- }
- );
-
- this->omega_nonlin(1);
- CLOOP_K2(
- this,
- [&](ptrdiff_t cindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/, ptrdiff_t /*zindex*/, double k2){
- if (k2 <= this->kM2)
- {
- double factor0 = exp(-this->nu * k2 * dt/2);
- double factor1 = exp( this->nu * k2 * dt/2);
- // cindex indexing is thread safe so there is no overlap between threads
- for (int cc=0; cc<3; cc++) for (int i=0; i<2; i++)
- this->cv[2][3*cindex+cc][i] = (3*this->cv[0][3*cindex+cc][i]*factor0 +
- (this->cv[1][3*cindex+cc][i] +
- dt*this->cu[3*cindex+cc][i])*factor1)*0.25;
- }
- }
- );
-
- this->omega_nonlin(2);
- CLOOP_K2(
- this,
- [&](ptrdiff_t cindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/, ptrdiff_t /*zindex*/, double k2){
- if (k2 <= this->kM2)
- {
- double factor0 = exp(-this->nu * k2 * dt * 0.5);
- // cindex indexing is thread safe so there is no overlap between threads
- for (int cc=0; cc<3; cc++) for (int i=0; i<2; i++)
- this->cv[3][3*cindex+cc][i] = (this->cv[0][3*cindex+cc][i]*factor0 +
- 2*(this->cv[2][3*cindex+cc][i] +
- dt*this->cu[3*cindex+cc][i]))*factor0/3;
- }
- }
- );
-
- this->force_divfree(this->cvorticity);
- this->symmetrize(this->cvorticity, 3);
- this->iteration++;
-}
-
-template <class rnumber>
-int fluid_solver<rnumber>::read(char field, char representation)
-{
- TIMEZONE("fluid_solver::read");
- char fname[512];
- int read_result;
- if (field == 'v')
- {
- if (representation == 'c')
- {
- this->fill_up_filename("cvorticity", fname);
- read_result = this->cd->read(fname, (void*)this->cvorticity);
- if (read_result != EXIT_SUCCESS)
- return read_result;
- }
- if (representation == 'r')
- {
- read_result = this->read_base("rvorticity", this->rvorticity);
- if (read_result != EXIT_SUCCESS)
- return read_result;
- else
- fftw_interface<rnumber>::execute(*(this->r2c_vorticity ));
- }
- this->low_pass_Fourier(this->cvorticity, 3, this->kM);
- this->force_divfree(this->cvorticity);
- this->symmetrize(this->cvorticity, 3);
- return EXIT_SUCCESS;
- }
- if ((field == 'u') && (representation == 'c'))
- {
- read_result = this->read_base("cvelocity", this->cvelocity);
- this->low_pass_Fourier(this->cvelocity, 3, this->kM);
- this->force_divfree(this->cvorticity);
- this->symmetrize(this->cvorticity, 3);
- return read_result;
- }
- if ((field == 'u') && (representation == 'r'))
- return this->read_base("rvelocity", this->rvelocity);
- return EXIT_FAILURE;
-}
-
-template <class rnumber>
-int fluid_solver<rnumber>::write(char field, char representation)
-{
- TIMEZONE("fluid_solver::write");
- char fname[512];
- if ((field == 'v') && (representation == 'c'))
- {
- this->fill_up_filename("cvorticity", fname);
- return this->cd->write(fname, (void*)this->cvorticity);
- }
- if ((field == 'v') && (representation == 'r'))
- {
- fftw_interface<rnumber>::execute(*(this->c2r_vorticity ));
- clip_zero_padding<rnumber>(this->rd, this->rvorticity, 3);
- this->fill_up_filename("rvorticity", fname);
- return this->rd->write(fname, this->rvorticity);
- }
- this->compute_velocity(this->cvorticity);
- if ((field == 'u') && (representation == 'c'))
- {
- this->fill_up_filename("cvelocity", fname);
- return this->cd->write(fname, this->cvelocity);
- }
- if ((field == 'u') && (representation == 'r'))
- {
- this->ift_velocity();
- clip_zero_padding<rnumber>(this->rd, this->rvelocity, 3);
- this->fill_up_filename("rvelocity", fname);
- return this->rd->write(fname, this->rvelocity);
- }
- return EXIT_FAILURE;
-}
-
-template <class rnumber>
-int fluid_solver<rnumber>::write_rTrS2()
-{
- TIMEZONE("fluid_solver::write_rTrS2");
- char fname[512];
- this->fill_up_filename("rTrS2", fname);
- typename fftw_interface<rnumber>::complex *ca;
- rnumber *ra;
- ca = fftw_interface<rnumber>::alloc_complex(this->cd->local_size*3);
- ra = (rnumber*)(ca);
- this->compute_velocity(this->cvorticity);
- this->compute_vector_gradient(ca, this->cvelocity);
- for (int cc=0; cc<3; cc++)
- {
- std::copy(
- (rnumber*)(ca + cc*this->cd->local_size),
- (rnumber*)(ca + (cc+1)*this->cd->local_size),
- (rnumber*)this->cv[1]);
- fftw_interface<rnumber>::execute(*(this->vc2r[1]));
- std::copy(
- this->rv[1],
- this->rv[1] + this->cd->local_size*2,
- ra + cc*this->cd->local_size*2);
- }
- /* velocity gradient is now stored, in real space, in ra */
- rnumber *dx_u, *dy_u, *dz_u;
- dx_u = ra;
- dy_u = ra + 2*this->cd->local_size;
- dz_u = ra + 4*this->cd->local_size;
- rnumber *trS2 = fftw_interface<rnumber>::alloc_real((this->cd->local_size/3)*2);
- shared_array<double> average_local(1, [&](double* data){
- data[0] = 0;
- });
-
- RLOOP(
- this,
- [&](ptrdiff_t rindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/, ptrdiff_t /*zindex*/){
- rnumber AxxAxx;
- rnumber AyyAyy;
- rnumber AzzAzz;
- rnumber Sxy;
- rnumber Syz;
- rnumber Szx;
- ptrdiff_t tindex = 3*rindex;
- AxxAxx = dx_u[tindex+0]*dx_u[tindex+0];
- AyyAyy = dy_u[tindex+1]*dy_u[tindex+1];
- AzzAzz = dz_u[tindex+2]*dz_u[tindex+2];
- Sxy = dx_u[tindex+1]+dy_u[tindex+0];
- Syz = dy_u[tindex+2]+dz_u[tindex+1];
- Szx = dz_u[tindex+0]+dx_u[tindex+2];
- // rindex is thread safe + No overlap between thread it is a write
- trS2[rindex] = (AxxAxx + AyyAyy + AzzAzz +
- (Sxy*Sxy + Syz*Syz + Szx*Szx)/2);
- average_local.getMine()[0] += trS2[rindex];
- }
- );
- average_local.mergeParallel();
- double average;
- MPI_Allreduce(
- average_local.getMasterData(),
- &average,
- 1,
- MPI_DOUBLE, MPI_SUM, this->cd->comm);
- DEBUG_MSG("average TrS2 is %g\n", average);
- fftw_interface<rnumber>::free(ca);
- /* output goes here */
- int ntmp[3];
- ntmp[0] = this->rd->sizes[0];
- ntmp[1] = this->rd->sizes[1];
- ntmp[2] = this->rd->sizes[2];
- field_descriptor<rnumber> *scalar_descriptor = new field_descriptor<rnumber>(3, ntmp, mpi_real_type<rnumber>::real(), this->cd->comm);
- clip_zero_padding<rnumber>(scalar_descriptor, trS2, 1);
- int return_value = scalar_descriptor->write(fname, trS2);
- delete scalar_descriptor;
- fftw_interface<rnumber>::free(trS2);
- return return_value;
-}
-
-template <class rnumber>
-int fluid_solver<rnumber>::write_renstrophy()
-{
- TIMEZONE("fluid_solver::write_renstrophy");
- char fname[512];
- this->fill_up_filename("renstrophy", fname);
- rnumber *enstrophy = fftw_interface<rnumber>::alloc_real((this->cd->local_size/3)*2);
- this->ift_vorticity();
- shared_array<double> average_local(1, [&](double* data){
- data[0] = 0;
- });
-
- RLOOP(
- this,
- [&](ptrdiff_t rindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/, ptrdiff_t /*zindex*/){
- ptrdiff_t tindex = 3*rindex;
- // rindex indexing is thread safe so there is no overlap between threads
- enstrophy[rindex] = (
- this->rvorticity[tindex+0]*this->rvorticity[tindex+0] +
- this->rvorticity[tindex+1]*this->rvorticity[tindex+1] +
- this->rvorticity[tindex+2]*this->rvorticity[tindex+2]
- )/2;
- average_local.getMine()[0] += enstrophy[rindex];
- }
- );
- average_local.mergeParallel();
- double average;
- MPI_Allreduce(
- average_local.getMasterData(),
- &average,
- 1,
- MPI_DOUBLE, MPI_SUM, this->cd->comm);
- DEBUG_MSG("average enstrophy is %g\n", average);
- /* output goes here */
- int ntmp[3];
- ntmp[0] = this->rd->sizes[0];
- ntmp[1] = this->rd->sizes[1];
- ntmp[2] = this->rd->sizes[2];
- field_descriptor<rnumber> *scalar_descriptor = new field_descriptor<rnumber>(3, ntmp, mpi_real_type<rnumber>::real(), this->cd->comm);
- clip_zero_padding<rnumber>(scalar_descriptor, enstrophy, 1);
- int return_value = scalar_descriptor->write(fname, enstrophy);
- delete scalar_descriptor;
- fftw_interface<rnumber>::free(enstrophy);
- return return_value;
-}
-
-template <class rnumber>
-void fluid_solver<rnumber>::compute_pressure(rnumber (*__restrict__ pressure)[2])
-{
- TIMEZONE("fluid_solver::compute_pressure");
- /* assume velocity is already in real space representation */
- /* diagonal terms 11 22 33 */
- RLOOP (
- this,
- [&](ptrdiff_t rindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/, ptrdiff_t /*zindex*/){
- // rindex indexing is thread safe so there is no overlap between threads
- ptrdiff_t tindex = 3*rindex;
- for (int cc=0; cc<3; cc++)
- this->rv[1][tindex+cc] = this->ru[tindex+cc]*this->ru[tindex+cc];
- }
- );
- this->clean_up_real_space(this->rv[1], 3);
- {
- TIMEZONE("fftw_interface<rnumber>::execute");
- fftw_interface<rnumber>::execute(*(this->vr2c[1]));
- }
- this->dealias(this->cv[1], 3);
- CLOOP_K2(
- this,
- [&](ptrdiff_t cindex, ptrdiff_t xindex, ptrdiff_t yindex, ptrdiff_t zindex, double k2){
- if (k2 <= this->kM2 && k2 > 0)
- {
- // cindex indexing is thread safe so there is no overlap between threads
- ptrdiff_t tindex = 3*cindex;
- for (int i=0; i<2; i++)
- {
- pressure[cindex][i] = -(this->kx[xindex]*this->kx[xindex]*this->cv[1][tindex+0][i] +
- this->ky[yindex]*this->ky[yindex]*this->cv[1][tindex+1][i] +
- this->kz[zindex]*this->kz[zindex]*this->cv[1][tindex+2][i]);
- }
- }
- else
- std::fill_n((rnumber*)(pressure+cindex), 2, 0.0);
- }
- );
- /* off-diagonal terms 12 23 31 */
- RLOOP (
- this,
- [&](ptrdiff_t rindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/, ptrdiff_t /*zindex*/){
- // rindex indexing is thread safe so there is no overlap between threads
- ptrdiff_t tindex = 3*rindex;
- for (int cc=0; cc<3; cc++)
- this->rv[1][tindex+cc] = this->ru[tindex+cc]*this->ru[tindex+(cc+1)%3];
- }
- );
- this->clean_up_real_space(this->rv[1], 3);
- {
- TIMEZONE("fftw_interface<rnumber>::execute");
- fftw_interface<rnumber>::execute(*(this->vr2c[1]));
- }
- this->dealias(this->cv[1], 3);
- CLOOP_K2(
- this,
- [&](ptrdiff_t cindex, ptrdiff_t xindex, ptrdiff_t yindex, ptrdiff_t zindex, double k2){
- if (k2 <= this->kM2 && k2 > 0)
- {
- // cindex indexing is thread safe so there is no overlap between threads
- ptrdiff_t tindex = 3*cindex;
- for (int i=0; i<2; i++)
- {
- pressure[cindex][i] -= 2*(this->kx[xindex]*this->ky[yindex]*this->cv[1][tindex+0][i] +
- this->ky[yindex]*this->kz[zindex]*this->cv[1][tindex+1][i] +
- this->kz[zindex]*this->kx[xindex]*this->cv[1][tindex+2][i]);
- pressure[cindex][i] /= this->normalization_factor*k2;
- }
- }
- }
- );
-}
-
-template <class rnumber>
-void fluid_solver<rnumber>::compute_gradient_statistics(
- rnumber (*__restrict__ vec)[2],
-double *gradu_moments,
-double *trS2QR_moments,
-ptrdiff_t *gradu_hist,
-ptrdiff_t *trS2QR_hist,
-ptrdiff_t *QR2D_hist,
-double trS2QR_max_estimates[],
-double gradu_max_estimates[],
-int nbins,
-int QR2D_nbins)
-{
- TIMEZONE("fluid_solver::compute_gradient_statistics");
- typename fftw_interface<rnumber>::complex *ca;
- rnumber *ra;
- ca = fftw_interface<rnumber>::alloc_complex(this->cd->local_size*3);
- ra = (rnumber*)(ca);
- this->compute_vector_gradient(ca, vec);
- for (int cc=0; cc<3; cc++)
- {
- std::copy(
- (rnumber*)(ca + cc*this->cd->local_size),
- (rnumber*)(ca + (cc+1)*this->cd->local_size),
- (rnumber*)this->cv[1]);
- fftw_interface<rnumber>::execute(*(this->vc2r[1]));
- std::copy(
- this->rv[1],
- this->rv[1] + this->cd->local_size*2,
- ra + cc*this->cd->local_size*2);
- }
- /* velocity gradient is now stored, in real space, in ra */
- std::fill_n(this->rv[1], 2*this->cd->local_size, 0.0);
- rnumber *dx_u, *dy_u, *dz_u;
- dx_u = ra;
- dy_u = ra + 2*this->cd->local_size;
- dz_u = ra + 4*this->cd->local_size;
- double binsize[2];
- double tmp_max_estimate[3];
- tmp_max_estimate[0] = trS2QR_max_estimates[0];
- tmp_max_estimate[1] = trS2QR_max_estimates[1];
- tmp_max_estimate[2] = trS2QR_max_estimates[2];
- binsize[0] = 2*tmp_max_estimate[2] / QR2D_nbins;
- binsize[1] = 2*tmp_max_estimate[1] / QR2D_nbins;
- ptrdiff_t *local_hist = new ptrdiff_t[QR2D_nbins*QR2D_nbins];
- std::fill_n(local_hist, QR2D_nbins*QR2D_nbins, 0);
- RLOOP(
- this,
- [&](ptrdiff_t rindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/, ptrdiff_t /*zindex*/){
- rnumber AxxAxx;
- rnumber AyyAyy;
- rnumber AzzAzz;
- rnumber AxyAyx;
- rnumber AyzAzy;
- rnumber AzxAxz;
- rnumber Sxy;
- rnumber Syz;
- rnumber Szx;
- // rindex indexing is thread safe so there is no overlap between threads
- // tindex[0:2] is thread safe too
- ptrdiff_t tindex = 3*rindex;
- AxxAxx = dx_u[tindex+0]*dx_u[tindex+0];
- AyyAyy = dy_u[tindex+1]*dy_u[tindex+1];
- AzzAzz = dz_u[tindex+2]*dz_u[tindex+2];
- AxyAyx = dx_u[tindex+1]*dy_u[tindex+0];
- AyzAzy = dy_u[tindex+2]*dz_u[tindex+1];
- AzxAxz = dz_u[tindex+0]*dx_u[tindex+2];
- this->rv[1][tindex+1] = - (AxxAxx + AyyAyy + AzzAzz)/2 - AxyAyx - AyzAzy - AzxAxz;
- this->rv[1][tindex+2] = - (dx_u[tindex+0]*(AxxAxx/3 + AxyAyx + AzxAxz) +
- dy_u[tindex+1]*(AyyAyy/3 + AxyAyx + AyzAzy) +
- dz_u[tindex+2]*(AzzAzz/3 + AzxAxz + AyzAzy) +
- dx_u[tindex+1]*dy_u[tindex+2]*dz_u[tindex+0] +
- dx_u[tindex+2]*dy_u[tindex+0]*dz_u[tindex+1]);
- int bin0 = int(floor((this->rv[1][tindex+2] + tmp_max_estimate[2]) / binsize[0]));
- int bin1 = int(floor((this->rv[1][tindex+1] + tmp_max_estimate[1]) / binsize[1]));
- if ((bin0 >= 0 && bin0 < QR2D_nbins) &&
- (bin1 >= 0 && bin1 < QR2D_nbins))
- local_hist[bin1*QR2D_nbins + bin0]++;
- Sxy = dx_u[tindex+1]+dy_u[tindex+0];
- Syz = dy_u[tindex+2]+dz_u[tindex+1];
- Szx = dz_u[tindex+0]+dx_u[tindex+2];
- this->rv[1][tindex] = (AxxAxx + AyyAyy + AzzAzz +
- (Sxy*Sxy + Syz*Syz + Szx*Szx)/2);
- }
- );
- MPI_Allreduce(
- local_hist,
- QR2D_hist,
- QR2D_nbins * QR2D_nbins,
- MPI_INT64_T, MPI_SUM, this->cd->comm);
- delete[] local_hist;
- this->compute_rspace_stats3(
- this->rv[1],
- trS2QR_moments,
- trS2QR_hist,
- tmp_max_estimate,
- nbins);
- double *tmp_moments = new double[10*3];
- ptrdiff_t *tmp_hist = new ptrdiff_t[nbins*3];
- for (int cc=0; cc<3; cc++)
- {
- tmp_max_estimate[0] = gradu_max_estimates[cc*3 + 0];
- tmp_max_estimate[1] = gradu_max_estimates[cc*3 + 1];
- tmp_max_estimate[2] = gradu_max_estimates[cc*3 + 2];
- this->compute_rspace_stats3(
- dx_u + cc*2*this->cd->local_size,
- tmp_moments,
- tmp_hist,
- tmp_max_estimate,
- nbins);
- for (int n = 0; n < 10; n++)
- for (int i = 0; i < 3 ; i++)
- {
- gradu_moments[(n*3 + cc)*3 + i] = tmp_moments[n*3 + i];
- }
- for (int n = 0; n < nbins; n++)
- for (int i = 0; i < 3; i++)
- {
- gradu_hist[(n*3 + cc)*3 + i] = tmp_hist[n*3 + i];
- }
- }
- delete[] tmp_moments;
- delete[] tmp_hist;
- fftw_interface<rnumber>::free(ca);
-}
-
-template <class rnumber>
-void fluid_solver<rnumber>::compute_Lagrangian_acceleration(rnumber (*acceleration)[2])
-{
- TIMEZONE("fluid_solver::compute_Lagrangian_acceleration");
- typename fftw_interface<rnumber>::complex *pressure;
- pressure = fftw_interface<rnumber>::alloc_complex(this->cd->local_size/3);
- this->compute_velocity(this->cvorticity);
- this->ift_velocity();
- this->compute_pressure(pressure);
- this->compute_velocity(this->cvorticity);
- std::fill_n((rnumber*)this->cv[1], 2*this->cd->local_size, 0.0);
- CLOOP_K2(
- this,
- [&](ptrdiff_t cindex, ptrdiff_t xindex, ptrdiff_t yindex, ptrdiff_t zindex, double k2){
- if (k2 <= this->kM2)
- {
- // cindex indexing is thread safe so there is no overlap between threads
- ptrdiff_t tindex = 3*cindex;
- for (int cc=0; cc<3; cc++)
- for (int i=0; i<2; i++)
- this->cv[1][tindex+cc][i] = - this->nu*k2*this->cu[tindex+cc][i];
- if (strcmp(this->forcing_type, "linear") == 0)
- {
- double knorm = sqrt(k2);
- if ((this->fk0 <= knorm) &&
- (this->fk1 >= knorm))
- for (int c=0; c<3; c++)
- for (int i=0; i<2; i++)
- this->cv[1][tindex+c][i] += this->famplitude*this->cu[tindex+c][i];
- }
- this->cv[1][tindex+0][0] += this->kx[xindex]*pressure[cindex][1];
- this->cv[1][tindex+1][0] += this->ky[yindex]*pressure[cindex][1];
- this->cv[1][tindex+2][0] += this->kz[zindex]*pressure[cindex][1];
- this->cv[1][tindex+0][1] -= this->kx[xindex]*pressure[cindex][0];
- this->cv[1][tindex+1][1] -= this->ky[yindex]*pressure[cindex][0];
- this->cv[1][tindex+2][1] -= this->kz[zindex]*pressure[cindex][0];
- }
- }
- );
- std::copy(
- (rnumber*)this->cv[1],
- (rnumber*)(this->cv[1] + this->cd->local_size),
- (rnumber*)acceleration);
- fftw_interface<rnumber>::free(pressure);
-}
-
-template <class rnumber>
-void fluid_solver<rnumber>::compute_Eulerian_acceleration(rnumber (*__restrict__ acceleration)[2])
-{
- TIMEZONE("fluid_solver::compute_Eulerian_acceleration");
- std::fill_n((rnumber*)(acceleration), 2*this->cd->local_size, 0.0);
- this->compute_velocity(this->cvorticity);
- /* put in linear terms */
- CLOOP_K2(
- this,
- [&](ptrdiff_t cindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/, ptrdiff_t /*zindex*/, double k2){
- if (k2 <= this->kM2)
- {
- // cindex indexing is thread safe so there is no overlap between threads
- ptrdiff_t tindex = 3*cindex;
- for (int cc=0; cc<3; cc++)
- for (int i=0; i<2; i++)
- acceleration[tindex+cc][i] = - this->nu*k2*this->cu[tindex+cc][i];
- if (strcmp(this->forcing_type, "linear") == 0)
- {
- double knorm = sqrt(k2);
- if ((this->fk0 <= knorm) &&
- (this->fk1 >= knorm))
- {
- for (int c=0; c<3; c++)
- for (int i=0; i<2; i++)
- acceleration[tindex+c][i] += this->famplitude*this->cu[tindex+c][i];
- }
- }
- }
- }
- );
- this->ift_velocity();
- /* compute uu */
- /* 11 22 33 */
- RLOOP (
- this,
- [&](ptrdiff_t rindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/, ptrdiff_t /*zindex*/){
- // cindex indexing is thread safe so there is no overlap between threads
- ptrdiff_t tindex = 3*rindex;
- for (int cc=0; cc<3; cc++)
- this->rv[1][tindex+cc] = this->ru[tindex+cc]*this->ru[tindex+cc] / this->normalization_factor;
- }
- );
- this->clean_up_real_space(this->rv[1], 3);
- fftw_interface<rnumber>::execute(*(this->vr2c[1]));
- this->dealias(this->cv[1], 3);
- CLOOP_K2(
- this,
- [&](ptrdiff_t cindex, ptrdiff_t xindex, ptrdiff_t yindex, ptrdiff_t zindex, double k2){
- if (k2 <= this->kM2)
- {
- // cindex indexing is thread safe so there is no overlap between threads
- ptrdiff_t tindex = 3*cindex;
- acceleration[tindex+0][0] +=
- this->kx[xindex]*this->cv[1][tindex+0][1];
- acceleration[tindex+0][1] +=
- -this->kx[xindex]*this->cv[1][tindex+0][0];
- acceleration[tindex+1][0] +=
- this->ky[yindex]*this->cv[1][tindex+1][1];
- acceleration[tindex+1][1] +=
- -this->ky[yindex]*this->cv[1][tindex+1][0];
- acceleration[tindex+2][0] +=
- this->kz[zindex]*this->cv[1][tindex+2][1];
- acceleration[tindex+2][1] +=
- -this->kz[zindex]*this->cv[1][tindex+2][0];
- }
- }
- );
- /* 12 23 31 */
- RLOOP (
- this,
- [&](ptrdiff_t rindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/, ptrdiff_t /*zindex*/){
- // cindex indexing is thread safe so there is no overlap between threads
- ptrdiff_t tindex = 3*rindex;
- for (int cc=0; cc<3; cc++)
- this->rv[1][tindex+cc] = this->ru[tindex+cc]*this->ru[tindex+(cc+1)%3] / this->normalization_factor;
- }
- );
- this->clean_up_real_space(this->rv[1], 3);
- fftw_interface<rnumber>::execute(*(this->vr2c[1]));
- this->dealias(this->cv[1], 3);
- CLOOP_K2(
- this,
- [&](ptrdiff_t cindex, ptrdiff_t xindex, ptrdiff_t yindex, ptrdiff_t zindex, double k2){
- if (k2 <= this->kM2)
- {
- // cindex indexing is thread safe so there is no overlap between threads
- ptrdiff_t tindex = 3*cindex;
- acceleration[tindex+0][0] +=
- (this->ky[yindex]*this->cv[1][tindex+0][1] +
- this->kz[zindex]*this->cv[1][tindex+2][1]);
- acceleration[tindex+0][1] +=
- - (this->ky[yindex]*this->cv[1][tindex+0][0] +
- this->kz[zindex]*this->cv[1][tindex+2][0]);
- acceleration[tindex+1][0] +=
- (this->kz[zindex]*this->cv[1][tindex+1][1] +
- this->kx[xindex]*this->cv[1][tindex+0][1]);
- acceleration[tindex+1][1] +=
- - (this->kz[zindex]*this->cv[1][tindex+1][0] +
- this->kx[xindex]*this->cv[1][tindex+0][0]);
- acceleration[tindex+2][0] +=
- (this->kx[xindex]*this->cv[1][tindex+2][1] +
- this->ky[yindex]*this->cv[1][tindex+1][1]);
- acceleration[tindex+2][1] +=
- - (this->kx[xindex]*this->cv[1][tindex+2][0] +
- this->ky[yindex]*this->cv[1][tindex+1][0]);
- }
- }
- );
- if (this->cd->myrank == this->cd->rank[0])
- std::fill_n((rnumber*)(acceleration), 6, 0.0);
- this->force_divfree(acceleration);
-}
-
-template <class rnumber>
-void fluid_solver<rnumber>::compute_Lagrangian_acceleration(rnumber *__restrict__ acceleration)
-{
- TIMEZONE("fluid_solver::compute_Lagrangian_acceleration");
- this->compute_Lagrangian_acceleration((typename fftw_interface<rnumber>::complex*)acceleration);
- fftw_interface<rnumber>::execute(*(this->vc2r[1]));
- std::copy(
- this->rv[1],
- this->rv[1] + 2*this->cd->local_size,
- acceleration);
-}
-
-template <class rnumber>
-int fluid_solver<rnumber>::write_rpressure()
-{
- TIMEZONE("fluid_solver::write_rpressure");
- char fname[512];
- typename fftw_interface<rnumber>::complex *pressure;
- pressure = fftw_interface<rnumber>::alloc_complex(this->cd->local_size/3);
- this->compute_velocity(this->cvorticity);
- this->ift_velocity();
- this->compute_pressure(pressure);
- this->fill_up_filename("rpressure", fname);
- rnumber *rpressure = fftw_interface<rnumber>::alloc_real((this->cd->local_size/3)*2);
- typename fftw_interface<rnumber>::plan c2r;
- c2r = fftw_interface<rnumber>::mpi_plan_dft_c2r_3d(
- this->rd->sizes[0], this->rd->sizes[1], this->rd->sizes[2],
- pressure, rpressure, this->cd->comm,
- this->fftw_plan_rigor | FFTW_MPI_TRANSPOSED_IN);
- fftw_interface<rnumber>::execute(c2r);
- /* output goes here */
- int ntmp[3];
- ntmp[0] = this->rd->sizes[0];
- ntmp[1] = this->rd->sizes[1];
- ntmp[2] = this->rd->sizes[2];
- field_descriptor<rnumber> *scalar_descriptor = new field_descriptor<rnumber>(3, ntmp, mpi_real_type<rnumber>::real(), this->cd->comm);
- clip_zero_padding<rnumber>(scalar_descriptor, rpressure, 1);
- int return_value = scalar_descriptor->write(fname, rpressure);
- delete scalar_descriptor;
- fftw_interface<rnumber>::destroy_plan(c2r);
- fftw_interface<rnumber>::free(pressure);
- fftw_interface<rnumber>::free(rpressure);
- return return_value;
-}
-
-/*****************************************************************************/
-
-
-
-
-/*****************************************************************************/
-/* finally, force generation of code for single precision */
-template class fluid_solver<float>;
-template class fluid_solver<double>;
-/*****************************************************************************/
-
diff --git a/bfps/cpp/fluid_solver.hpp b/bfps/cpp/fluid_solver.hpp
deleted file mode 100644
index aaddbb59..00000000
--- a/bfps/cpp/fluid_solver.hpp
+++ /dev/null
@@ -1,120 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <iostream>
-#include "field_descriptor.hpp"
-#include "fluid_solver_base.hpp"
-
-#ifndef FLUID_SOLVER
-
-#define FLUID_SOLVER
-
-extern int myrank, nprocs;
-
-
-/* container for field descriptor, fields themselves, parameters, etc
- * using the same big macro idea that they're using in fftw3.h
- * I feel like I should quote: Ugh.
- * */
-
-template <class rnumber>
-class fluid_solver:public fluid_solver_base<rnumber>
-{
- public:
- /* fields */
- rnumber *rvorticity;
- rnumber *rvelocity ;
- typename fluid_solver_base<rnumber>::cnumber *cvorticity;
- typename fluid_solver_base<rnumber>::cnumber *cvelocity ;
-
- /* short names for velocity, and 4 vorticity fields */
- rnumber *ru, *rv[4];
- typename fluid_solver_base<rnumber>::cnumber *cu, *cv[4];
-
- /* plans */
- typename fftw_interface<rnumber>::many_plan *c2r_vorticity;
- typename fftw_interface<rnumber>::many_plan *r2c_vorticity;
- typename fftw_interface<rnumber>::many_plan *c2r_velocity;
- typename fftw_interface<rnumber>::many_plan *r2c_velocity;
- typename fftw_interface<rnumber>::many_plan *uc2r, *ur2c;
- typename fftw_interface<rnumber>::many_plan *vr2c[3], *vc2r[3];
-
- /* physical parameters */
- double nu;
- int fmode; // for Kolmogorov flow
- double famplitude; // both for Kflow and band forcing
- double fk0, fk1; // for band forcing
- char forcing_type[128];
-
- /* methods */
- fluid_solver(
- const char *NAME,
- int nx,
- int ny,
- int nz,
- double DKX = 1.0,
- double DKY = 1.0,
- double DKZ = 1.0,
- int DEALIAS_TYPE = 1,
- unsigned FFTW_PLAN_RIGOR = FFTW_MEASURE);
- ~fluid_solver(void);
-
- void compute_gradient_statistics(
- rnumber (*__restrict__ vec)[2],
- double *__restrict__ gradu_moments,
- double *__restrict__ trS2_Q_R_moments,
- ptrdiff_t *__restrict__ gradu_histograms,
- ptrdiff_t *__restrict__ trS2_Q_R_histograms,
- ptrdiff_t *__restrict__ QR2D_histogram,
- double trS2_Q_R_max_estimates[3],
- double gradu_max_estimates[9],
- const int nbins_1D = 256,
- const int nbins_2D = 64);
-
- void compute_vorticity(void);
- void compute_velocity(rnumber (*__restrict__ vorticity)[2]);
- void compute_pressure(rnumber (*__restrict__ pressure)[2]);
- void compute_Eulerian_acceleration(rnumber (*__restrict__ dst)[2]);
- void compute_Lagrangian_acceleration(rnumber (*__restrict__ dst)[2]);
- void compute_Lagrangian_acceleration(rnumber *__restrict__ dst);
- void ift_velocity();
- void dft_velocity();
- void ift_vorticity();
- void dft_vorticity();
- void omega_nonlin(int src);
- void step(double dt);
- void impose_zero_modes(void);
- void add_forcing(rnumber (*__restrict__ acc_field)[2], rnumber (*__restrict__ vort_field)[2], rnumber factor);
-
- int read(char field, char representation);
- int write(char field, char representation);
- int write_rTrS2();
- int write_renstrophy();
- int write_rpressure();
-};
-
-#endif//FLUID_SOLVER
-
diff --git a/bfps/cpp/fluid_solver_base.cpp b/bfps/cpp/fluid_solver_base.cpp
deleted file mode 100644
index b1d64ef5..00000000
--- a/bfps/cpp/fluid_solver_base.cpp
+++ /dev/null
@@ -1,834 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#define NDEBUG
-
-#include <cassert>
-#include <cmath>
-#include <cstring>
-#include "base.hpp"
-#include "fluid_solver_base.hpp"
-#include "fftw_tools.hpp"
-#include "scope_timer.hpp"
-#include "shared_array.hpp"
-
-template <class rnumber>
-void fluid_solver_base<rnumber>::fill_up_filename(const char *base_name, char *destination)
-{
- sprintf(destination, "%s_%s_i%.5x", this->name, base_name, this->iteration);
-}
-
-template <class rnumber>
-void fluid_solver_base<rnumber>::clean_up_real_space(rnumber *a, int howmany)
-{
- TIMEZONE("fluid_solver_base::clean_up_real_space");
- for (ptrdiff_t rindex = 0; rindex < this->cd->local_size*2; rindex += howmany*(this->rd->subsizes[2]+2))
- std::fill_n(a+rindex+this->rd->subsizes[2]*howmany, 2*howmany, 0.0);
-}
-
-template <class rnumber>
-double fluid_solver_base<rnumber>::autocorrel(cnumber *a)
-{
- double *spec = fftw_interface<double>::alloc_real(this->nshells*9);
- double sum_local;
- this->cospectrum(a, a, spec);
- sum_local = 0.0;
- for (unsigned int n = 0; n < this->nshells; n++)
- {
- sum_local += spec[n*9] + spec[n*9 + 4] + spec[n*9 + 8];
- }
- fftw_interface<double>::free(spec);
- return sum_local;
-}
-
-template <class rnumber>
-void fluid_solver_base<rnumber>::cospectrum(cnumber *a, cnumber *b, double *spec)
-{
- TIMEZONE("fluid_solver_base::cospectrum");
- shared_array<double> cospec_local_thread(this->nshells*9,[&](double* cospec_local){
- std::fill_n(cospec_local, this->nshells*9, 0);
- });
-
- CLOOP_K2_NXMODES(
- this,
-
- [&](ptrdiff_t cindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/,
- ptrdiff_t /*zindex*/, double k2, int nxmodes){
- if (k2 <= this->kMspec2)
- {
- int tmp_int = int(sqrt(k2)/this->dk)*9;
- double* cospec_local = cospec_local_thread.getMine();
- for (int i=0; i<3; i++)
- for (int j=0; j<3; j++)
- {
- cospec_local[tmp_int+i*3+j] += nxmodes * (
- (*(a + 3*cindex+i))[0] * (*(b + 3*cindex+j))[0] +
- (*(a + 3*cindex+i))[1] * (*(b + 3*cindex+j))[1]);
- }
- }}
- );
- cospec_local_thread.mergeParallel();
- MPI_Allreduce(
- cospec_local_thread.getMasterData(),
- (void*)spec,
- this->nshells*9,
- MPI_DOUBLE, MPI_SUM, this->cd->comm);
-}
-
-template <class rnumber>
-void fluid_solver_base<rnumber>::cospectrum(cnumber *a, cnumber *b, double *spec, const double k2exponent)
-{
- TIMEZONE("fluid_solver_base::cospectrum2");
- shared_array<double> cospec_local_thread(this->nshells*9,[&](double* cospec_local){
- std::fill_n(cospec_local, this->nshells*9, 0);
- });
-
- CLOOP_K2_NXMODES(
- this,
-
- [&](ptrdiff_t cindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/,
- ptrdiff_t /*zindex*/, double k2, int nxmodes){
- if (k2 <= this->kMspec2)
- {
- double factor = nxmodes*pow(k2, k2exponent);
- int tmp_int = int(sqrt(k2)/this->dk)*9;
- double* cospec_local = cospec_local_thread.getMine();
- for (int i=0; i<3; i++)
- for (int j=0; j<3; j++)
- {
- cospec_local[tmp_int+i*3+j] += factor * (
- (*(a + 3*cindex+i))[0] * (*(b + 3*cindex+j))[0] +
- (*(a + 3*cindex+i))[1] * (*(b + 3*cindex+j))[1]);
- }
- }}
- );
- cospec_local_thread.mergeParallel();
- MPI_Allreduce(
- cospec_local_thread.getMasterData(),
- (void*)spec,
- this->nshells*9,
- MPI_DOUBLE, MPI_SUM, this->cd->comm);
- //for (int n=0; n<this->nshells; n++)
- //{
- // spec[n] *= 12.5663706144*pow(this->kshell[n], 2) / this->nshell[n];
- // /*is normalization needed?
- // * spec[n] /= this->normalization_factor*/
- //}
-}
-
-template <class rnumber>
-void fluid_solver_base<rnumber>::compute_rspace_stats(
- const rnumber *a,
- const hid_t group,
- const std::string dset_name,
- const hsize_t toffset,
- const std::vector<double> max_estimate)
-{
- TIMEZONE("fluid_solver_base::compute_rspace_stats");
- const int nmoments = 10;
- int nvals, nbins;
- if (this->rd->myrank == 0)
- {
- hid_t dset, wspace;
- hsize_t dims[3];
- int ndims;
- dset = H5Dopen(group, ("moments/" + dset_name).c_str(), H5P_DEFAULT);
- wspace = H5Dget_space(dset);
- ndims = H5Sget_simple_extent_dims(wspace, dims, NULL);
- assert(ndims == 3);
- variable_used_only_in_assert(ndims);
- assert(dims[1] == nmoments);
- nvals = dims[2];
- H5Sclose(wspace);
- H5Dclose(dset);
- dset = H5Dopen(group, ("histograms/" + dset_name).c_str(), H5P_DEFAULT);
- wspace = H5Dget_space(dset);
- ndims = H5Sget_simple_extent_dims(wspace, dims, NULL);
- assert(ndims == 3);
- nbins = dims[1];
- assert(nvals == dims[2]);
- H5Sclose(wspace);
- H5Dclose(dset);
- }
- MPI_Bcast(&nvals, 1, MPI_INT, 0, this->rd->comm);
- MPI_Bcast(&nbins, 1, MPI_INT, 0, this->rd->comm);
- assert(nvals == max_estimate.size());
- shared_array<double> threaded_local_moments(nmoments*nvals, [&](double* local_moments){
- std::fill_n(local_moments, nmoments*nvals, 0);
- if (nvals == 4) local_moments[3] = max_estimate[3];
- });
-
- shared_array<double> threaded_val_tmp(nvals);
-
- shared_array<ptrdiff_t> threaded_local_hist(nbins*nvals, [&](ptrdiff_t* local_hist){
- std::fill_n(local_hist, nbins*nvals, 0);
- });
-
- // Not written by threads
- double *binsize = new double[nvals];
- for (int i=0; i<nvals; i++)
- binsize[i] = 2*max_estimate[i] / nbins;
-
- RLOOP(
- this,
- [&](ptrdiff_t rindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/, ptrdiff_t /*zindex*/){
- double *val_tmp = threaded_val_tmp.getMine();
- ptrdiff_t* local_hist = threaded_local_hist.getMine();
- double *local_moments = threaded_local_moments.getMine();
-
- if (nvals == 4) val_tmp[3] = 0.0;
- for (int i=0; i<3; i++)
- {
- val_tmp[i] = a[rindex*3+i];
- if (nvals == 4) val_tmp[3] += val_tmp[i]*val_tmp[i];
- }
- if (nvals == 4)
- {
- val_tmp[3] = sqrt(val_tmp[3]);
- if (val_tmp[3] < local_moments[0*nvals+3])
- local_moments[0*nvals+3] = val_tmp[3];
- if (val_tmp[3] > local_moments[9*nvals+3])
- local_moments[9*nvals+3] = val_tmp[3];
- int bin = int(floor(val_tmp[3]*2/binsize[3]));
- if (bin >= 0 && bin < nbins)
- local_hist[bin*nvals+3]++;
- }
- for (int i=0; i<3; i++)
- {
- if (val_tmp[i] < local_moments[0*nvals+i])
- local_moments[0*nvals+i] = val_tmp[i];
- if (val_tmp[i] > local_moments[(nmoments-1)*nvals+i])
- local_moments[(nmoments-1)*nvals+i] = val_tmp[i];
- int bin = int(floor((val_tmp[i] + max_estimate[i]) / binsize[i]));
- if (bin >= 0 && bin < nbins)
- local_hist[bin*nvals+i]++;
- }
- for (int n=1; n < nmoments-1; n++){
- double pow_tmp = 1.;
- for (int i=0; i<nvals; i++){
- local_moments[n*nvals + i] += (pow_tmp = val_tmp[i]*pow_tmp);
- }
- }
- }
- );
-
- threaded_local_hist.mergeParallel();
- threaded_local_moments.mergeParallel([&](const int idx, const double& v1, const double& v2) -> double {
- if(nvals == int(4) && idx == 0*nvals+3){
- return std::min(v1, v2);
- }
- if(nvals == int(4) && idx == 9*nvals+3){
- return std::max(v1, v2);
- }
- if(idx < 3){
- return std::min(v1, v2);
- }
- if((nmoments-1)*nvals <= idx && idx < (nmoments-1)*nvals+3){
- return std::max(v1, v2);
- }
- return v1 + v2;
- });
-
-
- double *moments = new double[nmoments*nvals];
- MPI_Allreduce(
- threaded_local_moments.getMasterData(),
- (void*)moments,
- nvals,
- MPI_DOUBLE, MPI_MIN, this->cd->comm);
- MPI_Allreduce(
- (threaded_local_moments.getMasterData() + nvals),
- (void*)(moments+nvals),
- (nmoments-2)*nvals,
- MPI_DOUBLE, MPI_SUM, this->cd->comm);
- MPI_Allreduce(
- (threaded_local_moments.getMasterData() + (nmoments-1)*nvals),
- (void*)(moments+(nmoments-1)*nvals),
- nvals,
- MPI_DOUBLE, MPI_MAX, this->cd->comm);
- ptrdiff_t *hist = new ptrdiff_t[nbins*nvals];
- MPI_Allreduce(
- threaded_local_hist.getMasterData(),
- (void*)hist,
- nbins*nvals,
- MPI_INT64_T, MPI_SUM, this->cd->comm);
- for (int n=1; n < nmoments-1; n++)
- for (int i=0; i<nvals; i++)
- moments[n*nvals + i] /= this->normalization_factor;
- delete[] binsize;
- if (this->rd->myrank == 0)
- {
- hid_t dset, wspace, mspace;
- hsize_t count[3], offset[3], dims[3];
- dset = H5Dopen(group, ("moments/" + dset_name).c_str(), H5P_DEFAULT);
- wspace = H5Dget_space(dset);
- H5Sget_simple_extent_dims(wspace, dims, NULL);
- offset[0] = toffset;
- offset[1] = 0;
- offset[2] = 0;
- count[0] = 1;
- count[1] = nmoments;
- count[2] = nvals;
- mspace = H5Screate_simple(3, count, NULL);
- H5Sselect_hyperslab(wspace, H5S_SELECT_SET, offset, NULL, count, NULL);
- H5Dwrite(dset, H5T_NATIVE_DOUBLE, mspace, wspace, H5P_DEFAULT, moments);
- H5Sclose(wspace);
- H5Sclose(mspace);
- H5Dclose(dset);
- dset = H5Dopen(group, ("histograms/" + dset_name).c_str(), H5P_DEFAULT);
- wspace = H5Dget_space(dset);
- count[1] = nbins;
- mspace = H5Screate_simple(3, count, NULL);
- H5Sselect_hyperslab(wspace, H5S_SELECT_SET, offset, NULL, count, NULL);
- H5Dwrite(dset, H5T_NATIVE_INT64, mspace, wspace, H5P_DEFAULT, hist);
- H5Sclose(wspace);
- H5Sclose(mspace);
- H5Dclose(dset);
- }
- delete[] moments;
- delete[] hist;
-}
-
-
-
-template <class rnumber>
-template<int nvals>
-void fluid_solver_base<rnumber>::compute_rspace_stats(
- rnumber *a,
- double *moments,
- ptrdiff_t *hist,
- double max_estimate[],
- const int nbins)
-{
- TIMEZONE("fluid_solver_base::compute_rspace_stats");
- shared_array<double> threaded_local_moments(10*nvals,[&](double* local_moments){
- std::fill_n(local_moments, 10*nvals, 0);
- if (nvals == 4) local_moments[3] = max_estimate[3];
- });
-
- shared_array<ptrdiff_t> threaded_local_hist(nbins*nvals, [&](ptrdiff_t* local_hist){
- std::fill_n(local_hist, nbins*nvals, 0);
- });
-
- // Will not be modified by the threads
- double binsize[nvals];
- for (int i=0; i<nvals; i++)
- binsize[i] = 2*max_estimate[i] / nbins;
-
- RLOOP(
- this,
- [&](ptrdiff_t rindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/, ptrdiff_t /*zindex*/){
- ptrdiff_t *local_hist = threaded_local_hist.getMine();
- double *local_moments = threaded_local_moments.getMine();
-
- double val_tmp[nvals];
- if (nvals == 4) val_tmp[3] = 0.0;
- for (int i=0; i<3; i++)
- {
- val_tmp[i] = a[rindex*3+i];
- if (nvals == 4) val_tmp[3] += val_tmp[i]*val_tmp[i];
- }
- if (nvals == 4)
- {
- val_tmp[3] = sqrt(val_tmp[3]);
- if (val_tmp[3] < local_moments[0*nvals+3])
- local_moments[0*nvals+3] = val_tmp[3];
- if (val_tmp[3] > local_moments[9*nvals+3])
- local_moments[9*nvals+3] = val_tmp[3];
- int bin = int(floor(val_tmp[3]*2/binsize[3]));
- if (bin >= 0 && bin < nbins)
- local_hist[bin*nvals+3]++;
- }
- for (int i=0; i<3; i++)
- {
- if (val_tmp[i] < local_moments[0*nvals+i])
- local_moments[0*nvals+i] = val_tmp[i];
- if (val_tmp[i] > local_moments[9*nvals+i])
- local_moments[9*nvals+i] = val_tmp[i];
- int bin = int(floor((val_tmp[i] + max_estimate[i]) / binsize[i]));
- if (bin >= 0 && bin < nbins)
- local_hist[bin*nvals+i]++;
- }
- for (int n=1; n<9; n++){
- double pow_tmp = 1;
- for (int i=0; i<nvals; i++){
- local_moments[n*nvals + i] += (pow_tmp = val_tmp[i]*pow_tmp);
- }
- }
- }
- );
-
- threaded_local_moments.mergeParallel([&](const int idx, const double& v1, const double& v2) -> double {
- if(nvals == int(4) && idx == 0*nvals+3){
- return std::min(v1, v2);
- }
- if(nvals == int(4) && idx == 9*nvals+3){
- return std::max(v1, v2);
- }
- if(idx < 3){
- return std::min(v1, v2);
- }
- if(9*nvals <= idx && idx < 9*nvals+3){
- return std::max(v1, v2);
- }
- return v1 + v2;
- });
- threaded_local_hist.mergeParallel();
-
- MPI_Allreduce(
- threaded_local_moments.getMasterData(),
- (void*)moments,
- nvals,
- MPI_DOUBLE, MPI_MIN, this->cd->comm);
- MPI_Allreduce(
- (threaded_local_moments.getMasterData() + nvals),
- (void*)(moments+nvals),
- 8*nvals,
- MPI_DOUBLE, MPI_SUM, this->cd->comm);
- MPI_Allreduce(
- (threaded_local_moments.getMasterData() + 9*nvals),
- (void*)(moments+9*nvals),
- nvals,
- MPI_DOUBLE, MPI_MAX, this->cd->comm);
- MPI_Allreduce(
- (void*)threaded_local_hist.getMasterData(),
- (void*)hist,
- nbins*nvals,
- MPI_INT64_T, MPI_SUM, this->cd->comm);
- for (int n=1; n<9; n++)
- for (int i=0; i<nvals; i++)
- moments[n*nvals + i] /= this->normalization_factor;
-}
-
-template <class rnumber>
-void fluid_solver_base<rnumber>::write_spectrum(const char *fname, cnumber *a, const double k2exponent)
-{
- TIMEZONE("fluid_solver_base::write_spectrum");
- double *spec = fftw_interface<double>::alloc_real(this->nshells);
- this->cospectrum(a, a, spec, k2exponent);
- if (this->cd->myrank == 0)
- {
- FILE *spec_file;
- char full_name[512];
- sprintf(full_name, "%s_%s_spec", this->name, fname);
- spec_file = fopen(full_name, "ab");
- fwrite((void*)&this->iteration, sizeof(int), 1, spec_file);
- fwrite((void*)spec, sizeof(double), this->nshells, spec_file);
- fclose(spec_file);
- }
- fftw_interface<double>::free(spec);
-}
-
-/*****************************************************************************/
-/* macro for specializations to numeric types compatible with FFTW */
-
-template <class rnumber>
-fluid_solver_base<rnumber>::fluid_solver_base(
- const char *NAME,
- int nx,
- int ny,
- int nz,
- double DKX,
- double DKY,
- double DKZ,
- int DEALIAS_TYPE,
- unsigned FFTW_PLAN_RIGOR)
-{
- TIMEZONE("fluid_solver_base::fluid_solver_base");
- strncpy(this->name, NAME, 256);
- this->name[255] = '\0';
- this->iteration = 0;
- this->fftw_plan_rigor = FFTW_PLAN_RIGOR;
-
- int ntmp[4];
- ntmp[0] = nz;
- ntmp[1] = ny;
- ntmp[2] = nx;
- ntmp[3] = 3;
- this->rd = new field_descriptor<rnumber>(
- 4, ntmp, mpi_real_type<rnumber>::real(), MPI_COMM_WORLD);
- this->normalization_factor = (this->rd->full_size/3);
- ntmp[0] = ny;
- ntmp[1] = nz;
- ntmp[2] = nx/2 + 1;
- ntmp[3] = 3;
- this->cd = new field_descriptor<rnumber>(
- 4, ntmp, mpi_real_type<rnumber>::complex(), this->rd->comm);
-
- this->dkx = DKX;
- this->dky = DKY;
- this->dkz = DKZ;
- this->kx = new double[this->cd->sizes[2]];
- this->ky = new double[this->cd->subsizes[0]];
- this->kz = new double[this->cd->sizes[1]];
- this->dealias_type = DEALIAS_TYPE;
- switch(this->dealias_type)
- {
- /* HL07 smooth filter */
- case 1:
- this->kMx = this->dkx*(int(this->rd->sizes[2] / 2)-1);
- this->kMy = this->dky*(int(this->rd->sizes[1] / 2)-1);
- this->kMz = this->dkz*(int(this->rd->sizes[0] / 2)-1);
- break;
- default:
- this->kMx = this->dkx*(int(this->rd->sizes[2] / 3)-1);
- this->kMy = this->dky*(int(this->rd->sizes[1] / 3)-1);
- this->kMz = this->dkz*(int(this->rd->sizes[0] / 3)-1);
- }
- int i, ii;
- for (i = 0; i<this->cd->sizes[2]; i++)
- this->kx[i] = i*this->dkx;
- for (i = 0; i<this->cd->subsizes[0]; i++)
- {
- ii = i + this->cd->starts[0];
- if (ii <= this->rd->sizes[1]/2)
- this->ky[i] = this->dky*ii;
- else
- this->ky[i] = this->dky*(ii - this->rd->sizes[1]);
- }
- for (i = 0; i<this->cd->sizes[1]; i++)
- {
- if (i <= this->rd->sizes[0]/2)
- this->kz[i] = this->dkz*i;
- else
- this->kz[i] = this->dkz*(i - this->rd->sizes[0]);
- }
- this->kM = this->kMx;
- if (this->kM < this->kMy) this->kM = this->kMy;
- if (this->kM < this->kMz) this->kM = this->kMz;
- this->kM2 = this->kM * this->kM;
- this->kMspec = this->kM;
- this->kMspec2 = this->kM2;
- this->dk = this->dkx;
- if (this->dk > this->dky) this->dk = this->dky;
- if (this->dk > this->dkz) this->dk = this->dkz;
- this->dk2 = this->dk*this->dk;
- DEBUG_MSG(
- "kM = %g, kM2 = %g, dk = %g, dk2 = %g\n",
- this->kM, this->kM2, this->dk, this->dk2);
- /* spectra stuff */
- this->nshells = int(this->kMspec / this->dk) + 2;
- DEBUG_MSG(
- "kMspec = %g, kMspec2 = %g, nshells = %ld\n",
- this->kMspec, this->kMspec2, this->nshells);
- this->kshell = new double[this->nshells];
- std::fill_n(this->kshell, this->nshells, 0.0);
- this->nshell = new int64_t[this->nshells];
- std::fill_n(this->nshell, this->nshells, 0);
- DEBUG_MSG("fluid_solver_base::fluid_solver_base before declaring shared_array\n");
-
- shared_array<double> kshell_local_threaded(this->nshells,[&](double* kshell_local){
- std::fill_n(kshell_local, this->nshells, 0.0);
- });
- DEBUG_MSG("fluid_solver_base::fluid_solver_base before declaring shared_array\n");
- shared_array<int64_t> nshell_local_threaded(this->nshells,[&](int64_t* nshell_local){
- std::fill_n(nshell_local, this->nshells, 0);
- });
-
- std::vector<std::unordered_map<int, double>> Fourier_filter_threaded(omp_get_max_threads());
-
- DEBUG_MSG("fluid_solver_base::fluid_solver_base before cloop_k2_nxmodes\n");
- CLOOP_K2_NXMODES(
- this,
-
- [&](ptrdiff_t /*cindex*/, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/,
- ptrdiff_t /*zindex*/, double k2, int nxmodes){
- if (k2 < this->kM2)
- {
- double knorm = sqrt(k2);
- nshell_local_threaded.getMine()[int(knorm/this->dk)] += nxmodes;
- kshell_local_threaded.getMine()[int(knorm/this->dk)] += nxmodes*knorm;
- }
- Fourier_filter_threaded[omp_get_thread_num()][int(round(k2 / this->dk2))] = exp(-36.0 * pow(k2/this->kM2, 18.));}
- );
-
- // Merge results
- nshell_local_threaded.mergeParallel();
- kshell_local_threaded.mergeParallel();
- for(int idxMerge = 0 ; idxMerge < int(Fourier_filter_threaded.size()) ; ++idxMerge){
- for(const auto kv : Fourier_filter_threaded[idxMerge]){
- this->Fourier_filter[kv.first] = kv.second;
- }
- }
-
- MPI_Allreduce(
- (void*)(nshell_local_threaded.getMasterData()),
- (void*)(this->nshell),
- this->nshells,
- MPI_INT64_T, MPI_SUM, this->cd->comm);
- MPI_Allreduce(
- (void*)(kshell_local_threaded.getMasterData()),
- (void*)(this->kshell),
- this->nshells,
- MPI_DOUBLE, MPI_SUM, this->cd->comm);
- for (unsigned int n=0; n<this->nshells; n++)
- {
- if (this->nshell[n] != 0)
- this->kshell[n] /= this->nshell[n];
- else
- this->kshell[n] = -1;
- }
- DEBUG_MSG("exiting fluid_solver_base::fluid_solver_base\n");
-}
-
-template <class rnumber>
-fluid_solver_base<rnumber>::~fluid_solver_base()
-{
- delete[] this->kshell;
- delete[] this->nshell;
-
- delete[] this->kx;
- delete[] this->ky;
- delete[] this->kz;
-
- delete this->cd;
- delete this->rd;
-}
-
-template <class rnumber>
-void fluid_solver_base<rnumber>::low_pass_Fourier(cnumber *a, const int howmany, const double kmax)
-{
- TIMEZONE("fluid_solver_base::low_pass_Fourier");
- const double km2 = kmax*kmax;
- const int howmany2 = 2*howmany;
- /*DEBUG_MSG("entered low_pass_Fourier, kmax=%lg km2=%lg howmany2=%d\n", kmax, km2, howmany2);*/
- CLOOP_K2(
- this,
- /*DEBUG_MSG("kx=%lg ky=%lg kz=%lg k2=%lg\n",
- this->kx[xindex],
- this->ky[yindex],
- this->kz[zindex],
- k2);*/
-
- [&](ptrdiff_t cindex, ptrdiff_t xindex, ptrdiff_t yindex,
- ptrdiff_t zindex, double k2){
- if (k2 >= km2)
- std::fill_n((rnumber*)(a + howmany*cindex), howmany2, 0.0);}
- );
-}
-
-template <class rnumber>
-void fluid_solver_base<rnumber>::dealias(cnumber *a, const int howmany)
-{
- TIMEZONE("fluid_solver_base::dealias");
- if (this->dealias_type == 0)
- {
- this->low_pass_Fourier(a, howmany, this->kM);
- return;
- }
-
- CLOOP_K2(
- this,
- [&](ptrdiff_t cindex, ptrdiff_t /*xindex*/, ptrdiff_t /*yindex*/,
- ptrdiff_t /*zindex*/, double k2){
- double tval = this->Fourier_filter[int(round(k2/this->dk2))];
- // It is thread safe on the index cindex
- for (int tcounter = 0; tcounter < howmany; tcounter++)
- for (int i=0; i<2; i++)
- a[howmany*cindex+tcounter][i] *= tval;
- }
- );
-}
-
-template <class rnumber>
-void fluid_solver_base<rnumber>::force_divfree(cnumber *a)
-{
- TIMEZONE("fluid_solver_base::force_divfree");
- CLOOP_K2(
- this,
-
- [&](ptrdiff_t cindex, ptrdiff_t xindex, ptrdiff_t yindex,
- ptrdiff_t zindex, double k2){
- if (k2 > 0)
- {
- // It is thread safe on index cindex
- cnumber tval;
- tval[0] = (this->kx[xindex]*((*(a + cindex*3 ))[0]) +
- this->ky[yindex]*((*(a + cindex*3+1))[0]) +
- this->kz[zindex]*((*(a + cindex*3+2))[0]) ) / k2;
- tval[1] = (this->kx[xindex]*((*(a + cindex*3 ))[1]) +
- this->ky[yindex]*((*(a + cindex*3+1))[1]) +
- this->kz[zindex]*((*(a + cindex*3+2))[1]) ) / k2;
- for (int imag_part=0; imag_part<2; imag_part++)
- {
- a[cindex*3 ][imag_part] -= tval[imag_part]*this->kx[xindex];
- a[cindex*3+1][imag_part] -= tval[imag_part]*this->ky[yindex];
- a[cindex*3+2][imag_part] -= tval[imag_part]*this->kz[zindex];
- }
- }}
- );
- if (this->cd->myrank == this->cd->rank[0])
- std::fill_n((rnumber*)(a), 6, 0.0);
-}
-
-template <class rnumber>
-void fluid_solver_base<rnumber>::compute_vector_gradient(cnumber *A, cnumber *cvec)
-{
- TIMEZONE("fluid_solver_base::compute_vector_gradient");
- std::fill_n((rnumber*)A, 3*2*this->cd->local_size, 0.0);
- cnumber *dx_u, *dy_u, *dz_u;
- dx_u = A;
- dy_u = A + this->cd->local_size;
- dz_u = A + 2*this->cd->local_size;
- CLOOP_K2(
- this,
-
- [&](ptrdiff_t cindex, ptrdiff_t xindex, ptrdiff_t yindex,
- ptrdiff_t zindex, double k2){
- if (k2 <= this->kM2)
- {
- // It is thread safe on cindex
- ptrdiff_t tindex = 3*cindex;
- for (int cc=0; cc<3; cc++)
- {
- dx_u[tindex + cc][0] = -this->kx[xindex]*cvec[tindex+cc][1];
- dx_u[tindex + cc][1] = this->kx[xindex]*cvec[tindex+cc][0];
- dy_u[tindex + cc][0] = -this->ky[yindex]*cvec[tindex+cc][1];
- dy_u[tindex + cc][1] = this->ky[yindex]*cvec[tindex+cc][0];
- dz_u[tindex + cc][0] = -this->kz[zindex]*cvec[tindex+cc][1];
- dz_u[tindex + cc][1] = this->kz[zindex]*cvec[tindex+cc][0];
- }
- }}
- );
-}
-
-template <class rnumber>
-void fluid_solver_base<rnumber>::symmetrize(cnumber *data, const int howmany)
-{
- TIMEZONE("fluid_solver_base::symmetrize");
- ptrdiff_t ii, cc;
- MPI_Status *mpistatus = new MPI_Status;
- if (this->cd->myrank == this->cd->rank[0])
- {
- for (cc = 0; cc < howmany; cc++)
- data[cc][1] = 0.0;
- for (ii = 1; ii < this->cd->sizes[1]/2; ii++)
- for (cc = 0; cc < howmany; cc++) {
- ( *(data + cc + howmany*(this->cd->sizes[1] - ii)*this->cd->sizes[2]))[0] =
- (*(data + cc + howmany*( ii)*this->cd->sizes[2]))[0];
- ( *(data + cc + howmany*(this->cd->sizes[1] - ii)*this->cd->sizes[2]))[1] =
- -(*(data + cc + howmany*( ii)*this->cd->sizes[2]))[1];
- }
- }
- cnumber *buffer;
- buffer = fftw_interface<rnumber>::alloc_complex(howmany*this->cd->sizes[1]);
- ptrdiff_t yy;
- /*ptrdiff_t tindex;*/
- int ranksrc, rankdst;
- for (yy = 1; yy < this->cd->sizes[0]/2; yy++) {
- ranksrc = this->cd->rank[yy];
- rankdst = this->cd->rank[this->cd->sizes[0] - yy];
- if (this->cd->myrank == ranksrc)
- for (ii = 0; ii < this->cd->sizes[1]; ii++)
- for (cc = 0; cc < howmany; cc++)
- for (int imag_comp=0; imag_comp<2; imag_comp++)
- (*(buffer + howmany*ii+cc))[imag_comp] =
- (*(data + howmany*((yy - this->cd->starts[0])*this->cd->sizes[1] + ii)*this->cd->sizes[2] + cc))[imag_comp];
- if (ranksrc != rankdst)
- {
- if (this->cd->myrank == ranksrc)
- MPI_Send((void*)buffer,
- howmany*this->cd->sizes[1], mpi_real_type<rnumber>::complex(), rankdst, yy,
- this->cd->comm);
- if (this->cd->myrank == rankdst)
- MPI_Recv((void*)buffer,
- howmany*this->cd->sizes[1], mpi_real_type<rnumber>::complex(), ranksrc, yy,
- this->cd->comm, mpistatus);
- }
- if (this->cd->myrank == rankdst)
- {
- for (ii = 1; ii < this->cd->sizes[1]; ii++)
- for (cc = 0; cc < howmany; cc++)
- {
- (*(data + howmany*((this->cd->sizes[0] - yy - this->cd->starts[0])*this->cd->sizes[1] + ii)*this->cd->sizes[2] + cc))[0] =
- (*(buffer + howmany*(this->cd->sizes[1]-ii)+cc))[0];
- (*(data + howmany*((this->cd->sizes[0] - yy - this->cd->starts[0])*this->cd->sizes[1] + ii)*this->cd->sizes[2] + cc))[1] =
- -(*(buffer + howmany*(this->cd->sizes[1]-ii)+cc))[1];
- }
- for (cc = 0; cc < howmany; cc++)
- {
- (*((data + cc + howmany*(this->cd->sizes[0] - yy - this->cd->starts[0])*this->cd->sizes[1]*this->cd->sizes[2])))[0] = (*(buffer + cc))[0];
- (*((data + cc + howmany*(this->cd->sizes[0] - yy - this->cd->starts[0])*this->cd->sizes[1]*this->cd->sizes[2])))[1] = -(*(buffer + cc))[1];
- }
- }
- }
- fftw_interface<rnumber>::free(buffer);
- delete mpistatus;
- /* put asymmetric data to 0 */
- /*if (this->cd->myrank == this->cd->rank[this->cd->sizes[0]/2])
- {
- tindex = howmany*(this->cd->sizes[0]/2 - this->cd->starts[0])*this->cd->sizes[1]*this->cd->sizes[2];
- for (ii = 0; ii < this->cd->sizes[1]; ii++)
- {
- std::fill_n((rnumber*)(data + tindex), howmany*2*this->cd->sizes[2], 0.0);
- tindex += howmany*this->cd->sizes[2];
- }
- }
- tindex = howmany*();
- std::fill_n((rnumber*)(data + tindex), howmany*2, 0.0);*/
-}
-
-template <class rnumber>
-int fluid_solver_base<rnumber>::read_base(const char *fname, rnumber *data)
-{
- char full_name[512];
- sprintf(full_name, "%s_%s_i%.5x", this->name, fname, this->iteration);
- return this->rd->read(full_name, (void*)data);
-}
-
-template <class rnumber>
-int fluid_solver_base<rnumber>::read_base(const char *fname, cnumber *data)
-{
- char full_name[512];
- sprintf(full_name, "%s_%s_i%.5x", this->name, fname, this->iteration);
- return this->cd->read(full_name, (void*)data);
-}
-
-template <class rnumber>
-int fluid_solver_base<rnumber>::write_base(const char *fname, rnumber *data)
-{
- char full_name[512];
- sprintf(full_name, "%s_%s_i%.5x", this->name, fname, this->iteration);
- return this->rd->write(full_name, (void*)data);
-}
-
-template <class rnumber>
-int fluid_solver_base<rnumber>::write_base(const char *fname, cnumber *data)
-{
- char full_name[512];
- sprintf(full_name, "%s_%s_i%.5x", this->name, fname, this->iteration);
- return this->cd->write(full_name, (void*)data);
-}
-
-/* finally, force generation of code */
-template class fluid_solver_base<float>;
-template class fluid_solver_base<double>;
-
-/*****************************************************************************/
-
-
-
-
diff --git a/bfps/cpp/fluid_solver_base.hpp b/bfps/cpp/fluid_solver_base.hpp
deleted file mode 100644
index e4469560..00000000
--- a/bfps/cpp/fluid_solver_base.hpp
+++ /dev/null
@@ -1,272 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <hdf5.h>
-#include <iostream>
-#include <unordered_map>
-#include <vector>
-#include "base.hpp"
-#include "field_descriptor.hpp"
-#include "scope_timer.hpp"
-#include "omputils.hpp"
-
-#ifndef FLUID_SOLVER_BASE
-
-#define FLUID_SOLVER_BASE
-
-extern int myrank, nprocs;
-
-
-/* container for field descriptor, fields themselves, parameters, etc
- * using the same big macro idea that they're using in fftw3.h
- * I feel like I should quote: Ugh.
- * */
-
-template <class rnumber>
-class fluid_solver_base
-{
- protected:
- typedef rnumber cnumber[2];
- public:
- field_descriptor<rnumber> *cd, *rd;
- ptrdiff_t normalization_factor;
- unsigned fftw_plan_rigor;
-
- /* simulation parameters */
- char name[256];
- int iteration;
-
- /* physical parameters */
- double dkx, dky, dkz, dk, dk2;
-
- /* mode and dealiasing information */
- int dealias_type;
- double kMx, kMy, kMz, kM, kM2;
- double kMspec, kMspec2;
- double *kx, *ky, *kz;
- std::unordered_map<int, double> Fourier_filter;
- double *kshell;
- int64_t *nshell;
- unsigned int nshells;
-
-
- /* methods */
- fluid_solver_base(
- const char *NAME,
- int nx,
- int ny,
- int nz,
- double DKX = 1.0,
- double DKY = 1.0,
- double DKZ = 1.0,
- int DEALIAS_TYPE = 0,
- unsigned FFTW_PLAN_RIGOR = DEFAULT_FFTW_FLAG);
- ~fluid_solver_base();
-
- void low_pass_Fourier(cnumber *__restrict__ a, int howmany, double kmax);
- void dealias(cnumber *__restrict__ a, int howmany);
- void force_divfree(cnumber *__restrict__ a);
- void symmetrize(cnumber *__restrict__ a, int howmany);
- void clean_up_real_space(rnumber *__restrict__ a, int howmany);
- void cospectrum(cnumber *__restrict__ a, cnumber *__restrict__ b, double *__restrict__ spec);
- void cospectrum(cnumber *__restrict__ a, cnumber *__restrict__ b, double *__restrict__ spec, const double k2exponent);
- double autocorrel(cnumber *__restrict__ a);
- void compute_rspace_stats(
- const rnumber *__restrict__ a,
- const hid_t group,
- const std::string dset_name,
- const hsize_t toffset,
- const std::vector<double> max_estimate);
- template <int nvals>
- void compute_rspace_stats(rnumber *__restrict__ a,
- double *__restrict__ moments,
- ptrdiff_t *__restrict__ hist,
- double max_estimate[nvals],
- const int nbins = 256);
- inline void compute_rspace_stats3(rnumber *__restrict__ a,
- double *__restrict__ moments,
- ptrdiff_t *__restrict__ hist,
- double max_estimate[3],
- const int nbins = 256)
- {
- this->compute_rspace_stats<3>(a, moments, hist, max_estimate, nbins);
- }
- inline void compute_rspace_stats4(rnumber *__restrict__ a,
- double *__restrict__ moments,
- ptrdiff_t *__restrict__ hist,
- double max_estimate[4],
- const int nbins = 256)
- {
- this->compute_rspace_stats<4>(a, moments, hist, max_estimate, nbins);
- }
- void compute_vector_gradient(rnumber (*__restrict__ A)[2], rnumber(*__restrict__ source)[2]);
- void write_spectrum(const char *fname, cnumber *a, const double k2exponent = 0.0);
- void fill_up_filename(const char *base_name, char *full_name);
- int read_base(const char *fname, rnumber *data);
- int read_base(const char *fname, cnumber *data);
- int write_base(const char *fname, rnumber *data);
- int write_base(const char *fname, cnumber *data);
-};
-
-
-
-/*****************************************************************************/
-/* macros for loops */
-
-/* Fourier space loop */
-template <class ObjectType, class FuncType>
-void CLOOP(ObjectType* obj, FuncType expression)
-{
- TIMEZONE("CLOOP");
- #pragma omp parallel
- {
- const hsize_t start = OmpUtils::ForIntervalStart(obj->cd->subsizes[0]);
- const hsize_t end = OmpUtils::ForIntervalEnd(obj->cd->subsizes[0]);
- for (ptrdiff_t yindex = start; yindex < ptrdiff_t(end); yindex++){
- ptrdiff_t cindex = yindex*obj->cd->subsizes[1]*obj->cd->subsizes[2];
- for (ptrdiff_t zindex = 0; zindex < obj->cd->subsizes[1]; zindex++)
- for (ptrdiff_t xindex = 0; xindex < obj->cd->subsizes[2]; xindex++)
- {
- expression(cindex, xindex, yindex, zindex);
- cindex++;
- }
- }
- }
-}
-
-template <class ObjectType, class FuncType>
-void CLOOP_NXMODES(ObjectType* obj, FuncType expression)
-{
- TIMEZONE("CLOOP_NXMODES");
- #pragma omp parallel
- {
- const hsize_t start = OmpUtils::ForIntervalStart(obj->cd->subsizes[1]);
- const hsize_t end = OmpUtils::ForIntervalEnd(obj->cd->subsizes[1]);
- for (ptrdiff_t yindex = 0; yindex < obj->cd->subsizes[0]; yindex++){
- for (ptrdiff_t zindex = start; zindex < ptrdiff_t(end); zindex++)
- {
- ptrdiff_t cindex = yindex*obj->cd->subsizes[1]*obj->cd->subsizes[2]
- + zindex*obj->cd->subsizes[2];
- int nxmodes = 1;
- ptrdiff_t xindex = 0;
- expression();
- cindex++;
- nxmodes = 2;
- for (xindex = 1; xindex < obj->cd->subsizes[2]; xindex++)
- {
- expression();
- cindex++;
- }
- }
- }
- }
-}
-
-
-template <class ObjectType, class FuncType>
-void CLOOP_K2(ObjectType* obj, FuncType expression)
-{
- TIMEZONE("CLOOP_K2");
- #pragma omp parallel
- {
- const hsize_t start = OmpUtils::ForIntervalStart(obj->cd->subsizes[1]);
- const hsize_t end = OmpUtils::ForIntervalEnd(obj->cd->subsizes[1]);
- for (ptrdiff_t yindex = 0; yindex < obj->cd->subsizes[0]; yindex++){
- for (ptrdiff_t zindex = start; zindex < ptrdiff_t(end); zindex++){
- ptrdiff_t cindex = yindex*obj->cd->subsizes[1]*obj->cd->subsizes[2]
- + zindex*obj->cd->subsizes[2];
- for (ptrdiff_t xindex = 0; xindex < obj->cd->subsizes[2]; xindex++)
- {
- double k2 = (obj->kx[xindex]*obj->kx[xindex] +
- obj->ky[yindex]*obj->ky[yindex] +
- obj->kz[zindex]*obj->kz[zindex]);
- expression(cindex, xindex, yindex, zindex, k2);
- cindex++;
- }
- }
- }
- }
-}
-
-
-template <class ObjectType, class FuncType>
-void CLOOP_K2_NXMODES(ObjectType* obj, FuncType expression)
-{
- #pragma omp parallel
- {
- const hsize_t start = OmpUtils::ForIntervalStart(obj->cd->subsizes[1]);
- const hsize_t end = OmpUtils::ForIntervalEnd(obj->cd->subsizes[1]);
- for (ptrdiff_t yindex = 0; yindex < obj->cd->subsizes[0]; yindex++){
- for (ptrdiff_t zindex = start; zindex < ptrdiff_t(end); zindex++)
- {
- ptrdiff_t cindex = yindex*obj->cd->subsizes[1]*obj->cd->subsizes[2]
- + zindex*obj->cd->subsizes[2];
- int nxmodes = 1;
- ptrdiff_t xindex = 0;
- double k2 = (obj->kx[xindex]*obj->kx[xindex] +
- obj->ky[yindex]*obj->ky[yindex] +
- obj->kz[zindex]*obj->kz[zindex]);
- expression(cindex, xindex, yindex, zindex, k2, nxmodes);
- cindex++;
- nxmodes = 2;
- for (xindex = 1; xindex < obj->cd->subsizes[2]; xindex++)
- {
- double k2 = (obj->kx[xindex]*obj->kx[xindex] +
- obj->ky[yindex]*obj->ky[yindex] +
- obj->kz[zindex]*obj->kz[zindex]);
- expression(cindex, xindex, yindex, zindex, k2, nxmodes);
- cindex++;
- }
- }
- }
- }
-}
-
-
-template <class ObjectType, class FuncType>
-void RLOOP(ObjectType* obj, FuncType expression)
-{
- #pragma omp parallel
- {
- const hsize_t start = OmpUtils::ForIntervalStart(obj->rd->subsizes[1]);
- const hsize_t end = OmpUtils::ForIntervalEnd(obj->rd->subsizes[1]);
- for (int zindex = 0; zindex < obj->rd->subsizes[0] ; zindex++)
- for (int yindex = start; yindex < ptrdiff_t(end); yindex++)
- {
- ptrdiff_t rindex = (zindex * obj->rd->subsizes[1] + yindex)*(obj->rd->subsizes[2]+2);
- for (int xindex = 0; xindex < obj->rd->subsizes[2]; xindex++)
- {
- expression(rindex, xindex, yindex, zindex);
- rindex++;
- }
- }
- }
-}
-
-/*****************************************************************************/
-
-#endif//FLUID_SOLVER_BASE
-
diff --git a/bfps/cpp/interpolator.cpp b/bfps/cpp/interpolator.cpp
deleted file mode 100644
index a0b38c40..00000000
--- a/bfps/cpp/interpolator.cpp
+++ /dev/null
@@ -1,214 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#define NDEBUG
-
-#include "interpolator.hpp"
-
-template <class rnumber, int interp_neighbours>
-interpolator<rnumber, interp_neighbours>::interpolator(
- fluid_solver_base<rnumber> *fs,
- base_polynomial_values BETA_POLYS,
- ...) : interpolator_base<rnumber, interp_neighbours>(fs, BETA_POLYS)
-{
- int tdims[4];
- this->compute_beta = BETA_POLYS;
- tdims[0] = (interp_neighbours+1)*2*this->descriptor->nprocs + this->descriptor->sizes[0];
- tdims[1] = this->descriptor->sizes[1];
- tdims[2] = this->descriptor->sizes[2]+2;
- tdims[3] = this->descriptor->sizes[3];
- this->buffered_descriptor = new field_descriptor<rnumber>(
- 4, tdims,
- this->descriptor->mpi_dtype,
- this->descriptor->comm);
- this->buffer_size = (interp_neighbours+1)*this->buffered_descriptor->slice_size;
- this->field = new rnumber[this->buffered_descriptor->local_size];
-}
-
-template <class rnumber, int interp_neighbours>
-interpolator<rnumber, interp_neighbours>::~interpolator()
-{
- delete[] this->field;
- delete this->buffered_descriptor;
-}
-
-template <class rnumber, int interp_neighbours>
-int interpolator<rnumber, interp_neighbours>::read_rFFTW(const void *void_src)
-{
- rnumber *src = (rnumber*)void_src;
- rnumber *dst = this->field;
- /* do big copy of middle stuff */
- std::copy(src,
- src + this->buffered_descriptor->slice_size*this->descriptor->subsizes[0],
- dst + this->buffer_size);
- MPI_Datatype MPI_RNUM = (sizeof(rnumber) == 4) ? MPI_FLOAT : MPI_DOUBLE;
- int rsrc;
- /* get upper slices */
- for (int rdst = 0; rdst < this->descriptor->nprocs; rdst++)
- {
- rsrc = this->descriptor->rank[(this->descriptor->all_start0[rdst] +
- this->descriptor->all_size0[rdst]) %
- this->descriptor->sizes[0]];
- if (this->descriptor->myrank == rsrc)
- MPI_Send(
- src,
- this->buffer_size,
- MPI_RNUM,
- rdst,
- 2*(rsrc*this->descriptor->nprocs + rdst),
- this->buffered_descriptor->comm);
- if (this->descriptor->myrank == rdst)
- MPI_Recv(
- dst + this->buffer_size + this->buffered_descriptor->slice_size*this->descriptor->subsizes[0],
- this->buffer_size,
- MPI_RNUM,
- rsrc,
- 2*(rsrc*this->descriptor->nprocs + rdst),
- this->buffered_descriptor->comm,
- MPI_STATUS_IGNORE);
- }
- /* get lower slices */
- for (int rdst = 0; rdst < this->descriptor->nprocs; rdst++)
- {
- rsrc = this->descriptor->rank[MOD(this->descriptor->all_start0[rdst] - 1,
- this->descriptor->sizes[0])];
- if (this->descriptor->myrank == rsrc)
- MPI_Send(
- src + this->buffered_descriptor->slice_size*this->descriptor->subsizes[0] - this->buffer_size,
- this->buffer_size,
- MPI_RNUM,
- rdst,
- 2*(rsrc*this->descriptor->nprocs + rdst)+1,
- this->descriptor->comm);
- if (this->descriptor->myrank == rdst)
- MPI_Recv(
- dst,
- this->buffer_size,
- MPI_RNUM,
- rsrc,
- 2*(rsrc*this->descriptor->nprocs + rdst)+1,
- this->descriptor->comm,
- MPI_STATUS_IGNORE);
- }
- return EXIT_SUCCESS;
-}
-
-template <class rnumber, int interp_neighbours>
-void interpolator<rnumber, interp_neighbours>::sample(
- const int nparticles,
- const int pdimension,
- const double *__restrict__ x,
- double *__restrict__ y,
- const int *deriv)
-{
- /* get grid coordinates */
- int *xg = new int[3*nparticles];
- double *xx = new double[3*nparticles];
- double *yy = new double[3*nparticles];
- std::fill_n(yy, 3*nparticles, 0.0);
- this->get_grid_coordinates(nparticles, pdimension, x, xg, xx);
- /* perform interpolation */
- for (int p=0; p<nparticles; p++)
- if (this->descriptor->rank[MOD(xg[p*3+2], this->descriptor->sizes[0])] == this->descriptor->myrank)
- this->operator()(xg + p*3, xx + p*3, yy + p*3, deriv);
- MPI_Allreduce(
- yy,
- y,
- 3*nparticles,
- MPI_DOUBLE,
- MPI_SUM,
- this->descriptor->comm);
- delete[] yy;
- delete[] xg;
- delete[] xx;
-}
-
-template <class rnumber, int interp_neighbours>
-void interpolator<rnumber, interp_neighbours>::operator()(
- const int *xg,
- const double *xx,
- double *__restrict__ dest,
- const int *deriv)
-{
- double bx[interp_neighbours*2+2], by[interp_neighbours*2+2], bz[interp_neighbours*2+2];
- if (deriv == NULL)
- {
- this->compute_beta(0, xx[0], bx);
- this->compute_beta(0, xx[1], by);
- this->compute_beta(0, xx[2], bz);
- }
- else
- {
- this->compute_beta(deriv[0], xx[0], bx);
- this->compute_beta(deriv[1], xx[1], by);
- this->compute_beta(deriv[2], xx[2], bz);
- }
- std::fill_n(dest, 3, 0);
- ptrdiff_t bigiz, bigiy, bigix;
- for (int iz = -interp_neighbours; iz <= interp_neighbours+1; iz++)
- {
- bigiz = ptrdiff_t(xg[2]+iz)-this->descriptor->starts[0];
- for (int iy = -interp_neighbours; iy <= interp_neighbours+1; iy++)
- {
- bigiy = ptrdiff_t(MOD(xg[1]+iy, this->descriptor->sizes[1]));
- for (int ix = -interp_neighbours; ix <= interp_neighbours+1; ix++)
- {
- bigix = ptrdiff_t(MOD(xg[0]+ix, this->descriptor->sizes[2]));
- ptrdiff_t tindex = ((bigiz *this->buffered_descriptor->sizes[1] +
- bigiy)*this->buffered_descriptor->sizes[2] +
- bigix)*3 + this->buffer_size;
- for (int c=0; c<3; c++)
- {
- dest[c] += this->field[tindex+c]*(bz[iz+interp_neighbours]*
- by[iy+interp_neighbours]*
- bx[ix+interp_neighbours]);
- }
- }
- }
- }
-}
-
-template class interpolator<float, 1>;
-template class interpolator<float, 2>;
-template class interpolator<float, 3>;
-template class interpolator<float, 4>;
-template class interpolator<float, 5>;
-template class interpolator<float, 6>;
-template class interpolator<float, 7>;
-template class interpolator<float, 8>;
-template class interpolator<float, 9>;
-template class interpolator<float, 10>;
-template class interpolator<double, 1>;
-template class interpolator<double, 2>;
-template class interpolator<double, 3>;
-template class interpolator<double, 4>;
-template class interpolator<double, 5>;
-template class interpolator<double, 6>;
-template class interpolator<double, 7>;
-template class interpolator<double, 8>;
-template class interpolator<double, 9>;
-template class interpolator<double, 10>;
-
diff --git a/bfps/cpp/interpolator.hpp b/bfps/cpp/interpolator.hpp
deleted file mode 100644
index 7e56ebe1..00000000
--- a/bfps/cpp/interpolator.hpp
+++ /dev/null
@@ -1,79 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#include <cmath>
-#include "field_descriptor.hpp"
-#include "fftw_tools.hpp"
-#include "fluid_solver_base.hpp"
-#include "interpolator_base.hpp"
-
-#ifndef INTERPOLATOR
-
-#define INTERPOLATOR
-
-template <class rnumber, int interp_neighbours>
-class interpolator:public interpolator_base<rnumber, interp_neighbours>
-{
- private:
- /* pointer to buffered field */
- rnumber *field;
-
- public:
- using interpolator_base<rnumber, interp_neighbours>::operator();
- ptrdiff_t buffer_size;
-
- /* descriptor for buffered field */
- field_descriptor<rnumber> *buffered_descriptor;
-
- interpolator(
- fluid_solver_base<rnumber> *FSOLVER,
- base_polynomial_values BETA_POLYS,
- ...);
- ~interpolator();
-
- int read_rFFTW(const void *src);
-
- inline int get_rank(double z)
- {
- return this->descriptor->rank[MOD(int(floor(z/this->dz)), this->descriptor->sizes[0])];
- }
-
- /* interpolate field at an array of locations */
- void sample(
- const int nparticles,
- const int pdimension,
- const double *__restrict__ x,
- double *__restrict__ y,
- const int *deriv = NULL);
- void operator()(
- const int *__restrict__ xg,
- const double *__restrict__ xx,
- double *__restrict__ dest,
- const int *deriv = NULL);
-};
-
-#endif//INTERPOLATOR
-
diff --git a/bfps/cpp/interpolator_base.cpp b/bfps/cpp/interpolator_base.cpp
deleted file mode 100644
index 668a965c..00000000
--- a/bfps/cpp/interpolator_base.cpp
+++ /dev/null
@@ -1,113 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#define NDEBUG
-
-#include <cmath>
-#include "interpolator_base.hpp"
-
-template <class rnumber, int interp_neighbours>
-interpolator_base<rnumber, interp_neighbours>::interpolator_base(
- fluid_solver_base<rnumber> *fs,
- base_polynomial_values BETA_POLYS)
-{
- this->descriptor = fs->rd;
- this->compute_beta = BETA_POLYS;
-
- // compute dx, dy, dz;
- this->dx = 4*acos(0) / (fs->dkx*this->descriptor->sizes[2]);
- this->dy = 4*acos(0) / (fs->dky*this->descriptor->sizes[1]);
- this->dz = 4*acos(0) / (fs->dkz*this->descriptor->sizes[0]);
-}
-
-template <class rnumber, int interp_neighbours>
-interpolator_base<rnumber, interp_neighbours>::interpolator_base(
- vorticity_equation<rnumber, FFTW> *fs,
- base_polynomial_values BETA_POLYS)
-{
-// this->descriptor = fs->rd;
-// this->compute_beta = BETA_POLYS;
-//
-// // compute dx, dy, dz;
-// this->dx = 4*acos(0) / (fs->kk->dkx*this->descriptor->sizes[2]);
-// this->dy = 4*acos(0) / (fs->kk->dky*this->descriptor->sizes[1]);
-// this->dz = 4*acos(0) / (fs->kk->dkz*this->descriptor->sizes[0]);
-}
-
-template <class rnumber, int interp_neighbours>
-void interpolator_base<rnumber, interp_neighbours>::get_grid_coordinates(
- const int nparticles,
- const int pdimension,
- const double *x,
- int *xg,
- double *xx)
-{
- for (int p=0; p<nparticles; p++)
- this->get_grid_coordinates(
- x + p*pdimension,
- xg + p*3,
- xx + p*3);
-}
-
-template <class rnumber, int interp_neighbours>
-void interpolator_base<rnumber, interp_neighbours>::get_grid_coordinates(
- const double *x,
- int *xg,
- double *xx)
-{
- static double grid_size[] = {this->dx, this->dy, this->dz};
- double tval;
- for (int c=0; c<3; c++)
- {
- tval = floor(x[c]/grid_size[c]);
- xg[c] = MOD(int(tval), this->descriptor->sizes[2-c]);
- xx[c] = (x[c] - tval*grid_size[c]) / grid_size[c];
- }
-}
-
-
-
-template class interpolator_base<float, 1>;
-template class interpolator_base<float, 2>;
-template class interpolator_base<float, 3>;
-template class interpolator_base<float, 4>;
-template class interpolator_base<float, 5>;
-template class interpolator_base<float, 6>;
-template class interpolator_base<float, 7>;
-template class interpolator_base<float, 8>;
-template class interpolator_base<float, 9>;
-template class interpolator_base<float, 10>;
-template class interpolator_base<double, 1>;
-template class interpolator_base<double, 2>;
-template class interpolator_base<double, 3>;
-template class interpolator_base<double, 4>;
-template class interpolator_base<double, 5>;
-template class interpolator_base<double, 6>;
-template class interpolator_base<double, 7>;
-template class interpolator_base<double, 8>;
-template class interpolator_base<double, 9>;
-template class interpolator_base<double, 10>;
-
diff --git a/bfps/cpp/interpolator_base.hpp b/bfps/cpp/interpolator_base.hpp
deleted file mode 100644
index f4c28db7..00000000
--- a/bfps/cpp/interpolator_base.hpp
+++ /dev/null
@@ -1,114 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#include "fluid_solver_base.hpp"
-#include "vorticity_equation.hpp"
-#include "spline_n1.hpp"
-#include "spline_n2.hpp"
-#include "spline_n3.hpp"
-#include "spline_n4.hpp"
-#include "spline_n5.hpp"
-#include "spline_n6.hpp"
-#include "spline_n7.hpp"
-#include "spline_n8.hpp"
-#include "spline_n9.hpp"
-#include "spline_n10.hpp"
-#include "Lagrange_polys.hpp"
-
-#ifndef INTERPOLATOR_BASE
-
-#define INTERPOLATOR_BASE
-
-typedef void (*base_polynomial_values)(
- const int derivative,
- const double fraction,
- double *__restrict__ destination);
-
-template <class rnumber, int interp_neighbours>
-class interpolator_base
-{
- public:
- /* pointer to polynomial function */
- base_polynomial_values compute_beta;
-
- /* descriptor of field to interpolate */
- field_descriptor<rnumber> *descriptor;
-
- /* physical parameters of field */
- double dx, dy, dz;
-
- interpolator_base(
- fluid_solver_base<rnumber> *FSOLVER,
- base_polynomial_values BETA_POLYS);
-
- interpolator_base(
- vorticity_equation<rnumber, FFTW> *FSOLVER,
- base_polynomial_values BETA_POLYS);
- virtual ~interpolator_base(){}
-
- /* may not destroy input */
- virtual int read_rFFTW(const void *src) = 0;
-
- /* map real locations to grid coordinates */
- void get_grid_coordinates(
- const int nparticles,
- const int pdimension,
- const double *__restrict__ x,
- int *__restrict__ xg,
- double *__restrict__ xx);
- void get_grid_coordinates(
- const double *__restrict__ x,
- int *__restrict__ xg,
- double *__restrict__ xx);
- /* interpolate field at an array of locations */
- virtual void sample(
- const int nparticles,
- const int pdimension,
- const double *__restrict__ x,
- double *__restrict__ y,
- const int *deriv = NULL) = 0;
- /* interpolate 1 point */
- virtual void operator()(
- const int *__restrict__ xg,
- const double *__restrict__ xx,
- double *__restrict__ dest,
- const int *deriv = NULL) = 0;
-
- /* interpolate 1 point */
- inline void operator()(
- const double *__restrict__ x,
- double *__restrict__ dest,
- const int *deriv = NULL)
- {
- int xg[3];
- double xx[3];
- this->get_grid_coordinates(x, xg, xx);
- (*this)(xg, xx, dest, deriv);
- }
-};
-
-#endif//INTERPOLATOR_BASE
-
diff --git a/bfps/cpp/rFFTW_distributed_particles.cpp b/bfps/cpp/rFFTW_distributed_particles.cpp
deleted file mode 100644
index 265975f8..00000000
--- a/bfps/cpp/rFFTW_distributed_particles.cpp
+++ /dev/null
@@ -1,804 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#define NDEBUG
-
-#include <cmath>
-#include <cassert>
-#include <cstring>
-#include <string>
-#include <sstream>
-#include <set>
-#include <algorithm>
-#include <ctime>
-
-#include "base.hpp"
-#include "rFFTW_distributed_particles.hpp"
-#include "fftw_tools.hpp"
-#include "scope_timer.hpp"
-
-
-extern int myrank, nprocs;
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-rFFTW_distributed_particles<particle_type, rnumber, interp_neighbours>::rFFTW_distributed_particles(
- const char *NAME,
- const hid_t data_file_id,
- rFFTW_interpolator<rnumber, interp_neighbours> *VEL,
- const int TRAJ_SKIP,
- const int INTEGRATION_STEPS) : particles_io_base<particle_type>(
- NAME,
- TRAJ_SKIP,
- data_file_id,
- VEL->descriptor->comm)
-{
- TIMEZONE("rFFTW_distributed_particles::rFFTW_distributed_particles");
- /* check that integration_steps has a valid value.
- * If NDEBUG is defined, "assert" doesn't do anything.
- * With NDEBUG defined, and an invalid INTEGRATION_STEPS,
- * the particles will simply sit still.
- * */
- assert((INTEGRATION_STEPS <= 6) &&
- (INTEGRATION_STEPS >= 1));
- /* check that the field layout is compatible with this class.
- * if it's not, the code will fail in bad ways, most likely ending up
- * with various CPUs locked in some MPI send/receive.
- * therefore I prefer to just kill the code at this point,
- * no matter whether or not NDEBUG is present.
- * */
- if (interp_neighbours*2+2 > VEL->descriptor->subsizes[0])
- {
- DEBUG_MSG("parameters incompatible with rFFTW_distributed_particles.\n"
- "interp kernel size is %d, local_z_size is %d\n",
- interp_neighbours*2+2, VEL->descriptor->subsizes[0]);
- if (VEL->descriptor->myrank == 0)
- std::cerr << "parameters incompatible with rFFTW_distributed_particles." << std::endl;
- exit(0);
- }
- this->vel = VEL;
- this->rhs.resize(INTEGRATION_STEPS);
- this->integration_steps = INTEGRATION_STEPS;
- /* the particles are expected to be evenly distributed among processes.
- * therefore allocating twice that amount of memory seems enough.
- * */
- this->state.reserve(2*this->nparticles / this->nprocs);
- for (unsigned int i=0; i<this->rhs.size(); i++)
- this->rhs[i].reserve(2*this->nparticles / this->nprocs);
-
- /* build communicators and stuff for interpolation */
-
- /* number of processors per domain */
- this->domain_nprocs[-1] = 2; // domain in common with lower z CPU
- this->domain_nprocs[ 0] = 1; // local domain
- this->domain_nprocs[ 1] = 2; // domain in common with higher z CPU
-
- /* initialize domain bins */
- this->domain_particles[-1] = std::unordered_set<int>();
- this->domain_particles[ 0] = std::unordered_set<int>();
- this->domain_particles[ 1] = std::unordered_set<int>();
- this->domain_particles[-1].reserve(unsigned(
- 1.5*(interp_neighbours*2+2)*
- float(this->nparticles) /
- this->nprocs));
- this->domain_particles[ 1].reserve(unsigned(
- 1.5*(interp_neighbours*2+2)*
- float(this->nparticles) /
- this->nprocs));
- this->domain_particles[ 0].reserve(unsigned(
- 1.5*(this->vel->descriptor->subsizes[0] - interp_neighbours*2-2)*
- float(this->nparticles) /
- this->nprocs));
-
- int color, key;
- MPI_Comm tmpcomm;
- for (int rank=0; rank<this->nprocs; rank++)
- {
- color = MPI_UNDEFINED;
- key = MPI_UNDEFINED;
- if (this->myrank == rank)
- {
- color = rank;
- key = 0;
- }
- if (this->myrank == MOD(rank + 1, this->nprocs))
- {
- color = rank;
- key = 1;
- }
- MPI_Comm_split(this->comm, color, key, &tmpcomm);
- if (this->myrank == rank)
- this->domain_comm[ 1] = tmpcomm;
- if (this->myrank == MOD(rank+1, this->nprocs))
- this->domain_comm[-1] = tmpcomm;
-
- }
-
- /* following code may be useful in the future for the general case */
- //this->interp_comm.resize(this->vel->descriptor->sizes[0]);
- //this->interp_nprocs.resize(this->vel->descriptor->sizes[0]);
- //for (int zg=0; zg<this->vel->descriptor->sizes[0]; zg++)
- //{
- // color = (this->vel->get_rank_info(
- // (zg+.5)*this->vel->dz, rminz, rmaxz) ? zg : MPI_UNDEFINED);
- // key = zg - this->vel->descriptor->starts[0] + interp_neighbours;
- // MPI_Comm_split(this->comm, color, key, &this->interp_comm[zg]);
- // if (this->interp_comm[zg] != MPI_COMM_NULL)
- // MPI_Comm_size(this->interp_comm[zg], &this->interp_nprocs[zg]);
- // else
- // this->interp_nprocs[zg] = 0;
- //}
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-rFFTW_distributed_particles<particle_type, rnumber, interp_neighbours>::~rFFTW_distributed_particles()
-{
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void rFFTW_distributed_particles<particle_type, rnumber, interp_neighbours>::sample(
- rFFTW_interpolator<rnumber, interp_neighbours> *field,
- const std::unordered_map<int, single_particle_state<particle_type>> &x,
- const std::unordered_map<int, std::unordered_set<int>> &dp,
- std::unordered_map<int, single_particle_state<POINT3D>> &y)
-{
- TIMEZONE("rFFTW_distributed_particles::sample");
- double *yyy;
- double *yy;
- y.clear();
- /* local z domain */
- yy = new double[3];
- for (auto p: dp.at(0))
- {
- (*field)(x.find(p)->second.data, yy);
- y[p] = yy;
- }
- delete[] yy;
- /* boundary z domains */
- int domain_index;
- for (int rankpair = 0; rankpair < this->nprocs; rankpair++)
- {
- if (this->myrank == rankpair)
- domain_index = 1;
- if (this->myrank == MOD(rankpair+1, this->nprocs))
- domain_index = -1;
- if (this->myrank == rankpair ||
- this->myrank == MOD(rankpair+1, this->nprocs))
- {
- yy = new double[3*dp.at(domain_index).size()];
- yyy = new double[3*dp.at(domain_index).size()];
- int tindex;
- tindex = 0;
- // can this sorting be done more efficiently?
- std::vector<int> ordered_dp;
- {
- TIMEZONE("rFFTW_distributed_particles::sample::ordered_dp");
- ordered_dp.reserve(dp.at(domain_index).size());
- for (auto p: dp.at(domain_index))
- ordered_dp.push_back(p);
- //std::set<int> ordered_dp(dp.at(domain_index));
- std::sort(ordered_dp.begin(), ordered_dp.end());
- }
-
- for (auto p: ordered_dp)
- //for (auto p: dp.at(domain_index))
- {
- (*field)(x.at(p).data, yy + tindex*3);
- tindex++;
- }
- {
- TIMEZONE("rFFTW_distributed_particles::sample::MPI_Allreduce");
- MPI_Allreduce(
- yy,
- yyy,
- 3*dp.at(domain_index).size(),
- MPI_DOUBLE,
- MPI_SUM,
- this->domain_comm[domain_index]);
- }
- tindex = 0;
- for (auto p: ordered_dp)
- //for (auto p: dp.at(domain_index))
- {
- y[p] = yyy + tindex*3;
- tindex++;
- }
- delete[] yy;
- delete[] yyy;
- }
- }
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void rFFTW_distributed_particles<particle_type, rnumber, interp_neighbours>::get_rhs(
- const std::unordered_map<int, single_particle_state<particle_type>> &x,
- const std::unordered_map<int, std::unordered_set<int>> &dp,
- std::unordered_map<int, single_particle_state<particle_type>> &y)
-{
- std::unordered_map<int, single_particle_state<POINT3D>> yy;
- switch(particle_type)
- {
- case VELOCITY_TRACER:
- this->sample(this->vel, x, dp, yy);
- y.clear();
- y.reserve(yy.size());
- y.rehash(this->nparticles);
- for (auto &pp: yy)
- y[pp.first] = pp.second.data;
- break;
- }
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void rFFTW_distributed_particles<particle_type, rnumber, interp_neighbours>::sample(
- rFFTW_interpolator<rnumber, interp_neighbours> *field,
- const char *dset_name)
-{
- std::unordered_map<int, single_particle_state<POINT3D>> y;
- this->sample(field, this->state, this->domain_particles, y);
- this->write(dset_name, y);
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void rFFTW_distributed_particles<particle_type, rnumber, interp_neighbours>::roll_rhs()
-{
- for (int i=this->integration_steps-2; i>=0; i--)
- rhs[i+1] = rhs[i];
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void rFFTW_distributed_particles<particle_type, rnumber, interp_neighbours>::redistribute(
- std::unordered_map<int, single_particle_state<particle_type>> &x,
- std::vector<std::unordered_map<int, single_particle_state<particle_type>>> &vals,
- std::unordered_map<int, std::unordered_set<int>> &dp)
-{
- TIMEZONE("rFFTW_distributed_particles::redistribute");
- //DEBUG_MSG("entered redistribute\n");
- /* get new distribution of particles */
- std::unordered_map<int, std::unordered_set<int>> newdp;
- {
- TIMEZONE("sort_into_domains");
- this->sort_into_domains(x, newdp);
- }
- /* take care of particles that are leaving the shared domains */
- int dindex[2] = {-1, 1};
- // for each D of the 2 shared domains
- {
- TIMEZONE("Loop1");
- for (int di=0; di<2; di++)
- // for all particles previously in D
- for (auto p: dp[dindex[di]])
- {
- // if the particle is no longer in D
- if (newdp[dindex[di]].find(p) == newdp[dindex[di]].end())
- {
- // and the particle is not in the local domain
- if (newdp[0].find(p) == newdp[0].end())
- {
- // remove the particle from the local list
- x.erase(p);
- for (unsigned int i=0; i<vals.size(); i++)
- vals[i].erase(p);
- }
- // if the particle is in the local domain, do nothing
- }
- }
- }
- /* take care of particles that are entering the shared domains */
- /* neighbouring rank offsets */
- int ro[2];
- ro[0] = -1;
- ro[1] = 1;
- /* particles to send, particles to receive */
- std::vector<int> ps[2], pr[2];
- for (int tcounter = 0; tcounter < 2; tcounter++)
- {
- ps[tcounter].reserve(newdp[dindex[tcounter]].size());
- }
- /* number of particles to send, number of particles to receive */
- int nps[2], npr[2];
- int rsrc, rdst;
- /* get list of id-s to send */
- {
- TIMEZONE("Loop2");
- for (auto &p: dp[0])
- {
- for (int di=0; di<2; di++)
- {
- if (newdp[dindex[di]].find(p) != newdp[dindex[di]].end())
- ps[di].push_back(p);
- }
- }
- }
- /* prepare data for send recv */
- for (int i=0; i<2; i++)
- nps[i] = ps[i].size();
- for (rsrc = 0; rsrc<this->nprocs; rsrc++)
- for (int i=0; i<2; i++)
- {
- rdst = MOD(rsrc+ro[i], this->nprocs);
- if (this->myrank == rsrc){
- TIMEZONE("MPI_Send");
- MPI_Send(
- nps+i,
- 1,
- MPI_INTEGER,
- rdst,
- 2*(rsrc*this->nprocs + rdst)+i,
- this->comm);
- }
- if (this->myrank == rdst){
- TIMEZONE("MPI_Recv");
- MPI_Recv(
- npr+1-i,
- 1,
- MPI_INTEGER,
- rsrc,
- 2*(rsrc*this->nprocs + rdst)+i,
- this->comm,
- MPI_STATUS_IGNORE);
- }
- }
- //DEBUG_MSG("I have to send %d %d particles\n", nps[0], nps[1]);
- //DEBUG_MSG("I have to recv %d %d particles\n", npr[0], npr[1]);
- for (int i=0; i<2; i++)
- pr[i].resize(npr[i]);
-
- int buffer_size = (nps[0] > nps[1]) ? nps[0] : nps[1];
- buffer_size = (buffer_size > npr[0])? buffer_size : npr[0];
- buffer_size = (buffer_size > npr[1])? buffer_size : npr[1];
- //DEBUG_MSG("buffer size is %d\n", buffer_size);
- double *buffer = new double[buffer_size*state_dimension(particle_type)*(1+vals.size())];
- for (rsrc = 0; rsrc<this->nprocs; rsrc++)
- for (int i=0; i<2; i++)
- {
- rdst = MOD(rsrc+ro[i], this->nprocs);
- if (this->myrank == rsrc && nps[i] > 0)
- {
- TIMEZONE("this->myrank == rsrc && nps[i] > 0");
- MPI_Send(
- &ps[i].front(),
- nps[i],
- MPI_INTEGER,
- rdst,
- 2*(rsrc*this->nprocs + rdst),
- this->comm);
- int pcounter = 0;
- for (int p: ps[i])
- {
- std::copy(x[p].data,
- x[p].data + state_dimension(particle_type),
- buffer + pcounter*(1+vals.size())*state_dimension(particle_type));
- for (unsigned int tindex=0; tindex<vals.size(); tindex++)
- {
- std::copy(vals[tindex][p].data,
- vals[tindex][p].data + state_dimension(particle_type),
- buffer + (pcounter*(1+vals.size()) + tindex+1)*state_dimension(particle_type));
- }
- pcounter++;
- }
- MPI_Send(
- buffer,
- nps[i]*(1+vals.size())*state_dimension(particle_type),
- MPI_DOUBLE,
- rdst,
- 2*(rsrc*this->nprocs + rdst)+1,
- this->comm);
- }
- if (this->myrank == rdst && npr[1-i] > 0)
- {
- TIMEZONE("this->myrank == rdst && npr[1-i] > 0");
- MPI_Recv(
- &pr[1-i].front(),
- npr[1-i],
- MPI_INTEGER,
- rsrc,
- 2*(rsrc*this->nprocs + rdst),
- this->comm,
- MPI_STATUS_IGNORE);
- MPI_Recv(
- buffer,
- npr[1-i]*(1+vals.size())*state_dimension(particle_type),
- MPI_DOUBLE,
- rsrc,
- 2*(rsrc*this->nprocs + rdst)+1,
- this->comm,
- MPI_STATUS_IGNORE);
- int pcounter = 0;
- for (int p: pr[1-i])
- {
- x[p] = buffer + (pcounter*(1+vals.size()))*state_dimension(particle_type);
- newdp[1-i].insert(p);
- for (unsigned int tindex=0; tindex<vals.size(); tindex++)
- {
- vals[tindex][p] = buffer + (pcounter*(1+vals.size()) + tindex+1)*state_dimension(particle_type);
- }
- pcounter++;
- }
- }
- }
- delete[] buffer;
- // x has been changed, so newdp is obsolete
- // we need to sort into domains again
- {
- TIMEZONE("sort_into_domains2");
- this->sort_into_domains(x, dp);
- }
-
-#ifndef NDEBUG
- /* check that all particles at x are local */
- //for (auto &pp: x)
- // if (this->vel->get_rank(pp.second.data[2]) != this->myrank)
- // {
- // DEBUG_MSG("found particle %d with rank %d\n",
- // pp.first,
- // this->vel->get_rank(pp.second.data[2]));
- // assert(false);
- // }
-#endif
- //DEBUG_MSG("exiting redistribute\n");
-}
-
-
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void rFFTW_distributed_particles<particle_type, rnumber, interp_neighbours>::AdamsBashforth(
- const int nsteps)
-{
- this->get_rhs(this->state, this->domain_particles, this->rhs[0]);
-#define AdamsBashforth_LOOP_PREAMBLE \
- for (auto &pp: this->state) \
- for (unsigned int i=0; i<state_dimension(particle_type); i++)
- switch(nsteps)
- {
- case 1:
- AdamsBashforth_LOOP_PREAMBLE
- pp.second[i] += this->dt*this->rhs[0][pp.first][i];
- break;
- case 2:
- AdamsBashforth_LOOP_PREAMBLE
- pp.second[i] += this->dt*(3*this->rhs[0][pp.first][i]
- - this->rhs[1][pp.first][i])/2;
- break;
- case 3:
- AdamsBashforth_LOOP_PREAMBLE
- pp.second[i] += this->dt*(23*this->rhs[0][pp.first][i]
- - 16*this->rhs[1][pp.first][i]
- + 5*this->rhs[2][pp.first][i])/12;
- break;
- case 4:
- AdamsBashforth_LOOP_PREAMBLE
- pp.second[i] += this->dt*(55*this->rhs[0][pp.first][i]
- - 59*this->rhs[1][pp.first][i]
- + 37*this->rhs[2][pp.first][i]
- - 9*this->rhs[3][pp.first][i])/24;
- break;
- case 5:
- AdamsBashforth_LOOP_PREAMBLE
- pp.second[i] += this->dt*(1901*this->rhs[0][pp.first][i]
- - 2774*this->rhs[1][pp.first][i]
- + 2616*this->rhs[2][pp.first][i]
- - 1274*this->rhs[3][pp.first][i]
- + 251*this->rhs[4][pp.first][i])/720;
- break;
- case 6:
- AdamsBashforth_LOOP_PREAMBLE
- pp.second[i] += this->dt*(4277*this->rhs[0][pp.first][i]
- - 7923*this->rhs[1][pp.first][i]
- + 9982*this->rhs[2][pp.first][i]
- - 7298*this->rhs[3][pp.first][i]
- + 2877*this->rhs[4][pp.first][i]
- - 475*this->rhs[5][pp.first][i])/1440;
- break;
- }
- this->redistribute(this->state, this->rhs, this->domain_particles);
- this->roll_rhs();
-}
-
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void rFFTW_distributed_particles<particle_type, rnumber, interp_neighbours>::step()
-{
- TIMEZONE("rFFTW_distributed_particles::step");
- this->AdamsBashforth((this->iteration < this->integration_steps) ?
- this->iteration+1 :
- this->integration_steps);
- this->iteration++;
-}
-
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void rFFTW_distributed_particles<particle_type, rnumber, interp_neighbours>::sort_into_domains(
- const std::unordered_map<int, single_particle_state<particle_type>> &x,
- std::unordered_map<int, std::unordered_set<int>> &dp)
-{
- TIMEZONE("rFFTW_distributed_particles::sort_into_domains");
- int tmpint1, tmpint2;
- dp.clear();
- dp[-1] = std::unordered_set<int>();
- dp[ 0] = std::unordered_set<int>();
- dp[ 1] = std::unordered_set<int>();
- dp[-1].reserve(unsigned(
- 1.5*(interp_neighbours*2+2)*
- float(this->nparticles) /
- this->nprocs));
- dp[ 1].reserve(unsigned(
- 1.5*(interp_neighbours*2+2)*
- float(this->nparticles) /
- this->nprocs));
- dp[ 0].reserve(unsigned(
- 1.5*(this->vel->descriptor->subsizes[0] - interp_neighbours*2-2)*
- float(this->nparticles) /
- this->nprocs));
- for (auto &xx: x)
- {
- if (this->vel->get_rank_info(xx.second.data[2], tmpint1, tmpint2))
- {
- if (tmpint1 == tmpint2)
- dp[0].insert(xx.first);
- else
- {
- if (this->myrank == tmpint1)
- dp[-1].insert(xx.first);
- else
- dp[ 1].insert(xx.first);
- }
- }
- }
-}
-
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void rFFTW_distributed_particles<particle_type, rnumber, interp_neighbours>::read()
-{
- TIMEZONE("rFFTW_distributed_particles::read");
- double *temp = new double[this->chunk_size*state_dimension(particle_type)];
- int tmpint1, tmpint2;
- for (unsigned int cindex=0; cindex<this->get_number_of_chunks(); cindex++)
- {
- //read state
- if (this->myrank == 0){
- TIMEZONE("read_state_chunk");
- this->read_state_chunk(cindex, temp);
- }
- {
- TIMEZONE("MPI_Bcast");
- MPI_Bcast(
- temp,
- this->chunk_size*state_dimension(particle_type),
- MPI_DOUBLE,
- 0,
- this->comm);
- }
- for (unsigned int p=0; p<this->chunk_size; p++)
- {
- if (this->vel->get_rank_info(temp[state_dimension(particle_type)*p+2], tmpint1, tmpint2))
- {
- this->state[p+cindex*this->chunk_size] = temp + state_dimension(particle_type)*p;
- }
- }
- //read rhs
- if (this->iteration > 0){
- TIMEZONE("this->iteration > 0");
- for (int i=0; i<this->integration_steps; i++)
- {
- if (this->myrank == 0){
- TIMEZONE("read_rhs_chunk");
- this->read_rhs_chunk(cindex, i, temp);
- }
- {
- TIMEZONE("MPI_Bcast");
- MPI_Bcast(
- temp,
- this->chunk_size*state_dimension(particle_type),
- MPI_DOUBLE,
- 0,
- this->comm);
- }
- for (unsigned int p=0; p<this->chunk_size; p++)
- {
- auto pp = this->state.find(p+cindex*this->chunk_size);
- if (pp != this->state.end())
- this->rhs[i][p+cindex*this->chunk_size] = temp + state_dimension(particle_type)*p;
- }
- }
- }
- }
- this->sort_into_domains(this->state, this->domain_particles);
- DEBUG_MSG("%s->state.size = %ld\n", this->name.c_str(), this->state.size());
- for (int domain=-1; domain<=1; domain++)
- {
- DEBUG_MSG("domain %d nparticles = %ld\n", domain, this->domain_particles[domain].size());
- }
- delete[] temp;
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void rFFTW_distributed_particles<particle_type, rnumber, interp_neighbours>::write(
- const char *dset_name,
- std::unordered_map<int, single_particle_state<POINT3D>> &y)
-{
- TIMEZONE("rFFTW_distributed_particles::write");
- double *data = new double[this->chunk_size*3];
- double *yy = new double[this->chunk_size*3];
- //int pindex = 0;
- for (unsigned int cindex=0; cindex<this->get_number_of_chunks(); cindex++)
- {
- std::fill_n(yy, this->chunk_size*3, 0);
- //for (unsigned int p=0; p<this->chunk_size; p++, pindex++)
- //{
- // if (this->domain_particles[-1].find(pindex) != this->domain_particles[-1].end() ||
- // this->domain_particles[ 0].find(pindex) != this->domain_particles[ 0].end())
- // {
- // std::copy(y[pindex].data,
- // y[pindex].data + 3,
- // yy + p*3);
- // }
- //}
- for (int s = -1; s <= 0; s++)
- for (auto &pp: this->domain_particles[s])
- {
- if (pp >= int(cindex*this->chunk_size) &&
- pp < int((cindex+1)*this->chunk_size))
- {
- std::copy(y[pp].data,
- y[pp].data + 3,
- yy + (pp-cindex*this->chunk_size)*3);
- }
- }
- {
- TIMEZONE("MPI_Allreduce");
- MPI_Allreduce(
- yy,
- data,
- 3*this->chunk_size,
- MPI_DOUBLE,
- MPI_SUM,
- this->comm);
- }
- if (this->myrank == 0){
- TIMEZONE("write_point3D_chunk");
- this->write_point3D_chunk(dset_name, cindex, data);
- }
- }
- delete[] yy;
- delete[] data;
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void rFFTW_distributed_particles<particle_type, rnumber, interp_neighbours>::write(
- const bool write_rhs)
-{
- TIMEZONE("rFFTW_distributed_particles::write2");
- double *temp0 = new double[this->chunk_size*state_dimension(particle_type)];
- double *temp1 = new double[this->chunk_size*state_dimension(particle_type)];
- //int pindex = 0;
- for (unsigned int cindex=0; cindex<this->get_number_of_chunks(); cindex++)
- {
- //write state
- std::fill_n(temp0, state_dimension(particle_type)*this->chunk_size, 0);
- //pindex = cindex*this->chunk_size;
- //for (unsigned int p=0; p<this->chunk_size; p++, pindex++)
- //{
- // if (this->domain_particles[-1].find(pindex) != this->domain_particles[-1].end() ||
- // this->domain_particles[ 0].find(pindex) != this->domain_particles[ 0].end())
- // {
- // TIMEZONE("std::copy");
- // std::copy(this->state[pindex].data,
- // this->state[pindex].data + state_dimension(particle_type),
- // temp0 + p*state_dimension(particle_type));
- // }
- //}
- for (int s = -1; s <= 0; s++)
- for (auto &pp: this->domain_particles[s])
- {
- if (pp >= int(cindex*this->chunk_size) &&
- pp < int((cindex+1)*this->chunk_size))
- {
- std::copy(this->state[pp].data,
- this->state[pp].data + state_dimension(particle_type),
- temp0 + (pp-cindex*this->chunk_size)*state_dimension(particle_type));
- }
- }
- {
- TIMEZONE("MPI_Allreduce");
- MPI_Allreduce(
- temp0,
- temp1,
- state_dimension(particle_type)*this->chunk_size,
- MPI_DOUBLE,
- MPI_SUM,
- this->comm);
- }
- if (this->myrank == 0){
- TIMEZONE("write_state_chunk");
- this->write_state_chunk(cindex, temp1);
- }
- //write rhs
- if (write_rhs){
- TIMEZONE("write_rhs");
- for (int i=0; i<this->integration_steps; i++)
- {
- std::fill_n(temp0, state_dimension(particle_type)*this->chunk_size, 0);
- //pindex = cindex*this->chunk_size;
- //for (unsigned int p=0; p<this->chunk_size; p++, pindex++)
- //{
- // if (this->domain_particles[-1].find(pindex) != this->domain_particles[-1].end() ||
- // this->domain_particles[ 0].find(pindex) != this->domain_particles[ 0].end())
- // {
- // TIMEZONE("std::copy");
- // std::copy(this->rhs[i][pindex].data,
- // this->rhs[i][pindex].data + state_dimension(particle_type),
- // temp0 + p*state_dimension(particle_type));
- // }
- //}
- for (int s = -1; s <= 0; s++)
- for (auto &pp: this->domain_particles[s])
- {
- if (pp >= int(cindex*this->chunk_size) &&
- pp < int((cindex+1)*this->chunk_size))
- {
- std::copy(this->rhs[i][pp].data,
- this->rhs[i][pp].data + state_dimension(particle_type),
- temp0 + (pp-cindex*this->chunk_size)*state_dimension(particle_type));
- }
- }
- {
- TIMEZONE("MPI_Allreduce");
- MPI_Allreduce(
- temp0,
- temp1,
- state_dimension(particle_type)*this->chunk_size,
- MPI_DOUBLE,
- MPI_SUM,
- this->comm);
- }
- if (this->myrank == 0){
- TIMEZONE("write_rhs_chunk");
- this->write_rhs_chunk(cindex, i, temp1);
- }
- }
- }
- }
- delete[] temp0;
- delete[] temp1;
-}
-
-
-/*****************************************************************************/
-template class rFFTW_distributed_particles<VELOCITY_TRACER, float, 1>;
-template class rFFTW_distributed_particles<VELOCITY_TRACER, float, 2>;
-template class rFFTW_distributed_particles<VELOCITY_TRACER, float, 3>;
-template class rFFTW_distributed_particles<VELOCITY_TRACER, float, 4>;
-template class rFFTW_distributed_particles<VELOCITY_TRACER, float, 5>;
-template class rFFTW_distributed_particles<VELOCITY_TRACER, float, 6>;
-template class rFFTW_distributed_particles<VELOCITY_TRACER, double, 1>;
-template class rFFTW_distributed_particles<VELOCITY_TRACER, double, 2>;
-template class rFFTW_distributed_particles<VELOCITY_TRACER, double, 3>;
-template class rFFTW_distributed_particles<VELOCITY_TRACER, double, 4>;
-template class rFFTW_distributed_particles<VELOCITY_TRACER, double, 5>;
-template class rFFTW_distributed_particles<VELOCITY_TRACER, double, 6>;
-/*****************************************************************************/
-
diff --git a/bfps/cpp/rFFTW_distributed_particles.hpp b/bfps/cpp/rFFTW_distributed_particles.hpp
deleted file mode 100644
index 400411d5..00000000
--- a/bfps/cpp/rFFTW_distributed_particles.hpp
+++ /dev/null
@@ -1,144 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <iostream>
-#include <unordered_map>
-#include <unordered_set>
-#include <vector>
-#include <hdf5.h>
-#include "base.hpp"
-#include "particles_base.hpp"
-#include "fluid_solver_base.hpp"
-#include "rFFTW_interpolator.hpp"
-
-#ifndef RFFTW_DISTRIBUTED_PARTICLES
-
-#define RFFTW_DISTRIBUTED_PARTICLES
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-class rFFTW_distributed_particles: public particles_io_base<particle_type>
-{
- private:
- // a "domain" corresponds to a region in 3D real space where a fixed set
- // of MPI processes are required to participate in the interpolation
- // formula (i.e. they all contain required information).
- // we need to know how many processes there are for each of the domains
- // to which the local process belongs.
- std::unordered_map<int, int> domain_nprocs;
- // each domain has an associated communicator, and we keep a list of the
- // communicators to which the local process belongs
- std::unordered_map<int, MPI_Comm> domain_comm;
- // for each domain, we need a list of the IDs of the particles located
- // in that domain
- std::unordered_map<int, std::unordered_set<int>> domain_particles;
-
- // for each domain, we need the state of each particle
- std::unordered_map<int, single_particle_state<particle_type>> state;
- // for each domain, we also need the last few values of the right hand
- // side of the ODE, since we use Adams-Bashforth integration
- std::vector<std::unordered_map<int, single_particle_state<particle_type>>> rhs;
-
- public:
- int integration_steps;
- // this class only works with rFFTW interpolator
- rFFTW_interpolator<rnumber, interp_neighbours> *vel;
-
- /* simulation parameters */
- double dt;
-
- /* methods */
-
- /* constructor and destructor.
- * allocate and deallocate:
- * this->state
- * this->rhs
- * */
- rFFTW_distributed_particles(
- const char *NAME,
- const hid_t data_file_id,
- rFFTW_interpolator<rnumber, interp_neighbours> *FIELD,
- const int TRAJ_SKIP,
- const int INTEGRATION_STEPS = 2);
- ~rFFTW_distributed_particles();
-
- void sample(
- rFFTW_interpolator<rnumber, interp_neighbours> *field,
- const char *dset_name);
- void sample(
- rFFTW_interpolator<rnumber, interp_neighbours> *field,
- const std::unordered_map<int, single_particle_state<particle_type>> &x,
- const std::unordered_map<int, std::unordered_set<int>> &dp,
- std::unordered_map<int, single_particle_state<POINT3D>> &y);
- void get_rhs(
- const std::unordered_map<int, single_particle_state<particle_type>> &x,
- const std::unordered_map<int, std::unordered_set<int>> &dp,
- std::unordered_map<int, single_particle_state<particle_type>> &y);
-
-
- /* given a list of particle positions,
- * figure out which go into what local domain, and construct the relevant
- * map of ID lists "dp" (for domain particles).
- * */
- void sort_into_domains(
- const std::unordered_map<int, single_particle_state<particle_type>> &x,
- std::unordered_map<int, std::unordered_set<int>> &dp);
- /* suppose the particles are currently badly distributed, and some
- * arbitrary quantities (stored in "vals") are associated to the particles,
- * and we need to properly distribute them among processes.
- * that's what this function does.
- * In practice it's only used to redistribute the rhs values (and it
- * automatically redistributes the state x being passed).
- * Some more comments are present in the .cpp file, but, in brief: the
- * particles are simply moved from one domain to another.
- * If it turns out that the new domain contains a process which does not
- * know about a particle, that information is sent from the closest process.
- * */
- void redistribute(
- std::unordered_map<int, single_particle_state<particle_type>> &x,
- std::vector<std::unordered_map<int, single_particle_state<particle_type>>> &vals,
- std::unordered_map<int, std::unordered_set<int>> &dp);
-
-
- /* input/output */
- void read();
- void write(
- const char *dset_name,
- std::unordered_map<int, single_particle_state<POINT3D>> &y);
- void write(
- const char *dset_name,
- std::unordered_map<int, single_particle_state<particle_type>> &y);
- void write(const bool write_rhs = true);
-
- /* solvers */
- void step();
- void roll_rhs();
- void AdamsBashforth(const int nsteps);
-};
-
-#endif//RFFTW_DISTRIBUTED_PARTICLES
-
diff --git a/bfps/cpp/rFFTW_interpolator.cpp b/bfps/cpp/rFFTW_interpolator.cpp
deleted file mode 100644
index b8b21e88..00000000
--- a/bfps/cpp/rFFTW_interpolator.cpp
+++ /dev/null
@@ -1,210 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#define NDEBUG
-
-#include <cmath>
-#include "rFFTW_interpolator.hpp"
-#include "scope_timer.hpp"
-
-template <class rnumber, int interp_neighbours>
-rFFTW_interpolator<rnumber, interp_neighbours>::rFFTW_interpolator(
- fluid_solver_base<rnumber> *fs,
- base_polynomial_values BETA_POLYS,
- rnumber *FIELD_POINTER) : interpolator_base<rnumber, interp_neighbours>(fs, BETA_POLYS)
-{
- this->field = FIELD_POINTER;
-
-
- // generate compute array
- this->compute = new bool[this->descriptor->sizes[0]];
- std::fill_n(this->compute, this->descriptor->sizes[0], false);
- for (int iz = this->descriptor->starts[0]-interp_neighbours-1;
- iz <= this->descriptor->starts[0]+this->descriptor->subsizes[0]+interp_neighbours;
- iz++)
- this->compute[((iz + this->descriptor->sizes[0]) % this->descriptor->sizes[0])] = true;
-}
-
-template <class rnumber, int interp_neighbours>
-rFFTW_interpolator<rnumber, interp_neighbours>::rFFTW_interpolator(
- vorticity_equation<rnumber, FFTW> *fs,
- base_polynomial_values BETA_POLYS,
- rnumber *FIELD_POINTER) : interpolator_base<rnumber, interp_neighbours>(fs, BETA_POLYS)
-{
-// this->field = FIELD_POINTER;
-//
-//
-// // generate compute array
-// this->compute = new bool[this->descriptor->sizes[0]];
-// std::fill_n(this->compute, this->descriptor->sizes[0], false);
-// for (int iz = this->descriptor->starts[0]-interp_neighbours-1;
-// iz <= this->descriptor->starts[0]+this->descriptor->subsizes[0]+interp_neighbours;
-// iz++)
-// this->compute[((iz + this->descriptor->sizes[0]) % this->descriptor->sizes[0])] = true;
-}
-
-template <class rnumber, int interp_neighbours>
-rFFTW_interpolator<rnumber, interp_neighbours>::~rFFTW_interpolator()
-{
- delete[] this->compute;
-}
-
-template <class rnumber, int interp_neighbours>
-bool rFFTW_interpolator<rnumber, interp_neighbours>::get_rank_info(double z, int &maxz_rank, int &minz_rank)
-{
- int zg = int(floor(z/this->dz));
- minz_rank = this->descriptor->rank[MOD(
- zg - interp_neighbours,
- this->descriptor->sizes[0])];
- maxz_rank = this->descriptor->rank[MOD(
- zg + 1 + interp_neighbours,
- this->descriptor->sizes[0])];
- bool is_here = false;
- for (int iz = -interp_neighbours; iz <= interp_neighbours+1; iz++)
- is_here = (is_here ||
- (this->descriptor->myrank ==
- this->descriptor->rank[MOD(zg+iz, this->descriptor->sizes[0])]));
- return is_here;
-}
-
-template <class rnumber, int interp_neighbours>
-void rFFTW_interpolator<rnumber, interp_neighbours>::sample(
- const int nparticles,
- const int pdimension,
- const double *__restrict__ x,
- double *__restrict__ y,
- const int *deriv)
-{
- TIMEZONE("rFFTW_interpolator::sample");
- /* get grid coordinates */
- int *xg = new int[3*nparticles];
- double *xx = new double[3*nparticles];
- double *yy = new double[3*nparticles];
- std::fill_n(yy, 3*nparticles, 0.0);
- this->get_grid_coordinates(nparticles, pdimension, x, xg, xx);
- /* perform interpolation */
- for (int p=0; p<nparticles; p++)
- if (this->compute[xg[p*3+2]])
- this->operator()(xg + p*3, xx + p*3, yy + p*3, deriv);
- MPI_Allreduce(
- yy,
- y,
- 3*nparticles,
- MPI_DOUBLE,
- MPI_SUM,
- this->descriptor->comm);
- delete[] yy;
- delete[] xg;
- delete[] xx;
-}
-
-template <class rnumber, int interp_neighbours>
-void rFFTW_interpolator<rnumber, interp_neighbours>::operator()(
- const int *xg,
- const double *xx,
- double *dest,
- const int *deriv)
-{
- TIMEZONE("rFFTW_interpolator::operator()");
- double bx[interp_neighbours*2+2], by[interp_neighbours*2+2], bz[interp_neighbours*2+2];
- /* please note that the polynomials in z are computed for all the different
- * iz values, independently of whether or not "myrank" will perform the
- * computation for all the different iz slices.
- * I don't know how big a deal this really is, but it is something that we can
- * optimize.
- * */
- if (deriv == NULL)
- {
- this->compute_beta(0, xx[0], bx);
- this->compute_beta(0, xx[1], by);
- this->compute_beta(0, xx[2], bz);
- }
- else
- {
- this->compute_beta(deriv[0], xx[0], bx);
- this->compute_beta(deriv[1], xx[1], by);
- this->compute_beta(deriv[2], xx[2], bz);
- }
- std::fill_n(dest, 3, 0);
- ptrdiff_t bigiz, bigiy, bigix;
- // loop over the 2*interp_neighbours + 2 z slices
- for (int iz = -interp_neighbours; iz <= interp_neighbours+1; iz++)
- {
- // bigiz is the z index of the cell containing the particles
- // this->descriptor->sizes[0] is added before taking the modulo
- // because we want to be sure that "bigiz" is a positive number.
- // I'm no longer sure why I don't use the MOD function here.
- bigiz = ptrdiff_t(((xg[2]+iz) + this->descriptor->sizes[0]) % this->descriptor->sizes[0]);
- // once we know bigiz, we know whether "myrank" has the relevant slice.
- // if not, go to next value of bigiz
- if (this->descriptor->myrank == this->descriptor->rank[bigiz])
- {
- for (int iy = -interp_neighbours; iy <= interp_neighbours+1; iy++)
- {
- // bigiy is the y index of the cell
- // since we have all the y indices in myrank, we can safely use the
- // modulo value
- bigiy = ptrdiff_t(MOD(xg[1]+iy, this->descriptor->sizes[1]));
- for (int ix = -interp_neighbours; ix <= interp_neighbours+1; ix++)
- {
- // bigix is the x index of the cell
- bigix = ptrdiff_t(MOD(xg[0]+ix, this->descriptor->sizes[2]));
- // here we create the index to the current grid node
- // note the removal of local_z_start from bigiz.
- ptrdiff_t tindex = (((bigiz-this->descriptor->starts[0])*this->descriptor->sizes[1] +
- bigiy)*(this->descriptor->sizes[2]+2) +
- bigix)*3;
- for (int c=0; c<3; c++)
- dest[c] += this->field[tindex+c]*(bz[iz+interp_neighbours]*
- by[iy+interp_neighbours]*
- bx[ix+interp_neighbours]);
- }
- }
- }
- }
-}
-
-template class rFFTW_interpolator<float, 1>;
-template class rFFTW_interpolator<float, 2>;
-template class rFFTW_interpolator<float, 3>;
-template class rFFTW_interpolator<float, 4>;
-template class rFFTW_interpolator<float, 5>;
-template class rFFTW_interpolator<float, 6>;
-template class rFFTW_interpolator<float, 7>;
-template class rFFTW_interpolator<float, 8>;
-template class rFFTW_interpolator<float, 9>;
-template class rFFTW_interpolator<float, 10>;
-template class rFFTW_interpolator<double, 1>;
-template class rFFTW_interpolator<double, 2>;
-template class rFFTW_interpolator<double, 3>;
-template class rFFTW_interpolator<double, 4>;
-template class rFFTW_interpolator<double, 5>;
-template class rFFTW_interpolator<double, 6>;
-template class rFFTW_interpolator<double, 7>;
-template class rFFTW_interpolator<double, 8>;
-template class rFFTW_interpolator<double, 9>;
-template class rFFTW_interpolator<double, 10>;
-
diff --git a/bfps/cpp/rFFTW_interpolator.hpp b/bfps/cpp/rFFTW_interpolator.hpp
deleted file mode 100644
index 5088be8b..00000000
--- a/bfps/cpp/rFFTW_interpolator.hpp
+++ /dev/null
@@ -1,118 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#include "field_descriptor.hpp"
-#include "fftw_tools.hpp"
-#include "fluid_solver_base.hpp"
-#include "vorticity_equation.hpp"
-#include "interpolator_base.hpp"
-
-#ifndef RFFTW_INTERPOLATOR
-
-#define RFFTW_INTERPOLATOR
-
-template <class rnumber, int interp_neighbours>
-class rFFTW_interpolator:public interpolator_base<rnumber, interp_neighbours>
-{
- public:
- using interpolator_base<rnumber, interp_neighbours>::operator();
-
- /* pointer to field that has to be interpolated
- * The reason this is a member variable is because I want this class
- * to be consistent with the "interpolator" class, where a member
- * variable is absolutely required (since that class uses padding).
- * */
- rnumber *field;
-
- /* compute[iz] is an array that says whether or not the current MPI
- * process is involved in the interpolation formula for a particle
- * located in cell "iz".
- * It is mostly used in the formula itself.
- * This translates as the following condition:
- * local_zstart - neighbours <= iz <= local_zend + 1 + neighbours
- * I think it's cleaner to keep things in an array, especially since
- * "local_zend" is shorthand for another arithmetic operation anyway.
- * */
- bool *compute;
-
-
- /* Constructors */
- rFFTW_interpolator(
- fluid_solver_base<rnumber> *FSOLVER,
- base_polynomial_values BETA_POLYS,
- rnumber *FIELD_DATA);
-
- /* this constructor is empty, I just needed for a quick hack of the
- * "vorticity_equation" class.
- * It should be removed soon.
- * */
- rFFTW_interpolator(
- vorticity_equation<rnumber, FFTW> *FSOLVER,
- base_polynomial_values BETA_POLYS,
- rnumber *FIELD_DATA);
- ~rFFTW_interpolator();
-
- /* This method is provided for consistency with "interpolator", and it
- * does not destroy input */
- inline int read_rFFTW(const void *src)
- {
- this->field = (rnumber*)src;
- return EXIT_SUCCESS;
- }
-
- /* This is used when "compute" is not enough.
- * For a given z location, it gives the outermost ranks that are relevant
- * for the interpolation formula.
- * */
- bool get_rank_info(double z, int &maxz_rank, int &minz_rank);
-
- /* interpolate field at an array of locations.
- * After interpolation is performed, call Allreduce for "y", over
- * this->descriptor->comm --- generally MPI_COMM_WORLD.
- * This is useful for the simple "particles" class, where particle
- * information is synchronized across all processes.
- * */
- void sample(
- const int nparticles,
- const int pdimension,
- const double *__restrict__ x,
- double *__restrict__ y,
- const int *deriv = NULL);
- /* interpolate 1 point.
- * Result is kept local.
- * This is used in the "rFFTW_distributed_particles" class, with the
- * result being synchronized across the relevant "local particle
- * communicator".
- * */
- void operator()(
- const int *__restrict__ xg,
- const double *__restrict__ xx,
- double *__restrict__ dest,
- const int *deriv = NULL);
-};
-
-#endif//RFFTW_INTERPOLATOR
-
diff --git a/bfps/cpp/slab_field_particles.cpp b/bfps/cpp/slab_field_particles.cpp
deleted file mode 100644
index e3c84574..00000000
--- a/bfps/cpp/slab_field_particles.cpp
+++ /dev/null
@@ -1,799 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#define NDEBUG
-
-
-#include <cmath>
-#include <cassert>
-#include <cstring>
-#include <string>
-#include <sstream>
-
-#include "base.hpp"
-#include "slab_field_particles.hpp"
-#include "fftw_tools.hpp"
-
-
-extern int myrank, nprocs;
-
-template <class rnumber>
-slab_field_particles<rnumber>::slab_field_particles(
- const char *NAME,
- fluid_solver_base<rnumber> *FSOLVER,
- const int NPARTICLES,
- const int NCOMPONENTS,
- base_polynomial_values BETA_POLYS,
- const int INTERP_NEIGHBOURS,
- const int TRAJ_SKIP,
- const int INTEGRATION_STEPS)
-{
- assert((NCOMPONENTS % 3) == 0);
- assert((INTERP_NEIGHBOURS >= 1) ||
- (INTERP_NEIGHBOURS <= 8));
- assert((INTEGRATION_STEPS <= 6) &&
- (INTEGRATION_STEPS >= 1));
- strncpy(this->name, NAME, 256);
- this->fs = FSOLVER;
- this->nparticles = NPARTICLES;
- this->ncomponents = NCOMPONENTS;
- this->integration_steps = INTEGRATION_STEPS;
- this->interp_neighbours = INTERP_NEIGHBOURS;
- this->traj_skip = TRAJ_SKIP;
- this->compute_beta = BETA_POLYS;
- // in principle only the buffer width at the top needs the +1,
- // but things are simpler if buffer_width is the same
- this->buffer_width = this->interp_neighbours+1;
- this->buffer_size = this->buffer_width*this->fs->rd->slice_size;
- this->array_size = this->nparticles * this->ncomponents;
- this->state = fftw_interface<rnumber>::alloc_real(this->array_size);
- std::fill_n(this->state, this->array_size, 0.0);
- for (int i=0; i < this->integration_steps; i++)
- {
- this->rhs[i] = fftw_interface<rnumber>::alloc_real(this->array_size);
- std::fill_n(this->rhs[i], this->array_size, 0.0);
- }
- this->watching = new bool[this->fs->rd->nprocs*nparticles];
- std::fill_n(this->watching, this->fs->rd->nprocs*this->nparticles, false);
- this->computing = new int[nparticles];
-
- int tdims[4];
- tdims[0] = this->buffer_width*2*this->fs->rd->nprocs + this->fs->rd->sizes[0];
- tdims[1] = this->fs->rd->sizes[1];
- tdims[2] = this->fs->rd->sizes[2];
- tdims[3] = this->fs->rd->sizes[3];
- this->buffered_field_descriptor = new field_descriptor<rnumber>(
- 4, tdims,
- this->fs->rd->mpi_dtype,
- this->fs->rd->comm);
-
- // compute dx, dy, dz;
- this->dx = 4*acos(0) / (this->fs->dkx*this->fs->rd->sizes[2]);
- this->dy = 4*acos(0) / (this->fs->dky*this->fs->rd->sizes[1]);
- this->dz = 4*acos(0) / (this->fs->dkz*this->fs->rd->sizes[0]);
-
- // compute lower and upper bounds
- this->lbound = new double[nprocs];
- this->ubound = new double[nprocs];
- double *tbound = new double[nprocs];
- std::fill_n(tbound, nprocs, 0.0);
- tbound[this->fs->rd->myrank] = this->fs->rd->starts[0]*this->dz;
- MPI_Allreduce(
- tbound,
- this->lbound,
- nprocs,
- MPI_DOUBLE,
- MPI_SUM,
- this->fs->rd->comm);
- std::fill_n(tbound, nprocs, 0.0);
- tbound[this->fs->rd->myrank] = (this->fs->rd->starts[0] + this->fs->rd->subsizes[0])*this->dz;
- MPI_Allreduce(
- tbound,
- this->ubound,
- nprocs,
- MPI_DOUBLE,
- MPI_SUM,
- this->fs->rd->comm);
- delete[] tbound;
- //for (int r = 0; r<nprocs; r++)
- // DEBUG_MSG(
- // "lbound[%d] = %lg, ubound[%d] = %lg\n",
- // r, this->lbound[r],
- // r, this->ubound[r]
- // );
-}
-
-template <class rnumber>
-slab_field_particles<rnumber>::~slab_field_particles()
-{
- delete[] this->computing;
- delete[] this->watching;
- fftw_interface<rnumber>::free(this->state);
- for (int i=0; i < this->integration_steps; i++)
- {
- fftw_interface<rnumber>::free(this->rhs[i]);
- }
- delete[] this->lbound;
- delete[] this->ubound;
- delete this->buffered_field_descriptor;
-}
-
-template <class rnumber>
-void slab_field_particles<rnumber>::get_rhs(double *x, double *y)
-{
- std::fill_n(y, this->array_size, 0.0);
-}
-
-template <class rnumber>
-void slab_field_particles<rnumber>::jump_estimate(double *dest)
-{
- std::fill_n(dest, this->nparticles, 0.0);
-}
-
-template <class rnumber>
-int slab_field_particles<rnumber>::get_rank(double z)
-{
- int tmp = this->fs->rd->rank[MOD(int(floor(z/this->dz)), this->fs->rd->sizes[0])];
- assert(tmp >= 0 && tmp < this->fs->rd->nprocs);
- return tmp;
-}
-
-template <class rnumber>
-void slab_field_particles<rnumber>::synchronize_single_particle_state(int p, double *x, int source)
-{
- if (source == -1) source = this->computing[p];
- if (this->watching[this->fs->rd->myrank*this->nparticles+p]) for (int r=0; r<this->fs->rd->nprocs; r++)
- if (r != source &&
- this->watching[r*this->nparticles+p])
- {
- //DEBUG_MSG("synchronizing state %d from %d to %d\n", p, this->computing[p], r);
- if (this->fs->rd->myrank == source)
- MPI_Send(
- x+p*this->ncomponents,
- this->ncomponents,
- MPI_DOUBLE,
- r,
- p+this->computing[p]*this->nparticles,
- this->fs->rd->comm);
- if (this->fs->rd->myrank == r)
- MPI_Recv(
- x+p*this->ncomponents,
- this->ncomponents,
- MPI_DOUBLE,
- source,
- p+this->computing[p]*this->nparticles,
- this->fs->rd->comm,
- MPI_STATUS_IGNORE);
- }
-}
-
-template <class rnumber>
-void slab_field_particles<rnumber>::synchronize()
-{
- double *tstate = fftw_interface<double>::alloc_real(this->array_size);
- // first, synchronize state and jump across CPUs
- std::fill_n(tstate, this->array_size, 0.0);
- for (int p=0; p<this->nparticles; p++)
- {
- //if (this->watching[this->fs->rd->myrank*this->nparticles + p])
- //DEBUG_MSG(
- // "in synchronize, position for particle %d is %g %g %g\n",
- // p,
- // this->state[p*this->ncomponents],
- // this->state[p*this->ncomponents+1],
- // this->state[p*this->ncomponents+2]);
- if (this->fs->rd->myrank == this->computing[p])
- std::copy(this->state + p*this->ncomponents,
- this->state + (p+1)*this->ncomponents,
- tstate + p*this->ncomponents);
- }
- MPI_Allreduce(
- tstate,
- this->state,
- this->array_size,
- MPI_DOUBLE,
- MPI_SUM,
- this->fs->rd->comm);
- if (this->integration_steps >= 1)
- {
- for (int i=0; i<this->integration_steps; i++)
- {
- std::fill_n(tstate, this->array_size, 0.0);
- for (int p=0; p<this->nparticles; p++) if (this->fs->rd->myrank == this->computing[p])
- std::copy(this->rhs[i] + p*this->ncomponents,
- this->rhs[i] + (p+1)*this->ncomponents,
- tstate + p*this->ncomponents);
- std::fill_n(this->rhs[i], this->array_size, 0.0);
- MPI_Allreduce(
- tstate,
- this->rhs[i],
- this->array_size,
- MPI_DOUBLE,
- MPI_SUM,
- this->fs->rd->comm);
- }
- }
- fftw_interface<double>::free(tstate);
- // assignment of particles
- for (int p=0; p<this->nparticles; p++)
- {
- this->computing[p] = this->get_rank(this->state[p*this->ncomponents + 2]);
- //DEBUG_MSG("synchronizing particles, particle %d computing is %d\n", p, this->computing[p]);
- }
- double *jump = fftw_interface<double>::alloc_real(this->nparticles);
- this->jump_estimate(jump);
- // now, see who needs to watch
- bool *local_watching = new bool[this->fs->rd->nprocs*this->nparticles];
- std::fill_n(local_watching, this->fs->rd->nprocs*this->nparticles, false);
- for (int p=0; p<this->nparticles; p++)
- if (this->fs->rd->myrank == this->computing[p])
- {
- local_watching[this->get_rank(this->state[this->ncomponents*p+2] )*this->nparticles+p] = true;
- local_watching[this->get_rank(this->state[this->ncomponents*p+2]-jump[p])*this->nparticles+p] = true;
- local_watching[this->get_rank(this->state[this->ncomponents*p+2]+jump[p])*this->nparticles+p] = true;
- }
- fftw_interface<double>::free(jump);
- MPI_Allreduce(
- local_watching,
- this->watching,
- this->nparticles*this->fs->rd->nprocs,
- MPI_C_BOOL,
- MPI_LOR,
- this->fs->rd->comm);
- delete[] local_watching;
- for (int p=0; p<this->nparticles; p++)
- DEBUG_MSG("watching = %d for particle %d\n", this->watching[this->fs->rd->myrank*nparticles+p], p);
-}
-
-
-
-template <class rnumber>
-void slab_field_particles<rnumber>::roll_rhs()
-{
- for (int i=this->integration_steps-2; i>=0; i--)
- std::copy(this->rhs[i],
- this->rhs[i] + this->array_size,
- this->rhs[i+1]);
-}
-
-
-
-template <class rnumber>
-void slab_field_particles<rnumber>::AdamsBashforth(int nsteps)
-{
- ptrdiff_t ii;
- this->get_rhs(this->state, this->rhs[0]);
- //if (myrank == 0)
- //{
- // DEBUG_MSG(
- // "in AdamsBashforth for particles %s, integration_steps = %d, nsteps = %d, iteration = %d\n",
- // this->name,
- // this->integration_steps,
- // nsteps,
- // this->iteration);
- // std::stringstream tstring;
- // for (int p=0; p<this->nparticles; p++)
- // tstring << " " << this->computing[p];
- // DEBUG_MSG("%s\n", tstring.str().c_str());
- // for (int i=0; i<this->integration_steps; i++)
- // {
- // std::stringstream tstring;
- // for (int p=0; p<this->nparticles; p++)
- // tstring << " " << this->rhs[i][p*3];
- // DEBUG_MSG("%s\n", tstring.str().c_str());
- // }
- //}
- switch(nsteps)
- {
- case 1:
- for (int p=0; p<this->nparticles; p++) if (this->fs->rd->myrank == this->computing[p])
- for (int i=0; i<this->ncomponents; i++)
- {
- ii = p*this->ncomponents+i;
- this->state[ii] += this->dt*this->rhs[0][ii];
- }
- break;
- case 2:
- for (int p=0; p<this->nparticles; p++) if (this->fs->rd->myrank == this->computing[p])
- for (int i=0; i<this->ncomponents; i++)
- {
- ii = p*this->ncomponents+i;
- this->state[ii] += this->dt*(3*this->rhs[0][ii]
- - this->rhs[1][ii])/2;
- }
- break;
- case 3:
- for (int p=0; p<this->nparticles; p++) if (this->fs->rd->myrank == this->computing[p])
- for (int i=0; i<this->ncomponents; i++)
- {
- ii = p*this->ncomponents+i;
- this->state[ii] += this->dt*(23*this->rhs[0][ii]
- - 16*this->rhs[1][ii]
- + 5*this->rhs[2][ii])/12;
- }
- break;
- case 4:
- for (int p=0; p<this->nparticles; p++) if (this->fs->rd->myrank == this->computing[p])
- for (int i=0; i<this->ncomponents; i++)
- {
- ii = p*this->ncomponents+i;
- this->state[ii] += this->dt*(55*this->rhs[0][ii]
- - 59*this->rhs[1][ii]
- + 37*this->rhs[2][ii]
- - 9*this->rhs[3][ii])/24;
- }
- break;
- case 5:
- for (int p=0; p<this->nparticles; p++) if (this->fs->rd->myrank == this->computing[p])
- for (int i=0; i<this->ncomponents; i++)
- {
- ii = p*this->ncomponents+i;
- this->state[ii] += this->dt*(1901*this->rhs[0][ii]
- - 2774*this->rhs[1][ii]
- + 2616*this->rhs[2][ii]
- - 1274*this->rhs[3][ii]
- + 251*this->rhs[4][ii])/720;
- }
- break;
- case 6:
- for (int p=0; p<this->nparticles; p++) if (this->fs->rd->myrank == this->computing[p])
- for (int i=0; i<this->ncomponents; i++)
- {
- ii = p*this->ncomponents+i;
- this->state[ii] += this->dt*(4277*this->rhs[0][ii]
- - 7923*this->rhs[1][ii]
- + 9982*this->rhs[2][ii]
- - 7298*this->rhs[3][ii]
- + 2877*this->rhs[4][ii]
- - 475*this->rhs[5][ii])/1440;
- }
- break;
- }
- this->roll_rhs();
-}
-
-
-template <class rnumber>
-void slab_field_particles<rnumber>::step()
-{
- this->AdamsBashforth((this->iteration < this->integration_steps) ? this->iteration+1 : this->integration_steps);
- //this->cRK4();
- this->iteration++;
- this->synchronize();
-}
-
-
-template <class rnumber>
-void slab_field_particles<rnumber>::Euler()
-{
- double *y = fftw_interface<double>::alloc_real(this->array_size);
- this->get_rhs(this->state, y);
- for (int p=0; p<this->nparticles; p++) if (this->fs->rd->myrank == this->computing[p])
- {
- for (int i=0; i<this->ncomponents; i++)
- this->state[p*this->ncomponents+i] += this->dt*y[p*this->ncomponents+i];
- //DEBUG_MSG(
- // "particle %d state is %lg %lg %lg\n",
- // p, this->state[p*this->ncomponents], this->state[p*this->ncomponents+1], this->state[p*this->ncomponents+2]);
- }
- fftw_interface<double>::free(y);
-}
-
-
-template <class rnumber>
-void slab_field_particles<rnumber>::Heun()
-{
- double *y = new double[this->array_size];
- double dtfactor[] = {0.0, this->dt};
- this->get_rhs(this->state, this->rhs[0]);
- for (int p=0; p<this->nparticles; p++)
- {
- this->synchronize_single_particle_state(p, this->rhs[0]);
- //int crank = this->get_rank(this->state[p*3 + 2]);
- //DEBUG_MSG(
- // "k 0 iteration %d particle is %d, crank is %d, computing rank is %d, position is %g %g %g, rhs is %g %g %g\n",
- // this->iteration, p,
- // crank, this->computing[p],
- // this->state[p*3], this->state[p*3+1], this->state[p*3+2],
- // this->rhs[0][p*3], this->rhs[0][p*3+1], this->rhs[0][p*3+2]);
- }
- for (int kindex = 1; kindex < 2; kindex++)
- {
- for (int p=0; p<this->nparticles; p++)
- {
- if (this->watching[this->fs->rd->myrank*this->nparticles+p])
- for (int i=0; i<this->ncomponents; i++)
- {
- ptrdiff_t tindex = ptrdiff_t(p)*this->ncomponents + i;
- y[tindex] = this->state[tindex] + dtfactor[kindex]*this->rhs[kindex-1][tindex];
- }
- }
- for (int p=0; p<this->nparticles; p++)
- this->synchronize_single_particle_state(p, y);
- this->get_rhs(y, this->rhs[kindex]);
- for (int p=0; p<this->nparticles; p++)
- {
- this->synchronize_single_particle_state(p, this->rhs[kindex]);
- DEBUG_MSG(
- "k %d iteration %d particle is %d, position is %g %g %g, rhs is %g %g %g\n",
- kindex, this->iteration, p,
- y[p*3], y[p*3+1], y[p*3+2],
- this->rhs[kindex][p*3], this->rhs[kindex][p*3+1], this->rhs[kindex][p*3+2]);
- }
- }
- for (int p=0; p<this->nparticles; p++)
- {
- if (this->watching[this->fs->rd->myrank*this->nparticles+p])
- {
- for (int i=0; i<this->ncomponents; i++)
- {
- ptrdiff_t tindex = ptrdiff_t(p)*this->ncomponents + i;
- this->state[tindex] += this->dt*(this->rhs[0][tindex] + this->rhs[1][tindex])/2;
- }
- //int crank = this->get_rank(this->state[p*3 + 2]);
- //if (crank != this->computing[p])
- // DEBUG_MSG(
- // "k _ iteration %d particle is %d, crank is %d, computing rank is %d, position is %g %g %g\n",
- // this->iteration, p,
- // crank, this->computing[p],
- // this->state[p*3], this->state[p*3+1], this->state[p*3+2]);
- }
- }
- delete[] y;
- DEBUG_MSG("exiting Heun\n");
-}
-
-
-template <class rnumber>
-void slab_field_particles<rnumber>::cRK4()
-{
- double *y = new double[this->array_size];
- double dtfactor[] = {0.0, this->dt/2, this->dt/2, this->dt};
- this->get_rhs(this->state, this->rhs[0]);
- for (int p=0; p<this->nparticles; p++)
- this->synchronize_single_particle_state(p, this->rhs[0]);
- for (int kindex = 1; kindex < 4; kindex++)
- {
- for (int p=0; p<this->nparticles; p++)
- {
- if (this->watching[this->fs->rd->myrank*this->nparticles+p])
- for (int i=0; i<this->ncomponents; i++)
- {
- ptrdiff_t tindex = ptrdiff_t(p)*this->ncomponents + i;
- y[tindex] = this->state[tindex] + dtfactor[kindex]*this->rhs[kindex-1][tindex];
- }
- }
- for (int p=0; p<this->nparticles; p++)
- this->synchronize_single_particle_state(p, y);
- this->get_rhs(y, this->rhs[kindex]);
- for (int p=0; p<this->nparticles; p++)
- this->synchronize_single_particle_state(p, this->rhs[kindex]);
- }
- for (int p=0; p<this->nparticles; p++)
- {
- if (this->watching[this->fs->rd->myrank*this->nparticles+p])
- for (int i=0; i<this->ncomponents; i++)
- {
- ptrdiff_t tindex = ptrdiff_t(p)*this->ncomponents + i;
- this->state[tindex] += this->dt*(this->rhs[0][tindex] +
- 2*(this->rhs[1][tindex] + this->rhs[2][tindex]) +
- this->rhs[3][tindex])/6;
- }
- }
- delete[] y;
-}
-
-template <class rnumber>
-void slab_field_particles<rnumber>::get_grid_coordinates(double *x, int *xg, double *xx)
-{
- static double grid_size[] = {this->dx, this->dy, this->dz};
- double tval;
- std::fill_n(xg, this->nparticles*3, 0);
- std::fill_n(xx, this->nparticles*3, 0.0);
- for (int p=0; p<this->nparticles; p++) if (this->watching[this->fs->rd->myrank*this->nparticles+p])
- {
- for (int c=0; c<3; c++)
- {
- tval = floor(x[p*this->ncomponents+c]/grid_size[c]);
- xg[p*3+c] = MOD(int(tval), this->fs->rd->sizes[2-c]);
- xx[p*3+c] = (x[p*this->ncomponents+c] - tval*grid_size[c]) / grid_size[c];
- }
- xg[p*3+2] -= this->fs->rd->starts[0];
- if (this->fs->rd->myrank == this->fs->rd->rank[0] &&
- xg[p*3+2] > this->fs->rd->subsizes[0])
- xg[p*3+2] -= this->fs->rd->sizes[0];
- //DEBUG_MSG(
- // "particle %d x is %lg %lg %lg xx is %lg %lg %lg xg is %d %d %d\n",
- // p,
- // x[p*3], x[p*3+1], x[p*3+2],
- // xx[p*3], xx[p*3+1], xx[p*3+2],
- // xg[p*3], xg[p*3+1], xg[p*3+2]);
- }
-}
-
-template <class rnumber>
-void slab_field_particles<rnumber>::interpolation_formula(rnumber *field, int *xg, double *xx, double *dest, int *deriv)
-{
- double bx[this->interp_neighbours*2+2], by[this->interp_neighbours*2+2], bz[this->interp_neighbours*2+2];
- this->compute_beta(deriv[0], xx[0], bx);
- this->compute_beta(deriv[1], xx[1], by);
- this->compute_beta(deriv[2], xx[2], bz);
- //DEBUG_MSG("computed beta polynomials\n");
- std::fill_n(dest, 3, 0);
- for (int iz = -this->interp_neighbours; iz <= this->interp_neighbours+1; iz++)
- for (int iy = -this->interp_neighbours; iy <= this->interp_neighbours+1; iy++)
- for (int ix = -this->interp_neighbours; ix <= this->interp_neighbours+1; ix++)
- for (int c=0; c<3; c++)
- {
- //DEBUG_MSG(
- // "%d %d %d %d %d %d %d %ld %ld\n",
- // xg[2], xg[1], xg[0], iz, iy, ix, c,
- // ((ptrdiff_t(xg[2]+iz) *this->fs->rd->subsizes[1] +
- // ptrdiff_t(xg[1]+iy))*this->fs->rd->subsizes[2] +
- // ptrdiff_t(xg[0]+ix))*3+c,
- // this->buffered_field_descriptor->local_size
- // );
- dest[c] += field[((ptrdiff_t( xg[2]+iz ) *this->fs->rd->subsizes[1] +
- ptrdiff_t(MOD(xg[1]+iy, this->fs->rd->sizes[1])))*this->fs->rd->subsizes[2] +
- ptrdiff_t(MOD(xg[0]+ix, this->fs->rd->sizes[2])))*3+c]*(bz[iz+this->interp_neighbours]*
- by[iy+this->interp_neighbours]*
- bx[ix+this->interp_neighbours]);
- }
-}
-
-template <class rnumber>
-void slab_field_particles<rnumber>::linear_interpolation(rnumber *field, int *xg, double *xx, double *dest, int *deriv)
-{
- //ptrdiff_t tindex, tmp;
- //tindex = ((ptrdiff_t(xg[2] )*this->fs->rd->subsizes[1]+xg[1] )*this->fs->rd->subsizes[2]+xg[0] )*3;
- //tmp = ptrdiff_t(xg[2]);
- //DEBUG_MSG(
- // "linear interpolation xx is %lg %lg %lg xg is %d %d %d,"
- // " corner index is ((%ld*%d+%d)*%d+%d)*3 = %ld\n",
- // xx[0], xx[1], xx[2],
- // xg[0], xg[1], xg[2],
- // tmp, this->fs->rd->subsizes[1], xg[1], this->fs->rd->subsizes[2], xg[0],
- // tindex);
- for (int c=0; c<3; c++)
- dest[c] = (field[((ptrdiff_t(xg[2] )*this->fs->rd->subsizes[1]+xg[1] )*this->fs->rd->subsizes[2]+xg[0] )*3+c]*((1-xx[0])*(1-xx[1])*(1-xx[2])) +
- field[((ptrdiff_t(xg[2] )*this->fs->rd->subsizes[1]+xg[1] )*this->fs->rd->subsizes[2]+xg[0]+1)*3+c]*(( xx[0])*(1-xx[1])*(1-xx[2])) +
- field[((ptrdiff_t(xg[2] )*this->fs->rd->subsizes[1]+xg[1]+1)*this->fs->rd->subsizes[2]+xg[0] )*3+c]*((1-xx[0])*( xx[1])*(1-xx[2])) +
- field[((ptrdiff_t(xg[2] )*this->fs->rd->subsizes[1]+xg[1]+1)*this->fs->rd->subsizes[2]+xg[0]+1)*3+c]*(( xx[0])*( xx[1])*(1-xx[2])) +
- field[((ptrdiff_t(xg[2]+1)*this->fs->rd->subsizes[1]+xg[1] )*this->fs->rd->subsizes[2]+xg[0] )*3+c]*((1-xx[0])*(1-xx[1])*( xx[2])) +
- field[((ptrdiff_t(xg[2]+1)*this->fs->rd->subsizes[1]+xg[1] )*this->fs->rd->subsizes[2]+xg[0]+1)*3+c]*(( xx[0])*(1-xx[1])*( xx[2])) +
- field[((ptrdiff_t(xg[2]+1)*this->fs->rd->subsizes[1]+xg[1]+1)*this->fs->rd->subsizes[2]+xg[0] )*3+c]*((1-xx[0])*( xx[1])*( xx[2])) +
- field[((ptrdiff_t(xg[2]+1)*this->fs->rd->subsizes[1]+xg[1]+1)*this->fs->rd->subsizes[2]+xg[0]+1)*3+c]*(( xx[0])*( xx[1])*( xx[2])));
-}
-
-template <class rnumber>
-void slab_field_particles<rnumber>::read(hid_t data_file_id)
-{
- //DEBUG_MSG("aloha\n");
- if (this->fs->rd->myrank == 0)
- {
- std::string temp_string = (std::string("/particles/") +
- std::string(this->name) +
- std::string("/state"));
- hid_t Cdset = H5Dopen(data_file_id, temp_string.c_str(), H5P_DEFAULT);
- hid_t mspace, rspace;
- hsize_t count[4], offset[4];
- rspace = H5Dget_space(Cdset);
- H5Sget_simple_extent_dims(rspace, count, NULL);
- count[0] = 1;
- offset[0] = this->iteration / this->traj_skip;
- offset[1] = 0;
- offset[2] = 0;
- mspace = H5Screate_simple(3, count, NULL);
- H5Sselect_hyperslab(rspace, H5S_SELECT_SET, offset, NULL, count, NULL);
- H5Dread(Cdset, H5T_NATIVE_DOUBLE, mspace, rspace, H5P_DEFAULT, this->state);
- H5Sclose(mspace);
- H5Sclose(rspace);
- H5Dclose(Cdset);
- if (this->iteration > 0)
- {
- temp_string = (std::string("/particles/") +
- std::string(this->name) +
- std::string("/rhs"));
- Cdset = H5Dopen(data_file_id, temp_string.c_str(), H5P_DEFAULT);
- rspace = H5Dget_space(Cdset);
- H5Sget_simple_extent_dims(rspace, count, NULL);
- //reading from last available position
- offset[0] = count[0] - 1;
- offset[3] = 0;
- count[0] = 1;
- count[1] = 1;
- mspace = H5Screate_simple(4, count, NULL);
- for (int i=0; i<this->integration_steps; i++)
- {
- offset[1] = i;
- H5Sselect_hyperslab(rspace, H5S_SELECT_SET, offset, NULL, count, NULL);
- H5Dread(Cdset, H5T_NATIVE_DOUBLE, mspace, rspace, H5P_DEFAULT, this->rhs[i]);
- }
- H5Sclose(mspace);
- H5Sclose(rspace);
- H5Dclose(Cdset);
- }
- }
- MPI_Bcast(
- this->state,
- this->array_size,
- MPI_DOUBLE,
- 0,
- this->fs->rd->comm);
- for (int i = 0; i<this->integration_steps; i++)
- {
- MPI_Bcast(
- this->rhs[i],
- this->array_size,
- MPI_DOUBLE,
- 0,
- this->fs->rd->comm);
- }
- // initial assignment of particles
- for (int p=0; p<this->nparticles; p++)
- {
- this->computing[p] = this->get_rank(this->state[p*this->ncomponents + 2]);
- //DEBUG_MSG("reading particles, particle %d computing is %d\n", p, this->computing[p]);
- }
- // now actual synchronization
- this->synchronize();
-}
-
-template <class rnumber>
-void slab_field_particles<rnumber>::write(hid_t data_file_id, bool write_rhs)
-{
- if (this->fs->rd->myrank == 0)
- {
- std::string temp_string = (std::string("/particles/") +
- std::string(this->name) +
- std::string("/state"));
- hid_t Cdset = H5Dopen(data_file_id, temp_string.c_str(), H5P_DEFAULT);
- hid_t mspace, wspace;
- hsize_t count[4], offset[4];
- wspace = H5Dget_space(Cdset);
- H5Sget_simple_extent_dims(wspace, count, NULL);
- count[0] = 1;
- offset[0] = this->iteration / this->traj_skip;
- offset[1] = 0;
- offset[2] = 0;
- mspace = H5Screate_simple(3, count, NULL);
- H5Sselect_hyperslab(wspace, H5S_SELECT_SET, offset, NULL, count, NULL);
- H5Dwrite(Cdset, H5T_NATIVE_DOUBLE, mspace, wspace, H5P_DEFAULT, this->state);
- H5Sclose(mspace);
- H5Sclose(wspace);
- H5Dclose(Cdset);
- if (write_rhs)
- {
- temp_string = (std::string("/particles/") +
- std::string(this->name) +
- std::string("/rhs"));
- Cdset = H5Dopen(data_file_id, temp_string.c_str(), H5P_DEFAULT);
- wspace = H5Dget_space(Cdset);
- H5Sget_simple_extent_dims(wspace, count, NULL);
- //writing to last available position
- offset[0] = count[0] - 1;
- count[0] = 1;
- count[1] = 1;
- offset[3] = 0;
- mspace = H5Screate_simple(4, count, NULL);
- for (int i=0; i<this->integration_steps; i++)
- {
- offset[1] = i;
- H5Sselect_hyperslab(wspace, H5S_SELECT_SET, offset, NULL, count, NULL);
- H5Dwrite(Cdset, H5T_NATIVE_DOUBLE, mspace, wspace, H5P_DEFAULT, this->rhs[i]);
- }
- H5Sclose(mspace);
- H5Sclose(wspace);
- H5Dclose(Cdset);
- }
- }
-}
-
-
-
-/*****************************************************************************/
-/* macro for specializations to numeric types compatible with FFTW */
-#define SLAB_FIELD_PARTICLES_DEFINITIONS(FFTW, R, MPI_RNUM) \
- \
-template <> \
-void slab_field_particles<R>::rFFTW_to_buffered(R *src, R *dst) \
-{ \
- /* do big copy of middle stuff */ \
- std::copy(src, \
- src + this->fs->rd->local_size, \
- dst + this->buffer_size); \
- int rsrc; \
- /* get upper slices */ \
- for (int rdst = 0; rdst < this->fs->rd->nprocs; rdst++) \
- { \
- rsrc = this->fs->rd->rank[(this->fs->rd->all_start0[rdst] + \
- this->fs->rd->all_size0[rdst]) % \
- this->fs->rd->sizes[0]]; \
- if (this->fs->rd->myrank == rsrc) \
- MPI_Send( \
- (void*)(src), \
- this->buffer_size, \
- MPI_RNUM, \
- rdst, \
- 2*(rsrc*this->fs->rd->nprocs + rdst), \
- this->fs->rd->comm); \
- if (this->fs->rd->myrank == rdst) \
- MPI_Recv( \
- (void*)(dst + this->buffer_size + this->fs->rd->local_size), \
- this->buffer_size, \
- MPI_RNUM, \
- rsrc, \
- 2*(rsrc*this->fs->rd->nprocs + rdst), \
- this->fs->rd->comm, \
- MPI_STATUS_IGNORE); \
- } \
- /* get lower slices */ \
- for (int rdst = 0; rdst < this->fs->rd->nprocs; rdst++) \
- { \
- rsrc = this->fs->rd->rank[MOD(this->fs->rd->all_start0[rdst] - 1, \
- this->fs->rd->sizes[0])]; \
- if (this->fs->rd->myrank == rsrc) \
- MPI_Send( \
- (void*)(src + this->fs->rd->local_size - this->buffer_size), \
- this->buffer_size, \
- MPI_RNUM, \
- rdst, \
- 2*(rsrc*this->fs->rd->nprocs + rdst)+1, \
- this->fs->rd->comm); \
- if (this->fs->rd->myrank == rdst) \
- MPI_Recv( \
- (void*)(dst), \
- this->buffer_size, \
- MPI_RNUM, \
- rsrc, \
- 2*(rsrc*this->fs->rd->nprocs + rdst)+1, \
- this->fs->rd->comm, \
- MPI_STATUS_IGNORE); \
- } \
-} \
-/*****************************************************************************/
-
-
-
-/*****************************************************************************/
-/* now actually use the macro defined above */
-SLAB_FIELD_PARTICLES_DEFINITIONS(
- FFTW_MANGLE_FLOAT,
- float,
- MPI_FLOAT)
-SLAB_FIELD_PARTICLES_DEFINITIONS(
- FFTW_MANGLE_DOUBLE,
- double,
- MPI_DOUBLE)
-/*****************************************************************************/
-
-
-
-/*****************************************************************************/
-/* finally, force generation of code for single precision */
-template class slab_field_particles<float>;
-template class slab_field_particles<double>;
-/*****************************************************************************/
diff --git a/bfps/cpp/slab_field_particles.hpp b/bfps/cpp/slab_field_particles.hpp
deleted file mode 100644
index 15f9477b..00000000
--- a/bfps/cpp/slab_field_particles.hpp
+++ /dev/null
@@ -1,149 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <iostream>
-#include <hdf5.h>
-#include "base.hpp"
-#include "fluid_solver_base.hpp"
-#include "interpolator.hpp"
-
-#ifndef SLAB_FIELD_PARTICLES
-
-#define SLAB_FIELD_PARTICLES
-
-extern int myrank, nprocs;
-
-template <class rnumber>
-class slab_field_particles
-{
- protected:
- //typedef void (slab_field_particles<rnumber>::*tensor_product_interpolation_formula)(
- // rnumber *field,
- // int *xg,
- // double *xx,
- // double *dest,
- // int *deriv);
- public:
- fluid_solver_base<rnumber> *fs;
- field_descriptor<rnumber> *buffered_field_descriptor;
-
- /* watching is an array of shape [nparticles], with
- * watching[p] being true if particle p is in the domain of myrank
- * or in the buffer regions.
- * watching is not really being used right now, since I don't do partial
- * synchronizations of particles.
- * we may do this at some point in the future, if it seems needed...
- * */
- bool *watching;
- /* computing is an array of shape [nparticles], with
- * computing[p] being the rank that is currently working on particle p
- * */
- int *computing;
-
- /* state will generally hold all the information about the particles.
- * in the beginning, we will only need to solve 3D ODEs, but I figured
- * a general ncomponents is better, since we may change our minds.
- * */
- double *state;
- double *rhs[6];
- int nparticles;
- int ncomponents;
- int array_size;
- int interp_neighbours;
- int buffer_width;
- int integration_steps;
- int traj_skip;
- ptrdiff_t buffer_size;
- double *lbound;
- double *ubound;
- //tensor_product_interpolation_formula spline_formula;
- base_polynomial_values compute_beta;
-
- /* simulation parameters */
- char name[256];
- int iteration;
- double dt;
-
- /* physical parameters of field */
- rnumber dx, dy, dz;
-
- /* methods */
-
- /* constructor and destructor.
- * allocate and deallocate:
- * this->state
- * this->lbound
- * this->ubound
- * this->watching
- * */
- slab_field_particles(
- const char *NAME,
- fluid_solver_base<rnumber> *FSOLVER,
- const int NPARTICLES,
- const int NCOMPONENTS,
- base_polynomial_values BETA_POLYS,
- const int INTERP_NEIGHBOURS,
- const int TRAJ_SKIP,
- const int INTEGRATION_STEPS = 2);
- ~slab_field_particles();
-
- /* an Euler step is needed to compute an estimate of future positions,
- * which is needed for synchronization.
- * */
- virtual void jump_estimate(double *jump_length);
- /* function get_rhs is virtual since we want children to do different things,
- * depending on the type of particle.
- * */
- virtual void get_rhs(double *x, double *rhs);
-
- /* generic methods, should work for all children of this class */
- int get_rank(double z); // get rank for given value of z
- void synchronize();
- void synchronize_single_particle_state(int p, double *x, int source_id = -1);
- void get_grid_coordinates(double *x, int *xg, double *xx);
- void linear_interpolation(rnumber *field, int *xg, double *xx, double *dest, int *deriv);
- void interpolation_formula(rnumber *field, int *xg, double *xx, double *dest, int *deriv);
-
- void rFFTW_to_buffered(rnumber *src, rnumber *dst);
-
- /* generic methods, should work for all children of this class */
- void read(hid_t data_file_id);
- void write(hid_t data_file_id, bool write_rhs = true);
-
- /* solver stuff */
- void step();
- void roll_rhs();
- void AdamsBashforth(int nsteps);
- void Euler();
- void Heun();
- void cRK4();
-};
-
-
-#endif//SLAB_FIELD_PARTICLES
-
diff --git a/bfps/cpp/tracers.cpp b/bfps/cpp/tracers.cpp
deleted file mode 100644
index 3d9fbfb6..00000000
--- a/bfps/cpp/tracers.cpp
+++ /dev/null
@@ -1,204 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#define NDEBUG
-
-
-#include <cmath>
-#include "base.hpp"
-#include "fftw_tools.hpp"
-#include "tracers.hpp"
-
-template <class rnumber>
-void tracers<rnumber>::jump_estimate(double *jump)
-{
- int deriv[] = {0, 0, 0};
- int *xg = new int[this->array_size];
- double *xx = new double[this->array_size];
- rnumber *vel = this->data + this->buffer_size;
- double tmp[3];
- /* get grid coordinates */
- this->get_grid_coordinates(this->state, xg, xx);
-
- /* perform interpolation */
- for (int p=0; p<this->nparticles; p++) if (this->fs->rd->myrank == this->computing[p])
- {
- this->interpolation_formula(vel, xg + p*3, xx + p*3, tmp, deriv);
- jump[p] = fabs(3*this->dt * tmp[2]);
- if (jump[p] < this->dz*1.01)
- jump[p] = this->dz*1.01;
- }
- delete[] xg;
- delete[] xx;
-}
-
-template <class rnumber>
-void tracers<rnumber>::get_rhs(double *x, double *y)
-{
- std::fill_n(y, this->array_size, 0.0);
- int deriv[] = {0, 0, 0};
- /* get grid coordinates */
- int *xg = new int[this->array_size];
- double *xx = new double[this->array_size];
- rnumber *vel = this->data + this->buffer_size;
- this->get_grid_coordinates(x, xg, xx);
- //DEBUG_MSG(
- // "position is %g %g %g, grid_coords are %d %d %d %g %g %g\n",
- // x[0], x[1], x[2],
- // xg[0], xg[1], xg[2],
- // xx[0], xx[1], xx[2]);
- /* perform interpolation */
- for (int p=0; p<this->nparticles; p++)
- {
- if (this->watching[this->fs->rd->myrank*this->nparticles+p])
- {
- int crank = this->get_rank(x[p*3 + 2]);
- if (this->fs->rd->myrank == crank)
- {
- this->interpolation_formula(vel, xg + p*3, xx + p*3, y + p*3, deriv);
- DEBUG_MSG(
- "position is %g %g %g %d %d %d %g %g %g, result is %g %g %g\n",
- x[p*3], x[p*3+1], x[p*3+2],
- xg[p*3], xg[p*3+1], xg[p*3+2],
- xx[p*3], xx[p*3+1], xx[p*3+2],
- y[p*3], y[p*3+1], y[p*3+2]);
- }
- if (crank != this->computing[p])
- {
- this->synchronize_single_particle_state(p, y, crank);
- }
- //DEBUG_MSG(
- // "after synch crank is %d, computing rank is %d, position is %g %g %g, result is %g %g %g\n",
- // this->iteration, p,
- // crank, this->computing[p],
- // x[p*3], x[p*3+1], x[p*3+2],
- // y[p*3], y[p*3+1], y[p*3+2]);
- }
- }
- delete[] xg;
- delete[] xx;
-}
-
-template<class rnumber>
-void tracers<rnumber>::update_field(bool clip_on)
-{
- if (clip_on)
- clip_zero_padding<rnumber>(this->fs->rd, this->source_data, 3);
- this->rFFTW_to_buffered(this->source_data, this->data);
-}
-
-/*****************************************************************************/
-/* macro for specializations to numeric types compatible with FFTW */
-
-#define TRACERS_DEFINITIONS(FFTW, R, MPI_RNUM, MPI_CNUM) \
- \
-template <> \
-tracers<R>::tracers( \
- const char *NAME, \
- fluid_solver_base<R> *FSOLVER, \
- const int NPARTICLES, \
- base_polynomial_values BETA_POLYS, \
- const int NEIGHBOURS, \
- const int TRAJ_SKIP, \
- const int INTEGRATION_STEPS, \
- R *SOURCE_DATA) : slab_field_particles<R>( \
- NAME, \
- FSOLVER, \
- NPARTICLES, \
- 3, \
- BETA_POLYS, \
- NEIGHBOURS, \
- TRAJ_SKIP, \
- INTEGRATION_STEPS) \
-{ \
- this->source_data = SOURCE_DATA; \
- this->data = FFTW(alloc_real)(this->buffered_field_descriptor->local_size); \
-} \
- \
-template<> \
-tracers<R>::~tracers() \
-{ \
- FFTW(free)(this->data); \
-} \
- \
-template <> \
-void tracers<R>::sample_vec_field(R *vec_field, double *vec_values) \
-{ \
- vec_field += this->buffer_size; \
- double *vec_local = new double[this->array_size]; \
- std::fill_n(vec_local, this->array_size, 0.0); \
- int deriv[] = {0, 0, 0}; \
- /* get grid coordinates */ \
- int *xg = new int[this->array_size]; \
- double *xx = new double[this->array_size]; \
- this->get_grid_coordinates(this->state, xg, xx); \
- /* perform interpolation */ \
- for (int p=0; p<this->nparticles; p++) \
- if (this->fs->rd->myrank == this->computing[p]) \
- this->interpolation_formula( \
- vec_field, \
- xg + p*3, \
- xx + p*3, \
- vec_local + p*3, \
- deriv); \
- MPI_Allreduce( \
- vec_local, \
- vec_values, \
- this->array_size, \
- MPI_DOUBLE, \
- MPI_SUM, \
- this->fs->rd->comm); \
- delete[] xg; \
- delete[] xx; \
- delete[] vec_local; \
-} \
-
-/*****************************************************************************/
-
-
-
-/*****************************************************************************/
-/* now actually use the macro defined above */
-TRACERS_DEFINITIONS(
- FFTW_MANGLE_FLOAT,
- float,
- MPI_FLOAT,
- MPI_COMPLEX)
-TRACERS_DEFINITIONS(
- FFTW_MANGLE_DOUBLE,
- double,
- MPI_DOUBLE,
- BFPS_MPICXX_DOUBLE_COMPLEX)
-/*****************************************************************************/
-
-
-
-/*****************************************************************************/
-/* finally, force generation of code */
-template class tracers<float>;
-template class tracers<double>;
-/*****************************************************************************/
-
diff --git a/bfps/cpp/tracers.hpp b/bfps/cpp/tracers.hpp
deleted file mode 100644
index 1a063e02..00000000
--- a/bfps/cpp/tracers.hpp
+++ /dev/null
@@ -1,63 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#include "slab_field_particles.hpp"
-
-#ifndef TRACERS
-
-#define TRACERS
-
-extern int myrank, nprocs;
-
-template <class rnumber>
-class tracers final:public slab_field_particles<rnumber>
-{
- public:
- rnumber *source_data;
- rnumber *data;
-
- /* methods */
- tracers(
- const char *NAME,
- fluid_solver_base<rnumber> *FSOLVER,
- const int NPARTICLES,
- base_polynomial_values BETA_POLYS,
- const int NEIGHBOURS,
- const int TRAJ_SKIP,
- const int INTEGRATION_STEPS,
- rnumber *SOURCE_DATA);
- ~tracers();
-
- void update_field(bool clip_on = true);
- virtual void get_rhs(double *x, double *rhs);
- virtual void jump_estimate(double *jump_length);
-
- void sample_vec_field(rnumber *vec_field, double *vec_values);
-};
-
-
-#endif//TRACERS
-
diff --git a/setup.py b/setup.py
index afb25f70..3094c692 100644
--- a/setup.py
+++ b/setup.py
@@ -123,8 +123,6 @@ src_file_list = [
'spline_n10',
'Lagrange_polys',
'scope_timer',
- 'interpolator',
- 'interpolator_base',
'full_code/test_interpolation',
'full_code/NSVEparticles',
'full_code/NSVEcomplex_particles',
--
GitLab
From d89dd1fe3c17a22e5b33e04a41f28d1d5fcb1b73 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 21 Sep 2018 11:03:24 +0200
Subject: [PATCH 230/342] remove old python files
---
bfps/FluidConvert.py | 140 ----
bfps/NSManyParticles.py | 92 ---
bfps/NSVorticityEquation.py | 864 --------------------
bfps/NavierStokes.py | 1263 ------------------------------
bfps/__main__.py | 45 +-
bfps/_fluid_base.py | 503 ------------
tests/base.py | 23 +-
tests/test_plain.py | 156 ----
tests/test_vorticity_equation.py | 10 +-
9 files changed, 8 insertions(+), 3088 deletions(-)
delete mode 100644 bfps/FluidConvert.py
delete mode 100644 bfps/NSManyParticles.py
delete mode 100644 bfps/NSVorticityEquation.py
delete mode 100644 bfps/NavierStokes.py
delete mode 100644 bfps/_fluid_base.py
delete mode 100644 tests/test_plain.py
diff --git a/bfps/FluidConvert.py b/bfps/FluidConvert.py
deleted file mode 100644
index 58d19116..00000000
--- a/bfps/FluidConvert.py
+++ /dev/null
@@ -1,140 +0,0 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
-
-
-
-import numpy as np
-import pickle
-import os
-from ._fluid_base import _fluid_particle_base
-from ._base import _base
-import bfps
-
-class FluidConvert(_fluid_particle_base):
- """This class is meant to be used for conversion of native DNS field
- representations to real-space representations of velocity/vorticity
- fields.
- It may be superseeded by streamlined functionality in the future...
- """
- def __init__(
- self,
- name = 'FluidConvert-v' + bfps.__version__,
- work_dir = './',
- simname = 'test',
- fluid_precision = 'single',
- use_fftw_wisdom = False):
- _fluid_particle_base.__init__(
- self,
- name = name + '-' + fluid_precision,
- work_dir = work_dir,
- simname = simname,
- dtype = fluid_precision,
- use_fftw_wisdom = use_fftw_wisdom)
- self.spec_parameters = {}
- self.spec_parameters['write_rvelocity'] = 1
- self.spec_parameters['write_rvorticity'] = 1
- self.spec_parameters['write_rTrS2'] = 1
- self.spec_parameters['write_renstrophy'] = 1
- self.spec_parameters['write_rpressure'] = 1
- self.spec_parameters['iter0'] = 0
- self.spec_parameters['iter1'] = -1
- self.fill_up_fluid_code()
- self.finalize_code(postprocess_mode = True)
- return None
- def fill_up_fluid_code(self):
- self.definitions += self.cread_pars(
- parameters = self.spec_parameters,
- function_suffix = '_specific',
- file_group = 'conversion_parameters')
- self.variables += self.cdef_pars(
- parameters = self.spec_parameters)
- self.main_start += 'read_parameters_specific();\n'
- self.fluid_includes += '#include <cstring>\n'
- self.fluid_variables += ('double t;\n' +
- 'fluid_solver<{0}> *fs;\n').format(self.C_dtype)
- self.fluid_definitions += """
- //begincpp
- void do_conversion(fluid_solver<{0}> *bla)
- {{
- bla->read('v', 'c');
- if (write_rvelocity)
- bla->write('u', 'r');
- if (write_rvorticity)
- bla->write('v', 'r');
- if (write_rTrS2)
- bla->write_rTrS2();
- if (write_renstrophy)
- bla->write_renstrophy();
- if (write_rpressure)
- bla->write_rpressure();
- }}
- //endcpp
- """.format(self.C_dtype)
- self.fluid_start += """
- //begincpp
- fs = new fluid_solver<{0}>(
- simname,
- nx, ny, nz,
- dkx, dky, dkz,
- dealias_type,
- DEFAULT_FFTW_FLAG);
- //endcpp
- """.format(self.C_dtype)
- self.fluid_loop += """
- //begincpp
- fs->iteration = frame_index;
- do_conversion(fs);
- //endcpp
- """
- self.fluid_end += 'delete fs;\n'
- return None
- def specific_parser_arguments(
- self,
- parser):
- _fluid_particle_base.specific_parser_arguments(self, parser)
- self.parameters_to_parser_arguments(
- parser,
- parameters = self.spec_parameters)
- return None
- def launch(
- self,
- args = [],
- **kwargs):
- opt = self.prepare_launch(args)
- if opt.iter1 == -1:
- opt.iter1 = self.get_data_file()['iteration'].value
- self.pars_from_namespace(
- opt,
- parameters = self.spec_parameters)
- self.rewrite_par(
- group = 'conversion_parameters',
- parameters = self.spec_parameters)
- self.run(opt.nb_processes,
- 1,
- hours = opt.minutes // 60,
- minutes = opt.minutes % 60,
- err_file = 'err_convert',
- out_file = 'out_convert')
- return None
-
diff --git a/bfps/NSManyParticles.py b/bfps/NSManyParticles.py
deleted file mode 100644
index 03f7345f..00000000
--- a/bfps/NSManyParticles.py
+++ /dev/null
@@ -1,92 +0,0 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
-
-
-
-import bfps
-
-class NSManyParticles(bfps.NavierStokes):
- def specific_parser_arguments(
- self,
- parser):
- bfps.NavierStokes.specific_parser_arguments(self, parser)
- parser.add_argument(
- '--particle-class',
- default = 'rFFTW_distributed_particles',
- dest = 'particle_class',
- type = str)
- parser.add_argument(
- '--interpolator-class',
- default = 'rFFTW_interpolator',
- dest = 'interpolator_class',
- type = str)
- parser.add_argument('--neighbours',
- type = int,
- dest = 'neighbours',
- default = 3)
- parser.add_argument('--smoothness',
- type = int,
- dest = 'smoothness',
- default = 2)
- return None
- def launch(
- self,
- args = [],
- **kwargs):
- opt = self.prepare_launch(args = args)
- self.fill_up_fluid_code()
- if type(opt.nparticles) == int:
- if opt.nparticles > 0:
- self.add_3D_rFFTW_field(
- name = 'rFFTW_acc')
- interp_list = []
- for n in range(1, opt.neighbours):
- interp_list.append('Lagrange_n{0}'.format(n))
- self.add_interpolator(
- interp_type = 'Lagrange',
- name = interp_list[-1],
- neighbours = n,
- class_name = opt.interpolator_class)
- for m in range(1, opt.smoothness):
- interp_list.append('spline_n{0}m{1}'.format(n, m))
- self.add_interpolator(
- interp_type = 'spline',
- name = interp_list[-1],
- neighbours = n,
- smoothness = m,
- class_name = opt.interpolator_class)
- self.add_particles(
- integration_steps = 2,
- interpolator = interp_list,
- acc_name = 'rFFTW_acc',
- class_name = opt.particle_class)
- self.add_particles(
- integration_steps = 4,
- interpolator = interp_list,
- acc_name = 'rFFTW_acc',
- class_name = opt.particle_class)
- self.finalize_code()
- self.launch_jobs(opt = opt)
- return None
-
diff --git a/bfps/NSVorticityEquation.py b/bfps/NSVorticityEquation.py
deleted file mode 100644
index 5f87097f..00000000
--- a/bfps/NSVorticityEquation.py
+++ /dev/null
@@ -1,864 +0,0 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
-
-
-
-import sys
-import os
-import numpy as np
-import h5py
-import argparse
-
-import bfps
-import bfps.tools
-from bfps._code import _code
-from bfps._fluid_base import _fluid_particle_base
-
-class NSVorticityEquation(_fluid_particle_base):
- def __init__(
- self,
- name = 'NSVorticityEquation-v' + bfps.__version__,
- work_dir = './',
- simname = 'test',
- fluid_precision = 'single',
- fftw_plan_rigor = 'FFTW_MEASURE',
- use_fftw_wisdom = True):
- """
- This code uses checkpoints for DNS restarts, and it can be stopped
- by creating the file "stop_<simname>" in the working directory.
- For postprocessing of field snapshots, consider creating a separate
- HDF5 file (from the python wrapper) which contains links to all the
- different snapshots.
- """
- self.fftw_plan_rigor = fftw_plan_rigor
- _fluid_particle_base.__init__(
- self,
- name = name + '-' + fluid_precision,
- work_dir = work_dir,
- simname = simname,
- dtype = fluid_precision,
- use_fftw_wisdom = use_fftw_wisdom)
- self.parameters['nu'] = float(0.1)
- self.parameters['fmode'] = 1
- self.parameters['famplitude'] = float(0.5)
- self.parameters['fk0'] = float(2.0)
- self.parameters['fk1'] = float(4.0)
- self.parameters['forcing_type'] = 'linear'
- self.parameters['histogram_bins'] = int(256)
- self.parameters['max_velocity_estimate'] = float(1)
- self.parameters['max_vorticity_estimate'] = float(1)
- self.parameters['checkpoints_per_file'] = int(1)
- self.file_datasets_grow = """
- //begincpp
- hid_t group;
- group = H5Gopen(stat_file, "/statistics", H5P_DEFAULT);
- H5Ovisit(group, H5_INDEX_NAME, H5_ITER_NATIVE, grow_statistics_dataset, NULL);
- H5Gclose(group);
- //endcpp
- """
- self.style = {}
- self.statistics = {}
- self.fluid_output = """
- fs->io_checkpoint(false);
- """
- # vorticity_equation specific things
- self.includes += '#include "vorticity_equation.hpp"\n'
- self.store_kspace = """
- //begincpp
- if (myrank == 0 && iteration == 0)
- {
- TIMEZONE("fluid_base::store_kspace");
- hsize_t dims[4];
- hid_t space, dset;
- // store kspace information
- dset = H5Dopen(stat_file, "/kspace/kshell", H5P_DEFAULT);
- space = H5Dget_space(dset);
- H5Sget_simple_extent_dims(space, dims, NULL);
- H5Sclose(space);
- if (fs->kk->nshells != dims[0])
- {
- DEBUG_MSG(
- "ERROR: computed nshells %d not equal to data file nshells %d\\n",
- fs->kk->nshells, dims[0]);
- }
- H5Dwrite(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &fs->kk->kshell.front());
- H5Dclose(dset);
- dset = H5Dopen(stat_file, "/kspace/nshell", H5P_DEFAULT);
- H5Dwrite(dset, H5T_NATIVE_INT64, H5S_ALL, H5S_ALL, H5P_DEFAULT, &fs->kk->nshell.front());
- H5Dclose(dset);
- dset = H5Dopen(stat_file, "/kspace/kM", H5P_DEFAULT);
- H5Dwrite(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &fs->kk->kM);
- H5Dclose(dset);
- dset = H5Dopen(stat_file, "/kspace/dk", H5P_DEFAULT);
- H5Dwrite(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &fs->kk->dk);
- H5Dclose(dset);
- }
- //endcpp
- """
- return None
- def add_particles(
- self,
- integration_steps = 2,
- neighbours = 1,
- smoothness = 1):
- assert(integration_steps > 0 and integration_steps < 6)
- self.particle_species = 1
- self.parameters['tracers0_integration_steps'] = int(integration_steps)
- self.parameters['tracers0_neighbours'] = int(neighbours)
- self.parameters['tracers0_smoothness'] = int(smoothness)
- self.parameters['tracers0_interpolator'] = 'spline'
- self.particle_includes += """
- #include "particles/particles_system_builder.hpp"
- #include "particles/particles_output_hdf5.hpp"
- """
- ## initialize
- self.particle_start += """
- DEBUG_MSG(
- "current fname is %s\\n and iteration is %d",
- fs->get_current_fname().c_str(),
- fs->iteration);
- std::unique_ptr<abstract_particles_system<long long int, double>> ps = particles_system_builder(
- fs->cvelocity, // (field object)
- fs->kk, // (kspace object, contains dkx, dky, dkz)
- tracers0_integration_steps, // to check coherency between parameters and hdf input file (nb rhs)
- (long long int)nparticles, // to check coherency between parameters and hdf input file
- fs->get_current_fname(), // particles input filename
- std::string("/tracers0/state/") + std::to_string(fs->iteration), // dataset name for initial input
- std::string("/tracers0/rhs/") + std::to_string(fs->iteration), // dataset name for initial input
- tracers0_neighbours, // parameter (interpolation no neighbours)
- tracers0_smoothness, // parameter
- MPI_COMM_WORLD,
- fs->iteration+1);
- particles_output_hdf5<long long int, double,3,3> particles_output_writer_mpi(
- MPI_COMM_WORLD,
- "tracers0",
- nparticles,
- tracers0_integration_steps);
- """
- self.particle_loop += """
- fs->compute_velocity(fs->cvorticity);
- fs->cvelocity->ift();
- ps->completeLoop(dt);
- """
- self.particle_output = """
- {
- particles_output_writer_mpi.open_file(fs->get_current_fname());
- particles_output_writer_mpi.save(ps->getParticlesPositions(),
- ps->getParticlesRhs(),
- ps->getParticlesIndexes(),
- ps->getLocalNbParticles(),
- fs->iteration);
- particles_output_writer_mpi.close_file();
- }
- """
- self.particle_end += 'ps.release();\n'
- return None
- def create_stat_output(
- self,
- dset_name,
- data_buffer,
- data_type = 'H5T_NATIVE_DOUBLE',
- size_setup = None,
- close_spaces = True):
- new_stat_output_txt = 'Cdset = H5Dopen(stat_file, "{0}", H5P_DEFAULT);\n'.format(dset_name)
- if not type(size_setup) == type(None):
- new_stat_output_txt += (
- size_setup +
- 'wspace = H5Dget_space(Cdset);\n' +
- 'ndims = H5Sget_simple_extent_dims(wspace, dims, NULL);\n' +
- 'mspace = H5Screate_simple(ndims, count, NULL);\n' +
- 'H5Sselect_hyperslab(wspace, H5S_SELECT_SET, offset, NULL, count, NULL);\n')
- new_stat_output_txt += ('H5Dwrite(Cdset, {0}, mspace, wspace, H5P_DEFAULT, {1});\n' +
- 'H5Dclose(Cdset);\n').format(data_type, data_buffer)
- if close_spaces:
- new_stat_output_txt += ('H5Sclose(mspace);\n' +
- 'H5Sclose(wspace);\n')
- return new_stat_output_txt
- def write_fluid_stats(self):
- self.fluid_includes += '#include <cmath>\n'
- self.fluid_includes += '#include "fftw_tools.hpp"\n'
- self.stat_src += """
- //begincpp
- hid_t stat_group;
- if (myrank == 0)
- stat_group = H5Gopen(stat_file, "statistics", H5P_DEFAULT);
- fs->compute_velocity(fs->cvorticity);
- *tmp_vec_field = fs->cvelocity->get_cdata();
- tmp_vec_field->compute_stats(
- fs->kk,
- stat_group,
- "velocity",
- fs->iteration / niter_stat,
- max_velocity_estimate/sqrt(3));
- //endcpp
- """
- self.stat_src += """
- //begincpp
- *tmp_vec_field = fs->cvorticity->get_cdata();
- tmp_vec_field->compute_stats(
- fs->kk,
- stat_group,
- "vorticity",
- fs->iteration / niter_stat,
- max_vorticity_estimate/sqrt(3));
- //endcpp
- """
- self.stat_src += """
- //begincpp
- if (myrank == 0)
- H5Gclose(stat_group);
- if (myrank == 0)
- {{
- hid_t Cdset, wspace, mspace;
- int ndims;
- hsize_t count[4], offset[4], dims[4];
- offset[0] = fs->iteration/niter_stat;
- offset[1] = 0;
- offset[2] = 0;
- offset[3] = 0;
- //endcpp
- """.format(self.C_dtype)
- if self.dtype == np.float32:
- field_H5T = 'H5T_NATIVE_FLOAT'
- elif self.dtype == np.float64:
- field_H5T = 'H5T_NATIVE_DOUBLE'
- self.stat_src += self.create_stat_output(
- '/statistics/xlines/velocity',
- 'fs->rvelocity->get_rdata()',
- data_type = field_H5T,
- size_setup = """
- count[0] = 1;
- count[1] = nx;
- count[2] = 3;
- """,
- close_spaces = False)
- self.stat_src += self.create_stat_output(
- '/statistics/xlines/vorticity',
- 'fs->rvorticity->get_rdata()',
- data_type = field_H5T)
- self.stat_src += '}\n'
- ## checkpoint
- self.stat_src += """
- //begincpp
- if (myrank == 0)
- {
- std::string fname = (
- std::string("stop_") +
- std::string(simname));
- {
- struct stat file_buffer;
- stop_code_now = (stat(fname.c_str(), &file_buffer) == 0);
- }
- }
- MPI_Bcast(&stop_code_now, 1, MPI_C_BOOL, 0, MPI_COMM_WORLD);
- //endcpp
- """
- return None
- def fill_up_fluid_code(self):
- self.fluid_includes += '#include <cstring>\n'
- self.fluid_variables += (
- 'vorticity_equation<{0}, FFTW> *fs;\n'.format(self.C_dtype) +
- 'field<{0}, FFTW, THREE> *tmp_vec_field;\n'.format(self.C_dtype) +
- 'field<{0}, FFTW, ONE> *tmp_scal_field;\n'.format(self.C_dtype))
- self.fluid_definitions += """
- typedef struct {{
- {0} re;
- {0} im;
- }} tmp_complex_type;
- """.format(self.C_dtype)
- self.write_fluid_stats()
- if self.dtype == np.float32:
- field_H5T = 'H5T_NATIVE_FLOAT'
- elif self.dtype == np.float64:
- field_H5T = 'H5T_NATIVE_DOUBLE'
- self.variables += 'int checkpoint;\n'
- self.variables += 'bool stop_code_now;\n'
- self.read_checkpoint = """
- //begincpp
- if (myrank == 0)
- {
- hid_t dset = H5Dopen(stat_file, "checkpoint", H5P_DEFAULT);
- H5Dread(
- dset,
- H5T_NATIVE_INT,
- H5S_ALL,
- H5S_ALL,
- H5P_DEFAULT,
- &checkpoint);
- H5Dclose(dset);
- }
- MPI_Bcast(&checkpoint, 1, MPI_INT, 0, MPI_COMM_WORLD);
- fs->checkpoint = checkpoint;
- //endcpp
- """
- self.store_checkpoint = """
- //begincpp
- checkpoint = fs->checkpoint;
- if (myrank == 0)
- {
- hid_t dset = H5Dopen(stat_file, "checkpoint", H5P_DEFAULT);
- H5Dwrite(
- dset,
- H5T_NATIVE_INT,
- H5S_ALL,
- H5S_ALL,
- H5P_DEFAULT,
- &checkpoint);
- H5Dclose(dset);
- }
- //endcpp
- """
- self.fluid_start += """
- //begincpp
- char fname[512];
- fs = new vorticity_equation<{0}, FFTW>(
- simname,
- nx, ny, nz,
- dkx, dky, dkz,
- {1});
- tmp_vec_field = new field<{0}, FFTW, THREE>(
- nx, ny, nz,
- MPI_COMM_WORLD,
- {1});
- tmp_scal_field = new field<{0}, FFTW, ONE>(
- nx, ny, nz,
- MPI_COMM_WORLD,
- {1});
- fs->checkpoints_per_file = checkpoints_per_file;
- fs->nu = nu;
- fs->fmode = fmode;
- fs->famplitude = famplitude;
- fs->fk0 = fk0;
- fs->fk1 = fk1;
- strncpy(fs->forcing_type, forcing_type, 128);
- fs->iteration = iteration;
- {2}
- fs->cvorticity->real_space_representation = false;
- fs->io_checkpoint();
- //endcpp
- """.format(
- self.C_dtype,
- self.fftw_plan_rigor,
- self.read_checkpoint)
- self.fluid_start += self.store_kspace
- self.fluid_start += 'stop_code_now = false;\n'
- self.fluid_loop = 'fs->step(dt);\n'
- self.fluid_loop += ('if (fs->iteration % niter_out == 0)\n{\n' +
- self.fluid_output +
- self.particle_output +
- self.store_checkpoint +
- '\n}\n' +
- 'if (stop_code_now){\n' +
- 'iteration = fs->iteration;\n' +
- 'break;\n}\n')
- self.fluid_end = ('if (fs->iteration % niter_out != 0)\n{\n' +
- self.fluid_output +
- self.particle_output +
- self.store_checkpoint +
- 'DEBUG_MSG("checkpoint value is %d\\n", checkpoint);\n' +
- '\n}\n' +
- 'delete fs;\n' +
- 'delete tmp_vec_field;\n' +
- 'delete tmp_scal_field;\n')
- return None
- def get_postprocess_file_name(self):
- return os.path.join(self.work_dir, self.simname + '_postprocess.h5')
- def get_postprocess_file(self):
- return h5py.File(self.get_postprocess_file_name(), 'r')
- def compute_statistics(self, iter0 = 0, iter1 = None):
- """Run basic postprocessing on raw data.
- The energy spectrum :math:`E(t, k)` and the enstrophy spectrum
- :math:`\\frac{1}{2}\omega^2(t, k)` are computed from the
-
- .. math::
-
- \sum_{k \\leq \\|\\mathbf{k}\\| \\leq k+dk}\\hat{u_i} \\hat{u_j}^*, \\hskip .5cm
- \sum_{k \\leq \\|\\mathbf{k}\\| \\leq k+dk}\\hat{\omega_i} \\hat{\\omega_j}^*
-
- tensors, and the enstrophy spectrum is also used to
- compute the dissipation :math:`\\varepsilon(t)`.
- These basic quantities are stored in a newly created HDF5 file,
- ``simname_postprocess.h5``.
- """
- if len(list(self.statistics.keys())) > 0:
- return None
- self.read_parameters()
- with self.get_data_file() as data_file:
- if 'moments' not in data_file['statistics'].keys():
- return None
- iter0 = min((data_file['statistics/moments/velocity'].shape[0] *
- self.parameters['niter_stat']-1),
- iter0)
- if type(iter1) == type(None):
- iter1 = data_file['iteration'].value
- else:
- iter1 = min(data_file['iteration'].value, iter1)
- ii0 = iter0 // self.parameters['niter_stat']
- ii1 = iter1 // self.parameters['niter_stat']
- self.statistics['kshell'] = data_file['kspace/kshell'].value
- self.statistics['kM'] = data_file['kspace/kM'].value
- self.statistics['dk'] = data_file['kspace/dk'].value
- computation_needed = True
- pp_file = h5py.File(self.get_postprocess_file_name(), 'a')
- if 'ii0' in pp_file.keys():
- computation_needed = not (ii0 == pp_file['ii0'].value and
- ii1 == pp_file['ii1'].value)
- if computation_needed:
- for k in pp_file.keys():
- del pp_file[k]
- if computation_needed:
- pp_file['iter0'] = iter0
- pp_file['iter1'] = iter1
- pp_file['ii0'] = ii0
- pp_file['ii1'] = ii1
- pp_file['t'] = (self.parameters['dt']*
- self.parameters['niter_stat']*
- (np.arange(ii0, ii1+1).astype(np.float)))
- pp_file['energy(t, k)'] = (
- data_file['statistics/spectra/velocity_velocity'][ii0:ii1+1, :, 0, 0] +
- data_file['statistics/spectra/velocity_velocity'][ii0:ii1+1, :, 1, 1] +
- data_file['statistics/spectra/velocity_velocity'][ii0:ii1+1, :, 2, 2])/2
- pp_file['enstrophy(t, k)'] = (
- data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 0, 0] +
- data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 1, 1] +
- data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 2, 2])/2
- pp_file['vel_max(t)'] = data_file['statistics/moments/velocity'] [ii0:ii1+1, 9, 3]
- pp_file['renergy(t)'] = data_file['statistics/moments/velocity'][ii0:ii1+1, 2, 3]/2
- for k in ['t',
- 'energy(t, k)',
- 'enstrophy(t, k)',
- 'vel_max(t)',
- 'renergy(t)']:
- if k in pp_file.keys():
- self.statistics[k] = pp_file[k].value
- self.compute_time_averages()
- return None
- def compute_time_averages(self):
- """Compute easy stats.
-
- Further computation of statistics based on the contents of
- ``simname_postprocess.h5``.
- Standard quantities are as follows
- (consistent with [Ishihara]_):
-
- .. math::
-
- U_{\\textrm{int}}(t) = \\sqrt{\\frac{2E(t)}{3}}, \\hskip .5cm
- L_{\\textrm{int}}(t) = \\frac{\pi}{2U_{int}^2(t)} \\int \\frac{dk}{k} E(t, k), \\hskip .5cm
- T_{\\textrm{int}}(t) =
- \\frac{L_{\\textrm{int}}(t)}{U_{\\textrm{int}}(t)}
-
- \\eta_K = \\left(\\frac{\\nu^3}{\\varepsilon}\\right)^{1/4}, \\hskip .5cm
- \\tau_K = \\left(\\frac{\\nu}{\\varepsilon}\\right)^{1/2}, \\hskip .5cm
- \\lambda = \\sqrt{\\frac{15 \\nu U_{\\textrm{int}}^2}{\\varepsilon}}
-
- Re = \\frac{U_{\\textrm{int}} L_{\\textrm{int}}}{\\nu}, \\hskip
- .5cm
- R_{\\lambda} = \\frac{U_{\\textrm{int}} \\lambda}{\\nu}
-
- .. [Ishihara] T. Ishihara et al,
- *Small-scale statistics in high-resolution direct numerical
- simulation of turbulence: Reynolds number dependence of
- one-point velocity gradient statistics*.
- J. Fluid Mech.,
- **592**, 335-366, 2007
- """
- for key in ['energy', 'enstrophy']:
- self.statistics[key + '(t)'] = (self.statistics['dk'] *
- np.sum(self.statistics[key + '(t, k)'], axis = 1))
- self.statistics['Uint(t)'] = np.sqrt(2*self.statistics['energy(t)'] / 3)
- self.statistics['Lint(t)'] = ((self.statistics['dk']*np.pi /
- (2*self.statistics['Uint(t)']**2)) *
- np.nansum(self.statistics['energy(t, k)'] /
- self.statistics['kshell'][None, :], axis = 1))
- for key in ['energy',
- 'enstrophy',
- 'vel_max',
- 'Uint',
- 'Lint']:
- if key + '(t)' in self.statistics.keys():
- self.statistics[key] = np.average(self.statistics[key + '(t)'], axis = 0)
- for suffix in ['', '(t)']:
- self.statistics['diss' + suffix] = (self.parameters['nu'] *
- self.statistics['enstrophy' + suffix]*2)
- self.statistics['etaK' + suffix] = (self.parameters['nu']**3 /
- self.statistics['diss' + suffix])**.25
- self.statistics['tauK' + suffix] = (self.parameters['nu'] /
- self.statistics['diss' + suffix])**.5
- self.statistics['Re' + suffix] = (self.statistics['Uint' + suffix] *
- self.statistics['Lint' + suffix] /
- self.parameters['nu'])
- self.statistics['lambda' + suffix] = (15 * self.parameters['nu'] *
- self.statistics['Uint' + suffix]**2 /
- self.statistics['diss' + suffix])**.5
- self.statistics['Rlambda' + suffix] = (self.statistics['Uint' + suffix] *
- self.statistics['lambda' + suffix] /
- self.parameters['nu'])
- self.statistics['kMeta' + suffix] = (self.statistics['kM'] *
- self.statistics['etaK' + suffix])
- if self.parameters['dealias_type'] == 1:
- self.statistics['kMeta' + suffix] *= 0.8
- self.statistics['Tint'] = self.statistics['Lint'] / self.statistics['Uint']
- self.statistics['Taylor_microscale'] = self.statistics['lambda']
- return None
- def set_plt_style(
- self,
- style = {'dashes' : (None, None)}):
- self.style.update(style)
- return None
- def convert_complex_from_binary(
- self,
- field_name = 'vorticity',
- iteration = 0,
- file_name = None):
- """read the Fourier representation of a vector field.
-
- Read the binary file containing iteration ``iteration`` of the
- field ``field_name``, and write it in a ``.h5`` file.
- """
- data = np.memmap(
- os.path.join(self.work_dir,
- self.simname + '_{0}_i{1:0>5x}'.format('c' + field_name, iteration)),
- dtype = self.ctype,
- mode = 'r',
- shape = (self.parameters['ny'],
- self.parameters['nz'],
- self.parameters['nx']//2+1,
- 3))
- if type(file_name) == type(None):
- file_name = self.simname + '_{0}_i{1:0>5x}.h5'.format('c' + field_name, iteration)
- file_name = os.path.join(self.work_dir, file_name)
- f = h5py.File(file_name, 'a')
- f[field_name + '/complex/{0}'.format(iteration)] = data
- f.close()
- return None
- def write_par(
- self,
- iter0 = 0,
- particle_ic = None):
- _fluid_particle_base.write_par(self, iter0 = iter0)
- with h5py.File(self.get_data_file_name(), 'r+') as ofile:
- kspace = self.get_kspace()
- nshells = kspace['nshell'].shape[0]
- vec_stat_datasets = ['velocity', 'vorticity']
- scal_stat_datasets = []
- for k in vec_stat_datasets:
- time_chunk = 2**20//(8*3*self.parameters['nx']) # FIXME: use proper size of self.dtype
- time_chunk = max(time_chunk, 1)
- ofile.create_dataset('statistics/xlines/' + k,
- (1, self.parameters['nx'], 3),
- chunks = (time_chunk, self.parameters['nx'], 3),
- maxshape = (None, self.parameters['nx'], 3),
- dtype = self.dtype)
- for k in vec_stat_datasets:
- time_chunk = 2**20//(8*3*3*nshells)
- time_chunk = max(time_chunk, 1)
- ofile.create_dataset('statistics/spectra/' + k + '_' + k,
- (1, nshells, 3, 3),
- chunks = (time_chunk, nshells, 3, 3),
- maxshape = (None, nshells, 3, 3),
- dtype = np.float64)
- time_chunk = 2**20//(8*4*10)
- time_chunk = max(time_chunk, 1)
- a = ofile.create_dataset('statistics/moments/' + k,
- (1, 10, 4),
- chunks = (time_chunk, 10, 4),
- maxshape = (None, 10, 4),
- dtype = np.float64)
- time_chunk = 2**20//(8*4*self.parameters['histogram_bins'])
- time_chunk = max(time_chunk, 1)
- ofile.create_dataset('statistics/histograms/' + k,
- (1,
- self.parameters['histogram_bins'],
- 4),
- chunks = (time_chunk,
- self.parameters['histogram_bins'],
- 4),
- maxshape = (None,
- self.parameters['histogram_bins'],
- 4),
- dtype = np.int64)
- ofile['checkpoint'] = int(0)
- if self.particle_species == 0:
- return None
-
- if type(particle_ic) == type(None):
- pbase_shape = (self.parameters['nparticles'],)
- number_of_particles = self.parameters['nparticles']
- else:
- pbase_shape = particle_ic.shape[:-1]
- assert(particle_ic.shape[-1] == 3)
- number_of_particles = 1
- for val in pbase_shape[1:]:
- number_of_particles *= val
- with h5py.File(self.get_checkpoint_0_fname(), 'a') as ofile:
- s = 0
- ofile.create_group('tracers{0}'.format(s))
- ofile.create_group('tracers{0}/rhs'.format(s))
- ofile.create_group('tracers{0}/state'.format(s))
- ofile['tracers{0}/rhs'.format(s)].create_dataset(
- '0',
- shape = (
- (self.parameters['tracers{0}_integration_steps'.format(s)],) +
- pbase_shape +
- (3,)),
- dtype = np.float)
- ofile['tracers{0}/state'.format(s)].create_dataset(
- '0',
- shape = (
- pbase_shape +
- (3,)),
- dtype = np.float)
- return None
- def specific_parser_arguments(
- self,
- parser):
- _fluid_particle_base.specific_parser_arguments(self, parser)
- parser.add_argument(
- '--src-wd',
- type = str,
- dest = 'src_work_dir',
- default = '')
- parser.add_argument(
- '--src-simname',
- type = str,
- dest = 'src_simname',
- default = '')
- parser.add_argument(
- '--src-iteration',
- type = int,
- dest = 'src_iteration',
- default = 0)
- parser.add_argument(
- '--njobs',
- type = int, dest = 'njobs',
- default = 1)
- parser.add_argument(
- '--kMeta',
- type = float,
- dest = 'kMeta',
- default = 2.0)
- parser.add_argument(
- '--dtfactor',
- type = float,
- dest = 'dtfactor',
- default = 0.5,
- help = 'dt is computed as DTFACTOR / N')
- parser.add_argument(
- '--particle-rand-seed',
- type = int,
- dest = 'particle_rand_seed',
- default = None)
- parser.add_argument(
- '--pclouds',
- type = int,
- dest = 'pclouds',
- default = 1,
- help = ('number of particle clouds. Particle "clouds" '
- 'consist of particles distributed according to '
- 'pcloud-type.'))
- parser.add_argument(
- '--pcloud-type',
- choices = ['random-cube',
- 'regular-cube'],
- dest = 'pcloud_type',
- default = 'random-cube')
- parser.add_argument(
- '--particle-cloud-size',
- type = float,
- dest = 'particle_cloud_size',
- default = 2*np.pi)
- parser.add_argument(
- '--neighbours',
- type = int,
- dest = 'neighbours',
- default = 1)
- parser.add_argument(
- '--smoothness',
- type = int,
- dest = 'smoothness',
- default = 1)
- return None
- def prepare_launch(
- self,
- args = []):
- """Set up reasonable parameters.
-
- With the default Lundgren forcing applied in the band [2, 4],
- we can estimate the dissipation, therefore we can estimate
- :math:`k_M \\eta_K` and constrain the viscosity.
-
- In brief, the command line parameter :math:`k_M \\eta_K` is
- used in the following formula for :math:`\\nu` (:math:`N` is the
- number of real space grid points per coordinate):
-
- .. math::
-
- \\nu = \\left(\\frac{2 k_M \\eta_K}{N} \\right)^{4/3}
-
- With this choice, the average dissipation :math:`\\varepsilon`
- will be close to 0.4, and the integral scale velocity will be
- close to 0.77, yielding the approximate value for the Taylor
- microscale and corresponding Reynolds number:
-
- .. math::
-
- \\lambda \\approx 4.75\\left(\\frac{2 k_M \\eta_K}{N} \\right)^{4/6}, \\hskip .5in
- R_\\lambda \\approx 3.7 \\left(\\frac{N}{2 k_M \\eta_K} \\right)^{4/6}
-
- """
- opt = _code.prepare_launch(self, args = args)
- self.parameters['nu'] = (opt.kMeta * 2 / opt.n)**(4./3)
- self.parameters['dt'] = (opt.dtfactor / opt.n)
- # custom famplitude for 288 and 576
- if opt.n == 288:
- self.parameters['famplitude'] = 0.45
- elif opt.n == 576:
- self.parameters['famplitude'] = 0.47
- if ((self.parameters['niter_todo'] % self.parameters['niter_out']) != 0):
- self.parameters['niter_out'] = self.parameters['niter_todo']
- if len(opt.src_work_dir) == 0:
- opt.src_work_dir = os.path.realpath(opt.work_dir)
- self.pars_from_namespace(opt)
- return opt
- def launch(
- self,
- args = [],
- **kwargs):
- opt = self.prepare_launch(args = args)
- if type(opt.nparticles) != type(None):
- if opt.nparticles > 0:
- self.name += '-particles'
- self.add_particles(
- integration_steps = 4,
- neighbours = opt.neighbours,
- smoothness = opt.smoothness)
- self.fill_up_fluid_code()
- self.finalize_code()
- self.launch_jobs(opt = opt, **kwargs)
- return None
- def get_checkpoint_0_fname(self):
- return os.path.join(
- self.work_dir,
- self.simname + '_checkpoint_0.h5')
- def generate_tracer_state(
- self,
- rseed = None,
- iteration = 0,
- species = 0,
- write_to_file = False,
- ncomponents = 3,
- testing = False,
- data = None):
- if (type(data) == type(None)):
- if not type(rseed) == type(None):
- np.random.seed(rseed)
- #point with problems: 5.37632864e+00, 6.10414710e+00, 6.25256493e+00]
- data = np.zeros(self.parameters['nparticles']*ncomponents).reshape(-1, ncomponents)
- data[:, :3] = np.random.random((self.parameters['nparticles'], 3))*2*np.pi
- if testing:
- #data[0] = np.array([3.26434, 4.24418, 3.12157])
- data[:] = np.array([ 0.72086101, 2.59043666, 6.27501953])
- with h5py.File(self.get_checkpoint_0_fname(), 'a') as data_file:
- data_file['tracers{0}/state/0'.format(species)][:] = data
- if write_to_file:
- data.tofile(
- os.path.join(
- self.work_dir,
- "tracers{0}_state_i{1:0>5x}".format(species, iteration)))
- return data
- def launch_jobs(
- self,
- opt = None,
- particle_initial_condition = None):
- if not os.path.exists(os.path.join(self.work_dir, self.simname + '.h5')):
- # take care of fields' initial condition
- if not os.path.exists(self.get_checkpoint_0_fname()):
- f = h5py.File(self.get_checkpoint_0_fname(), 'w')
- if len(opt.src_simname) > 0:
- source_cp = 0
- src_file = 'not_a_file'
- while True:
- src_file = os.path.join(
- os.path.realpath(opt.src_work_dir),
- opt.src_simname + '_checkpoint_{0}.h5'.format(source_cp))
- f0 = h5py.File(src_file, 'r')
- if '{0}'.format(opt.src_iteration) in f0['vorticity/complex'].keys():
- f0.close()
- break
- source_cp += 1
- f['vorticity/complex/{0}'.format(0)] = h5py.ExternalLink(
- src_file,
- 'vorticity/complex/{0}'.format(opt.src_iteration))
- else:
- data = self.generate_vector_field(
- write_to_file = False,
- spectra_slope = 2.0,
- amplitude = 0.05)
- f['vorticity/complex/{0}'.format(0)] = data
- f.close()
- # take care of particles' initial condition
- if opt.pclouds > 1:
- np.random.seed(opt.particle_rand_seed)
- if opt.pcloud_type == 'random-cube':
- particle_initial_condition = (
- np.random.random((opt.pclouds, 1, 3))*2*np.pi +
- np.random.random((1, self.parameters['nparticles'], 3))*opt.particle_cloud_size)
- elif opt.pcloud_type == 'regular-cube':
- onedarray = np.linspace(
- -opt.particle_cloud_size/2,
- opt.particle_cloud_size/2,
- self.parameters['nparticles'])
- particle_initial_condition = np.zeros(
- (opt.pclouds,
- self.parameters['nparticles'],
- self.parameters['nparticles'],
- self.parameters['nparticles'], 3),
- dtype = np.float64)
- particle_initial_condition[:] = \
- np.random.random((opt.pclouds, 1, 1, 1, 3))*2*np.pi
- particle_initial_condition[..., 0] += onedarray[None, None, None, :]
- particle_initial_condition[..., 1] += onedarray[None, None, :, None]
- particle_initial_condition[..., 2] += onedarray[None, :, None, None]
- self.write_par(
- particle_ic = particle_initial_condition)
- if self.parameters['nparticles'] > 0:
- data = self.generate_tracer_state(
- species = 0,
- rseed = opt.particle_rand_seed,
- data = particle_initial_condition)
- for s in range(1, self.particle_species):
- self.generate_tracer_state(species = s, data = data)
- self.run(
- nb_processes = opt.nb_processes,
- nb_threads_per_process = opt.nb_threads_per_process,
- njobs = opt.njobs,
- hours = opt.minutes // 60,
- minutes = opt.minutes % 60,
- no_submit = opt.no_submit)
- return None
-
-if __name__ == '__main__':
- pass
-
diff --git a/bfps/NavierStokes.py b/bfps/NavierStokes.py
deleted file mode 100644
index c30adbe2..00000000
--- a/bfps/NavierStokes.py
+++ /dev/null
@@ -1,1263 +0,0 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
-
-
-
-import sys
-import os
-import numpy as np
-import h5py
-import argparse
-
-import bfps
-import bfps.tools
-from ._code import _code
-from ._fluid_base import _fluid_particle_base
-
-class NavierStokes(_fluid_particle_base):
- """Objects of this class can be used to generate production DNS codes.
- Any functionality that users require should be available through this class,
- in the sense that they can implement whatever they need by simply inheriting
- this class.
- """
- def __init__(
- self,
- name = 'NavierStokes-v' + bfps.__version__,
- work_dir = './',
- simname = 'test',
- fluid_precision = 'single',
- fftw_plan_rigor = 'FFTW_MEASURE',
- frozen_fields = False,
- use_fftw_wisdom = True,
- QR_stats_on = False,
- Lag_acc_stats_on = False):
- self.QR_stats_on = QR_stats_on
- self.Lag_acc_stats_on = Lag_acc_stats_on
- self.frozen_fields = frozen_fields
- self.fftw_plan_rigor = fftw_plan_rigor
- _fluid_particle_base.__init__(
- self,
- name = name + '-' + fluid_precision,
- work_dir = work_dir,
- simname = simname,
- dtype = fluid_precision,
- use_fftw_wisdom = use_fftw_wisdom)
- self.parameters['nu'] = 0.1
- self.parameters['fmode'] = 1
- self.parameters['famplitude'] = 0.5
- self.parameters['fk0'] = 2.0
- self.parameters['fk1'] = 4.0
- self.parameters['forcing_type'] = 'linear'
- self.parameters['histogram_bins'] = 256
- self.parameters['max_velocity_estimate'] = 1.0
- self.parameters['max_vorticity_estimate'] = 1.0
- self.parameters['max_Lag_acc_estimate'] = 1.0
- self.parameters['max_pressure_estimate'] = 1.0
- self.parameters['QR2D_histogram_bins'] = 64
- self.parameters['max_trS2_estimate'] = 1.0
- self.parameters['max_Q_estimate'] = 1.0
- self.parameters['max_R_estimate'] = 1.0
- self.file_datasets_grow = """
- //begincpp
- hid_t group;
- group = H5Gopen(stat_file, "/statistics", H5P_DEFAULT);
- H5Ovisit(group, H5_INDEX_NAME, H5_ITER_NATIVE, grow_statistics_dataset, NULL);
- H5Gclose(group);
- //endcpp
- """
- self.style = {}
- self.statistics = {}
- self.fluid_output = 'fs->write(\'v\', \'c\');\n'
- return None
- def create_stat_output(
- self,
- dset_name,
- data_buffer,
- data_type = 'H5T_NATIVE_DOUBLE',
- size_setup = None,
- close_spaces = True):
- new_stat_output_txt = 'Cdset = H5Dopen(stat_file, "{0}", H5P_DEFAULT);\n'.format(dset_name)
- if not type(size_setup) == type(None):
- new_stat_output_txt += (
- size_setup +
- 'wspace = H5Dget_space(Cdset);\n' +
- 'ndims = H5Sget_simple_extent_dims(wspace, dims, NULL);\n' +
- 'mspace = H5Screate_simple(ndims, count, NULL);\n' +
- 'H5Sselect_hyperslab(wspace, H5S_SELECT_SET, offset, NULL, count, NULL);\n')
- new_stat_output_txt += ('H5Dwrite(Cdset, {0}, mspace, wspace, H5P_DEFAULT, {1});\n' +
- 'H5Dclose(Cdset);\n').format(data_type, data_buffer)
- if close_spaces:
- new_stat_output_txt += ('H5Sclose(mspace);\n' +
- 'H5Sclose(wspace);\n')
- return new_stat_output_txt
- def write_fluid_stats(self):
- self.fluid_includes += '#include <cmath>\n'
- self.fluid_includes += '#include "fftw_tools.hpp"\n'
- self.stat_src += """
- //begincpp
- hid_t stat_group;
- if (myrank == 0)
- stat_group = H5Gopen(stat_file, "statistics", H5P_DEFAULT);
- fs->compute_velocity(fs->cvorticity);
- std::vector<double> max_estimate_vector;
- max_estimate_vector.resize(4);
- *tmp_vec_field = fs->cvelocity;
- switch(fs->dealias_type)
- {
- case 0:
- tmp_vec_field->compute_stats(
- kk_two_thirds,
- stat_group,
- "velocity",
- fs->iteration / niter_stat,
- max_velocity_estimate/sqrt(3));
- break;
- case 1:
- tmp_vec_field->compute_stats(
- kk_smooth,
- stat_group,
- "velocity",
- fs->iteration / niter_stat,
- max_velocity_estimate/sqrt(3));
- break;
- }
- //endcpp
- """
- if self.Lag_acc_stats_on:
- self.stat_src += """
- //begincpp
- tmp_vec_field->real_space_representation = false;
- fs->compute_Lagrangian_acceleration(tmp_vec_field->get_cdata());
- switch(fs->dealias_type)
- {
- case 0:
- tmp_vec_field->compute_stats(
- kk_two_thirds,
- stat_group,
- "Lagrangian_acceleration",
- fs->iteration / niter_stat,
- max_Lag_acc_estimate);
- break;
- case 1:
- tmp_vec_field->compute_stats(
- kk_smooth,
- stat_group,
- "Lagrangian_acceleration",
- fs->iteration / niter_stat,
- max_Lag_acc_estimate);
- break;
- }
- tmp_scal_field->real_space_representation = false;
- fs->compute_velocity(fs->cvorticity);
- fs->ift_velocity();
- fs->compute_pressure(tmp_scal_field->get_cdata());
- switch(fs->dealias_type)
- {
- case 0:
- tmp_scal_field->compute_stats(
- kk_two_thirds,
- stat_group,
- "pressure",
- fs->iteration / niter_stat,
- max_pressure_estimate);
- break;
- case 1:
- tmp_scal_field->compute_stats(
- kk_smooth,
- stat_group,
- "pressure",
- fs->iteration / niter_stat,
- max_pressure_estimate);
- break;
- }
- //endcpp
- """
- self.stat_src += """
- //begincpp
- *tmp_vec_field = fs->cvorticity;
- switch(fs->dealias_type)
- {
- case 0:
- tmp_vec_field->compute_stats(
- kk_two_thirds,
- stat_group,
- "vorticity",
- fs->iteration / niter_stat,
- max_vorticity_estimate/sqrt(3));
- break;
- case 1:
- tmp_vec_field->compute_stats(
- kk_smooth,
- stat_group,
- "vorticity",
- fs->iteration / niter_stat,
- max_vorticity_estimate/sqrt(3));
- break;
- }
- //endcpp
- """
- if self.QR_stats_on:
- self.stat_src += """
- //begincpp
- double *trS2_Q_R_moments = new double[10*3];
- double *gradu_moments = new double[10*9];
- ptrdiff_t *hist_trS2_Q_R = new ptrdiff_t[histogram_bins*3];
- ptrdiff_t *hist_gradu = new ptrdiff_t[histogram_bins*9];
- ptrdiff_t *hist_QR2D = new ptrdiff_t[QR2D_histogram_bins*QR2D_histogram_bins];
- double trS2QR_max_estimates[3];
- double gradu_max_estimates[9];
- trS2QR_max_estimates[0] = max_trS2_estimate;
- trS2QR_max_estimates[1] = max_Q_estimate;
- trS2QR_max_estimates[2] = max_R_estimate;
- std::fill_n(gradu_max_estimates, 9, sqrt(3*max_trS2_estimate));
- fs->compute_gradient_statistics(
- fs->cvelocity,
- gradu_moments,
- trS2_Q_R_moments,
- hist_gradu,
- hist_trS2_Q_R,
- hist_QR2D,
- trS2QR_max_estimates,
- gradu_max_estimates,
- histogram_bins,
- QR2D_histogram_bins);
- //endcpp
- """
- self.stat_src += """
- //begincpp
- if (myrank == 0)
- H5Gclose(stat_group);
- if (fs->cd->myrank == 0)
- {{
- hid_t Cdset, wspace, mspace;
- int ndims;
- hsize_t count[4], offset[4], dims[4];
- offset[0] = fs->iteration/niter_stat;
- offset[1] = 0;
- offset[2] = 0;
- offset[3] = 0;
- //endcpp
- """.format(self.C_dtype)
- if self.dtype == np.float32:
- field_H5T = 'H5T_NATIVE_FLOAT'
- elif self.dtype == np.float64:
- field_H5T = 'H5T_NATIVE_DOUBLE'
- if self.QR_stats_on:
- self.stat_src += self.create_stat_output(
- '/statistics/moments/trS2_Q_R',
- 'trS2_Q_R_moments',
- size_setup ="""
- count[0] = 1;
- count[1] = 10;
- count[2] = 3;
- """)
- self.stat_src += self.create_stat_output(
- '/statistics/moments/velocity_gradient',
- 'gradu_moments',
- size_setup ="""
- count[0] = 1;
- count[1] = 10;
- count[2] = 3;
- count[3] = 3;
- """)
- self.stat_src += self.create_stat_output(
- '/statistics/histograms/trS2_Q_R',
- 'hist_trS2_Q_R',
- data_type = 'H5T_NATIVE_INT64',
- size_setup = """
- count[0] = 1;
- count[1] = histogram_bins;
- count[2] = 3;
- """)
- self.stat_src += self.create_stat_output(
- '/statistics/histograms/velocity_gradient',
- 'hist_gradu',
- data_type = 'H5T_NATIVE_INT64',
- size_setup = """
- count[0] = 1;
- count[1] = histogram_bins;
- count[2] = 3;
- count[3] = 3;
- """)
- self.stat_src += self.create_stat_output(
- '/statistics/histograms/QR2D',
- 'hist_QR2D',
- data_type = 'H5T_NATIVE_INT64',
- size_setup = """
- count[0] = 1;
- count[1] = QR2D_histogram_bins;
- count[2] = QR2D_histogram_bins;
- """)
- self.stat_src += '}\n'
- if self.QR_stats_on:
- self.stat_src += """
- //begincpp
- delete[] trS2_Q_R_moments;
- delete[] gradu_moments;
- delete[] hist_trS2_Q_R;
- delete[] hist_gradu;
- delete[] hist_QR2D;
- //endcpp
- """
- return None
- def fill_up_fluid_code(self):
- self.fluid_includes += '#include <cstring>\n'
- self.fluid_variables += (
- 'fluid_solver<{0}> *fs;\n'.format(self.C_dtype) +
- 'field<{0}, FFTW, THREE> *tmp_vec_field;\n'.format(self.C_dtype) +
- 'field<{0}, FFTW, ONE> *tmp_scal_field;\n'.format(self.C_dtype) +
- 'kspace<FFTW, SMOOTH> *kk_smooth;\n' +
- 'kspace<FFTW, TWO_THIRDS> *kk_two_thirds;\n')
- self.fluid_definitions += """
- typedef struct {{
- {0} re;
- {0} im;
- }} tmp_complex_type;
- """.format(self.C_dtype)
- self.write_fluid_stats()
- if self.dtype == np.float32:
- field_H5T = 'H5T_NATIVE_FLOAT'
- elif self.dtype == np.float64:
- field_H5T = 'H5T_NATIVE_DOUBLE'
- self.fluid_start += """
- //begincpp
- char fname[512];
- fs = new fluid_solver<{0}>(
- simname,
- nx, ny, nz,
- dkx, dky, dkz,
- dealias_type,
- {1});
- tmp_vec_field = new field<{0}, FFTW, THREE>(
- nx, ny, nz,
- MPI_COMM_WORLD,
- {1});
- tmp_scal_field = new field<{0}, FFTW, ONE>(
- nx, ny, nz,
- MPI_COMM_WORLD,
- {1});
- kk_smooth = new kspace<FFTW, SMOOTH>(
- tmp_vec_field->clayout,
- fs->dkx, fs->dky, fs->dkz);
- kk_two_thirds = new kspace<FFTW, TWO_THIRDS>(
- tmp_vec_field->clayout,
- fs->dkx, fs->dky, fs->dkz);
- fs->nu = nu;
- fs->fmode = fmode;
- fs->famplitude = famplitude;
- fs->fk0 = fk0;
- fs->fk1 = fk1;
- strncpy(fs->forcing_type, forcing_type, 128);
- fs->iteration = iteration;
- fs->read('v', 'c');
- //endcpp
- """.format(self.C_dtype, self.fftw_plan_rigor, field_H5T)
- self.fluid_start += self.store_kspace
- if not self.frozen_fields:
- self.fluid_loop = 'fs->step(dt);\n'
- else:
- self.fluid_loop = ''
- self.fluid_loop += ('if (fs->iteration % niter_out == 0)\n{\n' +
- self.fluid_output + '\n}\n')
- self.fluid_end = ('if (fs->iteration % niter_out != 0)\n{\n' +
- self.fluid_output + '\n}\n' +
- 'delete fs;\n' +
- 'delete tmp_vec_field;\n' +
- 'delete tmp_scal_field;\n' +
- 'delete kk_smooth;\n' +
- 'delete kk_two_thirds;\n')
- return None
- def add_3D_rFFTW_field(
- self,
- name = 'rFFTW_acc'):
- if self.dtype == np.float32:
- FFTW = 'fftwf'
- elif self.dtype == np.float64:
- FFTW = 'fftw'
- self.fluid_variables += '{0} *{1};\n'.format(self.C_dtype, name)
- self.fluid_start += '{0} = {1}_alloc_real(2*fs->cd->local_size);\n'.format(name, FFTW)
- self.fluid_end += '{0}_free({1});\n'.format(FFTW, name)
- return None
- def add_interpolator(
- self,
- interp_type = 'spline',
- neighbours = 1,
- smoothness = 1,
- name = 'field_interpolator',
- field_name = 'fs->rvelocity',
- class_name = 'rFFTW_interpolator'):
- self.fluid_includes += '#include "{0}.hpp"\n'.format(class_name)
- self.fluid_variables += '{0} <{1}, {2}> *{3};\n'.format(
- class_name, self.C_dtype, neighbours, name)
- self.parameters[name + '_type'] = interp_type
- self.parameters[name + '_neighbours'] = neighbours
- if interp_type == 'spline':
- self.parameters[name + '_smoothness'] = smoothness
- beta_name = 'beta_n{0}_m{1}'.format(neighbours, smoothness)
- elif interp_type == 'Lagrange':
- beta_name = 'beta_Lagrange_n{0}'.format(neighbours)
- self.fluid_start += '{0} = new {1}<{2}, {3}>(fs, {4}, {5});\n'.format(
- name,
- class_name,
- self.C_dtype,
- neighbours,
- beta_name,
- field_name)
- self.fluid_end += 'delete {0};\n'.format(name)
- return None
- def add_particles(
- self,
- integration_steps = 2,
- kcut = None,
- interpolator = 'field_interpolator',
- frozen_particles = False,
- acc_name = None,
- class_name = 'particles'):
- """Adds code for tracking a series of particle species, each
- consisting of `nparticles` particles.
-
- :type integration_steps: int, list of int
- :type kcut: None (default), str, list of str
- :type interpolator: str, list of str
- :type frozen_particles: bool
- :type acc_name: str
-
- .. warning:: if not None, kcut must be a list of decreasing
- wavenumbers, since filtering is done sequentially
- on the same complex FFTW field.
- """
- if self.dtype == np.float32:
- FFTW = 'fftwf'
- elif self.dtype == np.float64:
- FFTW = 'fftw'
- s0 = self.particle_species
- if type(integration_steps) == int:
- integration_steps = [integration_steps]
- if type(kcut) == str:
- kcut = [kcut]
- if type(interpolator) == str:
- interpolator = [interpolator]
- nspecies = max(len(integration_steps), len(interpolator))
- if type(kcut) == list:
- nspecies = max(nspecies, len(kcut))
- if len(integration_steps) == 1:
- integration_steps = [integration_steps[0] for s in range(nspecies)]
- if len(interpolator) == 1:
- interpolator = [interpolator[0] for s in range(nspecies)]
- if type(kcut) == list:
- if len(kcut) == 1:
- kcut = [kcut[0] for s in range(nspecies)]
- assert(len(integration_steps) == nspecies)
- assert(len(interpolator) == nspecies)
- if type(kcut) == list:
- assert(len(kcut) == nspecies)
- for s in range(nspecies):
- neighbours = self.parameters[interpolator[s] + '_neighbours']
- if type(kcut) == list:
- self.parameters['tracers{0}_kcut'.format(s0 + s)] = kcut[s]
- self.parameters['tracers{0}_interpolator'.format(s0 + s)] = interpolator[s]
- self.parameters['tracers{0}_acc_on'.format(s0 + s)] = int(not type(acc_name) == type(None))
- self.parameters['tracers{0}_integration_steps'.format(s0 + s)] = integration_steps[s]
- self.file_datasets_grow += """
- //begincpp
- group = H5Gopen(particle_file, "/tracers{0}", H5P_DEFAULT);
- grow_particle_datasets(group, "", NULL, NULL);
- H5Gclose(group);
- //endcpp
- """.format(s0 + s)
-
- #### code that outputs statistics
- output_vel_acc = '{\n'
- # array for putting sampled velocity in
- # must compute velocity, just in case it was messed up by some
- # other particle species before the stats
- output_vel_acc += 'fs->compute_velocity(fs->cvorticity);\n'
- if not type(kcut) == list:
- output_vel_acc += 'fs->ift_velocity();\n'
- if not type(acc_name) == type(None):
- # array for putting sampled acceleration in
- # must compute acceleration
- output_vel_acc += 'fs->compute_Lagrangian_acceleration({0});\n'.format(acc_name)
- for s in range(nspecies):
- if type(kcut) == list:
- output_vel_acc += 'fs->low_pass_Fourier(fs->cvelocity, 3, {0});\n'.format(kcut[s])
- output_vel_acc += 'fs->ift_velocity();\n'
- output_vel_acc += """
- {0}->read_rFFTW(fs->rvelocity);
- ps{1}->sample({0}, "velocity");
- """.format(interpolator[s], s0 + s)
- if not type(acc_name) == type(None):
- output_vel_acc += """
- {0}->read_rFFTW({1});
- ps{2}->sample({0}, "acceleration");
- """.format(interpolator[s], acc_name, s0 + s)
- output_vel_acc += '}\n'
-
- #### initialize, stepping and finalize code
- if not type(kcut) == list:
- update_fields = ('fs->compute_velocity(fs->cvorticity);\n' +
- 'fs->ift_velocity();\n')
- self.particle_start += update_fields
- self.particle_loop += update_fields
- else:
- self.particle_loop += 'fs->compute_velocity(fs->cvorticity);\n'
- self.particle_includes += '#include "{0}.hpp"\n'.format(class_name)
- self.particle_stat_src += (
- 'if (ps0->iteration % niter_part == 0)\n' +
- '{\n')
- for s in range(nspecies):
- neighbours = self.parameters[interpolator[s] + '_neighbours']
- self.particle_start += 'sprintf(fname, "tracers{0}");\n'.format(s0 + s)
- self.particle_end += ('ps{0}->write();\n' +
- 'delete ps{0};\n').format(s0 + s)
- self.particle_variables += '{0}<VELOCITY_TRACER, {1}, {2}> *ps{3};\n'.format(
- class_name,
- self.C_dtype,
- neighbours,
- s0 + s)
- self.particle_start += ('ps{0} = new {1}<VELOCITY_TRACER, {2}, {3}>(\n' +
- 'fname, particle_file, {4},\n' +
- 'niter_part, tracers{0}_integration_steps);\n').format(
- s0 + s,
- class_name,
- self.C_dtype,
- neighbours,
- interpolator[s])
- self.particle_start += ('ps{0}->dt = dt;\n' +
- 'ps{0}->iteration = iteration;\n' +
- 'ps{0}->read();\n').format(s0 + s)
- if not frozen_particles:
- if type(kcut) == list:
- update_field = ('fs->low_pass_Fourier(fs->cvelocity, 3, {0});\n'.format(kcut[s]) +
- 'fs->ift_velocity();\n')
- self.particle_loop += update_field
- self.particle_loop += '{0}->read_rFFTW(fs->rvelocity);\n'.format(interpolator[s])
- self.particle_loop += 'ps{0}->step();\n'.format(s0 + s)
- self.particle_stat_src += 'ps{0}->write(false);\n'.format(s0 + s)
- self.particle_stat_src += output_vel_acc
- self.particle_stat_src += '}\n'
- self.particle_species += nspecies
- return None
- def get_cache_file_name(self):
- return os.path.join(self.work_dir, self.simname + '_cache.h5')
- def get_cache_file(self):
- return h5py.File(self.get_postprocess_file_name(), 'r')
- def get_postprocess_file_name(self):
- return self.get_cache_file_name()
- def get_postprocess_file(self):
- return h5py.File(self.get_postprocess_file_name(), 'r')
- def compute_statistics(self, iter0 = 0, iter1 = None):
- """Run basic postprocessing on raw data.
- The energy spectrum :math:`E(t, k)` and the enstrophy spectrum
- :math:`\\frac{1}{2}\omega^2(t, k)` are computed from the
-
- .. math::
-
- \sum_{k \\leq \\|\\mathbf{k}\\| \\leq k+dk}\\hat{u_i} \\hat{u_j}^*, \\hskip .5cm
- \sum_{k \\leq \\|\\mathbf{k}\\| \\leq k+dk}\\hat{\omega_i} \\hat{\\omega_j}^*
-
- tensors, and the enstrophy spectrum is also used to
- compute the dissipation :math:`\\varepsilon(t)`.
- These basic quantities are stored in a newly created HDF5 file,
- ``simname_cache.h5``.
- """
- if len(list(self.statistics.keys())) > 0:
- return None
- if not os.path.exists(self.get_data_file_name()):
- if os.path.exists(self.get_cache_file_name()):
- self.read_parameters(fname = self.get_cache_file_name())
- with self.get_cache_file() as pp_file:
- for k in ['t',
- 'energy(t)',
- 'energy(k)',
- 'enstrophy(t)',
- 'enstrophy(k)',
- 'R_ij(t)',
- 'vel_max(t)',
- 'renergy(t)']:
- if k in pp_file.keys():
- self.statistics[k] = pp_file[k].value
- self.statistics['kM'] = pp_file['kspace/kM'].value
- self.statistics['dk'] = pp_file['kspace/dk'].value
- self.statistics['kshell'] = pp_file['kspace/kshell'].value
- self.statistics['nshell'] = pp_file['kspace/nshell'].value
- else:
- self.read_parameters()
- with self.get_data_file() as data_file:
- if 'moments' not in data_file['statistics'].keys():
- return None
- iter0 = min((data_file['statistics/moments/velocity'].shape[0] *
- self.parameters['niter_stat']-1),
- iter0)
- if type(iter1) == type(None):
- iter1 = data_file['iteration'].value
- else:
- iter1 = min(data_file['iteration'].value, iter1)
- ii0 = iter0 // self.parameters['niter_stat']
- ii1 = iter1 // self.parameters['niter_stat']
- self.statistics['kshell'] = data_file['kspace/kshell'].value
- self.statistics['nshell'] = data_file['kspace/nshell'].value
- for kk in [-1, -2]:
- if (self.statistics['kshell'][kk] == 0):
- self.statistics['kshell'][kk] = np.nan
- self.statistics['kM'] = data_file['kspace/kM'].value
- self.statistics['dk'] = data_file['kspace/dk'].value
- computation_needed = True
- pp_file = h5py.File(self.get_postprocess_file_name(), 'a')
- if not ('parameters' in pp_file.keys()):
- data_file.copy('parameters', pp_file)
- data_file.copy('kspace', pp_file)
- if 'ii0' in pp_file.keys():
- computation_needed = not (ii0 == pp_file['ii0'].value and
- ii1 == pp_file['ii1'].value)
- if computation_needed:
- for k in ['t', 'vel_max(t)', 'renergy(t)',
- 'energy(t)', 'enstrophy(t)',
- 'energy(k)', 'enstrophy(k)',
- 'energy(t, k)',
- 'enstrophy(t, k)',
- 'R_ij(t)',
- 'ii0', 'ii1', 'iter0', 'iter1']:
- if k in pp_file.keys():
- del pp_file[k]
- if computation_needed:
- pp_file['iter0'] = iter0
- pp_file['iter1'] = iter1
- pp_file['ii0'] = ii0
- pp_file['ii1'] = ii1
- pp_file['t'] = (self.parameters['dt']*
- self.parameters['niter_stat']*
- (np.arange(ii0, ii1+1).astype(np.float)))
- phi_ij = data_file['statistics/spectra/velocity_velocity'][ii0:ii1+1]
- pp_file['R_ij(t)'] = np.sum(phi_ij, axis = 1)
- energy_tk = (
- phi_ij[:, :, 0, 0] +
- phi_ij[:, :, 1, 1] +
- phi_ij[:, :, 2, 2])/2
- pp_file['energy(t)'] = np.sum(energy_tk, axis = 1)
- pp_file['energy(k)'] = np.mean(energy_tk, axis = 0)*(4*np.pi*self.statistics['kshell']**2) / (self.statistics['dk']*self.statistics['nshell'])
- enstrophy_tk = (
- data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 0, 0] +
- data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 1, 1] +
- data_file['statistics/spectra/vorticity_vorticity'][ii0:ii1+1, :, 2, 2])/2
- pp_file['enstrophy(t)'] = np.sum(enstrophy_tk, axis = 1)
- pp_file['enstrophy(k)'] = np.mean(enstrophy_tk, axis = 0)*(4*np.pi*self.statistics['kshell']**2) / (self.statistics['dk']*self.statistics['nshell'])
- pp_file['vel_max(t)'] = data_file['statistics/moments/velocity'][ii0:ii1+1, 9, 3]
- pp_file['renergy(t)'] = data_file['statistics/moments/velocity'][ii0:ii1+1, 2, 3]/2
- if 'trS2_Q_R' in data_file['statistics/moments'].keys():
- pp_file['mean_trS2(t)'] = data_file['statistics/moments/trS2_Q_R'][:, 1, 0]
- for k in ['t',
- 'energy(t)',
- 'energy(k)',
- 'enstrophy(t)',
- 'enstrophy(k)',
- 'R_ij(t)',
- 'vel_max(t)',
- 'renergy(t)',
- 'mean_trS2(t)']:
- if k in pp_file.keys():
- self.statistics[k] = pp_file[k].value
- # sanity check --- Parseval theorem check
- assert(np.max(np.abs(
- self.statistics['renergy(t)'] -
- self.statistics['energy(t)']) / self.statistics['energy(t)']) < 1e-5)
- self.compute_time_averages()
- return None
- def compute_Reynolds_stress_invariants(
- self):
- Rij = self.statistics['R_ij(t)']
- Rij /= (2*self.statistics['energy(t)'][:, None, None])
- Rij[:, 0, 0] -= 1./3
- Rij[:, 1, 1] -= 1./3
- Rij[:, 2, 2] -= 1./3
- self.statistics['I2(t)'] = np.sqrt(np.einsum('...ij,...ij', Rij, Rij, optimize = True) / 6)
- self.statistics['I3(t)'] = np.cbrt(np.einsum('...ij,...jk,...ki', Rij, Rij, Rij, optimize = True) / 6)
- return None
- def compute_time_averages(self):
- """Compute easy stats.
-
- Further computation of statistics based on the contents of
- ``simname_cache.h5``.
- Standard quantities are as follows
- (consistent with [Ishihara]_):
-
- .. math::
-
- U_{\\textrm{int}}(t) = \\sqrt{\\frac{2E(t)}{3}}, \\hskip .5cm
- L_{\\textrm{int}} = \\frac{\pi}{2U_{int}^2} \\int \\frac{dk}{k} E(k), \\hskip .5cm
- T_{\\textrm{int}} =
- \\frac{L_{\\textrm{int}}}{U_{\\textrm{int}}}
-
- \\eta_K = \\left(\\frac{\\nu^3}{\\varepsilon}\\right)^{1/4}, \\hskip .5cm
- \\tau_K = \\left(\\frac{\\nu}{\\varepsilon}\\right)^{1/2}, \\hskip .5cm
- \\lambda = \\sqrt{\\frac{15 \\nu U_{\\textrm{int}}^2}{\\varepsilon}}
-
- Re = \\frac{U_{\\textrm{int}} L_{\\textrm{int}}}{\\nu}, \\hskip
- .5cm
- R_{\\lambda} = \\frac{U_{\\textrm{int}} \\lambda}{\\nu}
-
- .. [Ishihara] T. Ishihara et al,
- *Small-scale statistics in high-resolution direct numerical
- simulation of turbulence: Reynolds number dependence of
- one-point velocity gradient statistics*.
- J. Fluid Mech.,
- **592**, 335-366, 2007
- """
- self.statistics['Uint(t)'] = np.sqrt(2*self.statistics['energy(t)'] / 3)
- for key in ['energy',
- 'enstrophy',
- 'mean_trS2',
- 'Uint']:
- if key + '(t)' in self.statistics.keys():
- self.statistics[key] = np.average(self.statistics[key + '(t)'], axis = 0)
- self.statistics['vel_max'] = np.max(self.statistics['vel_max(t)'])
- for suffix in ['', '(t)']:
- self.statistics['diss' + suffix] = (self.parameters['nu'] *
- self.statistics['enstrophy' + suffix]*2)
- self.statistics['etaK' + suffix] = (self.parameters['nu']**3 /
- self.statistics['diss' + suffix])**.25
- self.statistics['tauK' + suffix] = (self.parameters['nu'] /
- self.statistics['diss' + suffix])**.5
- self.statistics['lambda' + suffix] = (15 * self.parameters['nu'] *
- self.statistics['Uint' + suffix]**2 /
- self.statistics['diss' + suffix])**.5
- self.statistics['Rlambda' + suffix] = (self.statistics['Uint' + suffix] *
- self.statistics['lambda' + suffix] /
- self.parameters['nu'])
- self.statistics['kMeta' + suffix] = (self.statistics['kM'] *
- self.statistics['etaK' + suffix])
- if self.parameters['dealias_type'] == 1:
- self.statistics['kMeta' + suffix] *= 0.8
- self.statistics['Lint'] = ((np.pi /
- (2*self.statistics['Uint']**2)) *
- np.nansum(self.statistics['energy(k)'] /
- self.statistics['kshell']))
- self.statistics['Re'] = (self.statistics['Uint'] *
- self.statistics['Lint'] /
- self.parameters['nu'])
- self.statistics['Tint'] = self.statistics['Lint'] / self.statistics['Uint']
- self.statistics['Taylor_microscale'] = self.statistics['lambda']
- return None
- def set_plt_style(
- self,
- style = {'dashes' : (None, None)}):
- self.style.update(style)
- return None
- def read_cfield(
- self,
- field_name = 'vorticity',
- iteration = 0):
- """read the Fourier representation of a vector field.
-
- Read the binary file containing iteration ``iteration`` of the
- field ``field_name``, and return it as a properly shaped
- ``numpy.memmap`` object.
- """
- return np.memmap(
- os.path.join(self.work_dir,
- self.simname + '_{0}_i{1:0>5x}'.format('c' + field_name, iteration)),
- dtype = self.ctype,
- mode = 'r',
- shape = (self.parameters['ny'],
- self.parameters['nz'],
- self.parameters['nx']//2+1,
- 3))
- def write_par(
- self,
- iter0 = 0,
- particle_ic = None):
- _fluid_particle_base.write_par(self, iter0 = iter0)
- with h5py.File(self.get_data_file_name(), 'r+') as ofile:
- kspace = self.get_kspace()
- nshells = kspace['nshell'].shape[0]
- vec_stat_datasets = ['velocity', 'vorticity']
- scal_stat_datasets = []
- for k in vec_stat_datasets:
- time_chunk = 2**20 // (
- self.dtype.itemsize*3*
- self.parameters['nx']*self.parameters['ny'])
- time_chunk = max(time_chunk, 1)
- ofile.create_dataset('statistics/0slices/' + k + '/real',
- (1, self.parameters['ny'], self.parameters['nx'], 3),
- chunks = (time_chunk, self.parameters['ny'], self.parameters['nx'], 3),
- maxshape = (None, self.parameters['ny'], self.parameters['nx'], 3),
- dtype = self.dtype)
- if self.Lag_acc_stats_on:
- vec_stat_datasets += ['Lagrangian_acceleration']
- scal_stat_datasets += ['pressure']
- for k in vec_stat_datasets:
- time_chunk = 2**20//(8*3*3*nshells)
- time_chunk = max(time_chunk, 1)
- ofile.create_dataset('statistics/spectra/' + k + '_' + k,
- (1, nshells, 3, 3),
- chunks = (time_chunk, nshells, 3, 3),
- maxshape = (None, nshells, 3, 3),
- dtype = np.float64)
- time_chunk = 2**20//(8*4*10)
- time_chunk = max(time_chunk, 1)
- a = ofile.create_dataset('statistics/moments/' + k,
- (1, 10, 4),
- chunks = (time_chunk, 10, 4),
- maxshape = (None, 10, 4),
- dtype = np.float64)
- time_chunk = 2**20//(8*4*self.parameters['histogram_bins'])
- time_chunk = max(time_chunk, 1)
- ofile.create_dataset('statistics/histograms/' + k,
- (1,
- self.parameters['histogram_bins'],
- 4),
- chunks = (time_chunk,
- self.parameters['histogram_bins'],
- 4),
- maxshape = (None,
- self.parameters['histogram_bins'],
- 4),
- dtype = np.int64)
- for k in scal_stat_datasets:
- time_chunk = 2**20//(8*nshells)
- time_chunk = max(time_chunk, 1)
- ofile.create_dataset('statistics/spectra/' + k + '_' + k,
- (1, nshells),
- chunks = (time_chunk, nshells),
- maxshape = (None, nshells),
- dtype = np.float64)
- time_chunk = 2**20//(8*10)
- time_chunk = max(time_chunk, 1)
- a = ofile.create_dataset('statistics/moments/' + k,
- (1, 10),
- chunks = (time_chunk, 10),
- maxshape = (None, 10),
- dtype = np.float64)
- time_chunk = 2**20//(8*self.parameters['histogram_bins'])
- time_chunk = max(time_chunk, 1)
- ofile.create_dataset('statistics/histograms/' + k,
- (1,
- self.parameters['histogram_bins']),
- chunks = (time_chunk,
- self.parameters['histogram_bins']),
- maxshape = (None,
- self.parameters['histogram_bins']),
- dtype = np.int64)
- if self.QR_stats_on:
- time_chunk = 2**20//(8*3*self.parameters['histogram_bins'])
- time_chunk = max(time_chunk, 1)
- ofile.create_dataset('statistics/histograms/trS2_Q_R',
- (1,
- self.parameters['histogram_bins'],
- 3),
- chunks = (time_chunk,
- self.parameters['histogram_bins'],
- 3),
- maxshape = (None,
- self.parameters['histogram_bins'],
- 3),
- dtype = np.int64)
- time_chunk = 2**20//(8*9*self.parameters['histogram_bins'])
- time_chunk = max(time_chunk, 1)
- ofile.create_dataset('statistics/histograms/velocity_gradient',
- (1,
- self.parameters['histogram_bins'],
- 3,
- 3),
- chunks = (time_chunk,
- self.parameters['histogram_bins'],
- 3,
- 3),
- maxshape = (None,
- self.parameters['histogram_bins'],
- 3,
- 3),
- dtype = np.int64)
- time_chunk = 2**20//(8*3*10)
- time_chunk = max(time_chunk, 1)
- a = ofile.create_dataset('statistics/moments/trS2_Q_R',
- (1, 10, 3),
- chunks = (time_chunk, 10, 3),
- maxshape = (None, 10, 3),
- dtype = np.float64)
- time_chunk = 2**20//(8*9*10)
- time_chunk = max(time_chunk, 1)
- a = ofile.create_dataset('statistics/moments/velocity_gradient',
- (1, 10, 3, 3),
- chunks = (time_chunk, 10, 3, 3),
- maxshape = (None, 10, 3, 3),
- dtype = np.float64)
- time_chunk = 2**20//(8*self.parameters['QR2D_histogram_bins']**2)
- time_chunk = max(time_chunk, 1)
- ofile.create_dataset('statistics/histograms/QR2D',
- (1,
- self.parameters['QR2D_histogram_bins'],
- self.parameters['QR2D_histogram_bins']),
- chunks = (time_chunk,
- self.parameters['QR2D_histogram_bins'],
- self.parameters['QR2D_histogram_bins']),
- maxshape = (None,
- self.parameters['QR2D_histogram_bins'],
- self.parameters['QR2D_histogram_bins']),
- dtype = np.int64)
- if self.particle_species == 0:
- return None
-
- if type(particle_ic) == type(None):
- pbase_shape = (self.parameters['nparticles'],)
- number_of_particles = self.parameters['nparticles']
- else:
- pbase_shape = particle_ic.shape[:-1]
- assert(particle_ic.shape[-1] == 3)
- if len(pbase_shape) == 1:
- number_of_particles = pbase_shape[0]
- else:
- number_of_particles = 1
- for val in pbase_shape[1:]:
- number_of_particles *= val
-
- with h5py.File(self.get_particle_file_name(), 'a') as ofile:
- for s in range(self.particle_species):
- ofile.create_group('tracers{0}'.format(s))
- time_chunk = 2**20 // (8*3*number_of_particles)
- time_chunk = max(time_chunk, 1)
- dims = ((1,
- self.parameters['tracers{0}_integration_steps'.format(s)]) +
- pbase_shape + (3,))
- maxshape = (h5py.h5s.UNLIMITED,) + dims[1:]
- if len(pbase_shape) > 1:
- chunks = (time_chunk, 1, 1) + dims[3:]
- else:
- chunks = (time_chunk, 1) + dims[2:]
- bfps.tools.create_alloc_early_dataset(
- ofile,
- '/tracers{0}/rhs'.format(s),
- dims, maxshape, chunks)
- if len(pbase_shape) > 1:
- chunks = (time_chunk, 1) + pbase_shape[1:] + (3,)
- else:
- chunks = (time_chunk, pbase_shape[0], 3)
- bfps.tools.create_alloc_early_dataset(
- ofile,
- '/tracers{0}/state'.format(s),
- (1,) + pbase_shape + (3,),
- (h5py.h5s.UNLIMITED,) + pbase_shape + (3,),
- chunks)
- # "velocity" is sampled, single precision is enough
- # for the results we are interested in.
- bfps.tools.create_alloc_early_dataset(
- ofile,
- '/tracers{0}/velocity'.format(s),
- (1,) + pbase_shape + (3,),
- (h5py.h5s.UNLIMITED,) + pbase_shape + (3,),
- chunks,
- dset_dtype = h5py.h5t.IEEE_F32LE)
- if self.parameters['tracers{0}_acc_on'.format(s)]:
- bfps.tools.create_alloc_early_dataset(
- ofile,
- '/tracers{0}/acceleration'.format(s),
- (1,) + pbase_shape + (3,),
- (h5py.h5s.UNLIMITED,) + pbase_shape + (3,),
- chunks,
- dset_dtype = h5py.h5t.IEEE_F32LE)
- return None
- def add_particle_fields(
- self,
- interp_type = 'spline',
- kcut = None,
- neighbours = 1,
- smoothness = 1,
- name = 'particle_field',
- field_class = 'rFFTW_interpolator',
- acc_field_name = 'rFFTW_acc'):
- self.fluid_includes += '#include "{0}.hpp"\n'.format(field_class)
- self.fluid_variables += field_class + '<{0}, {1}> *vel_{2}, *acc_{2};\n'.format(
- self.C_dtype, neighbours, name)
- self.parameters[name + '_type'] = interp_type
- self.parameters[name + '_neighbours'] = neighbours
- if interp_type == 'spline':
- self.parameters[name + '_smoothness'] = smoothness
- beta_name = 'beta_n{0}_m{1}'.format(neighbours, smoothness)
- elif interp_type == 'Lagrange':
- beta_name = 'beta_Lagrange_n{0}'.format(neighbours)
- if field_class == 'rFFTW_interpolator':
- self.fluid_start += ('vel_{0} = new {1}<{2}, {3}>(fs, {4}, fs->rvelocity);\n' +
- 'acc_{0} = new {1}<{2}, {3}>(fs, {4}, {5});\n').format(name,
- field_class,
- self.C_dtype,
- neighbours,
- beta_name,
- acc_field_name)
- elif field_class == 'interpolator':
- self.fluid_start += ('vel_{0} = new {1}<{2}, {3}>(fs, {4});\n' +
- 'acc_{0} = new {1}<{2}, {3}>(fs, {4});\n').format(name,
- field_class,
- self.C_dtype,
- neighbours,
- beta_name,
- acc_field_name)
- self.fluid_end += ('delete vel_{0};\n' +
- 'delete acc_{0};\n').format(name)
- update_fields = 'fs->compute_velocity(fs->cvorticity);\n'
- if not type(kcut) == type(None):
- update_fields += 'fs->low_pass_Fourier(fs->cvelocity, 3, {0});\n'.format(kcut)
- update_fields += ('fs->ift_velocity();\n' +
- 'fs->compute_Lagrangian_acceleration(acc_{0}->field);\n').format(name)
- self.fluid_start += update_fields
- self.fluid_loop += update_fields
- return None
- def specific_parser_arguments(
- self,
- parser):
- _fluid_particle_base.specific_parser_arguments(self, parser)
- parser.add_argument(
- '--src-wd',
- type = str,
- dest = 'src_work_dir',
- default = '')
- parser.add_argument(
- '--src-simname',
- type = str,
- dest = 'src_simname',
- default = '')
- parser.add_argument(
- '--src-iteration',
- type = int,
- dest = 'src_iteration',
- default = 0)
- parser.add_argument(
- '--njobs',
- type = int, dest = 'njobs',
- default = 1)
- parser.add_argument(
- '--QR-stats',
- action = 'store_true',
- dest = 'QR_stats',
- help = 'add this option if you want to compute velocity gradient and QR stats')
- parser.add_argument(
- '--Lag-acc-stats',
- action = 'store_true',
- dest = 'Lag_acc_stats',
- help = 'add this option if you want to compute Lagrangian acceleration statistics')
- parser.add_argument(
- '--kMeta',
- type = float,
- dest = 'kMeta',
- default = 2.0)
- parser.add_argument(
- '--dtfactor',
- type = float,
- dest = 'dtfactor',
- default = 0.5,
- help = 'dt is computed as DTFACTOR / N')
- parser.add_argument(
- '--particle-rand-seed',
- type = int,
- dest = 'particle_rand_seed',
- default = None)
- parser.add_argument(
- '--pclouds',
- type = int,
- dest = 'pclouds',
- default = 1,
- help = ('number of particle clouds. Particle "clouds" '
- 'consist of particles distributed according to '
- 'pcloud-type.'))
- parser.add_argument(
- '--pcloud-type',
- choices = ['random-cube',
- 'regular-cube'],
- dest = 'pcloud_type',
- default = 'random-cube')
- parser.add_argument(
- '--particle-cloud-size',
- type = float,
- dest = 'particle_cloud_size',
- default = 2*np.pi)
- parser.add_argument(
- '--neighbours',
- type = int,
- dest = 'neighbours',
- default = 1)
- parser.add_argument(
- '--smoothness',
- type = int,
- dest = 'smoothness',
- default = 1)
- return None
- def prepare_launch(
- self,
- args = []):
- """Set up reasonable parameters.
-
- With the default Lundgren forcing applied in the band [2, 4],
- we can estimate the dissipation, therefore we can estimate
- :math:`k_M \\eta_K` and constrain the viscosity.
- Also, if velocity gradient statistics are computed, the
- dissipation is used for estimating the bins of the QR histogram.
-
- In brief, the command line parameter :math:`k_M \\eta_K` is
- used in the following formula for :math:`\\nu` (:math:`N` is the
- number of real space grid points per coordinate):
-
- .. math::
-
- \\nu = \\left(\\frac{2 k_M \\eta_K}{N} \\right)^{4/3}
-
- With this choice, the average dissipation :math:`\\varepsilon`
- will be close to 0.4, and the integral scale velocity will be
- close to 0.77, yielding the approximate value for the Taylor
- microscale and corresponding Reynolds number:
-
- .. math::
-
- \\lambda \\approx 4.75\\left(\\frac{2 k_M \\eta_K}{N} \\right)^{4/6}, \\hskip .5in
- R_\\lambda \\approx 3.7 \\left(\\frac{N}{2 k_M \\eta_K} \\right)^{4/6}
-
- """
- opt = _code.prepare_launch(self, args = args)
- self.QR_stats_on = opt.QR_stats
- self.Lag_acc_stats_on = opt.Lag_acc_stats
- self.parameters['nu'] = (opt.kMeta * 2 / opt.n)**(4./3)
- self.parameters['dt'] = (opt.dtfactor / opt.n)
- # custom famplitude for 288 and 576
- if opt.n == 288:
- self.parameters['famplitude'] = 0.45
- elif opt.n == 576:
- self.parameters['famplitude'] = 0.47
- if ((self.parameters['niter_todo'] % self.parameters['niter_out']) != 0):
- self.parameters['niter_out'] = self.parameters['niter_todo']
- if self.QR_stats_on:
- # max_Q_estimate and max_R_estimate are just used for the 2D pdf
- # therefore I just want them to be small multiples of mean trS2
- # I'm already estimating the dissipation with kMeta...
- meantrS2 = (opt.n//2 / opt.kMeta)**4 * self.parameters['nu']**2
- self.parameters['max_Q_estimate'] = meantrS2
- self.parameters['max_R_estimate'] = .4*meantrS2**1.5
- # add QR suffix to code name, since we now expect additional
- # datasets in the .h5 file
- self.name += '-QR'
- if self.Lag_acc_stats_on:
- self.name += '-Lag_acc'
- if len(opt.src_work_dir) == 0:
- opt.src_work_dir = os.path.realpath(opt.work_dir)
- self.pars_from_namespace(opt)
- return opt
- def launch(
- self,
- args = [],
- noparticles = False,
- **kwargs):
- opt = self.prepare_launch(args = args)
- self.fill_up_fluid_code()
- if noparticles:
- opt.nparticles = 0
- elif type(opt.nparticles) == int:
- if opt.nparticles > 0:
- self.name += '-particles'
- self.add_3D_rFFTW_field(
- name = 'rFFTW_acc')
- self.add_interpolator(
- name = 'cubic_spline',
- neighbours = opt.neighbours,
- smoothness = opt.smoothness,
- class_name = 'rFFTW_interpolator')
- self.add_particles(
- integration_steps = [4],
- interpolator = 'cubic_spline',
- acc_name = 'rFFTW_acc',
- class_name = 'rFFTW_distributed_particles')
- self.variables += 'hid_t particle_file;\n'
- self.main_start += """
- if (myrank == 0)
- {
- // set caching parameters
- hid_t fapl = H5Pcreate(H5P_FILE_ACCESS);
- herr_t cache_err = H5Pset_cache(fapl, 0, 521, 134217728, 1.0);
- DEBUG_MSG("when setting cache for particles I got %d\\n", cache_err);
- sprintf(fname, "%s_particles.h5", simname);
- particle_file = H5Fopen(fname, H5F_ACC_RDWR, fapl);
- }
- """
- self.main_end = ('if (myrank == 0)\n' +
- '{\n' +
- 'H5Fclose(particle_file);\n' +
- '}\n') + self.main_end
- self.finalize_code()
- self.launch_jobs(opt = opt, **kwargs)
- return None
- def launch_jobs(
- self,
- opt = None,
- particle_initial_condition = None):
- if not os.path.exists(os.path.join(self.work_dir, self.simname + '.h5')):
- if opt.pclouds > 1:
- np.random.seed(opt.particle_rand_seed)
- if opt.pcloud_type == 'random-cube':
- particle_initial_condition = (
- np.random.random((opt.pclouds, 1, 3))*2*np.pi +
- np.random.random((1, self.parameters['nparticles'], 3))*opt.particle_cloud_size)
- elif opt.pcloud_type == 'regular-cube':
- onedarray = np.linspace(
- -opt.particle_cloud_size/2,
- opt.particle_cloud_size/2,
- self.parameters['nparticles'])
- particle_initial_condition = np.zeros(
- (opt.pclouds,
- self.parameters['nparticles'],
- self.parameters['nparticles'],
- self.parameters['nparticles'], 3),
- dtype = np.float64)
- particle_initial_condition[:] = \
- np.random.random((opt.pclouds, 1, 1, 1, 3))*2*np.pi
- particle_initial_condition[..., 0] += onedarray[None, None, None, :]
- particle_initial_condition[..., 1] += onedarray[None, None, :, None]
- particle_initial_condition[..., 2] += onedarray[None, :, None, None]
- self.write_par(
- particle_ic = particle_initial_condition)
- if self.parameters['nparticles'] > 0:
- data = self.generate_tracer_state(
- species = 0,
- rseed = opt.particle_rand_seed,
- data = particle_initial_condition)
- for s in range(1, self.particle_species):
- self.generate_tracer_state(species = s, data = data)
- init_condition_file = os.path.join(
- self.work_dir,
- self.simname + '_cvorticity_i{0:0>5x}'.format(0))
- if not os.path.exists(init_condition_file):
- if len(opt.src_simname) > 0:
- src_file = os.path.join(
- os.path.realpath(opt.src_work_dir),
- opt.src_simname + '_cvorticity_i{0:0>5x}'.format(opt.src_iteration))
- os.symlink(src_file, init_condition_file)
- else:
- self.generate_vector_field(
- write_to_file = True,
- spectra_slope = 2.0,
- amplitude = 0.05)
- self.run(
- nb_processes = opt.nb_processes,
- nb_threads_per_process = opt.nb_threads_per_process,
- njobs = opt.njobs,
- hours = opt.minutes // 60,
- minutes = opt.minutes % 60,
- no_submit = opt.no_submit)
- return None
-
diff --git a/bfps/__main__.py b/bfps/__main__.py
index 03f68a97..25d3d101 100644
--- a/bfps/__main__.py
+++ b/bfps/__main__.py
@@ -31,10 +31,6 @@ import bfps
from .DNS import DNS
from .PP import PP
from .TEST import TEST
-from .NavierStokes import NavierStokes
-from .NSVorticityEquation import NSVorticityEquation
-from .FluidConvert import FluidConvert
-from .NSManyParticles import NSManyParticles
def main():
parser = argparse.ArgumentParser(prog = 'bfps')
@@ -42,29 +38,9 @@ def main():
'-v', '--version',
action = 'version',
version = '%(prog)s ' + bfps.__version__)
- NSoptions = ['NavierStokes',
- 'NavierStokes-single',
- 'NavierStokes-double',
- 'NS',
- 'NS-single',
- 'NS-double']
- NSVEoptions = ['NSVorticityEquation',
- 'NSVorticityEquation-single',
- 'NSVorticityEquation-double',
- 'NSVE',
- 'NSVE-single',
- 'NSVE-double']
- FCoptions = ['FluidConvert']
- NSMPopt = ['NSManyParticles',
- 'NSManyParticles-single',
- 'NSManyParticles-double']
parser.add_argument(
'base_class',
- choices = ['DNS', 'PP', 'TEST'] +
- NSoptions +
- NSVEoptions +
- FCoptions +
- NSMPopt,
+ choices = ['DNS', 'PP', 'TEST']
type = str)
# first option is the choice of base class or -h or -v
# all other options are passed on to the base_class instance
@@ -73,29 +49,10 @@ def main():
# cannot be executed by mistake.
if opt.base_class == 'DNS':
c = DNS()
- c.launch(args = sys.argv[2:])
- return None
if opt.base_class == 'PP':
c = PP()
- c.launch(args = sys.argv[2:])
- return None
if opt.base_class == 'TEST':
c = TEST()
- c.launch(args = sys.argv[2:])
- return None
- if 'double' in opt.base_class:
- precision = 'double'
- else:
- precision = 'single'
- if opt.base_class in NSoptions:
- base_class = NavierStokes
- if opt.base_class in NSVEoptions:
- base_class = NSVorticityEquation
- elif opt.base_class in FCoptions:
- base_class = FluidConvert
- elif opt.base_class in NSMPopt:
- base_class = NSManyParticles
- c = base_class(fluid_precision = precision)
c.launch(args = sys.argv[2:])
return None
diff --git a/bfps/_fluid_base.py b/bfps/_fluid_base.py
deleted file mode 100644
index 757e6cb8..00000000
--- a/bfps/_fluid_base.py
+++ /dev/null
@@ -1,503 +0,0 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
-
-
-
-from ._code import _code
-from bfps import tools
-
-import os
-import numpy as np
-import h5py
-
-class _fluid_particle_base(_code):
- """This class is meant to put together all common code between the
- different C++ solvers/postprocessing tools, so that development of
- specific functionalities is not overwhelming.
- """
- def __init__(
- self,
- name = 'solver',
- work_dir = './',
- simname = 'test',
- dtype = np.float32,
- use_fftw_wisdom = True):
- _code.__init__(
- self,
- work_dir = work_dir,
- simname = simname)
- self.use_fftw_wisdom = use_fftw_wisdom
- self.name = name
- self.particle_species = 0
- if dtype in [np.float32, np.float64]:
- self.dtype = dtype
- elif dtype in ['single', 'double']:
- if dtype == 'single':
- self.dtype = np.dtype(np.float32)
- elif dtype == 'double':
- self.dtype = np.dtype(np.float64)
- self.rtype = self.dtype
- if self.rtype == np.float32:
- self.ctype = np.dtype(np.complex64)
- self.C_dtype = 'float'
- elif self.rtype == np.float64:
- self.ctype = np.dtype(np.complex128)
- self.C_dtype = 'double'
- self.parameters['dealias_type'] = 1
- self.parameters['dkx'] = 1.0
- self.parameters['dky'] = 1.0
- self.parameters['dkz'] = 1.0
- self.parameters['niter_todo'] = 8
- self.parameters['niter_part'] = 1
- self.parameters['niter_stat'] = 1
- self.parameters['niter_out'] = 1024
- self.parameters['nparticles'] = 0
- self.parameters['dt'] = 0.01
- self.fluid_includes = '#include "fluid_solver.hpp"\n'
- self.fluid_includes = '#include "field.hpp"\n'
- self.fluid_variables = ''
- self.fluid_definitions = ''
- self.fluid_start = ''
- self.fluid_loop = ''
- self.fluid_end = ''
- self.fluid_output = ''
- self.stat_src = ''
- self.particle_includes = ''
- self.particle_variables = ''
- self.particle_definitions = ''
- self.particle_start = ''
- self.particle_loop = ''
- self.particle_output = ''
- self.particle_end = ''
- self.particle_stat_src = ''
- self.file_datasets_grow = ''
- self.store_kspace = """
- //begincpp
- if (myrank == 0 && iteration == 0)
- {
- TIMEZONE("fuild_base::store_kspace");
- hsize_t dims[4];
- hid_t space, dset;
- // store kspace information
- hid_t parameter_file = stat_file;
- //char fname[256];
- //sprintf(fname, "%s.h5", simname);
- //parameter_file = H5Fopen(fname, H5F_ACC_RDWR, H5P_DEFAULT);
- dset = H5Dopen(parameter_file, "/kspace/kshell", H5P_DEFAULT);
- space = H5Dget_space(dset);
- H5Sget_simple_extent_dims(space, dims, NULL);
- H5Sclose(space);
- if (fs->nshells != dims[0])
- {
- DEBUG_MSG(
- "ERROR: computed nshells %d not equal to data file nshells %d\\n",
- fs->nshells, dims[0]);
- }
- H5Dwrite(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, fs->kshell);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/kspace/nshell", H5P_DEFAULT);
- H5Dwrite(dset, H5T_NATIVE_INT64, H5S_ALL, H5S_ALL, H5P_DEFAULT, fs->nshell);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/kspace/kM", H5P_DEFAULT);
- H5Dwrite(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &fs->kMspec);
- H5Dclose(dset);
- dset = H5Dopen(parameter_file, "/kspace/dk", H5P_DEFAULT);
- H5Dwrite(dset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &fs->dk);
- H5Dclose(dset);
- //H5Fclose(parameter_file);
- }
- //endcpp
- """
- return None
- def get_data_file_name(self):
- return os.path.join(self.work_dir, self.simname + '.h5')
- def get_data_file(self):
- return h5py.File(self.get_data_file_name(), 'r')
- def get_particle_file_name(self):
- return os.path.join(self.work_dir, self.simname + '_particles.h5')
- def get_particle_file(self):
- return h5py.File(self.get_particle_file_name(), 'r')
- def finalize_code(
- self,
- postprocess_mode = False):
- self.includes += self.fluid_includes
- self.includes += '#include <ctime>\n'
- self.variables += self.fluid_variables
- self.definitions += ('int grow_single_dataset(hid_t dset, int tincrement)\n{\n' +
- 'int ndims;\n' +
- 'hsize_t space;\n' +
- 'space = H5Dget_space(dset);\n' +
- 'ndims = H5Sget_simple_extent_ndims(space);\n' +
- 'hsize_t *dims = new hsize_t[ndims];\n' +
- 'H5Sget_simple_extent_dims(space, dims, NULL);\n' +
- 'dims[0] += tincrement;\n' +
- 'H5Dset_extent(dset, dims);\n' +
- 'H5Sclose(space);\n' +
- 'delete[] dims;\n' +
- 'return EXIT_SUCCESS;\n}\n')
- self.definitions+= self.fluid_definitions
- if self.particle_species > 0:
- self.includes += self.particle_includes
- self.variables += self.particle_variables
- self.definitions += self.particle_definitions
- self.definitions += ('herr_t grow_statistics_dataset(hid_t o_id, const char *name, const H5O_info_t *info, void *op_data)\n{\n' +
- 'if (info->type == H5O_TYPE_DATASET)\n{\n' +
- 'hsize_t dset = H5Dopen(o_id, name, H5P_DEFAULT);\n' +
- 'grow_single_dataset(dset, niter_todo/niter_stat);\n'
- 'H5Dclose(dset);\n}\n' +
- 'return 0;\n}\n')
- self.definitions += ('herr_t grow_particle_datasets(hid_t g_id, const char *name, const H5L_info_t *info, void *op_data)\n{\n' +
- 'hsize_t dset;\n')
- for key in ['state', 'velocity', 'acceleration']:
- self.definitions += ('if (H5Lexists(g_id, "{0}", H5P_DEFAULT))\n'.format(key) +
- '{\n' +
- 'dset = H5Dopen(g_id, "{0}", H5P_DEFAULT);\n'.format(key) +
- 'grow_single_dataset(dset, niter_todo/niter_part);\n' +
- 'H5Dclose(dset);\n}\n')
- self.definitions += ('if (H5Lexists(g_id, "rhs", H5P_DEFAULT))\n{\n' +
- 'dset = H5Dopen(g_id, "rhs", H5P_DEFAULT);\n' +
- 'grow_single_dataset(dset, 1);\n' +
- 'H5Dclose(dset);\n}\n' +
- 'return 0;\n}\n')
- self.definitions += ('int grow_file_datasets()\n{\n' +
- 'int file_problems = 0;\n' +
- self.file_datasets_grow +
- 'return file_problems;\n'
- '}\n')
- self.definitions += 'void do_stats()\n{\n' + self.stat_src + '}\n'
- self.definitions += 'void do_particle_stats()\n{\n' + self.particle_stat_src + '}\n'
- # take care of wisdom
- if self.use_fftw_wisdom:
- if self.dtype == np.float32:
- fftw_prefix = 'fftwf_'
- elif self.dtype == np.float64:
- fftw_prefix = 'fftw_'
- self.main_start += """
- //begincpp
- if (myrank == 0)
- {{
- char fname[256];
- sprintf(fname, "%s_fftw_wisdom.txt", simname);
- {0}import_wisdom_from_filename(fname);
- }}
- {0}mpi_broadcast_wisdom(MPI_COMM_WORLD);
- //endcpp
- """.format(fftw_prefix)
- self.main_end = """
- //begincpp
- {0}mpi_gather_wisdom(MPI_COMM_WORLD);
- MPI_Barrier(MPI_COMM_WORLD);
- if (myrank == 0)
- {{
- char fname[256];
- sprintf(fname, "%s_fftw_wisdom.txt", simname);
- {0}export_wisdom_to_filename(fname);
- }}
- //endcpp
- """.format(fftw_prefix) + self.main_end
- self.main = """
- //begincpp
- int data_file_problem;
- clock_t time0, time1;
- double time_difference, local_time_difference;
- time0 = clock();
- if (myrank == 0) data_file_problem = grow_file_datasets();
- MPI_Bcast(&data_file_problem, 1, MPI_INT, 0, MPI_COMM_WORLD);
- if (data_file_problem > 0)
- {
- std::cerr << data_file_problem << " problems growing file datasets.\\ntrying to exit now." << std::endl;
- MPI_Finalize();
- return EXIT_SUCCESS;
- }
- //endcpp
- """
- self.main += self.fluid_start
- if self.particle_species > 0:
- self.main += self.particle_start
- output_time_difference = ('time1 = clock();\n' +
- 'local_time_difference = ((unsigned int)(time1 - time0))/((double)CLOCKS_PER_SEC);\n' +
- 'time_difference = 0.0;\n' +
- 'MPI_Allreduce(&local_time_difference, &time_difference, ' +
- '1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);\n' +
- 'if (myrank == 0) std::cout << "iteration " ' +
- '<< {0} << " took " ' +
- '<< time_difference/nprocs << " seconds" << std::endl;\n' +
- 'if (myrank == 0) std::cerr << "iteration " ' +
- '<< {0} << " took " ' +
- '<< time_difference/nprocs << " seconds" << std::endl;\n' +
- 'time0 = time1;\n')
- if not postprocess_mode:
- self.main += 'for (int max_iter = iteration+niter_todo-iteration%niter_todo; iteration < max_iter; iteration++)\n'
- self.main += '{\n'
-
- self.main += """
- #ifdef USE_TIMINGOUTPUT
- const std::string loopLabel = "code::main_start::loop-" + std::to_string(iteration);
- TIMEZONE(loopLabel.c_str());
- #endif
- """
- self.main += 'if (iteration % niter_stat == 0) do_stats();\n'
- if self.particle_species > 0:
- self.main += 'if (iteration % niter_part == 0) do_particle_stats();\n'
- self.main += self.particle_loop
- self.main += self.fluid_loop
- self.main += output_time_difference.format('iteration')
- self.main += '}\n'
- self.main += 'do_stats();\n'
- self.main += 'do_particle_stats();\n'
- self.main += output_time_difference.format('iteration')
- else:
- self.main += 'for (int frame_index = iter0; frame_index <= iter1; frame_index += niter_out)\n'
- self.main += '{\n'
- self.main += """
- #ifdef USE_TIMINGOUTPUT
- const std::string loopLabel = "code::main_start::loop-" + std::to_string(frame_index);
- TIMEZONE(loopLabel.c_str());
- #endif
- """
- if self.particle_species > 0:
- self.main += self.particle_loop
- self.main += self.fluid_loop
- self.main += output_time_difference.format('frame_index')
- self.main += '}\n'
- self.main += self.fluid_end
- if self.particle_species > 0:
- self.main += self.particle_end
- return None
- def read_rfield(
- self,
- field = 'velocity',
- iteration = 0,
- filename = None):
- """
- :note: assumes field is a vector field
- """
- if type(filename) == type(None):
- filename = os.path.join(
- self.work_dir,
- self.simname + '_r' + field + '_i{0:0>5x}'.format(iteration))
- return np.memmap(
- filename,
- dtype = self.dtype,
- mode = 'r',
- shape = (self.parameters['nz'],
- self.parameters['ny'],
- self.parameters['nx'], 3))
- def transpose_frame(
- self,
- field = 'velocity',
- iteration = 0,
- filename = None,
- ofile = None):
- Rdata = self.read_rfield(
- field = field,
- iteration = iteration,
- filename = filename)
- new_data = np.zeros(
- (3,
- self.parameters['nz'],
- self.parameters['ny'],
- self.parameters['nx']),
- dtype = self.dtype)
- for i in range(3):
- new_data[i] = Rdata[..., i]
- if type(ofile) == type(None):
- ofile = os.path.join(
- self.work_dir,
- self.simname + '_r' + field + '_i{0:0>5x}_3xNZxNYxNX'.format(iteration))
- else:
- new_data.tofile(ofile)
- return new_data
- def plot_vel_cut(
- self,
- axis,
- field = 'velocity',
- iteration = 0,
- yval = 13,
- filename = None):
- axis.set_axis_off()
- Rdata0 = self.read_rfield(field = field, iteration = iteration, filename = filename)
- energy = np.sum(Rdata0[:, yval, :, :]**2, axis = 2)*.5
- axis.imshow(energy, interpolation='none')
- axis.set_title('{0}'.format(np.average(Rdata0[..., 0]**2 +
- Rdata0[..., 1]**2 +
- Rdata0[..., 2]**2)*.5))
- return Rdata0
- def generate_vector_field(
- self,
- rseed = 7547,
- spectra_slope = 1.,
- amplitude = 1.,
- iteration = 0,
- field_name = 'vorticity',
- write_to_file = False,
- # to switch to constant field, use generate_data_3D_uniform
- # for scalar_generator
- scalar_generator = tools.generate_data_3D):
- """generate vector field.
-
- The generated field is not divergence free, but it has the proper
- shape.
-
- :param rseed: seed for random number generator
- :param spectra_slope: spectrum of field will look like k^(-p)
- :param amplitude: all amplitudes are multiplied with this value
- :param iteration: the field is written at this iteration
- :param field_name: the name of the field being generated
- :param write_to_file: should we write the field to file?
- :param scalar_generator: which function to use for generating the
- individual components.
- Possible values: bfps.tools.generate_data_3D,
- bfps.tools.generate_data_3D_uniform
- :type rseed: int
- :type spectra_slope: float
- :type amplitude: float
- :type iteration: int
- :type field_name: str
- :type write_to_file: bool
- :type scalar_generator: function
-
- :returns: ``Kdata``, a complex valued 4D ``numpy.array`` that uses the
- transposed FFTW layout.
- Kdata[ky, kz, kx, i] is the amplitude of mode (kx, ky, kz) for
- the i-th component of the field.
- (i.e. x is the fastest index and z the slowest index in the
- real-space representation).
- """
- np.random.seed(rseed)
- Kdata00 = scalar_generator(
- self.parameters['nz']//2,
- self.parameters['ny']//2,
- self.parameters['nx']//2,
- p = spectra_slope,
- amplitude = amplitude).astype(self.ctype)
- Kdata01 = scalar_generator(
- self.parameters['nz']//2,
- self.parameters['ny']//2,
- self.parameters['nx']//2,
- p = spectra_slope,
- amplitude = amplitude).astype(self.ctype)
- Kdata02 = scalar_generator(
- self.parameters['nz']//2,
- self.parameters['ny']//2,
- self.parameters['nx']//2,
- p = spectra_slope,
- amplitude = amplitude).astype(self.ctype)
- Kdata0 = np.zeros(
- Kdata00.shape + (3,),
- Kdata00.dtype)
- Kdata0[..., 0] = Kdata00
- Kdata0[..., 1] = Kdata01
- Kdata0[..., 2] = Kdata02
- Kdata1 = tools.padd_with_zeros(
- Kdata0,
- self.parameters['nz'],
- self.parameters['ny'],
- self.parameters['nx'])
- if write_to_file:
- Kdata1.tofile(
- os.path.join(self.work_dir,
- self.simname + "_c{0}_i{1:0>5x}".format(field_name, iteration)))
- return Kdata1
- def generate_tracer_state(
- self,
- rseed = None,
- iteration = 0,
- species = 0,
- write_to_file = False,
- ncomponents = 3,
- testing = False,
- data = None):
- if (type(data) == type(None)):
- if not type(rseed) == type(None):
- np.random.seed(rseed)
- #point with problems: 5.37632864e+00, 6.10414710e+00, 6.25256493e+00]
- data = np.zeros(self.parameters['nparticles']*ncomponents).reshape(-1, ncomponents)
- data[:, :3] = np.random.random((self.parameters['nparticles'], 3))*2*np.pi
- if testing:
- #data[0] = np.array([3.26434, 4.24418, 3.12157])
- data[0] = np.array([ 0.72086101, 2.59043666, 6.27501953])
- with h5py.File(self.get_particle_file_name(), 'r+') as data_file:
- data_file['tracers{0}/state'.format(species)][0] = data
- if write_to_file:
- data.tofile(
- os.path.join(
- self.work_dir,
- "tracers{0}_state_i{1:0>5x}".format(species, iteration)))
- return data
- def generate_initial_condition(self):
- self.generate_vector_field(write_to_file = True)
- for species in range(self.particle_species):
- self.generate_tracer_state(
- species = species,
- write_to_file = False)
- return None
- def get_kspace(self):
- kspace = {}
- if self.parameters['dealias_type'] == 1:
- kMx = self.parameters['dkx']*(self.parameters['nx']//2 - 1)
- kMy = self.parameters['dky']*(self.parameters['ny']//2 - 1)
- kMz = self.parameters['dkz']*(self.parameters['nz']//2 - 1)
- else:
- kMx = self.parameters['dkx']*(self.parameters['nx']//3 - 1)
- kMy = self.parameters['dky']*(self.parameters['ny']//3 - 1)
- kMz = self.parameters['dkz']*(self.parameters['nz']//3 - 1)
- kspace['kM'] = max(kMx, kMy, kMz)
- kspace['dk'] = min(self.parameters['dkx'],
- self.parameters['dky'],
- self.parameters['dkz'])
- nshells = int(kspace['kM'] / kspace['dk']) + 2
- kspace['nshell'] = np.zeros(nshells, dtype = np.int64)
- kspace['kshell'] = np.zeros(nshells, dtype = np.float64)
- kspace['kx'] = np.arange( 0,
- self.parameters['nx']//2 + 1).astype(np.float64)*self.parameters['dkx']
- kspace['ky'] = np.arange(-self.parameters['ny']//2 + 1,
- self.parameters['ny']//2 + 1).astype(np.float64)*self.parameters['dky']
- kspace['ky'] = np.roll(kspace['ky'], self.parameters['ny']//2+1)
- kspace['kz'] = np.arange(-self.parameters['nz']//2 + 1,
- self.parameters['nz']//2 + 1).astype(np.float64)*self.parameters['dkz']
- kspace['kz'] = np.roll(kspace['kz'], self.parameters['nz']//2+1)
- return kspace
- def write_par(self, iter0 = 0):
- assert (self.parameters['niter_todo'] % self.parameters['niter_stat'] == 0)
- assert (self.parameters['niter_todo'] % self.parameters['niter_out'] == 0)
- assert (self.parameters['niter_todo'] % self.parameters['niter_part'] == 0)
- assert (self.parameters['niter_out'] % self.parameters['niter_stat'] == 0)
- assert (self.parameters['niter_out'] % self.parameters['niter_part'] == 0)
- _code.write_par(self, iter0 = iter0)
- with h5py.File(os.path.join(self.work_dir, self.simname + '.h5'), 'r+') as ofile:
- ofile['bfps_info/exec_name'] = self.name
- ofile['field_dtype'] = np.dtype(self.dtype).str
- kspace = self.get_kspace()
- for k in kspace.keys():
- ofile['kspace/' + k] = kspace[k]
- nshells = kspace['nshell'].shape[0]
- ofile.close()
- return None
- def specific_parser_arguments(
- self,
- parser):
- _code.specific_parser_arguments(self, parser)
- return None
-
diff --git a/tests/base.py b/tests/base.py
index 6f110716..54267973 100644
--- a/tests/base.py
+++ b/tests/base.py
@@ -33,7 +33,6 @@ import numpy as np
import matplotlib.pyplot as plt
import bfps
-from bfps import FluidResize
from bfps.tools import particle_finite_diff_test as acceleration_test
import argparse
@@ -92,33 +91,13 @@ parser.add_argument(
dest = 'kMeta',
default = 2.0)
-def double(opt):
- old_simname = 'N{0:0>3x}'.format(opt.n)
- new_simname = 'N{0:0>3x}'.format(opt.n*2)
- c = FluidResize(fluid_precision = opt.precision)
- c.launch(
- args = ['--simname', old_simname + '_double',
- '--wd', opt.work_dir,
- '--nx', '{0}'.format(opt.n),
- '--ny', '{0}'.format(opt.n),
- '--nz', '{0}'.format(opt.n),
- '--dst_nx', '{0}'.format(2*opt.n),
- '--dst_ny', '{0}'.format(2*opt.n),
- '--dst_nz', '{0}'.format(2*opt.n),
- '--dst_simname', new_simname,
- '--src_simname', old_simname,
- '--src_iteration', '0',
- '--src_wd', './',
- '--niter_todo', '0'])
- return None
-
def launch(
opt,
nu = None,
dt = None,
tracer_state_file = None,
vorticity_field = None,
- code_class = bfps.NavierStokes,
+ code_class = bfps.DNS,
particle_class = 'particles',
interpolator_class = 'rFFTW_interpolator'):
c = code_class(
diff --git a/tests/test_plain.py b/tests/test_plain.py
deleted file mode 100644
index ad30224f..00000000
--- a/tests/test_plain.py
+++ /dev/null
@@ -1,156 +0,0 @@
-#! /usr/bin/env python3
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
-
-
-
-#from base import *
-import bfps
-from bfps.tools import particle_finite_diff_test as acceleration_test
-import sys
-
-import numpy as np
-import matplotlib.pyplot as plt
-
-#parser.add_argument('--multiplejob',
-# dest = 'multiplejob', action = 'store_true')
-#
-#parser.add_argument(
-# '--particle-class',
-# default = 'particles',
-# dest = 'particle_class',
-# type = str)
-#
-#parser.add_argument(
-# '--interpolator-class',
-# default = 'interpolator',
-# dest = 'interpolator_class',
-# type = str)
-
-class NSPlain(bfps.NavierStokes):
- def specific_parser_arguments(
- self,
- parser):
- bfps.NavierStokes.specific_parser_arguments(self, parser)
- parser.add_argument(
- '--particle-class',
- default = 'rFFTW_distributed_particles',
- dest = 'particle_class',
- type = str)
- parser.add_argument(
- '--interpolator-class',
- default = 'rFFTW_interpolator',
- dest = 'interpolator_class',
- type = str)
- parser.add_argument('--neighbours',
- type = int,
- dest = 'neighbours',
- default = 3)
- parser.add_argument('--smoothness',
- type = int,
- dest = 'smoothness',
- default = 2)
- return None
- def launch(
- self,
- args = [],
- **kwargs):
- opt = self.prepare_launch(args = args)
- self.fill_up_fluid_code()
- if type(opt.nparticles) == int:
- if opt.nparticles > 0:
- self.add_3D_rFFTW_field(
- name = 'rFFTW_acc')
- self.add_interpolator(
- name = 'spline',
- neighbours = opt.neighbours,
- smoothness = opt.smoothness,
- class_name = opt.interpolator_class)
- self.add_particles(
- kcut = ['fs->kM/2', 'fs->kM/3'],
- integration_steps = 3,
- interpolator = 'spline',
- class_name = opt.particle_class)
- self.add_particles(
- integration_steps = [2, 3, 4, 6],
- interpolator = 'spline',
- acc_name = 'rFFTW_acc',
- class_name = opt.particle_class)
- self.finalize_code()
- self.launch_jobs(opt = opt)
- return None
-
-def plain(args):
- wd = opt.work_dir
- opt.work_dir = wd + '/N{0:0>3x}_1'.format(opt.n)
- c0 = launch(opt, dt = 0.2/opt.n,
- particle_class = opt.particle_class,
- interpolator_class = opt.interpolator_class)
- c0.compute_statistics()
- print ('Re = {0:.0f}'.format(c0.statistics['Re']))
- print ('Rlambda = {0:.0f}'.format(c0.statistics['Rlambda']))
- print ('Lint = {0:.4e}, etaK = {1:.4e}'.format(c0.statistics['Lint'], c0.statistics['etaK']))
- print ('Tint = {0:.4e}, tauK = {1:.4e}'.format(c0.statistics['Tint'], c0.statistics['tauK']))
- print ('kMetaK = {0:.4e}'.format(c0.statistics['kMeta']))
- for s in range(c0.particle_species):
- acceleration_test(c0, species = s, m = 1)
- if not opt.multiplejob:
- return None
- assert(opt.niter_todo % 3 == 0)
- opt.work_dir = wd + '/N{0:0>3x}_2'.format(opt.n)
- opt.njobs *= 2
- opt.niter_todo = opt.niter_todo//2
- c1 = launch(opt, dt = c0.parameters['dt'],
- particle_class = opt.particle_class,
- interpolator_class = opt.interpolator_class)
- c1.compute_statistics()
- opt.work_dir = wd + '/N{0:0>3x}_3'.format(opt.n)
- opt.njobs = 3*opt.njobs//2
- opt.niter_todo = 2*opt.niter_todo//3
- c2 = launch(opt, dt = c0.parameters['dt'],
- particle_class = opt.particle_class,
- interpolator_class = opt.interpolator_class)
- c2.compute_statistics()
- compare_stats(opt, c0, c1)
- compare_stats(opt, c0, c2)
- return None
-
-if __name__ == '__main__':
- c0 = NSPlain()
- c0.launch(
- ['-n', '32',
- '--ncpu', '4',
- '--nparticles', '1000',
- '--niter_todo', '48',
- '--wd', 'data/single'] +
- sys.argv[1:])
- c0.compute_statistics()
- print ('Re = {0:.0f}'.format(c0.statistics['Re']))
- print ('Rlambda = {0:.0f}'.format(c0.statistics['Rlambda']))
- print ('Lint = {0:.4e}, etaK = {1:.4e}'.format(c0.statistics['Lint'], c0.statistics['etaK']))
- print ('Tint = {0:.4e}, tauK = {1:.4e}'.format(c0.statistics['Tint'], c0.statistics['tauK']))
- print ('kMetaK = {0:.4e}'.format(c0.statistics['kMeta']))
- for s in range(c0.particle_species):
- acceleration_test(c0, species = s, m = 1)
-
diff --git a/tests/test_vorticity_equation.py b/tests/test_vorticity_equation.py
index dfaccb8b..e492bfa5 100644
--- a/tests/test_vorticity_equation.py
+++ b/tests/test_vorticity_equation.py
@@ -273,12 +273,13 @@ def main():
particle_initial_condition[..., 2] = yvals[None, :, None]
particle_initial_condition = particle_initial_condition.reshape(-1, 3)
nparticles = nparticles**2
- c = bfps.NavierStokes(simname = 'fluid_solver')
+ c = bfps.DNS(simname = 'fluid_solver')
if run_NS:
run_NSVE = True
subprocess.call('rm *fluid_solver* NavierStokes*', shell = True)
c.launch(
- ['-n', '32',
+ ['NSVE',
+ '-n', '32',
'--simname', 'fluid_solver',
'--ncpu', '4',
'--niter_todo', '{0}'.format(niterations),
@@ -298,9 +299,10 @@ def main():
f = h5py.File('vorticity_equation_checkpoint_0.h5', 'w')
f['vorticity/complex/0'] = data
f.close()
- c = bfps.NSVorticityEquation()
+ c = bfps.DNS()
c.launch(
- ['-n', '32',
+ ['NSVEparticles',
+ '-n', '32',
'--simname', 'vorticity_equation',
'--np', '4',
'--ntpp', '1',
--
GitLab
From f244d748dd1b8979b3e263c93aa705e13b4ba398 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 21 Sep 2018 12:19:54 +0200
Subject: [PATCH 231/342] don't import old documents
---
bfps/__init__.py | 3 ---
1 file changed, 3 deletions(-)
diff --git a/bfps/__init__.py b/bfps/__init__.py
index 29dc62a1..babbc203 100644
--- a/bfps/__init__.py
+++ b/bfps/__init__.py
@@ -49,8 +49,5 @@ from host_information import host_info
from .DNS import DNS
from .PP import PP
from .TEST import TEST
-from .FluidConvert import FluidConvert
-from .NavierStokes import NavierStokes
-from .NSVorticityEquation import NSVorticityEquation
#import test
--
GitLab
From 99e1c858876c38951eb66b103dca314f5e11010a Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Sat, 22 Sep 2018 17:36:20 +0200
Subject: [PATCH 232/342] use different precision initial conditions without
conversion
also, fix typo in __main__.py
---
bfps/DNS.py | 15 +++------------
bfps/__main__.py | 2 +-
2 files changed, 4 insertions(+), 13 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index b22a309a..48758395 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -930,18 +930,9 @@ class DNS(_code):
3)
src_file = h5py.File(src_file_name, 'r')
if (src_file[src_dset_name].shape == dst_shape):
- if make_link and (src_file[src_dset_name].dtype == self.ctype):
- dst_file[dst_dset_name] = h5py.ExternalLink(
- src_file_name,
- src_dset_name)
- else:
- dst_file.create_dataset(
- dst_dset_name,
- shape = dst_shape,
- dtype = self.ctype,
- fillvalue = 0.0)
- for kz in range(src_file[src_dset_name].shape[0]):
- dst_file[dst_dset_name][kz] = src_file[src_dset_name][kz]
+ dst_file[dst_dset_name] = h5py.ExternalLink(
+ src_file_name,
+ src_dset_name)
else:
min_shape = (min(dst_shape[0], src_file[src_dset_name].shape[0]),
min(dst_shape[1], src_file[src_dset_name].shape[1]),
diff --git a/bfps/__main__.py b/bfps/__main__.py
index 25d3d101..16a7cf7d 100644
--- a/bfps/__main__.py
+++ b/bfps/__main__.py
@@ -40,7 +40,7 @@ def main():
version = '%(prog)s ' + bfps.__version__)
parser.add_argument(
'base_class',
- choices = ['DNS', 'PP', 'TEST']
+ choices = ['DNS', 'PP', 'TEST'],
type = str)
# first option is the choice of base class or -h or -v
# all other options are passed on to the base_class instance
--
GitLab
From 0302289cb417693206e1cef2832f7b9263d90f1c Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Sun, 23 Sep 2018 07:24:10 +0200
Subject: [PATCH 233/342] rename parameter
---
bfps/DNS.py | 2 +-
bfps/PP.py | 2 +-
bfps/TEST.py | 2 +-
bfps/cpp/full_code/NSVE.cpp | 6 +++---
bfps/cpp/full_code/NSVE.hpp | 2 +-
bfps/cpp/full_code/NSVE_field_stats.cpp | 4 ++--
bfps/cpp/full_code/NSVE_field_stats.hpp | 2 +-
bfps/cpp/full_code/symmetrize_test.cpp | 6 +++---
bfps/cpp/full_code/symmetrize_test.hpp | 2 +-
9 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 48758395..eb51862e 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -120,7 +120,7 @@ class DNS(_code):
return None
def generate_default_parameters(self):
# these parameters are relevant for all DNS classes
- self.parameters['fftw_planner_type'] = 'FFTW_ESTIMATE'
+ self.parameters['fftw_plan_rigor'] = 'FFTW_ESTIMATE'
self.parameters['dealias_type'] = int(1)
self.parameters['dkx'] = float(1.0)
self.parameters['dky'] = float(1.0)
diff --git a/bfps/PP.py b/bfps/PP.py
index 867864e0..5716a7fe 100644
--- a/bfps/PP.py
+++ b/bfps/PP.py
@@ -118,7 +118,7 @@ class PP(_code):
return None
def generate_default_parameters(self):
# these parameters are relevant for all PP classes
- self.parameters['fftw_planner_type'] = 'FFTW_ESTIMATE'
+ self.parameters['fftw_plan_rigor'] = 'FFTW_ESTIMATE'
self.parameters['dealias_type'] = int(1)
self.parameters['dkx'] = float(1.0)
self.parameters['dky'] = float(1.0)
diff --git a/bfps/TEST.py b/bfps/TEST.py
index 2a8b37ba..66b2b4aa 100644
--- a/bfps/TEST.py
+++ b/bfps/TEST.py
@@ -119,7 +119,7 @@ class TEST(_code):
return None
def generate_default_parameters(self):
# these parameters are relevant for all TEST classes
- self.parameters['fftw_planner_type'] = 'FFTW_ESTIMATE'
+ self.parameters['fftw_plan_rigor'] = 'FFTW_ESTIMATE'
self.parameters['dealias_type'] = int(1)
self.parameters['dkx'] = float(1.0)
self.parameters['dky'] = float(1.0)
diff --git a/bfps/cpp/full_code/NSVE.cpp b/bfps/cpp/full_code/NSVE.cpp
index 74593b37..770ee120 100644
--- a/bfps/cpp/full_code/NSVE.cpp
+++ b/bfps/cpp/full_code/NSVE.cpp
@@ -38,11 +38,11 @@ int NSVE<rnumber>::initialize(void)
simname.c_str(),
nx, ny, nz,
dkx, dky, dkz,
- fftw_planner_string_to_flag[this->fftw_plan_type]);
+ fftw_planner_string_to_flag[this->fftw_plan_rigor]);
this->tmp_vec_field = new field<rnumber, FFTW, THREE>(
nx, ny, nz,
this->comm,
- fftw_planner_string_to_flag[this->fftw_plan_type]);
+ fftw_planner_string_to_flag[this->fftw_plan_rigor]);
this->fs->checkpoints_per_file = checkpoints_per_file;
@@ -162,7 +162,7 @@ int NSVE<rnumber>::read_parameters(void)
this->max_vorticity_estimate = hdf5_tools::read_value<double>(parameter_file, "parameters/max_vorticity_estimate");
std::string tmp = hdf5_tools::read_string(parameter_file, "parameters/forcing_type");
snprintf(this->forcing_type, 511, "%s", tmp.c_str());
- this->fftw_plan_type = hdf5_tools::read_string(parameter_file, "parameters/fftw_plan_type");
+ this->fftw_plan_rigor = hdf5_tools::read_string(parameter_file, "parameters/fftw_plan_rigor");
H5Fclose(parameter_file);
return EXIT_SUCCESS;
}
diff --git a/bfps/cpp/full_code/NSVE.hpp b/bfps/cpp/full_code/NSVE.hpp
index c3ad7a0a..83c63d35 100644
--- a/bfps/cpp/full_code/NSVE.hpp
+++ b/bfps/cpp/full_code/NSVE.hpp
@@ -53,7 +53,7 @@ class NSVE: public direct_numerical_simulation
double max_velocity_estimate;
double max_vorticity_estimate;
double nu;
- std::string fftw_plan_type;
+ std::string fftw_plan_rigor;
/* other stuff */
vorticity_equation<rnumber, FFTW> *fs;
diff --git a/bfps/cpp/full_code/NSVE_field_stats.cpp b/bfps/cpp/full_code/NSVE_field_stats.cpp
index 142c51e7..b1c8d567 100644
--- a/bfps/cpp/full_code/NSVE_field_stats.cpp
+++ b/bfps/cpp/full_code/NSVE_field_stats.cpp
@@ -13,7 +13,7 @@ int NSVE_field_stats<rnumber>::initialize(void)
this->vorticity = new field<rnumber, FFTW, THREE>(
nx, ny, nz,
this->comm,
- fftw_planner_string_to_flag[this->fftw_plan_type]);
+ fftw_planner_string_to_flag[this->fftw_plan_rigor]);
this->vorticity->real_space_representation = false;
hid_t parameter_file = H5Fopen(
(this->simname + std::string(".h5")).c_str(),
@@ -44,7 +44,7 @@ int NSVE_field_stats<rnumber>::initialize(void)
this->vorticity->clayout->starts,
this->vorticity->clayout->comm);
}
- this->fftw_plan_type = hdf5_tools::read_string(parameter_file, "parameters/fftw_plan_type");
+ this->fftw_plan_rigor = hdf5_tools::read_string(parameter_file, "parameters/fftw_plan_rigor");
H5Fclose(parameter_file);
return EXIT_SUCCESS;
}
diff --git a/bfps/cpp/full_code/NSVE_field_stats.hpp b/bfps/cpp/full_code/NSVE_field_stats.hpp
index ae519fc7..28a2376f 100644
--- a/bfps/cpp/full_code/NSVE_field_stats.hpp
+++ b/bfps/cpp/full_code/NSVE_field_stats.hpp
@@ -42,7 +42,7 @@ class NSVE_field_stats: public postprocess
private:
field_binary_IO<rnumber, COMPLEX, THREE> *bin_IO;
public:
- std::string fftw_plan_type;
+ std::string fftw_plan_rigor;
field<rnumber, FFTW, THREE> *vorticity;
diff --git a/bfps/cpp/full_code/symmetrize_test.cpp b/bfps/cpp/full_code/symmetrize_test.cpp
index 45225fcb..7cf96a71 100644
--- a/bfps/cpp/full_code/symmetrize_test.cpp
+++ b/bfps/cpp/full_code/symmetrize_test.cpp
@@ -32,7 +32,7 @@ int symmetrize_test<rnumber>::read_parameters()
H5P_DEFAULT);
this->random_seed = hdf5_tools::read_value<int>(
parameter_file, "/parameters/random_seed");
- this->fftw_plan_type = hdf5_tools::read_string(parameter_file, "parameters/fftw_plan_type");
+ this->fftw_plan_rigor = hdf5_tools::read_string(parameter_file, "parameters/fftw_plan_rigor");
H5Fclose(parameter_file);
return EXIT_SUCCESS;
}
@@ -46,13 +46,13 @@ int symmetrize_test<rnumber>::do_work(void)
field<rnumber, FFTW, THREE> *test_field0 = new field<rnumber, FFTW, THREE>(
this->nx, this->ny, this->nz,
this->comm,
- fftw_planner_string_to_flag[this->fftw_plan_type]);
+ fftw_planner_string_to_flag[this->fftw_plan_rigor]);
DEBUG_MSG("finished allocating field0\n");
DEBUG_MSG("about to allocate field1\n");
field<rnumber, FFTW, THREE> *test_field1 = new field<rnumber, FFTW, THREE>(
this->nx, this->ny, this->nz,
this->comm,
- fftw_planner_string_to_flag[this->fftw_plan_type]);
+ fftw_planner_string_to_flag[this->fftw_plan_rigor]);
DEBUG_MSG("finished allocating field1\n");
std::default_random_engine rgen;
std::normal_distribution<rnumber> rdist;
diff --git a/bfps/cpp/full_code/symmetrize_test.hpp b/bfps/cpp/full_code/symmetrize_test.hpp
index f674d365..628aee6f 100644
--- a/bfps/cpp/full_code/symmetrize_test.hpp
+++ b/bfps/cpp/full_code/symmetrize_test.hpp
@@ -42,7 +42,7 @@ template <typename rnumber>
class symmetrize_test: public test
{
public:
- std::string fftw_plan_type;
+ std::string fftw_plan_rigor;
int random_seed;
symmetrize_test(
--
GitLab
From f7109d3817a1f279682c19482a1b3743bce4bfc8 Mon Sep 17 00:00:00 2001
From: Cristian Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Sun, 23 Sep 2018 10:58:01 +0200
Subject: [PATCH 234/342] turn off some debug messages
---
bfps/cpp/field.cpp | 3 +++
bfps/cpp/field_layout.cpp | 5 +++++
bfps/cpp/kspace.cpp | 5 +++++
3 files changed, 13 insertions(+)
diff --git a/bfps/cpp/field.cpp b/bfps/cpp/field.cpp
index b70825cb..d5bc78a5 100644
--- a/bfps/cpp/field.cpp
+++ b/bfps/cpp/field.cpp
@@ -23,6 +23,9 @@
**********************************************************************/
+
+#define NDEBUG
+
#include <sys/stat.h>
#include <cmath>
#include <cstdlib>
diff --git a/bfps/cpp/field_layout.cpp b/bfps/cpp/field_layout.cpp
index 90890499..61dd3f2a 100644
--- a/bfps/cpp/field_layout.cpp
+++ b/bfps/cpp/field_layout.cpp
@@ -23,10 +23,15 @@
**********************************************************************/
+
+#define NDEBUG
+
#include <cassert>
#include "field_layout.hpp"
#include "scope_timer.hpp"
+
+
template <field_components fc>
field_layout<fc>::field_layout(
const hsize_t *SIZES,
diff --git a/bfps/cpp/kspace.cpp b/bfps/cpp/kspace.cpp
index 3fb25000..5ceb2a80 100644
--- a/bfps/cpp/kspace.cpp
+++ b/bfps/cpp/kspace.cpp
@@ -23,6 +23,9 @@
**********************************************************************/
+
+#define NDEBUG
+
#include <cmath>
#include <cstdlib>
#include <algorithm>
@@ -31,6 +34,8 @@
#include "scope_timer.hpp"
#include "shared_array.hpp"
+
+
template <field_backend be,
kspace_dealias_type dt>
template <field_components fc>
--
GitLab
From 93abd081118a91b1c15c496cb45f7dc34e055793 Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Sun, 23 Sep 2018 11:40:33 +0200
Subject: [PATCH 235/342] turn off more debug messages
---
bfps/cpp/full_code/NSVE.cpp | 2 ++
bfps/cpp/full_code/NSVEcomplex_particles.cpp | 2 ++
bfps/cpp/full_code/NSVEparticles.cpp | 3 +++
bfps/cpp/full_code/direct_numerical_simulation.cpp | 2 ++
4 files changed, 9 insertions(+)
diff --git a/bfps/cpp/full_code/NSVE.cpp b/bfps/cpp/full_code/NSVE.cpp
index 770ee120..ecec7db3 100644
--- a/bfps/cpp/full_code/NSVE.cpp
+++ b/bfps/cpp/full_code/NSVE.cpp
@@ -1,3 +1,5 @@
+#define NDEBUG
+
#include <string>
#include <cmath>
#include "NSVE.hpp"
diff --git a/bfps/cpp/full_code/NSVEcomplex_particles.cpp b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
index 9b910e9b..02a19931 100644
--- a/bfps/cpp/full_code/NSVEcomplex_particles.cpp
+++ b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
@@ -24,6 +24,8 @@
+#define NDEBUG
+
#include <string>
#include <cmath>
#include "NSVEcomplex_particles.hpp"
diff --git a/bfps/cpp/full_code/NSVEparticles.cpp b/bfps/cpp/full_code/NSVEparticles.cpp
index b09e3280..bcb2f435 100644
--- a/bfps/cpp/full_code/NSVEparticles.cpp
+++ b/bfps/cpp/full_code/NSVEparticles.cpp
@@ -1,6 +1,9 @@
+
+#define NDEBUG
+
#include <string>
#include <cmath>
#include "NSVEparticles.hpp"
diff --git a/bfps/cpp/full_code/direct_numerical_simulation.cpp b/bfps/cpp/full_code/direct_numerical_simulation.cpp
index c0b0441e..cacda323 100644
--- a/bfps/cpp/full_code/direct_numerical_simulation.cpp
+++ b/bfps/cpp/full_code/direct_numerical_simulation.cpp
@@ -1,3 +1,5 @@
+#define NDEBUG
+
#include <cstdlib>
#include <sys/types.h>
#include <sys/stat.h>
--
GitLab
From 2cb76bf330009ca37629cb191e9ee80948b5b59c Mon Sep 17 00:00:00 2001
From: Cristian Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Sun, 23 Sep 2018 16:04:21 +0200
Subject: [PATCH 236/342] don't prepend rank for mpiexec.hydra calls
---
bfps/_code.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bfps/_code.py b/bfps/_code.py
index fe7c35ab..0e9eeca5 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -443,7 +443,7 @@ class _code(_base):
script_file.write('mpiexec.hydra '
+ ' -np {} '.format(nb_mpi_processes)
+ ' -ppn {} '.format(nb_processes_per_node)
- + ' -ordered-output -prepend-rank '
+ #+ ' -ordered-output -prepend-rank '
+ os.path.join(
self.work_dir,
command_atoms[0]) +
--
GitLab
From a862c190576e7050f7f7bf1c6620eba3dd02923b Mon Sep 17 00:00:00 2001
From: Cristian Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Sun, 23 Sep 2018 22:11:02 +0200
Subject: [PATCH 237/342] tweak to avoid "unused variable" warnings
---
bfps/cpp/particles/particles_distr_mpi.hpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/bfps/cpp/particles/particles_distr_mpi.hpp b/bfps/cpp/particles/particles_distr_mpi.hpp
index 85f4f416..10a566e5 100644
--- a/bfps/cpp/particles/particles_distr_mpi.hpp
+++ b/bfps/cpp/particles/particles_distr_mpi.hpp
@@ -235,6 +235,7 @@ public:
}
const int nbProcToRecvUpper = int(neigDescriptors.size())-nbProcToRecvLower;
const int nbProcToRecv = nbProcToRecvUpper + nbProcToRecvLower;
+ variable_used_only_in_assert(nbProcToRecv);
assert(int(neigDescriptors.size()) == nbProcToRecv);
for(int idxDescr = 0 ; idxDescr < int(neigDescriptors.size()) ; ++idxDescr){
--
GitLab
From 038f9434fb66436d8cab762173ebc309832764ae Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 24 Sep 2018 09:53:32 +0200
Subject: [PATCH 238/342] identify more variables used only in assert
---
bfps/cpp/particles/p2p_distr_mpi.hpp | 1 +
bfps/cpp/particles/particles_distr_mpi.hpp | 1 +
bfps/cpp/particles/particles_inner_computer.hpp | 1 +
bfps/cpp/particles/particles_input_hdf5.hpp | 9 ++++++++-
bfps/cpp/particles/particles_output_hdf5.hpp | 9 +++++++++
bfps/cpp/particles/particles_output_sampling_hdf5.hpp | 8 ++++++++
bfps/cpp/particles/particles_system.hpp | 1 +
7 files changed, 29 insertions(+), 1 deletion(-)
diff --git a/bfps/cpp/particles/p2p_distr_mpi.hpp b/bfps/cpp/particles/p2p_distr_mpi.hpp
index 31d5f4f8..740aba18 100644
--- a/bfps/cpp/particles/p2p_distr_mpi.hpp
+++ b/bfps/cpp/particles/p2p_distr_mpi.hpp
@@ -351,6 +351,7 @@ public:
// Offset per cell layers
long int previous_index = 0;
+ variable_used_only_in_assert(previous_index);
std::unique_ptr<partsize_t[]> particles_offset_layers(new partsize_t[my_nb_cell_levels+1]());
for(int idxPartition = 0 ; idxPartition < current_partition_size ; ++idxPartition){
for(partsize_t idx_part = current_offset_particles_for_partition[idxPartition] ;
diff --git a/bfps/cpp/particles/particles_distr_mpi.hpp b/bfps/cpp/particles/particles_distr_mpi.hpp
index 10a566e5..251119be 100644
--- a/bfps/cpp/particles/particles_distr_mpi.hpp
+++ b/bfps/cpp/particles/particles_distr_mpi.hpp
@@ -523,6 +523,7 @@ public:
for(int idxPartition = 0 ; idxPartition < current_partition_size ; ++idxPartition){
for(partsize_t idx = 0 ; idx < current_my_nb_particles_per_partition[idxPartition] ; ++idx){
const int partition_level = in_computer.pbc_field_layer((*inout_positions_particles)[(idx+partOffset)*size_particle_positions+IDX_Z], IDX_Z);
+ variable_used_only_in_assert(partition_level);
assert(partition_level == current_partition_interval.first + idxPartition
|| partition_level == (current_partition_interval.first + idxPartition-1+int(field_grid_dim[IDX_Z]))%int(field_grid_dim[IDX_Z])
|| partition_level == (current_partition_interval.first + idxPartition+1)%int(field_grid_dim[IDX_Z]));
diff --git a/bfps/cpp/particles/particles_inner_computer.hpp b/bfps/cpp/particles/particles_inner_computer.hpp
index f1fe322a..b2eb95dd 100644
--- a/bfps/cpp/particles/particles_inner_computer.hpp
+++ b/bfps/cpp/particles/particles_inner_computer.hpp
@@ -46,6 +46,7 @@ public:
pos_part[idx0+IDX_X]*pos_part[idx0+IDX_X] +
pos_part[idx0+IDX_Y]*pos_part[idx0+IDX_Y] +
pos_part[idx0+IDX_Z]*pos_part[idx0+IDX_Z]);
+ variable_used_only_in_assert(orientation_size);
assert(orientation_size > 0.99);
assert(orientation_size < 1.01);
// I call "rotation" to be the right hand side of the orientation part of the ODE
diff --git a/bfps/cpp/particles/particles_input_hdf5.hpp b/bfps/cpp/particles/particles_input_hdf5.hpp
index 20239f06..40fef3c4 100644
--- a/bfps/cpp/particles/particles_input_hdf5.hpp
+++ b/bfps/cpp/particles/particles_input_hdf5.hpp
@@ -79,6 +79,7 @@ public:
assert(plist_id_par >= 0);
{
int retTest = H5Pset_fapl_mpio(plist_id_par, mpi_comm, MPI_INFO_NULL);
+ variable_used_only_in_assert(retTest);
assert(retTest >= 0);
}
@@ -98,6 +99,7 @@ public:
std::vector<hsize_t> state_dim_array(space_dim);
int hdfret = H5Sget_simple_extent_dims(dspace, &state_dim_array[0], NULL);
+ variable_used_only_in_assert(hdfret);
assert(hdfret >= 0);
// Last value is the position dim of the particles
assert(state_dim_array.back() == size_particle_positions);
@@ -126,6 +128,7 @@ public:
// Chichi comment: wouldn't &rhs_dim_array.front() be safer?
int hdfret = H5Sget_simple_extent_dims(dspace, &rhs_dim_array[0], NULL);
+ variable_used_only_in_assert(hdfret);
assert(hdfret >= 0);
assert(rhs_dim_array.back() == size_particle_rhs);
// Chichi comment: this assertion will fail in general
@@ -166,6 +169,7 @@ public:
int rethdf = H5Sselect_hyperslab(rspace, H5S_SELECT_SET, offset,
NULL, mem_dims, NULL);
+ variable_used_only_in_assert(rethdf);
assert(rethdf >= 0);
rethdf = H5Dread(dset, type_id, mspace, rspace, H5P_DEFAULT, split_particles_positions.get());
assert(rethdf >= 0);
@@ -197,6 +201,7 @@ public:
int rethdf = H5Sselect_hyperslab( rspace, H5S_SELECT_SET, offset,
NULL, mem_dims, NULL);
+ variable_used_only_in_assert(rethdf);
assert(rethdf >= 0);
rethdf = H5Dread(dset, type_id, mspace, rspace, H5P_DEFAULT, split_particles_rhs[idx_rhs].get());
assert(rethdf >= 0);
@@ -208,6 +213,7 @@ public:
assert(rethdf >= 0);
}
int rethdf = H5Dclose(dset);
+ variable_used_only_in_assert(rethdf);
assert(rethdf >= 0);
}
@@ -274,7 +280,7 @@ public:
split_particles_indexes.release();
my_particles_rhs.resize(nb_rhs);
- for(int idx_rhs = 0 ; idx_rhs < int(nb_rhs) ; ++idx_rhs){
+ for(int idx_rhs = 0 ; idx_rhs < int(nb_rhs) ; ++idx_rhs){
if(nb_particles_for_me){
my_particles_rhs[idx_rhs].reset(new real_number[exchanger.getTotalToRecv()*size_particle_rhs]);
}
@@ -285,6 +291,7 @@ public:
{
TIMEZONE("close");
int hdfret = H5Fclose(particle_file);
+ variable_used_only_in_assert(hdfret);
assert(hdfret >= 0);
hdfret = H5Pclose(plist_id_par);
assert(hdfret >= 0);
diff --git a/bfps/cpp/particles/particles_output_hdf5.hpp b/bfps/cpp/particles/particles_output_hdf5.hpp
index 22d2fa85..0098ba54 100644
--- a/bfps/cpp/particles/particles_output_hdf5.hpp
+++ b/bfps/cpp/particles/particles_output_hdf5.hpp
@@ -59,6 +59,7 @@ public:
plist_id_par,
Parent::getComWriter(),
MPI_INFO_NULL);
+ variable_used_only_in_assert(retTest);
assert(retTest >= 0);
// Parallel HDF5 write
@@ -97,6 +98,7 @@ public:
TIMEZONE("particles_output_hdf5::close_file");
int rethdf = H5Gclose(dset_id_state);
+ variable_used_only_in_assert(rethdf);
assert(rethdf >= 0);
rethdf = H5Gclose(dset_id_rhs);
@@ -197,6 +199,7 @@ public:
assert(plist_id >= 0);
{
int rethdf = H5Pset_dxpl_mpio(plist_id, use_collective_io ? H5FD_MPIO_COLLECTIVE : H5FD_MPIO_INDEPENDENT);
+ variable_used_only_in_assert(rethdf);
assert(rethdf >= 0);
}
@@ -233,6 +236,7 @@ public:
NULL,
count,
NULL);
+ variable_used_only_in_assert(rethdf);
assert(rethdf >= 0);
herr_t status = H5Dwrite(
@@ -242,6 +246,7 @@ public:
filespace,
plist_id,
particles_positions);
+ variable_used_only_in_assert(status);
assert(status >= 0);
rethdf = H5Sclose(memspace);
assert(rethdf >= 0);
@@ -289,6 +294,7 @@ public:
NULL,
count,
NULL);
+ variable_used_only_in_assert(rethdf);
assert(rethdf >= 0);
herr_t status = H5Dwrite(
@@ -298,6 +304,7 @@ public:
filespace,
plist_id,
particles_rhs[idx_rhs].get());
+ variable_used_only_in_assert(status);
assert(status >= 0);
rethdf = H5Sclose(filespace);
assert(rethdf >= 0);
@@ -305,11 +312,13 @@ public:
assert(rethdf >= 0);
}
int rethdf = H5Dclose(dataset_id);
+ variable_used_only_in_assert(rethdf);
assert(rethdf >= 0);
}
{
int rethdf = H5Pclose(plist_id);
+ variable_used_only_in_assert(rethdf);
assert(rethdf >= 0);
}
}
diff --git a/bfps/cpp/particles/particles_output_sampling_hdf5.hpp b/bfps/cpp/particles/particles_output_sampling_hdf5.hpp
index dc213226..22dafaed 100644
--- a/bfps/cpp/particles/particles_output_sampling_hdf5.hpp
+++ b/bfps/cpp/particles/particles_output_sampling_hdf5.hpp
@@ -69,6 +69,7 @@ public:
plist_id_par,
Parent::getComWriter(),
MPI_INFO_NULL);
+ variable_used_only_in_assert(retTest);
assert(retTest >= 0);
// open file for parallel HDF5 access
@@ -93,6 +94,7 @@ public:
if(Parent::isInvolved()){
// close group
int retTest = H5Gclose(pgroup_id);
+ variable_used_only_in_assert(retTest);
assert(retTest >= 0);
// close file
retTest = H5Fclose(file_id);
@@ -106,6 +108,7 @@ public:
if(Parent::isInvolved()){
// close old group
int retTest = H5Gclose(pgroup_id);
+ variable_used_only_in_assert(retTest);
assert(retTest >= 0);
// open new group
@@ -131,6 +134,7 @@ public:
// update group
int retTest = this->switch_to_group(
in_groupname);
+ variable_used_only_in_assert(retTest);
assert(retTest == EXIT_SUCCESS);
// update dataset name
dataset_name = in_dataset_name + "/" + std::to_string(idx_time_step);
@@ -182,6 +186,7 @@ public:
(use_collective_io ?
H5FD_MPIO_COLLECTIVE :
H5FD_MPIO_INDEPENDENT));
+ variable_used_only_in_assert(rethdf);
assert(rethdf >= 0);
}
{
@@ -219,6 +224,7 @@ public:
NULL,
count,
NULL);
+ variable_used_only_in_assert(rethdf);
assert(rethdf >= 0);
herr_t status = H5Dwrite(
@@ -228,6 +234,7 @@ public:
filespace,
plist_id,
particles_rhs[0].get());
+ variable_used_only_in_assert(status);
assert(status >= 0);
rethdf = H5Sclose(filespace);
assert(rethdf >= 0);
@@ -239,6 +246,7 @@ public:
{
int rethdf = H5Pclose(plist_id);
+ variable_used_only_in_assert(rethdf);
assert(rethdf >= 0);
}
}
diff --git a/bfps/cpp/particles/particles_system.hpp b/bfps/cpp/particles/particles_system.hpp
index e451f56a..f8688f61 100644
--- a/bfps/cpp/particles/particles_system.hpp
+++ b/bfps/cpp/particles/particles_system.hpp
@@ -101,6 +101,7 @@ public:
for(partsize_t idx_part = 0 ; idx_part < my_nb_particles ; ++idx_part){ // TODO remove me
const int partition_level = computer.pbc_field_layer(my_particles_positions[idx_part*size_particle_positions+IDX_Z], IDX_Z);
+ variable_used_only_in_assert(partition_level);
assert(partition_level >= current_partition_interval.first);
assert(partition_level < current_partition_interval.second);
}
--
GitLab
From 4379f6651a2ddc85551a7b20904deea2846a6ac3 Mon Sep 17 00:00:00 2001
From: Cristian Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 24 Sep 2018 12:54:43 +0200
Subject: [PATCH 239/342] fix sge script generation
---
bfps/_code.py | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/bfps/_code.py b/bfps/_code.py
index 143ef29a..c29dda63 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -281,11 +281,12 @@ class _code(_base):
nb_threads_per_process = nb_threads_per_process)
os.chdir(self.work_dir)
qsub_atoms = ['qsub']
- if len(job_name_list) >= 1:
- qsub_atoms += ['-hold_jid', job_name_list[-1]]
- subprocess.check_call(qsub_atoms + [qsub_script_name])
- os.chdir(current_dir)
- job_name_list.append(suffix)
+ if not no_submit:
+ if len(job_name_list) >= 1:
+ qsub_atoms += ['-hold_jid', job_name_list[-1]]
+ subprocess.check_call(qsub_atoms + [qsub_script_name])
+ os.chdir(current_dir)
+ job_name_list.append(suffix)
if self.host_info['type'] == 'SLURM':
job_id_list = []
for j in range(njobs):
@@ -584,7 +585,7 @@ class _code(_base):
if not type(out_file) == type(None):
script_file.write('#$ -o ' + out_file + '\n')
if not type(self.host_info['environment']) == type(None):
- envprocs = self.host_info['deltanprocs'] * int(math.ceil((nprocesses *1.0/ self.host_info['deltanprocs'])))
+ envprocs = nb_threads_per_process * nprocesses
script_file.write('#$ -pe {0} {1}\n'.format(
self.host_info['environment'],
envprocs))
--
GitLab
From 23ed2f4afc4d8ecaff1bcfc46b2f3cf53923188b Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 25 Sep 2018 11:30:48 +0200
Subject: [PATCH 240/342] split inner_computer into hpp and cpp
---
bfps/cpp/field.cpp | 7 +
bfps/cpp/full_code/NSVE.cpp | 1 +
.../particles/particles_inner_computer.cpp | 155 ++++++++++++++++++
.../particles/particles_inner_computer.hpp | 121 ++------------
setup.py | 8 +-
5 files changed, 185 insertions(+), 107 deletions(-)
create mode 100644 bfps/cpp/particles/particles_inner_computer.cpp
diff --git a/bfps/cpp/field.cpp b/bfps/cpp/field.cpp
index d5bc78a5..87e18b67 100644
--- a/bfps/cpp/field.cpp
+++ b/bfps/cpp/field.cpp
@@ -79,6 +79,7 @@ field<rnumber, be, fc>::field(
hsize_t tmp_local_size;
ptrdiff_t local_n0, local_0_start;
ptrdiff_t local_n1, local_1_start;
+ variable_used_only_in_assert(tmp_local_size);
tmp_local_size = fftw_interface<rnumber>::mpi_local_size_many_transposed(
3, nfftw, ncomp(fc),
FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK, this->comm,
@@ -227,6 +228,7 @@ int field<rnumber, be, fc>::io(
H5Tequal(dset_type, H5T_IEEE_F64LE) ||
H5Tequal(dset_type, H5T_INTEL_F64) ||
H5Tequal(dset_type, H5T_NATIVE_DOUBLE));
+ variable_used_only_in_assert(io_for_real);
H5Tclose(dset_type);
assert(this->real_space_representation == io_for_real);
}
@@ -307,6 +309,7 @@ int field<rnumber, be, fc>::io(
/* check file space */
int ndims_fspace = H5Sget_simple_extent_dims(fspace, dims, NULL);
+ variable_used_only_in_assert(ndims_fspace);
assert(((unsigned int)(ndims_fspace)) == ndim(fc));
if (this->real_space_representation)
{
@@ -417,6 +420,7 @@ int field<rnumber, be, fc>::io_database(
H5Tequal(dset_type, H5T_IEEE_F64LE) ||
H5Tequal(dset_type, H5T_INTEL_F64) ||
H5Tequal(dset_type, H5T_NATIVE_DOUBLE));
+ variable_used_only_in_assert(io_for_real);
H5Tclose(dset_type);
assert(this->real_space_representation == io_for_real);
}
@@ -493,6 +497,7 @@ int field<rnumber, be, fc>::io_database(
/* check file space */
int ndims_fspace = H5Sget_simple_extent_dims(fspace, dims, NULL);
+ variable_used_only_in_assert(ndims_fspace);
assert(ndims_fspace == int(ndim(fc) + 1));
offset[0] = toffset;
if (this->real_space_representation)
@@ -714,6 +719,7 @@ int field<rnumber, be, fc>::write_filtered(
}
/* check file space */
int ndims_fspace = H5Sget_simple_extent_dims(fspace, fdims, NULL);
+ variable_used_only_in_assert(ndims_fspace);
assert(((unsigned int)(ndims_fspace)) == ndim(fc));
for (unsigned int i=0; i<ndim(fc); i++)
{
@@ -1567,6 +1573,7 @@ int joint_rspace_PDF(
hid_t dset, wspace;
hsize_t dims[5];
int ndims;
+ variable_used_only_in_assert(ndims);
if (fc == THREE)
{
dset = H5Dopen(
diff --git a/bfps/cpp/full_code/NSVE.cpp b/bfps/cpp/full_code/NSVE.cpp
index ecec7db3..efd82fb3 100644
--- a/bfps/cpp/full_code/NSVE.cpp
+++ b/bfps/cpp/full_code/NSVE.cpp
@@ -18,6 +18,7 @@ int NSVE<rnumber>::initialize(void)
// set caching parameters
hid_t fapl = H5Pcreate(H5P_FILE_ACCESS);
herr_t cache_err = H5Pset_cache(fapl, 0, 521, 134217728, 1.0);
+ variable_used_only_in_assert(cache_err);
DEBUG_MSG("when setting stat_file cache I got %d\n", cache_err);
this->stat_file = H5Fopen(
(this->simname + ".h5").c_str(),
diff --git a/bfps/cpp/particles/particles_inner_computer.cpp b/bfps/cpp/particles/particles_inner_computer.cpp
new file mode 100644
index 00000000..85286190
--- /dev/null
+++ b/bfps/cpp/particles/particles_inner_computer.cpp
@@ -0,0 +1,155 @@
+#include "base.hpp"
+#include "particles_utils.hpp"
+#include "particles_inner_computer.hpp"
+
+#include <cmath>
+
+template <class real_number, class partsize_t>
+template <int size_particle_positions, int size_particle_rhs>
+void particles_inner_computer<real_number, partsize_t>::compute_interaction(
+ const partsize_t nb_particles,
+ const real_number pos_part[],
+ real_number rhs_part[]) const{
+ static_assert(size_particle_positions == 6, "This kernel works only with 6 values for one particle's position");
+ static_assert(size_particle_rhs == 6, "This kernel works only with 6 values per particle's rhs");
+
+ #pragma omp parallel for
+ for(partsize_t idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
+ // Add attr × V0 to the field interpolation
+ rhs_part[idx_part*size_particle_rhs + IDX_X] += pos_part[idx_part*size_particle_positions + 3+IDX_X]*v0;
+ rhs_part[idx_part*size_particle_rhs + IDX_Y] += pos_part[idx_part*size_particle_positions + 3+IDX_Y]*v0;
+ rhs_part[idx_part*size_particle_rhs + IDX_Z] += pos_part[idx_part*size_particle_positions + 3+IDX_Z]*v0;
+ }
+}
+
+ // for given orientation and right-hand-side, recompute right-hand-side such
+ // that it is perpendicular to the current orientation.
+ // this is the job of the Lagrange multiplier terms, hence the
+ // "add_Lagrange_multipliers" name of the method.
+template <class real_number, class partsize_t>
+template <int size_particle_positions, int size_particle_rhs>
+void particles_inner_computer<real_number, partsize_t>::add_Lagrange_multipliers(
+ const partsize_t nb_particles,
+ const real_number pos_part[],
+ real_number rhs_part[]) const{
+ static_assert(size_particle_positions == 6, "This kernel works only with 6 values for one particle's position");
+ static_assert(size_particle_rhs == 6, "This kernel works only with 6 values per particle's rhs");
+
+ #pragma omp parallel for
+ for(partsize_t idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
+ const partsize_t idx0 = idx_part*size_particle_positions + 3;
+ const partsize_t idx1 = idx_part*size_particle_rhs + 3;
+ // check that orientation is unit vector:
+ real_number orientation_size = sqrt(
+ pos_part[idx0+IDX_X]*pos_part[idx0+IDX_X] +
+ pos_part[idx0+IDX_Y]*pos_part[idx0+IDX_Y] +
+ pos_part[idx0+IDX_Z]*pos_part[idx0+IDX_Z]);
+ variable_used_only_in_assert(orientation_size);
+ assert(orientation_size > 0.99);
+ assert(orientation_size < 1.01);
+ // I call "rotation" to be the right hand side of the orientation part of the ODE
+ // project rotation on orientation:
+ real_number projection = (
+ pos_part[idx0+IDX_X]*rhs_part[idx1+IDX_X] +
+ pos_part[idx0+IDX_Y]*rhs_part[idx1+IDX_Y] +
+ pos_part[idx0+IDX_Z]*rhs_part[idx1+IDX_Z]);
+
+ // now remove parallel bit.
+ rhs_part[idx1+IDX_X] -= pos_part[idx0+IDX_X]*projection;
+ rhs_part[idx1+IDX_Y] -= pos_part[idx0+IDX_Y]*projection;
+ rhs_part[idx1+IDX_Z] -= pos_part[idx0+IDX_Z]*projection;
+
+ // DEBUG
+ // sanity check, for debugging purposes
+ // compute dot product between orientation and orientation change
+ //real_number dotproduct = (
+ // rhs_part[idx1 + IDX_X]*pos_part[idx0 + IDX_X] +
+ // rhs_part[idx1 + IDX_Y]*pos_part[idx0 + IDX_Y] +
+ // rhs_part[idx1 + IDX_Z]*pos_part[idx0 + IDX_Z]);
+ //if (dotproduct > 0.1)
+ //{
+ // DEBUG_MSG("dotproduct = %g, projection = %g\n"
+ // "pos_part[%d] = %g, pos_part[%d] = %g, pos_part[%d] = %g\n"
+ // "rhs_part[%d] = %g, rhs_part[%d] = %g, rhs_part[%d] = %g\n",
+ // dotproduct,
+ // projection,
+ // IDX_X, pos_part[idx0 + IDX_X],
+ // IDX_Y, pos_part[idx0 + IDX_Y],
+ // IDX_Z, pos_part[idx0 + IDX_Z],
+ // IDX_X, rhs_part[idx1 + IDX_X],
+ // IDX_Y, rhs_part[idx1 + IDX_Y],
+ // IDX_Z, rhs_part[idx1 + IDX_Z]);
+ // assert(false);
+ //}
+ //assert(dotproduct <= 0.1);
+ }
+ }
+
+template <class real_number, class partsize_t>
+template <int size_particle_positions, int size_particle_rhs, int size_particle_rhs_extra>
+void particles_inner_computer<real_number, partsize_t>::compute_interaction_with_extra(
+ const partsize_t nb_particles,
+ const real_number pos_part[],
+ real_number rhs_part[],
+ const real_number rhs_part_extra[]) const{
+ static_assert(size_particle_rhs_extra == 3, "This kernel works only with 3 values for one particle's rhs extra");
+
+ // call plain compute_interaction first
+ compute_interaction<size_particle_positions, size_particle_rhs>(nb_particles, pos_part, rhs_part);
+
+ // now add vorticity term
+ #pragma omp parallel for
+ for(partsize_t idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
+ // Cross product vorticity/orientation
+ rhs_part[idx_part*size_particle_rhs + 3+IDX_X] += 0.5*(rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Y]*pos_part[idx_part*size_particle_positions + 3+IDX_Z] -
+ rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Z]*pos_part[idx_part*size_particle_positions + 3+IDX_Y]);
+ rhs_part[idx_part*size_particle_rhs + 3+IDX_Y] += 0.5*(rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Z]*pos_part[idx_part*size_particle_positions + 3+IDX_X] -
+ rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_X]*pos_part[idx_part*size_particle_positions + 3+IDX_Z]);
+ rhs_part[idx_part*size_particle_rhs + 3+IDX_Z] += 0.5*(rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_X]*pos_part[idx_part*size_particle_positions + 3+IDX_Y] -
+ rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Y]*pos_part[idx_part*size_particle_positions + 3+IDX_X]);
+ }
+}
+
+// meant to be called AFTER executing the time-stepping operation.
+// once the particles have been moved, ensure that the orientation is a unit vector.
+template <class real_number, class partsize_t>
+template <int size_particle_positions>
+void particles_inner_computer<real_number, partsize_t>::enforce_unit_orientation(
+ const partsize_t nb_particles,
+ real_number pos_part[]) const{
+ static_assert(size_particle_positions == 6, "This kernel works only with 6 values for one particle's position");
+
+ #pragma omp parallel for
+ for(partsize_t idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
+ const partsize_t idx0 = idx_part*size_particle_positions + 3;
+ // compute orientation size:
+ real_number orientation_size = sqrt(
+ pos_part[idx0+IDX_X]*pos_part[idx0+IDX_X] +
+ pos_part[idx0+IDX_Y]*pos_part[idx0+IDX_Y] +
+ pos_part[idx0+IDX_Z]*pos_part[idx0+IDX_Z]);
+ // now renormalize
+ pos_part[idx0 + IDX_X] /= orientation_size;
+ pos_part[idx0 + IDX_Y] /= orientation_size;
+ pos_part[idx0 + IDX_Z] /= orientation_size;
+ }
+}
+
+template class particles_inner_computer<double, long long>;
+
+template void particles_inner_computer<double, long long>::compute_interaction<6, 6>(
+ const long long nb_particles,
+ const double pos_part[],
+ double rhs_part[]) const;
+template void particles_inner_computer<double, long long>::add_Lagrange_multipliers<6, 6>(
+ const long long nb_particles,
+ const double pos_part[],
+ double rhs_part[]) const;
+template void particles_inner_computer<double, long long>::compute_interaction_with_extra <6, 6, 3>(
+ const long long nb_particles,
+ const double pos_part[],
+ double rhs_part[],
+ const double rhs_part_extra[]) const;
+template void particles_inner_computer<double, long long>:: enforce_unit_orientation<6>(
+ const long long nb_particles,
+ double pos_part[]) const;
+
diff --git a/bfps/cpp/particles/particles_inner_computer.hpp b/bfps/cpp/particles/particles_inner_computer.hpp
index b2eb95dd..3e233370 100644
--- a/bfps/cpp/particles/particles_inner_computer.hpp
+++ b/bfps/cpp/particles/particles_inner_computer.hpp
@@ -15,120 +15,31 @@ public:
}
template <int size_particle_positions, int size_particle_rhs>
- void compute_interaction(const partsize_t nb_particles, const real_number pos_part[], real_number rhs_part[]) const{
- static_assert(size_particle_positions == 6, "This kernel works only with 6 values for one particle's position");
- static_assert(size_particle_rhs == 6, "This kernel works only with 6 values per particle's rhs");
-
- #pragma omp parallel for
- for(partsize_t idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
- // Add attr × V0 to the field interpolation
- rhs_part[idx_part*size_particle_rhs + IDX_X] += pos_part[idx_part*size_particle_positions + 3+IDX_X]*v0;
- rhs_part[idx_part*size_particle_rhs + IDX_Y] += pos_part[idx_part*size_particle_positions + 3+IDX_Y]*v0;
- rhs_part[idx_part*size_particle_rhs + IDX_Z] += pos_part[idx_part*size_particle_positions + 3+IDX_Z]*v0;
- }
- }
-
+ void compute_interaction(
+ const partsize_t nb_particles,
+ const real_number pos_part[],
+ real_number rhs_part[]) const;
// for given orientation and right-hand-side, recompute right-hand-side such
// that it is perpendicular to the current orientation.
// this is the job of the Lagrange multiplier terms, hence the
// "add_Lagrange_multipliers" name of the method.
template <int size_particle_positions, int size_particle_rhs>
- void add_Lagrange_multipliers(const partsize_t nb_particles, const real_number pos_part[], real_number rhs_part[]) const{
- static_assert(size_particle_positions == 6, "This kernel works only with 6 values for one particle's position");
- static_assert(size_particle_rhs == 6, "This kernel works only with 6 values per particle's rhs");
-
- #pragma omp parallel for
- for(partsize_t idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
- const partsize_t idx0 = idx_part*size_particle_positions + 3;
- const partsize_t idx1 = idx_part*size_particle_rhs + 3;
- // check that orientation is unit vector:
- real_number orientation_size = sqrt(
- pos_part[idx0+IDX_X]*pos_part[idx0+IDX_X] +
- pos_part[idx0+IDX_Y]*pos_part[idx0+IDX_Y] +
- pos_part[idx0+IDX_Z]*pos_part[idx0+IDX_Z]);
- variable_used_only_in_assert(orientation_size);
- assert(orientation_size > 0.99);
- assert(orientation_size < 1.01);
- // I call "rotation" to be the right hand side of the orientation part of the ODE
- // project rotation on orientation:
- real_number projection = (
- pos_part[idx0+IDX_X]*rhs_part[idx1+IDX_X] +
- pos_part[idx0+IDX_Y]*rhs_part[idx1+IDX_Y] +
- pos_part[idx0+IDX_Z]*rhs_part[idx1+IDX_Z]);
-
- // now remove parallel bit.
- rhs_part[idx1+IDX_X] -= pos_part[idx0+IDX_X]*projection;
- rhs_part[idx1+IDX_Y] -= pos_part[idx0+IDX_Y]*projection;
- rhs_part[idx1+IDX_Z] -= pos_part[idx0+IDX_Z]*projection;
-
- // DEBUG
- // sanity check, for debugging purposes
- // compute dot product between orientation and orientation change
- //real_number dotproduct = (
- // rhs_part[idx1 + IDX_X]*pos_part[idx0 + IDX_X] +
- // rhs_part[idx1 + IDX_Y]*pos_part[idx0 + IDX_Y] +
- // rhs_part[idx1 + IDX_Z]*pos_part[idx0 + IDX_Z]);
- //if (dotproduct > 0.1)
- //{
- // DEBUG_MSG("dotproduct = %g, projection = %g\n"
- // "pos_part[%d] = %g, pos_part[%d] = %g, pos_part[%d] = %g\n"
- // "rhs_part[%d] = %g, rhs_part[%d] = %g, rhs_part[%d] = %g\n",
- // dotproduct,
- // projection,
- // IDX_X, pos_part[idx0 + IDX_X],
- // IDX_Y, pos_part[idx0 + IDX_Y],
- // IDX_Z, pos_part[idx0 + IDX_Z],
- // IDX_X, rhs_part[idx1 + IDX_X],
- // IDX_Y, rhs_part[idx1 + IDX_Y],
- // IDX_Z, rhs_part[idx1 + IDX_Z]);
- // assert(false);
- //}
- //assert(dotproduct <= 0.1);
- }
- }
-
+ void add_Lagrange_multipliers(
+ const partsize_t nb_particles,
+ const real_number pos_part[],
+ real_number rhs_part[]) const;
template <int size_particle_positions, int size_particle_rhs, int size_particle_rhs_extra>
- void compute_interaction_with_extra(const partsize_t nb_particles, const real_number pos_part[], real_number rhs_part[],
- const real_number rhs_part_extra[]) const{
- static_assert(size_particle_rhs_extra == 3, "This kernel works only with 3 values for one particle's rhs extra");
-
- // call plain compute_interaction first
- compute_interaction<size_particle_positions, size_particle_rhs>(nb_particles, pos_part, rhs_part);
-
- // now add vorticity term
- #pragma omp parallel for
- for(partsize_t idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
- // Cross product vorticity/orientation
- rhs_part[idx_part*size_particle_rhs + 3+IDX_X] += 0.5*(rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Y]*pos_part[idx_part*size_particle_positions + 3+IDX_Z] -
- rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Z]*pos_part[idx_part*size_particle_positions + 3+IDX_Y]);
- rhs_part[idx_part*size_particle_rhs + 3+IDX_Y] += 0.5*(rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Z]*pos_part[idx_part*size_particle_positions + 3+IDX_X] -
- rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_X]*pos_part[idx_part*size_particle_positions + 3+IDX_Z]);
- rhs_part[idx_part*size_particle_rhs + 3+IDX_Z] += 0.5*(rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_X]*pos_part[idx_part*size_particle_positions + 3+IDX_Y] -
- rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Y]*pos_part[idx_part*size_particle_positions + 3+IDX_X]);
- }
- }
-
+ void compute_interaction_with_extra(
+ const partsize_t nb_particles,
+ const real_number pos_part[],
+ real_number rhs_part[],
+ const real_number rhs_part_extra[]) const;
// meant to be called AFTER executing the time-stepping operation.
// once the particles have been moved, ensure that the orientation is a unit vector.
template <int size_particle_positions>
- void enforce_unit_orientation(const partsize_t nb_particles, real_number pos_part[]) const{
- static_assert(size_particle_positions == 6, "This kernel works only with 6 values for one particle's position");
-
- #pragma omp parallel for
- for(partsize_t idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
- const partsize_t idx0 = idx_part*size_particle_positions + 3;
- // compute orientation size:
- real_number orientation_size = sqrt(
- pos_part[idx0+IDX_X]*pos_part[idx0+IDX_X] +
- pos_part[idx0+IDX_Y]*pos_part[idx0+IDX_Y] +
- pos_part[idx0+IDX_Z]*pos_part[idx0+IDX_Z]);
- // now renormalize
- pos_part[idx0 + IDX_X] /= orientation_size;
- pos_part[idx0 + IDX_Y] /= orientation_size;
- pos_part[idx0 + IDX_Z] /= orientation_size;
- }
- }
-
+ void enforce_unit_orientation(
+ const partsize_t nb_particles,
+ real_number pos_part[]) const;
bool isEnable() const {
return isActive;
diff --git a/setup.py b/setup.py
index 3094c692..34f63082 100644
--- a/setup.py
+++ b/setup.py
@@ -126,7 +126,8 @@ src_file_list = [
'full_code/test_interpolation',
'full_code/NSVEparticles',
'full_code/NSVEcomplex_particles',
- 'full_code/NSVEp_extra_sampling']
+ 'full_code/NSVEp_extra_sampling',
+ 'particles/particles_inner_computer']
particle_headers = [
'cpp/particles/abstract_particles_input.hpp',
@@ -144,7 +145,7 @@ particle_headers = [
'cpp/particles/particles_field_computer.hpp',
'cpp/particles/particles_generic_interp.hpp',
'cpp/particles/particles_inner_computer_empty.hpp',
- 'cpp/particles/particles_inner_computer.hpp',
+ #'cpp/particles/particles_inner_computer.hpp',
'cpp/particles/particles_input_hdf5.hpp',
'cpp/particles/particles_output_hdf5.hpp',
'cpp/particles/particles_output_mpiio.hpp',
@@ -210,6 +211,9 @@ class CompileLibCommand(distutils.cmd.Command):
if not os.path.isdir('obj/full_code'):
os.makedirs('obj/full_code')
need_to_compile = True
+ if not os.path.isdir('obj/particles'):
+ os.makedirs('obj/particles')
+ need_to_compile = True
if not os.path.isfile('bfps/libbfps.a'):
need_to_compile = True
else:
--
GitLab
From 6e37d91cfed994485e2ce5ae9c86f98d8944b595 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 25 Sep 2018 13:54:43 +0200
Subject: [PATCH 241/342] update fftw rigor for test
---
bfps/test/test_bfps_NSVEparticles.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/bfps/test/test_bfps_NSVEparticles.py b/bfps/test/test_bfps_NSVEparticles.py
index 7ceab8e5..f914ad7d 100644
--- a/bfps/test/test_bfps_NSVEparticles.py
+++ b/bfps/test/test_bfps_NSVEparticles.py
@@ -24,6 +24,7 @@ def main():
'--simname', 'dns_nsveparticles',
'--np', '4',
'--ntpp', '1',
+ '--fftw_plan_rigor', 'FFTW_PATIENT',
'--niter_todo', '{0}'.format(niterations),
'--niter_out', '{0}'.format(niterations),
'--niter_stat', '1',
--
GitLab
From 131642869a3cdcdf11a6b0a7f1533a18e3647299 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 25 Sep 2018 21:56:14 +0200
Subject: [PATCH 242/342] specialize particle inner computer
---
.../particles/particles_inner_computer.cpp | 96 +++++++------------
1 file changed, 36 insertions(+), 60 deletions(-)
diff --git a/bfps/cpp/particles/particles_inner_computer.cpp b/bfps/cpp/particles/particles_inner_computer.cpp
index 85286190..d68f3de6 100644
--- a/bfps/cpp/particles/particles_inner_computer.cpp
+++ b/bfps/cpp/particles/particles_inner_computer.cpp
@@ -26,21 +26,19 @@ void particles_inner_computer<real_number, partsize_t>::compute_interaction(
// that it is perpendicular to the current orientation.
// this is the job of the Lagrange multiplier terms, hence the
// "add_Lagrange_multipliers" name of the method.
-template <class real_number, class partsize_t>
-template <int size_particle_positions, int size_particle_rhs>
-void particles_inner_computer<real_number, partsize_t>::add_Lagrange_multipliers(
- const partsize_t nb_particles,
- const real_number pos_part[],
- real_number rhs_part[]) const{
- static_assert(size_particle_positions == 6, "This kernel works only with 6 values for one particle's position");
- static_assert(size_particle_rhs == 6, "This kernel works only with 6 values per particle's rhs");
+template <>
+template <>
+void particles_inner_computer<double, long long>::add_Lagrange_multipliers<6,6>(
+ const long long nb_particles,
+ const double pos_part[],
+ double rhs_part[]) const{
#pragma omp parallel for
- for(partsize_t idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
- const partsize_t idx0 = idx_part*size_particle_positions + 3;
- const partsize_t idx1 = idx_part*size_particle_rhs + 3;
+ for(long long idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
+ const long long idx0 = idx_part*6 + 3;
+ const long long idx1 = idx_part*6 + 3;
// check that orientation is unit vector:
- real_number orientation_size = sqrt(
+ double orientation_size = sqrt(
pos_part[idx0+IDX_X]*pos_part[idx0+IDX_X] +
pos_part[idx0+IDX_Y]*pos_part[idx0+IDX_Y] +
pos_part[idx0+IDX_Z]*pos_part[idx0+IDX_Z]);
@@ -49,7 +47,7 @@ void particles_inner_computer<real_number, partsize_t>::add_Lagrange_multipliers
assert(orientation_size < 1.01);
// I call "rotation" to be the right hand side of the orientation part of the ODE
// project rotation on orientation:
- real_number projection = (
+ double projection = (
pos_part[idx0+IDX_X]*rhs_part[idx1+IDX_X] +
pos_part[idx0+IDX_Y]*rhs_part[idx1+IDX_Y] +
pos_part[idx0+IDX_Z]*rhs_part[idx1+IDX_Z]);
@@ -62,7 +60,7 @@ void particles_inner_computer<real_number, partsize_t>::add_Lagrange_multipliers
// DEBUG
// sanity check, for debugging purposes
// compute dot product between orientation and orientation change
- //real_number dotproduct = (
+ //double dotproduct = (
// rhs_part[idx1 + IDX_X]*pos_part[idx0 + IDX_X] +
// rhs_part[idx1 + IDX_Y]*pos_part[idx0 + IDX_Y] +
// rhs_part[idx1 + IDX_Z]*pos_part[idx0 + IDX_Z]);
@@ -85,45 +83,42 @@ void particles_inner_computer<real_number, partsize_t>::add_Lagrange_multipliers
}
}
-template <class real_number, class partsize_t>
-template <int size_particle_positions, int size_particle_rhs, int size_particle_rhs_extra>
-void particles_inner_computer<real_number, partsize_t>::compute_interaction_with_extra(
- const partsize_t nb_particles,
- const real_number pos_part[],
- real_number rhs_part[],
- const real_number rhs_part_extra[]) const{
- static_assert(size_particle_rhs_extra == 3, "This kernel works only with 3 values for one particle's rhs extra");
-
+template <>
+template <>
+void particles_inner_computer<double, long long>::compute_interaction_with_extra<6,6,3>(
+ const long long nb_particles,
+ const double pos_part[],
+ double rhs_part[],
+ const double rhs_part_extra[]) const{
// call plain compute_interaction first
- compute_interaction<size_particle_positions, size_particle_rhs>(nb_particles, pos_part, rhs_part);
+ compute_interaction<6, 6>(nb_particles, pos_part, rhs_part);
// now add vorticity term
#pragma omp parallel for
- for(partsize_t idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
+ for(long long idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
// Cross product vorticity/orientation
- rhs_part[idx_part*size_particle_rhs + 3+IDX_X] += 0.5*(rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Y]*pos_part[idx_part*size_particle_positions + 3+IDX_Z] -
- rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Z]*pos_part[idx_part*size_particle_positions + 3+IDX_Y]);
- rhs_part[idx_part*size_particle_rhs + 3+IDX_Y] += 0.5*(rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Z]*pos_part[idx_part*size_particle_positions + 3+IDX_X] -
- rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_X]*pos_part[idx_part*size_particle_positions + 3+IDX_Z]);
- rhs_part[idx_part*size_particle_rhs + 3+IDX_Z] += 0.5*(rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_X]*pos_part[idx_part*size_particle_positions + 3+IDX_Y] -
- rhs_part_extra[idx_part*size_particle_rhs_extra + IDX_Y]*pos_part[idx_part*size_particle_positions + 3+IDX_X]);
+ rhs_part[idx_part*6 + 3+IDX_X] += 0.5*(rhs_part_extra[idx_part*3 + IDX_Y]*pos_part[idx_part*6 + 3+IDX_Z] -
+ rhs_part_extra[idx_part*3 + IDX_Z]*pos_part[idx_part*6 + 3+IDX_Y]);
+ rhs_part[idx_part*6 + 3+IDX_Y] += 0.5*(rhs_part_extra[idx_part*3 + IDX_Z]*pos_part[idx_part*6 + 3+IDX_X] -
+ rhs_part_extra[idx_part*3 + IDX_X]*pos_part[idx_part*6 + 3+IDX_Z]);
+ rhs_part[idx_part*6 + 3+IDX_Z] += 0.5*(rhs_part_extra[idx_part*3 + IDX_X]*pos_part[idx_part*6 + 3+IDX_Y] -
+ rhs_part_extra[idx_part*3 + IDX_Y]*pos_part[idx_part*6 + 3+IDX_X]);
}
}
+
// meant to be called AFTER executing the time-stepping operation.
// once the particles have been moved, ensure that the orientation is a unit vector.
-template <class real_number, class partsize_t>
-template <int size_particle_positions>
-void particles_inner_computer<real_number, partsize_t>::enforce_unit_orientation(
- const partsize_t nb_particles,
- real_number pos_part[]) const{
- static_assert(size_particle_positions == 6, "This kernel works only with 6 values for one particle's position");
-
+template <>
+template <>
+void particles_inner_computer<double, long long>::enforce_unit_orientation<6>(
+ const long long nb_particles,
+ double pos_part[]) const{
#pragma omp parallel for
- for(partsize_t idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
- const partsize_t idx0 = idx_part*size_particle_positions + 3;
+ for(long long idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
+ const long long idx0 = idx_part*6 + 3;
// compute orientation size:
- real_number orientation_size = sqrt(
+ double orientation_size = sqrt(
pos_part[idx0+IDX_X]*pos_part[idx0+IDX_X] +
pos_part[idx0+IDX_Y]*pos_part[idx0+IDX_Y] +
pos_part[idx0+IDX_Z]*pos_part[idx0+IDX_Z]);
@@ -134,22 +129,3 @@ void particles_inner_computer<real_number, partsize_t>::enforce_unit_orientation
}
}
-template class particles_inner_computer<double, long long>;
-
-template void particles_inner_computer<double, long long>::compute_interaction<6, 6>(
- const long long nb_particles,
- const double pos_part[],
- double rhs_part[]) const;
-template void particles_inner_computer<double, long long>::add_Lagrange_multipliers<6, 6>(
- const long long nb_particles,
- const double pos_part[],
- double rhs_part[]) const;
-template void particles_inner_computer<double, long long>::compute_interaction_with_extra <6, 6, 3>(
- const long long nb_particles,
- const double pos_part[],
- double rhs_part[],
- const double rhs_part_extra[]) const;
-template void particles_inner_computer<double, long long>:: enforce_unit_orientation<6>(
- const long long nb_particles,
- double pos_part[]) const;
-
--
GitLab
From 94f89084947730dd1c738398060ee0b20ba5a348 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 26 Sep 2018 13:42:24 +0200
Subject: [PATCH 243/342] add dumb version of ellipsoid inner computer
---
.../particles/particles_inner_computer.cpp | 89 ++++++++++++-------
1 file changed, 56 insertions(+), 33 deletions(-)
diff --git a/bfps/cpp/particles/particles_inner_computer.cpp b/bfps/cpp/particles/particles_inner_computer.cpp
index d68f3de6..f5967087 100644
--- a/bfps/cpp/particles/particles_inner_computer.cpp
+++ b/bfps/cpp/particles/particles_inner_computer.cpp
@@ -16,9 +16,9 @@ void particles_inner_computer<real_number, partsize_t>::compute_interaction(
#pragma omp parallel for
for(partsize_t idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
// Add attr × V0 to the field interpolation
- rhs_part[idx_part*size_particle_rhs + IDX_X] += pos_part[idx_part*size_particle_positions + 3+IDX_X]*v0;
- rhs_part[idx_part*size_particle_rhs + IDX_Y] += pos_part[idx_part*size_particle_positions + 3+IDX_Y]*v0;
- rhs_part[idx_part*size_particle_rhs + IDX_Z] += pos_part[idx_part*size_particle_positions + 3+IDX_Z]*v0;
+ rhs_part[idx_part*size_particle_rhs + IDXC_X] += pos_part[idx_part*size_particle_positions + 3+IDXC_X]*v0;
+ rhs_part[idx_part*size_particle_rhs + IDXC_Y] += pos_part[idx_part*size_particle_positions + 3+IDXC_Y]*v0;
+ rhs_part[idx_part*size_particle_rhs + IDXC_Z] += pos_part[idx_part*size_particle_positions + 3+IDXC_Z]*v0;
}
}
@@ -39,31 +39,31 @@ void particles_inner_computer<double, long long>::add_Lagrange_multipliers<6,6>(
const long long idx1 = idx_part*6 + 3;
// check that orientation is unit vector:
double orientation_size = sqrt(
- pos_part[idx0+IDX_X]*pos_part[idx0+IDX_X] +
- pos_part[idx0+IDX_Y]*pos_part[idx0+IDX_Y] +
- pos_part[idx0+IDX_Z]*pos_part[idx0+IDX_Z]);
+ pos_part[idx0+IDXC_X]*pos_part[idx0+IDXC_X] +
+ pos_part[idx0+IDXC_Y]*pos_part[idx0+IDXC_Y] +
+ pos_part[idx0+IDXC_Z]*pos_part[idx0+IDXC_Z]);
variable_used_only_in_assert(orientation_size);
assert(orientation_size > 0.99);
assert(orientation_size < 1.01);
// I call "rotation" to be the right hand side of the orientation part of the ODE
// project rotation on orientation:
double projection = (
- pos_part[idx0+IDX_X]*rhs_part[idx1+IDX_X] +
- pos_part[idx0+IDX_Y]*rhs_part[idx1+IDX_Y] +
- pos_part[idx0+IDX_Z]*rhs_part[idx1+IDX_Z]);
+ pos_part[idx0+IDXC_X]*rhs_part[idx1+IDXC_X] +
+ pos_part[idx0+IDXC_Y]*rhs_part[idx1+IDXC_Y] +
+ pos_part[idx0+IDXC_Z]*rhs_part[idx1+IDXC_Z]);
// now remove parallel bit.
- rhs_part[idx1+IDX_X] -= pos_part[idx0+IDX_X]*projection;
- rhs_part[idx1+IDX_Y] -= pos_part[idx0+IDX_Y]*projection;
- rhs_part[idx1+IDX_Z] -= pos_part[idx0+IDX_Z]*projection;
+ rhs_part[idx1+IDXC_X] -= pos_part[idx0+IDXC_X]*projection;
+ rhs_part[idx1+IDXC_Y] -= pos_part[idx0+IDXC_Y]*projection;
+ rhs_part[idx1+IDXC_Z] -= pos_part[idx0+IDXC_Z]*projection;
// DEBUG
// sanity check, for debugging purposes
// compute dot product between orientation and orientation change
//double dotproduct = (
- // rhs_part[idx1 + IDX_X]*pos_part[idx0 + IDX_X] +
- // rhs_part[idx1 + IDX_Y]*pos_part[idx0 + IDX_Y] +
- // rhs_part[idx1 + IDX_Z]*pos_part[idx0 + IDX_Z]);
+ // rhs_part[idx1 + IDXC_X]*pos_part[idx0 + IDXC_X] +
+ // rhs_part[idx1 + IDXC_Y]*pos_part[idx0 + IDXC_Y] +
+ // rhs_part[idx1 + IDXC_Z]*pos_part[idx0 + IDXC_Z]);
//if (dotproduct > 0.1)
//{
// DEBUG_MSG("dotproduct = %g, projection = %g\n"
@@ -71,12 +71,12 @@ void particles_inner_computer<double, long long>::add_Lagrange_multipliers<6,6>(
// "rhs_part[%d] = %g, rhs_part[%d] = %g, rhs_part[%d] = %g\n",
// dotproduct,
// projection,
- // IDX_X, pos_part[idx0 + IDX_X],
- // IDX_Y, pos_part[idx0 + IDX_Y],
- // IDX_Z, pos_part[idx0 + IDX_Z],
- // IDX_X, rhs_part[idx1 + IDX_X],
- // IDX_Y, rhs_part[idx1 + IDX_Y],
- // IDX_Z, rhs_part[idx1 + IDX_Z]);
+ // IDXC_X, pos_part[idx0 + IDXC_X],
+ // IDXC_Y, pos_part[idx0 + IDXC_Y],
+ // IDXC_Z, pos_part[idx0 + IDXC_Z],
+ // IDXC_X, rhs_part[idx1 + IDXC_X],
+ // IDXC_Y, rhs_part[idx1 + IDXC_Y],
+ // IDXC_Z, rhs_part[idx1 + IDXC_Z]);
// assert(false);
//}
//assert(dotproduct <= 0.1);
@@ -97,12 +97,35 @@ void particles_inner_computer<double, long long>::compute_interaction_with_extra
#pragma omp parallel for
for(long long idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
// Cross product vorticity/orientation
- rhs_part[idx_part*6 + 3+IDX_X] += 0.5*(rhs_part_extra[idx_part*3 + IDX_Y]*pos_part[idx_part*6 + 3+IDX_Z] -
- rhs_part_extra[idx_part*3 + IDX_Z]*pos_part[idx_part*6 + 3+IDX_Y]);
- rhs_part[idx_part*6 + 3+IDX_Y] += 0.5*(rhs_part_extra[idx_part*3 + IDX_Z]*pos_part[idx_part*6 + 3+IDX_X] -
- rhs_part_extra[idx_part*3 + IDX_X]*pos_part[idx_part*6 + 3+IDX_Z]);
- rhs_part[idx_part*6 + 3+IDX_Z] += 0.5*(rhs_part_extra[idx_part*3 + IDX_X]*pos_part[idx_part*6 + 3+IDX_Y] -
- rhs_part_extra[idx_part*3 + IDX_Y]*pos_part[idx_part*6 + 3+IDX_X]);
+ rhs_part[idx_part*6 + 3+IDXC_X] += 0.5*(rhs_part_extra[idx_part*3 + IDXC_Y]*pos_part[idx_part*6 + 3+IDXC_Z] -
+ rhs_part_extra[idx_part*3 + IDXC_Z]*pos_part[idx_part*6 + 3+IDXC_Y]);
+ rhs_part[idx_part*6 + 3+IDXC_Y] += 0.5*(rhs_part_extra[idx_part*3 + IDXC_Z]*pos_part[idx_part*6 + 3+IDXC_X] -
+ rhs_part_extra[idx_part*3 + IDXC_X]*pos_part[idx_part*6 + 3+IDXC_Z]);
+ rhs_part[idx_part*6 + 3+IDXC_Z] += 0.5*(rhs_part_extra[idx_part*3 + IDXC_X]*pos_part[idx_part*6 + 3+IDXC_Y] -
+ rhs_part_extra[idx_part*3 + IDXC_Y]*pos_part[idx_part*6 + 3+IDXC_X]);
+ }
+}
+
+template <>
+template <>
+void particles_inner_computer<double, long long>::compute_interaction_with_extra<6,6,9>(
+ const long long nb_particles,
+ const double pos_part[],
+ double rhs_part[],
+ const double rhs_part_extra[]) const{
+ // call plain compute_interaction first
+ compute_interaction<6, 6>(nb_particles, pos_part, rhs_part);
+
+ // now add vorticity term
+ #pragma omp parallel for
+ for(long long idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
+ // Cross product vorticity/orientation
+ rhs_part[idx_part*6 + 3+IDXC_X] += 0.5*(rhs_part_extra[idx_part*9 + IDXC_Y]*pos_part[idx_part*6 + 3+IDXC_Z] -
+ rhs_part_extra[idx_part*9 + IDXC_Z]*pos_part[idx_part*6 + 3+IDXC_Y]);
+ rhs_part[idx_part*6 + 3+IDXC_Y] += 0.5*(rhs_part_extra[idx_part*9 + IDXC_Z]*pos_part[idx_part*6 + 3+IDXC_X] -
+ rhs_part_extra[idx_part*9 + IDXC_X]*pos_part[idx_part*6 + 3+IDXC_Z]);
+ rhs_part[idx_part*6 + 3+IDXC_Z] += 0.5*(rhs_part_extra[idx_part*9 + IDXC_X]*pos_part[idx_part*6 + 3+IDXC_Y] -
+ rhs_part_extra[idx_part*9 + IDXC_Y]*pos_part[idx_part*6 + 3+IDXC_X]);
}
}
@@ -119,13 +142,13 @@ void particles_inner_computer<double, long long>::enforce_unit_orientation<6>(
const long long idx0 = idx_part*6 + 3;
// compute orientation size:
double orientation_size = sqrt(
- pos_part[idx0+IDX_X]*pos_part[idx0+IDX_X] +
- pos_part[idx0+IDX_Y]*pos_part[idx0+IDX_Y] +
- pos_part[idx0+IDX_Z]*pos_part[idx0+IDX_Z]);
+ pos_part[idx0+IDXC_X]*pos_part[idx0+IDXC_X] +
+ pos_part[idx0+IDXC_Y]*pos_part[idx0+IDXC_Y] +
+ pos_part[idx0+IDXC_Z]*pos_part[idx0+IDXC_Z]);
// now renormalize
- pos_part[idx0 + IDX_X] /= orientation_size;
- pos_part[idx0 + IDX_Y] /= orientation_size;
- pos_part[idx0 + IDX_Z] /= orientation_size;
+ pos_part[idx0 + IDXC_X] /= orientation_size;
+ pos_part[idx0 + IDXC_Y] /= orientation_size;
+ pos_part[idx0 + IDXC_Z] /= orientation_size;
}
}
--
GitLab
From 906f54b2140e42c5c9e355adff15cf2d42c8e8e1 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 26 Sep 2018 13:42:41 +0200
Subject: [PATCH 244/342] rename IDX_X etc to IDXC_X etc
I think it's more obvious when we talk about "index of component" and
"index of variable" what is meant, rather than having "FIELD_IDX" and
plain "IDX".
---
bfps/cpp/particles/p2p_computer.hpp | 18 +--
bfps/cpp/particles/p2p_distr_mpi.hpp | 128 +++++++++---------
bfps/cpp/particles/p2p_tree.hpp | 34 ++---
bfps/cpp/particles/particles_distr_mpi.hpp | 28 ++--
.../particles/particles_field_computer.hpp | 50 +++----
bfps/cpp/particles/particles_input_hdf5.hpp | 2 +-
bfps/cpp/particles/particles_system.hpp | 14 +-
.../particles/particles_system_builder.hpp | 70 +++++-----
bfps/cpp/particles/particles_utils.hpp | 32 +++--
9 files changed, 194 insertions(+), 182 deletions(-)
diff --git a/bfps/cpp/particles/p2p_computer.hpp b/bfps/cpp/particles/p2p_computer.hpp
index 46328c3a..922d65d1 100644
--- a/bfps/cpp/particles/p2p_computer.hpp
+++ b/bfps/cpp/particles/p2p_computer.hpp
@@ -62,15 +62,15 @@ public:
/// (4 / \tau) \sum_j W_\ell ( | x^i - x^j | ) (p^i \cdot p^j)p^j
/// \f]
///
- const double dot_product = (pos_part1[3+IDX_X]*pos_part2[3+IDX_X] +
- pos_part1[3+IDX_Y]*pos_part2[3+IDX_Y] +
- pos_part1[3+IDX_Z]*pos_part2[3+IDX_Z]);
- rhs_part1[3+IDX_X] += pos_part2[3+IDX_X] * 4 * ww * dot_product;
- rhs_part1[3+IDX_Y] += pos_part2[3+IDX_Y] * 4 * ww * dot_product;
- rhs_part1[3+IDX_Z] += pos_part2[3+IDX_Z] * 4 * ww * dot_product;
- rhs_part2[3+IDX_X] += pos_part1[3+IDX_X] * 4 * ww * dot_product;
- rhs_part2[3+IDX_Y] += pos_part1[3+IDX_Y] * 4 * ww * dot_product;
- rhs_part2[3+IDX_Z] += pos_part1[3+IDX_Z] * 4 * ww * dot_product;
+ const double dot_product = (pos_part1[3+IDXC_X]*pos_part2[3+IDXC_X] +
+ pos_part1[3+IDXC_Y]*pos_part2[3+IDXC_Y] +
+ pos_part1[3+IDXC_Z]*pos_part2[3+IDXC_Z]);
+ rhs_part1[3+IDXC_X] += pos_part2[3+IDXC_X] * 4 * ww * dot_product;
+ rhs_part1[3+IDXC_Y] += pos_part2[3+IDXC_Y] * 4 * ww * dot_product;
+ rhs_part1[3+IDXC_Z] += pos_part2[3+IDXC_Z] * 4 * ww * dot_product;
+ rhs_part2[3+IDXC_X] += pos_part1[3+IDXC_X] * 4 * ww * dot_product;
+ rhs_part2[3+IDXC_Y] += pos_part1[3+IDXC_Y] * 4 * ww * dot_product;
+ rhs_part2[3+IDXC_Z] += pos_part1[3+IDXC_Z] * 4 * ww * dot_product;
}
bool isEnable() const {
diff --git a/bfps/cpp/particles/p2p_distr_mpi.hpp b/bfps/cpp/particles/p2p_distr_mpi.hpp
index 740aba18..2758aecf 100644
--- a/bfps/cpp/particles/p2p_distr_mpi.hpp
+++ b/bfps/cpp/particles/p2p_distr_mpi.hpp
@@ -103,7 +103,7 @@ protected:
static int foundGridFactor(const real_number in_cutoff_radius, const std::array<real_number,3>& in_spatial_box_width){
int idx_factor = 1;
- while(in_cutoff_radius <= in_spatial_box_width[IDX_Z]/real_number(idx_factor+1)){
+ while(in_cutoff_radius <= in_spatial_box_width[IDXC_Z]/real_number(idx_factor+1)){
idx_factor += 1;
}
return idx_factor;
@@ -126,7 +126,7 @@ public:
spatial_box_width(in_spatial_box_width), spatial_box_offset(in_spatial_box_offset),
cutoff_radius_compute(in_cutoff_radius),
nb_cells_factor(foundGridFactor(in_cutoff_radius, in_spatial_box_width)),
- cutoff_radius(in_spatial_box_width[IDX_Z]/real_number(nb_cells_factor)){
+ cutoff_radius(in_spatial_box_width[IDXC_Z]/real_number(nb_cells_factor)){
AssertMpi(MPI_Comm_rank(current_com, &my_rank));
AssertMpi(MPI_Comm_size(current_com, &nb_processes));
@@ -154,11 +154,11 @@ public:
assert(partition_interval_size_per_proc[idx_proc_involved] != 0);
}
- assert(int(field_grid_dim[IDX_Z]) == partition_interval_offset_per_proc[nb_processes_involved]);
+ assert(int(field_grid_dim[IDXC_Z]) == partition_interval_offset_per_proc[nb_processes_involved]);
- nb_cell_levels[IDX_X] = nb_cells_factor;
- nb_cell_levels[IDX_Y] = nb_cells_factor;
- nb_cell_levels[IDX_Z] = nb_cells_factor;
+ nb_cell_levels[IDXC_X] = nb_cells_factor;
+ nb_cell_levels[IDXC_Y] = nb_cells_factor;
+ nb_cell_levels[IDXC_Z] = nb_cells_factor;
}
virtual ~p2p_distr_mpi(){}
@@ -174,25 +174,25 @@ public:
}
long int get_cell_coord_x_from_index(const long int index) const{
- return index % nb_cell_levels[IDX_X];
+ return index % nb_cell_levels[IDXC_X];
}
long int get_cell_coord_y_from_index(const long int index) const{
- return (index % (nb_cell_levels[IDX_X]*nb_cell_levels[IDX_Y]))
- / nb_cell_levels[IDX_X];
+ return (index % (nb_cell_levels[IDXC_X]*nb_cell_levels[IDXC_Y]))
+ / nb_cell_levels[IDXC_X];
}
long int get_cell_coord_z_from_index(const long int index) const{
- return index / (nb_cell_levels[IDX_X]*nb_cell_levels[IDX_Y]);
+ return index / (nb_cell_levels[IDXC_X]*nb_cell_levels[IDXC_Y]);
}
long int first_cell_level_proc(const int dest_proc) const{
- const real_number field_section_width_z = spatial_box_width[IDX_Z]/real_number(field_grid_dim[IDX_Z]);
+ const real_number field_section_width_z = spatial_box_width[IDXC_Z]/real_number(field_grid_dim[IDXC_Z]);
return static_cast<long int>((field_section_width_z*real_number(partition_interval_offset_per_proc[dest_proc]))/cutoff_radius);
}
long int last_cell_level_proc(const int dest_proc) const{
- const real_number field_section_width_z = spatial_box_width[IDX_Z]/real_number(field_grid_dim[IDX_Z]);
+ const real_number field_section_width_z = spatial_box_width[IDXC_Z]/real_number(field_grid_dim[IDXC_Z]);
const long int limite = static_cast<long int>((field_section_width_z*real_number(partition_interval_offset_per_proc[dest_proc+1])
- std::numeric_limits<real_number>::epsilon())/cutoff_radius);
if(static_cast<real_number>(limite)*cutoff_radius
@@ -202,7 +202,7 @@ public:
return limite;
}
- real_number apply_pbc(real_number pos, IDXS_3D dim) const{
+ real_number apply_pbc(real_number pos, COMPONENT_3D dim) const{
while( pos < spatial_box_offset[dim] ){
pos += spatial_box_width[dim];
}
@@ -214,32 +214,32 @@ public:
std::array<long int,3> get_cell_coordinate(const real_number pos_x, const real_number pos_y,
const real_number pos_z) const {
- const real_number diff_x = apply_pbc(pos_x,IDX_X) - spatial_box_offset[IDX_X];
- const real_number diff_y = apply_pbc(pos_y,IDX_Y) - spatial_box_offset[IDX_Y];
- const real_number diff_z = apply_pbc(pos_z,IDX_Z) - spatial_box_offset[IDX_Z];
+ const real_number diff_x = apply_pbc(pos_x,IDXC_X) - spatial_box_offset[IDXC_X];
+ const real_number diff_y = apply_pbc(pos_y,IDXC_Y) - spatial_box_offset[IDXC_Y];
+ const real_number diff_z = apply_pbc(pos_z,IDXC_Z) - spatial_box_offset[IDXC_Z];
std::array<long int,3> coord;
- coord[IDX_X] = static_cast<long int>(diff_x/cutoff_radius);
- coord[IDX_Y] = static_cast<long int>(diff_y/cutoff_radius);
- coord[IDX_Z] = static_cast<long int>(diff_z/cutoff_radius);
+ coord[IDXC_X] = static_cast<long int>(diff_x/cutoff_radius);
+ coord[IDXC_Y] = static_cast<long int>(diff_y/cutoff_radius);
+ coord[IDXC_Z] = static_cast<long int>(diff_z/cutoff_radius);
return coord;
}
long int get_cell_idx(const real_number pos_x, const real_number pos_y,
const real_number pos_z) const {
std::array<long int,3> coord = get_cell_coordinate(pos_x, pos_y, pos_z);
- return ((coord[IDX_Z]*nb_cell_levels[IDX_Y])+coord[IDX_Y])*nb_cell_levels[IDX_X]+coord[IDX_X];
+ return ((coord[IDXC_Z]*nb_cell_levels[IDXC_Y])+coord[IDXC_Y])*nb_cell_levels[IDXC_X]+coord[IDXC_X];
}
real_number compute_distance_r2(const real_number x1, const real_number y1, const real_number z1,
const real_number x2, const real_number y2, const real_number z2,
const real_number xshift_coef, const real_number yshift_coef, const real_number zshift_coef) const {
- real_number diff_x = std::abs(apply_pbc(x1,IDX_X)-apply_pbc(x2,IDX_X)+xshift_coef*spatial_box_width[IDX_X]);
+ real_number diff_x = std::abs(apply_pbc(x1,IDXC_X)-apply_pbc(x2,IDXC_X)+xshift_coef*spatial_box_width[IDXC_X]);
assert(diff_x <= 2*cutoff_radius);
- real_number diff_y = std::abs(apply_pbc(y1,IDX_X)-apply_pbc(y2,IDX_X)+yshift_coef*spatial_box_width[IDX_Y]);
+ real_number diff_y = std::abs(apply_pbc(y1,IDXC_X)-apply_pbc(y2,IDXC_X)+yshift_coef*spatial_box_width[IDXC_Y]);
assert(diff_y <= 2*cutoff_radius);
- real_number diff_z = std::abs(apply_pbc(z1,IDX_X)-apply_pbc(z2,IDX_X)+zshift_coef*spatial_box_width[IDX_Z]);
+ real_number diff_z = std::abs(apply_pbc(z1,IDXC_X)-apply_pbc(z2,IDXC_X)+zshift_coef*spatial_box_width[IDXC_Z]);
assert(diff_z <= 2*cutoff_radius);
return (diff_x*diff_x) + (diff_y*diff_y) + (diff_z*diff_z);
@@ -276,9 +276,9 @@ public:
for(int idxPartition = 0 ; idxPartition < current_partition_size ; ++idxPartition){
#pragma omp parallel for schedule(static)
for(partsize_t idxPart = current_offset_particles_for_partition[idxPartition] ; idxPart < current_offset_particles_for_partition[idxPartition+1] ; ++idxPart ){
- particles_coord[idxPart] = get_cell_idx(particles_positions[(idxPart)*size_particle_positions + IDX_X],
- particles_positions[(idxPart)*size_particle_positions + IDX_Y],
- particles_positions[(idxPart)*size_particle_positions + IDX_Z]);
+ particles_coord[idxPart] = get_cell_idx(particles_positions[(idxPart)*size_particle_positions + IDXC_X],
+ particles_positions[(idxPart)*size_particle_positions + IDXC_Y],
+ particles_positions[(idxPart)*size_particle_positions + IDXC_Z]);
assert(my_down_z_cell_level <= get_cell_coord_z_from_index(particles_coord[idxPart]));
assert(get_cell_coord_z_from_index(particles_coord[idxPart]) <= my_top_z_cell_level);
}
@@ -378,11 +378,11 @@ public:
int dest_proc = (my_rank+1)%nb_processes_involved;
while(dest_proc != my_rank
&& (my_top_z_cell_level == first_cell_level_proc(dest_proc)
- || (my_top_z_cell_level+1)%nb_cell_levels[IDX_Z] == first_cell_level_proc(dest_proc))){
+ || (my_top_z_cell_level+1)%nb_cell_levels[IDXC_Z] == first_cell_level_proc(dest_proc))){
// Find if we have to send 1 or 2 cell levels
int nb_levels_to_send = 1;
if(my_nb_cell_levels > 1 // I have more than one level
- && (my_top_z_cell_level-1+2)%nb_cell_levels[IDX_Z] <= last_cell_level_proc(dest_proc)){
+ && (my_top_z_cell_level-1+2)%nb_cell_levels[IDXC_Z] <= last_cell_level_proc(dest_proc)){
nb_levels_to_send += 1;
}
@@ -400,11 +400,11 @@ public:
int src_proc = (my_rank-1+nb_processes_involved)%nb_processes_involved;
while(src_proc != my_rank
&& (last_cell_level_proc(src_proc) == my_down_z_cell_level
- || (last_cell_level_proc(src_proc)+1)%nb_cell_levels[IDX_Z] == my_down_z_cell_level)){
+ || (last_cell_level_proc(src_proc)+1)%nb_cell_levels[IDXC_Z] == my_down_z_cell_level)){
// Find if we have to send 1 or 2 cell levels
int nb_levels_to_recv = 1;
if(my_nb_cell_levels > 1 // I have more than one level
- && first_cell_level_proc(src_proc) <= (my_down_z_cell_level-1+2)%nb_cell_levels[IDX_Z]){
+ && first_cell_level_proc(src_proc) <= (my_down_z_cell_level-1+2)%nb_cell_levels[IDXC_Z]){
nb_levels_to_recv += 1;
}
@@ -564,14 +564,14 @@ public:
// Compute
partsize_t idxPart = 0;
while(idxPart != NbParticlesToReceive){
- const long int current_cell_idx = get_cell_idx(descriptor.toCompute[idxPart*size_particle_positions + IDX_X],
- descriptor.toCompute[idxPart*size_particle_positions + IDX_Y],
- descriptor.toCompute[idxPart*size_particle_positions + IDX_Z]);
+ const long int current_cell_idx = get_cell_idx(descriptor.toCompute[idxPart*size_particle_positions + IDXC_X],
+ descriptor.toCompute[idxPart*size_particle_positions + IDXC_Y],
+ descriptor.toCompute[idxPart*size_particle_positions + IDXC_Z]);
partsize_t nb_parts_in_cell = 1;
while(idxPart+nb_parts_in_cell != NbParticlesToReceive
- && current_cell_idx == get_cell_idx(descriptor.toCompute[(idxPart+nb_parts_in_cell)*size_particle_positions + IDX_X],
- descriptor.toCompute[(idxPart+nb_parts_in_cell)*size_particle_positions + IDX_Y],
- descriptor.toCompute[(idxPart+nb_parts_in_cell)*size_particle_positions + IDX_Z])){
+ && current_cell_idx == get_cell_idx(descriptor.toCompute[(idxPart+nb_parts_in_cell)*size_particle_positions + IDXC_X],
+ descriptor.toCompute[(idxPart+nb_parts_in_cell)*size_particle_positions + IDXC_Y],
+ descriptor.toCompute[(idxPart+nb_parts_in_cell)*size_particle_positions + IDXC_Z])){
nb_parts_in_cell += 1;
}
@@ -589,20 +589,20 @@ public:
for(size_t idx_2 = 0 ; idx_2 < (*neighbors[idx_neighbor]).size() ; ++idx_2){
for(partsize_t idx_p1 = 0 ; idx_p1 < nb_parts_in_cell ; ++idx_p1){
for(partsize_t idx_p2 = 0 ; idx_p2 < (*neighbors[idx_neighbor])[idx_2].second ; ++idx_p2){
- const real_number dist_r2 = compute_distance_r2(descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_X],
- descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_Y],
- descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDX_Z],
- particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
- particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
- particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z],
- shift[idx_neighbor][IDX_X], shift[idx_neighbor][IDX_Y], shift[idx_neighbor][IDX_Z]);
+ const real_number dist_r2 = compute_distance_r2(descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDXC_X],
+ descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDXC_Y],
+ descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions + IDXC_Z],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDXC_X],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDXC_Y],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDXC_Z],
+ shift[idx_neighbor][IDXC_X], shift[idx_neighbor][IDXC_Y], shift[idx_neighbor][IDXC_Z]);
if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
in_computer.template compute_interaction<size_particle_positions, size_particle_rhs>(
&descriptor.toCompute[(idxPart+idx_p1)*size_particle_positions],
&descriptor.results[(idxPart+idx_p1)*size_particle_rhs],
&particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions],
&particles_current_rhs[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_rhs],
- dist_r2, cutoff_radius_compute, shift[idx_neighbor][IDX_X], shift[idx_neighbor][IDX_Y], shift[idx_neighbor][IDX_Z]);
+ dist_r2, cutoff_radius_compute, shift[idx_neighbor][IDXC_X], shift[idx_neighbor][IDXC_Y], shift[idx_neighbor][IDXC_Z]);
}
}
}
@@ -671,12 +671,12 @@ public:
// self interval
for(partsize_t idx_p1 = 0 ; idx_p1 < intervals[idx_1].second ; ++idx_p1){
for(partsize_t idx_p2 = idx_p1+1 ; idx_p2 < intervals[idx_1].second ; ++idx_p2){
- const real_number dist_r2 = compute_distance_r2(particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
- particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
- particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z],
- particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_X],
- particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_Y],
- particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDX_Z],
+ const real_number dist_r2 = compute_distance_r2(particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDXC_X],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDXC_Y],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDXC_Z],
+ particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDXC_X],
+ particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDXC_Y],
+ particles_positions[(intervals[idx_1].first+idx_p2)*size_particle_positions + IDXC_Z],
0, 0, 0);
if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
in_computer.template compute_interaction<size_particle_positions,size_particle_rhs>(
@@ -693,12 +693,12 @@ public:
for(size_t idx_2 = idx_1+1 ; idx_2 < intervals.size() ; ++idx_2){
for(partsize_t idx_p1 = 0 ; idx_p1 < intervals[idx_1].second ; ++idx_p1){
for(partsize_t idx_p2 = 0 ; idx_p2 < intervals[idx_2].second ; ++idx_p2){
- const real_number dist_r2 = compute_distance_r2(particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
- particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
- particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z],
- particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
- particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
- particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDX_Z],
+ const real_number dist_r2 = compute_distance_r2(particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDXC_X],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDXC_Y],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDXC_Z],
+ particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDXC_X],
+ particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDXC_Y],
+ particles_positions[(intervals[idx_2].first+idx_p2)*size_particle_positions + IDXC_Z],
0, 0, 0);
if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
in_computer.template compute_interaction<size_particle_positions,size_particle_rhs>(
@@ -727,20 +727,20 @@ public:
for(size_t idx_2 = 0 ; idx_2 < (*neighbors[idx_neighbor]).size() ; ++idx_2){
for(partsize_t idx_p1 = 0 ; idx_p1 < intervals[idx_1].second ; ++idx_p1){
for(partsize_t idx_p2 = 0 ; idx_p2 < (*neighbors[idx_neighbor])[idx_2].second ; ++idx_p2){
- const real_number dist_r2 = compute_distance_r2(particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_X],
- particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Y],
- particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDX_Z],
- particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_X],
- particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Y],
- particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDX_Z],
- shift[idx_neighbor][IDX_X], shift[idx_neighbor][IDX_Y], shift[idx_neighbor][IDX_Z]);
+ const real_number dist_r2 = compute_distance_r2(particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDXC_X],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDXC_Y],
+ particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions + IDXC_Z],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDXC_X],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDXC_Y],
+ particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions + IDXC_Z],
+ shift[idx_neighbor][IDXC_X], shift[idx_neighbor][IDXC_Y], shift[idx_neighbor][IDXC_Z]);
if(dist_r2 < cutoff_radius_compute*cutoff_radius_compute){
in_computer.template compute_interaction<size_particle_positions,size_particle_rhs>(
&particles_positions[(intervals[idx_1].first+idx_p1)*size_particle_positions],
&particles_current_rhs[(intervals[idx_1].first+idx_p1)*size_particle_rhs],
&particles_positions[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_positions],
&particles_current_rhs[((*neighbors[idx_neighbor])[idx_2].first+idx_p2)*size_particle_rhs],
- dist_r2, cutoff_radius_compute, shift[idx_neighbor][IDX_X], shift[idx_neighbor][IDX_Y], shift[idx_neighbor][IDX_Z]);
+ dist_r2, cutoff_radius_compute, shift[idx_neighbor][IDXC_X], shift[idx_neighbor][IDXC_Y], shift[idx_neighbor][IDXC_Z]);
}
}
}
diff --git a/bfps/cpp/particles/p2p_tree.hpp b/bfps/cpp/particles/p2p_tree.hpp
index 3d92c4e5..a4441543 100644
--- a/bfps/cpp/particles/p2p_tree.hpp
+++ b/bfps/cpp/particles/p2p_tree.hpp
@@ -11,21 +11,21 @@ class p2p_tree{
std::array<long int,3> nb_cell_levels;
long int get_cell_coord_x_from_index(const long int index) const{
- return index % nb_cell_levels[IDX_X];
+ return index % nb_cell_levels[IDXC_X];
}
long int get_cell_coord_y_from_index(const long int index) const{
- return (index % (nb_cell_levels[IDX_X]*nb_cell_levels[IDX_Y]))
- / nb_cell_levels[IDX_X];
+ return (index % (nb_cell_levels[IDXC_X]*nb_cell_levels[IDXC_Y]))
+ / nb_cell_levels[IDXC_X];
}
long int get_cell_coord_z_from_index(const long int index) const{
- return index / (nb_cell_levels[IDX_X]*nb_cell_levels[IDX_Y]);
+ return index / (nb_cell_levels[IDXC_X]*nb_cell_levels[IDXC_Y]);
}
long int get_cell_idx(const long int idx_x, const long int idx_y,
const long int idx_z) const {
- return (((idx_z*nb_cell_levels[IDX_Y])+idx_y)*nb_cell_levels[IDX_X])+idx_x;
+ return (((idx_z*nb_cell_levels[IDXC_Y])+idx_y)*nb_cell_levels[IDXC_X])+idx_x;
}
public:
@@ -61,11 +61,11 @@ public:
long int neigh_x_pbc = neigh_x+idx_x;
ShiftType shift_x = 0;
if(neigh_x_pbc < 0){
- neigh_x_pbc += nb_cell_levels[IDX_X];
+ neigh_x_pbc += nb_cell_levels[IDXC_X];
shift_x = 1;
}
- else if(nb_cell_levels[IDX_X] <= neigh_x_pbc){
- neigh_x_pbc -= nb_cell_levels[IDX_X];
+ else if(nb_cell_levels[IDXC_X] <= neigh_x_pbc){
+ neigh_x_pbc -= nb_cell_levels[IDXC_X];
shift_x = -1;
}
@@ -73,11 +73,11 @@ public:
long int neigh_y_pbc = neigh_y+idx_y;
ShiftType shift_y = 0;
if(neigh_y_pbc < 0){
- neigh_y_pbc += nb_cell_levels[IDX_Y];
+ neigh_y_pbc += nb_cell_levels[IDXC_Y];
shift_y = 1;
}
- else if(nb_cell_levels[IDX_Y] <= neigh_y_pbc){
- neigh_y_pbc -= nb_cell_levels[IDX_Y];
+ else if(nb_cell_levels[IDXC_Y] <= neigh_y_pbc){
+ neigh_y_pbc -= nb_cell_levels[IDXC_Y];
shift_y = -1;
}
@@ -85,11 +85,11 @@ public:
long int neigh_z_pbc = neigh_z+idx_z;
ShiftType shift_z = 0;
if(neigh_z_pbc < 0){
- neigh_z_pbc += nb_cell_levels[IDX_Z];
+ neigh_z_pbc += nb_cell_levels[IDXC_Z];
shift_z = 1;
}
- else if(nb_cell_levels[IDX_Z] <= neigh_z_pbc){
- neigh_z_pbc -= nb_cell_levels[IDX_Z];
+ else if(nb_cell_levels[IDXC_Z] <= neigh_z_pbc){
+ neigh_z_pbc -= nb_cell_levels[IDXC_Z];
shift_z = -1;
}
@@ -102,9 +102,9 @@ public:
output[nbNeighbors] = &(iter->second);
output_indexes[nbNeighbors] = idx_neigh;
- shift[nbNeighbors][IDX_X] = shift_x;
- shift[nbNeighbors][IDX_Y] = shift_y;
- shift[nbNeighbors][IDX_Z] = shift_z;
+ shift[nbNeighbors][IDXC_X] = shift_x;
+ shift[nbNeighbors][IDXC_Y] = shift_y;
+ shift[nbNeighbors][IDXC_Z] = shift_z;
nbNeighbors += 1;
}
diff --git a/bfps/cpp/particles/particles_distr_mpi.hpp b/bfps/cpp/particles/particles_distr_mpi.hpp
index 251119be..b03749fd 100644
--- a/bfps/cpp/particles/particles_distr_mpi.hpp
+++ b/bfps/cpp/particles/particles_distr_mpi.hpp
@@ -127,7 +127,7 @@ public:
assert(partition_interval_size_per_proc[idx_proc_involved] != 0);
}
- assert(int(field_grid_dim[IDX_Z]) == partition_interval_offset_per_proc[nb_processes_involved]);
+ assert(int(field_grid_dim[IDXC_Z]) == partition_interval_offset_per_proc[nb_processes_involved]);
}
virtual ~particles_distr_mpi(){}
@@ -522,11 +522,11 @@ public:
partsize_t partOffset = 0;
for(int idxPartition = 0 ; idxPartition < current_partition_size ; ++idxPartition){
for(partsize_t idx = 0 ; idx < current_my_nb_particles_per_partition[idxPartition] ; ++idx){
- const int partition_level = in_computer.pbc_field_layer((*inout_positions_particles)[(idx+partOffset)*size_particle_positions+IDX_Z], IDX_Z);
+ const int partition_level = in_computer.pbc_field_layer((*inout_positions_particles)[(idx+partOffset)*size_particle_positions+IDXC_Z], IDXC_Z);
variable_used_only_in_assert(partition_level);
assert(partition_level == current_partition_interval.first + idxPartition
- || partition_level == (current_partition_interval.first + idxPartition-1+int(field_grid_dim[IDX_Z]))%int(field_grid_dim[IDX_Z])
- || partition_level == (current_partition_interval.first + idxPartition+1)%int(field_grid_dim[IDX_Z]));
+ || partition_level == (current_partition_interval.first + idxPartition-1+int(field_grid_dim[IDXC_Z]))%int(field_grid_dim[IDXC_Z])
+ || partition_level == (current_partition_interval.first + idxPartition+1)%int(field_grid_dim[IDXC_Z]));
}
partOffset += current_my_nb_particles_per_partition[idxPartition];
}
@@ -543,11 +543,11 @@ public:
// Find particles outside my interval
const partsize_t nbOutLower = particles_utils::partition_extra<partsize_t, size_particle_positions>(&(*inout_positions_particles)[0], current_my_nb_particles_per_partition[0],
[&](const real_number val[]){
- const int partition_level = in_computer.pbc_field_layer(val[IDX_Z], IDX_Z);
+ const int partition_level = in_computer.pbc_field_layer(val[IDXC_Z], IDXC_Z);
assert(partition_level == current_partition_interval.first
- || partition_level == (current_partition_interval.first-1+int(field_grid_dim[IDX_Z]))%int(field_grid_dim[IDX_Z])
- || partition_level == (current_partition_interval.first+1)%int(field_grid_dim[IDX_Z]));
- const bool isLower = partition_level == (current_partition_interval.first-1+int(field_grid_dim[IDX_Z]))%int(field_grid_dim[IDX_Z]);
+ || partition_level == (current_partition_interval.first-1+int(field_grid_dim[IDXC_Z]))%int(field_grid_dim[IDXC_Z])
+ || partition_level == (current_partition_interval.first+1)%int(field_grid_dim[IDXC_Z]));
+ const bool isLower = partition_level == (current_partition_interval.first-1+int(field_grid_dim[IDXC_Z]))%int(field_grid_dim[IDXC_Z]);
return isLower;
},
[&](const partsize_t idx1, const partsize_t idx2){
@@ -569,11 +569,11 @@ public:
&(*inout_positions_particles)[(current_offset_particles_for_partition[current_partition_size-1]+offesetOutLow)*size_particle_positions],
myTotalNbParticles - (current_offset_particles_for_partition[current_partition_size-1]+offesetOutLow),
[&](const real_number val[]){
- const int partition_level = in_computer.pbc_field_layer(val[IDX_Z], IDX_Z);
+ const int partition_level = in_computer.pbc_field_layer(val[IDXC_Z], IDXC_Z);
assert(partition_level == (current_partition_interval.second-1)
- || partition_level == ((current_partition_interval.second-1)-1+int(field_grid_dim[IDX_Z]))%int(field_grid_dim[IDX_Z])
- || partition_level == ((current_partition_interval.second-1)+1)%int(field_grid_dim[IDX_Z]));
- const bool isUpper = (partition_level == ((current_partition_interval.second-1)+1)%int(field_grid_dim[IDX_Z]));
+ || partition_level == ((current_partition_interval.second-1)-1+int(field_grid_dim[IDXC_Z]))%int(field_grid_dim[IDXC_Z])
+ || partition_level == ((current_partition_interval.second-1)+1)%int(field_grid_dim[IDXC_Z]));
+ const bool isUpper = (partition_level == ((current_partition_interval.second-1)+1)%int(field_grid_dim[IDXC_Z]));
return !isUpper;
},
[&](const partsize_t idx1, const partsize_t idx2){
@@ -822,7 +822,7 @@ public:
myTotalNbParticles,current_partition_size,
current_my_nb_particles_per_partition, current_offset_particles_for_partition.get(),
[&](const real_number& z_pos){
- const int partition_level = in_computer.pbc_field_layer(z_pos, IDX_Z);
+ const int partition_level = in_computer.pbc_field_layer(z_pos, IDXC_Z);
assert(current_partition_interval.first <= partition_level && partition_level < current_partition_interval.second);
return partition_level - current_partition_interval.first;
},
@@ -845,7 +845,7 @@ public:
assert(current_my_nb_particles_per_partition[idxPartition] ==
current_offset_particles_for_partition[idxPartition+1] - current_offset_particles_for_partition[idxPartition]);
for(partsize_t idx = current_offset_particles_for_partition[idxPartition] ; idx < current_offset_particles_for_partition[idxPartition+1] ; ++idx){
- assert(in_computer.pbc_field_layer((*inout_positions_particles)[idx*size_particle_positions+IDX_Z], IDX_Z)-current_partition_interval.first == idxPartition);
+ assert(in_computer.pbc_field_layer((*inout_positions_particles)[idx*size_particle_positions+IDXC_Z], IDXC_Z)-current_partition_interval.first == idxPartition);
}
}
}
diff --git a/bfps/cpp/particles/particles_field_computer.hpp b/bfps/cpp/particles/particles_field_computer.hpp
index 330763c8..50d4df78 100644
--- a/bfps/cpp/particles/particles_field_computer.hpp
+++ b/bfps/cpp/particles/particles_field_computer.hpp
@@ -44,9 +44,9 @@ public:
: field_grid_dim({{int(in_field_grid_dim[0]),int(in_field_grid_dim[1]),int(in_field_grid_dim[2])}}), current_partition_interval(in_current_partitions),
interpolator(in_interpolator),
spatial_box_width(in_spatial_box_width), spatial_box_offset(in_spatial_box_offset), box_step_width(in_box_step_width){
- deriv[IDX_X] = 0;
- deriv[IDX_Y] = 0;
- deriv[IDX_Z] = 0;
+ deriv[IDXC_X] = 0;
+ deriv[IDXC_Y] = 0;
+ deriv[IDXC_Z] = 0;
}
////////////////////////////////////////////////////////////////////////
@@ -82,25 +82,25 @@ public:
TIMEZONE("particles_field_computer::apply_computation");
for(partsize_t idxPart = 0 ; idxPart < nb_particles ; ++idxPart){
- const real_number reltv_x = get_norm_pos_in_cell(particles_positions[idxPart*size_particle_positions+IDX_X], IDX_X);
- const real_number reltv_y = get_norm_pos_in_cell(particles_positions[idxPart*size_particle_positions+IDX_Y], IDX_Y);
- const real_number reltv_z = get_norm_pos_in_cell(particles_positions[idxPart*size_particle_positions+IDX_Z], IDX_Z);
+ const real_number reltv_x = get_norm_pos_in_cell(particles_positions[idxPart*size_particle_positions+IDXC_X], IDXC_X);
+ const real_number reltv_y = get_norm_pos_in_cell(particles_positions[idxPart*size_particle_positions+IDXC_Y], IDXC_Y);
+ const real_number reltv_z = get_norm_pos_in_cell(particles_positions[idxPart*size_particle_positions+IDXC_Z], IDXC_Z);
typename interpolator_class::real_number
bx[interp_neighbours*2+2],
by[interp_neighbours*2+2],
bz[interp_neighbours*2+2];
- interpolator.compute_beta(deriv[IDX_X], reltv_x, bx);
- interpolator.compute_beta(deriv[IDX_Y], reltv_y, by);
- interpolator.compute_beta(deriv[IDX_Z], reltv_z, bz);
+ interpolator.compute_beta(deriv[IDXC_X], reltv_x, bx);
+ interpolator.compute_beta(deriv[IDXC_Y], reltv_y, by);
+ interpolator.compute_beta(deriv[IDXC_Z], reltv_z, bz);
- const int partGridIdx_x = pbc_field_layer(particles_positions[idxPart*size_particle_positions+IDX_X], IDX_X);
- const int partGridIdx_y = pbc_field_layer(particles_positions[idxPart*size_particle_positions+IDX_Y], IDX_Y);
- const int partGridIdx_z = pbc_field_layer(particles_positions[idxPart*size_particle_positions+IDX_Z], IDX_Z);
+ const int partGridIdx_x = pbc_field_layer(particles_positions[idxPart*size_particle_positions+IDXC_X], IDXC_X);
+ const int partGridIdx_y = pbc_field_layer(particles_positions[idxPart*size_particle_positions+IDXC_Y], IDXC_Y);
+ const int partGridIdx_z = pbc_field_layer(particles_positions[idxPart*size_particle_positions+IDXC_Z], IDXC_Z);
- assert(0 <= partGridIdx_x && partGridIdx_x < int(field_grid_dim[IDX_X]));
- assert(0 <= partGridIdx_y && partGridIdx_y < int(field_grid_dim[IDX_Y]));
- assert(0 <= partGridIdx_z && partGridIdx_z < int(field_grid_dim[IDX_Z]));
+ assert(0 <= partGridIdx_x && partGridIdx_x < int(field_grid_dim[IDXC_X]));
+ assert(0 <= partGridIdx_y && partGridIdx_y < int(field_grid_dim[IDXC_Y]));
+ assert(0 <= partGridIdx_z && partGridIdx_z < int(field_grid_dim[IDXC_Z]));
const int interp_limit_mx = partGridIdx_x-interp_neighbours;
const int interp_limit_x = partGridIdx_x+interp_neighbours+1;
@@ -113,8 +113,8 @@ public:
int nb_z_intervals;
if((partGridIdx_z-interp_neighbours) < 0){
- assert(partGridIdx_z+interp_neighbours+1 < int(field_grid_dim[IDX_Z]));
- interp_limit_mz[0] = std::max(current_partition_interval.first, partGridIdx_z-interp_neighbours+int(field_grid_dim[IDX_Z]));
+ assert(partGridIdx_z+interp_neighbours+1 < int(field_grid_dim[IDXC_Z]));
+ interp_limit_mz[0] = std::max(current_partition_interval.first, partGridIdx_z-interp_neighbours+int(field_grid_dim[IDXC_Z]));
interp_limit_z[0] = current_partition_interval.second-1;
interp_limit_mz[1] = std::max(0, current_partition_interval.first);
@@ -122,12 +122,12 @@ public:
nb_z_intervals = 2;
}
- else if(int(field_grid_dim[IDX_Z]) <= (partGridIdx_z+interp_neighbours+1)){
+ else if(int(field_grid_dim[IDXC_Z]) <= (partGridIdx_z+interp_neighbours+1)){
interp_limit_mz[0] = std::max(current_partition_interval.first, partGridIdx_z-interp_neighbours);
- interp_limit_z[0] = std::min(int(field_grid_dim[IDX_Z])-1,current_partition_interval.second-1);
+ interp_limit_z[0] = std::min(int(field_grid_dim[IDXC_Z])-1,current_partition_interval.second-1);
interp_limit_mz[1] = std::max(0, current_partition_interval.first);
- interp_limit_z[1] = std::min(partGridIdx_z+interp_neighbours+1-int(field_grid_dim[IDX_Z]), current_partition_interval.second-1);
+ interp_limit_z[1] = std::min(partGridIdx_z+interp_neighbours+1-int(field_grid_dim[IDXC_Z]), current_partition_interval.second-1);
nb_z_intervals = 2;
}
@@ -139,19 +139,19 @@ public:
for(int idx_inter = 0 ; idx_inter < nb_z_intervals ; ++idx_inter){
for(int idx_z = interp_limit_mz[idx_inter] ; idx_z <= interp_limit_z[idx_inter] ; ++idx_z ){
- const int idx_z_pbc = (idx_z + field_grid_dim[IDX_Z])%field_grid_dim[IDX_Z];
+ const int idx_z_pbc = (idx_z + field_grid_dim[IDXC_Z])%field_grid_dim[IDXC_Z];
assert(current_partition_interval.first <= idx_z_pbc && idx_z_pbc < current_partition_interval.second);
- assert(((idx_z+field_grid_dim[IDX_Z]-interp_limit_mz_bz)%field_grid_dim[IDX_Z]) < interp_neighbours*2+2);
+ assert(((idx_z+field_grid_dim[IDXC_Z]-interp_limit_mz_bz)%field_grid_dim[IDXC_Z]) < interp_neighbours*2+2);
for(int idx_x = interp_limit_mx ; idx_x <= interp_limit_x ; ++idx_x ){
- const int idx_x_pbc = (idx_x + field_grid_dim[IDX_X])%field_grid_dim[IDX_X];
+ const int idx_x_pbc = (idx_x + field_grid_dim[IDXC_X])%field_grid_dim[IDXC_X];
assert(idx_x-interp_limit_mx < interp_neighbours*2+2);
for(int idx_y = interp_limit_my ; idx_y <= interp_limit_y ; ++idx_y ){
- const int idx_y_pbc = (idx_y + field_grid_dim[IDX_Y])%field_grid_dim[IDX_Y];
+ const int idx_y_pbc = (idx_y + field_grid_dim[IDXC_Y])%field_grid_dim[IDXC_Y];
assert(idx_y-interp_limit_my < interp_neighbours*2+2);
- const real_number coef = (bz[((idx_z+field_grid_dim[IDX_Z]-interp_limit_mz_bz)%field_grid_dim[IDX_Z])]
+ const real_number coef = (bz[((idx_z+field_grid_dim[IDXC_Z]-interp_limit_mz_bz)%field_grid_dim[IDXC_Z])]
* by[idx_y-interp_limit_my]
* bx[idx_x-interp_limit_mx]);
diff --git a/bfps/cpp/particles/particles_input_hdf5.hpp b/bfps/cpp/particles/particles_input_hdf5.hpp
index 40fef3c4..5231872d 100644
--- a/bfps/cpp/particles/particles_input_hdf5.hpp
+++ b/bfps/cpp/particles/particles_input_hdf5.hpp
@@ -240,7 +240,7 @@ public:
&split_particles_positions[previousOffset*size_particle_positions],
partsize_t(load_splitter.getMySize())-previousOffset,
[&](const real_number val[]){
- const real_number shiftPos = val[IDX_Z]-spatial_box_offset;
+ const real_number shiftPos = val[IDXC_Z]-spatial_box_offset;
const real_number nbRepeat = floor(shiftPos/spatial_box_width);
const real_number posInBox = shiftPos - (spatial_box_width*nbRepeat);
return posInBox < limitPartitionShifted;
diff --git a/bfps/cpp/particles/particles_system.hpp b/bfps/cpp/particles/particles_system.hpp
index f8688f61..460383c3 100644
--- a/bfps/cpp/particles/particles_system.hpp
+++ b/bfps/cpp/particles/particles_system.hpp
@@ -70,7 +70,7 @@ public:
particles_inner_computer_class in_computer_particules_inner,
const int in_current_iteration = 1)
: mpi_com(in_mpi_com),
- current_partition_interval({in_local_field_offset[IDX_Z], in_local_field_offset[IDX_Z] + in_local_field_dims[IDX_Z]}),
+ current_partition_interval({in_local_field_offset[IDXC_Z], in_local_field_offset[IDXC_Z] + in_local_field_dims[IDXC_Z]}),
partition_interval_size(current_partition_interval.second - current_partition_interval.first),
interpolator(),
particles_distr(in_mpi_com, current_partition_interval,field_grid_dim),
@@ -100,7 +100,7 @@ public:
my_nb_particles = particles_input.getLocalNbParticles();
for(partsize_t idx_part = 0 ; idx_part < my_nb_particles ; ++idx_part){ // TODO remove me
- const int partition_level = computer.pbc_field_layer(my_particles_positions[idx_part*size_particle_positions+IDX_Z], IDX_Z);
+ const int partition_level = computer.pbc_field_layer(my_particles_positions[idx_part*size_particle_positions+IDXC_Z], IDXC_Z);
variable_used_only_in_assert(partition_level);
assert(partition_level >= current_partition_interval.first);
assert(partition_level < current_partition_interval.second);
@@ -109,7 +109,7 @@ public:
particles_utils::partition_extra_z<partsize_t, size_particle_positions>(&my_particles_positions[0], my_nb_particles, partition_interval_size,
current_my_nb_particles_per_partition.get(), current_offset_particles_for_partition.get(),
[&](const real_number& z_pos){
- const int partition_level = computer.pbc_field_layer(z_pos, IDX_Z);
+ const int partition_level = computer.pbc_field_layer(z_pos, IDXC_Z);
assert(current_partition_interval.first <= partition_level && partition_level < current_partition_interval.second);
return partition_level - current_partition_interval.first;
},
@@ -128,7 +128,7 @@ public:
assert(current_my_nb_particles_per_partition[idxPartition] ==
current_offset_particles_for_partition[idxPartition+1] - current_offset_particles_for_partition[idxPartition]);
for(partsize_t idx = current_offset_particles_for_partition[idxPartition] ; idx < current_offset_particles_for_partition[idxPartition+1] ; ++idx){
- assert(computer.pbc_field_layer(my_particles_positions[idx*size_particle_positions+IDX_Z], IDX_Z)-current_partition_interval.first == idxPartition);
+ assert(computer.pbc_field_layer(my_particles_positions[idx*size_particle_positions+IDXC_Z], IDXC_Z)-current_partition_interval.first == idxPartition);
}
}
}
@@ -331,9 +331,9 @@ public:
void checkNan() const { // TODO remove
for(partsize_t idx_part = 0 ; idx_part < my_nb_particles ; ++idx_part){ // TODO remove me
- assert(std::isnan(my_particles_positions[idx_part*size_particle_positions+IDX_X]) == false);
- assert(std::isnan(my_particles_positions[idx_part*size_particle_positions+IDX_Y]) == false);
- assert(std::isnan(my_particles_positions[idx_part*size_particle_positions+IDX_Z]) == false);
+ assert(std::isnan(my_particles_positions[idx_part*size_particle_positions+IDXC_X]) == false);
+ assert(std::isnan(my_particles_positions[idx_part*size_particle_positions+IDXC_Y]) == false);
+ assert(std::isnan(my_particles_positions[idx_part*size_particle_positions+IDXC_Z]) == false);
for(int idx_rhs = 0 ; idx_rhs < my_particles_rhs.size() ; ++idx_rhs){
for(int idx_rhs_val = 0 ; idx_rhs_val < size_particle_rhs ; ++idx_rhs_val){
diff --git a/bfps/cpp/particles/particles_system_builder.hpp b/bfps/cpp/particles/particles_system_builder.hpp
index 3e72ceaa..a9a140ac 100644
--- a/bfps/cpp/particles/particles_system_builder.hpp
+++ b/bfps/cpp/particles/particles_system_builder.hpp
@@ -130,21 +130,21 @@ struct particles_system_build_container {
// The size of the field grid (global size) all_size seems
std::array<size_t,3> field_grid_dim;
- field_grid_dim[IDX_X] = fs_field->rlayout->sizes[FIELD_IDX_X];// nx
- field_grid_dim[IDX_Y] = fs_field->rlayout->sizes[FIELD_IDX_Y];// nx
- field_grid_dim[IDX_Z] = fs_field->rlayout->sizes[FIELD_IDX_Z];// nz
+ field_grid_dim[IDXC_X] = fs_field->rlayout->sizes[IDXV_X];// nx
+ field_grid_dim[IDXC_Y] = fs_field->rlayout->sizes[IDXV_Y];// nx
+ field_grid_dim[IDXC_Z] = fs_field->rlayout->sizes[IDXV_Z];// nz
// The size of the local field grid (the field nodes that belong to current process)
std::array<size_t,3> local_field_dims;
- local_field_dims[IDX_X] = fs_field->rlayout->subsizes[FIELD_IDX_X];
- local_field_dims[IDX_Y] = fs_field->rlayout->subsizes[FIELD_IDX_Y];
- local_field_dims[IDX_Z] = fs_field->rlayout->subsizes[FIELD_IDX_Z];
+ local_field_dims[IDXC_X] = fs_field->rlayout->subsizes[IDXV_X];
+ local_field_dims[IDXC_Y] = fs_field->rlayout->subsizes[IDXV_Y];
+ local_field_dims[IDXC_Z] = fs_field->rlayout->subsizes[IDXV_Z];
// The offset of the local field grid
std::array<size_t,3> local_field_offset;
- local_field_offset[IDX_X] = fs_field->rlayout->starts[FIELD_IDX_X];
- local_field_offset[IDX_Y] = fs_field->rlayout->starts[FIELD_IDX_Y];
- local_field_offset[IDX_Z] = fs_field->rlayout->starts[FIELD_IDX_Z];
+ local_field_offset[IDXC_X] = fs_field->rlayout->starts[IDXV_X];
+ local_field_offset[IDXC_Y] = fs_field->rlayout->starts[IDXV_Y];
+ local_field_offset[IDXC_Z] = fs_field->rlayout->starts[IDXV_Z];
// Retreive split from fftw to know processes that have no work
@@ -152,51 +152,51 @@ struct particles_system_build_container {
AssertMpi(MPI_Comm_rank(mpi_comm, &my_rank));
AssertMpi(MPI_Comm_size(mpi_comm, &nb_processes));
- const int split_step = (int(field_grid_dim[IDX_Z])+nb_processes-1)/nb_processes;
- const int nb_processes_involved = (int(field_grid_dim[IDX_Z])+split_step-1)/split_step;
+ const int split_step = (int(field_grid_dim[IDXC_Z])+nb_processes-1)/nb_processes;
+ const int nb_processes_involved = (int(field_grid_dim[IDXC_Z])+split_step-1)/split_step;
- assert((my_rank < nb_processes_involved && local_field_dims[IDX_Z] != 0)
- || (nb_processes_involved <= my_rank && local_field_dims[IDX_Z] == 0));
- assert(nb_processes_involved <= int(field_grid_dim[IDX_Z]));
+ assert((my_rank < nb_processes_involved && local_field_dims[IDXC_Z] != 0)
+ || (nb_processes_involved <= my_rank && local_field_dims[IDXC_Z] == 0));
+ assert(nb_processes_involved <= int(field_grid_dim[IDXC_Z]));
// Make the idle processes starting from the limit (and not 0 as set by fftw)
if(nb_processes_involved <= my_rank){
- local_field_offset[IDX_Z] = field_grid_dim[IDX_Z];
+ local_field_offset[IDXC_Z] = field_grid_dim[IDXC_Z];
}
// Ensure that 1D partitioning is used
{
- assert(local_field_offset[IDX_X] == 0);
- assert(local_field_offset[IDX_Y] == 0);
- assert(local_field_dims[IDX_X] == field_grid_dim[IDX_X]);
- assert(local_field_dims[IDX_Y] == field_grid_dim[IDX_Y]);
-
- assert(my_rank >= nb_processes_involved || ((my_rank == 0 && local_field_offset[IDX_Z] == 0)
- || (my_rank != 0 && local_field_offset[IDX_Z] != 0)));
- assert(my_rank >= nb_processes_involved || ((my_rank == nb_processes_involved-1 && local_field_offset[IDX_Z]+local_field_dims[IDX_Z] == field_grid_dim[IDX_Z])
- || (my_rank != nb_processes_involved-1 && local_field_offset[IDX_Z]+local_field_dims[IDX_Z] != field_grid_dim[IDX_Z])));
+ assert(local_field_offset[IDXC_X] == 0);
+ assert(local_field_offset[IDXC_Y] == 0);
+ assert(local_field_dims[IDXC_X] == field_grid_dim[IDXC_X]);
+ assert(local_field_dims[IDXC_Y] == field_grid_dim[IDXC_Y]);
+
+ assert(my_rank >= nb_processes_involved || ((my_rank == 0 && local_field_offset[IDXC_Z] == 0)
+ || (my_rank != 0 && local_field_offset[IDXC_Z] != 0)));
+ assert(my_rank >= nb_processes_involved || ((my_rank == nb_processes_involved-1 && local_field_offset[IDXC_Z]+local_field_dims[IDXC_Z] == field_grid_dim[IDXC_Z])
+ || (my_rank != nb_processes_involved-1 && local_field_offset[IDXC_Z]+local_field_dims[IDXC_Z] != field_grid_dim[IDXC_Z])));
}
// The spatial box size (all particles should be included inside)
std::array<particles_rnumber,3> spatial_box_width;
- spatial_box_width[IDX_X] = 4 * acos(0) / (fs_kk->dkx);
- spatial_box_width[IDX_Y] = 4 * acos(0) / (fs_kk->dky);
- spatial_box_width[IDX_Z] = 4 * acos(0) / (fs_kk->dkz);
+ spatial_box_width[IDXC_X] = 4 * acos(0) / (fs_kk->dkx);
+ spatial_box_width[IDXC_Y] = 4 * acos(0) / (fs_kk->dky);
+ spatial_box_width[IDXC_Z] = 4 * acos(0) / (fs_kk->dkz);
// Box is in the corner
std::array<particles_rnumber,3> spatial_box_offset;
- spatial_box_offset[IDX_X] = 0;
- spatial_box_offset[IDX_Y] = 0;
- spatial_box_offset[IDX_Z] = 0;
+ spatial_box_offset[IDXC_X] = 0;
+ spatial_box_offset[IDXC_Y] = 0;
+ spatial_box_offset[IDXC_Z] = 0;
// The distance between two field nodes in z
std::array<particles_rnumber,3> spatial_partition_width;
- spatial_partition_width[IDX_X] = spatial_box_width[IDX_X]/particles_rnumber(field_grid_dim[IDX_X]);
- spatial_partition_width[IDX_Y] = spatial_box_width[IDX_Y]/particles_rnumber(field_grid_dim[IDX_Y]);
- spatial_partition_width[IDX_Z] = spatial_box_width[IDX_Z]/particles_rnumber(field_grid_dim[IDX_Z]);
+ spatial_partition_width[IDXC_X] = spatial_box_width[IDXC_X]/particles_rnumber(field_grid_dim[IDXC_X]);
+ spatial_partition_width[IDXC_Y] = spatial_box_width[IDXC_Y]/particles_rnumber(field_grid_dim[IDXC_Y]);
+ spatial_partition_width[IDXC_Z] = spatial_box_width[IDXC_Z]/particles_rnumber(field_grid_dim[IDXC_Z]);
// The spatial interval of the current process
- const particles_rnumber my_spatial_low_limit_z = particles_rnumber(local_field_offset[IDX_Z])*spatial_partition_width[IDX_Z];
- const particles_rnumber my_spatial_up_limit_z = particles_rnumber(local_field_offset[IDX_Z]+local_field_dims[IDX_Z])*spatial_partition_width[IDX_Z];
+ const particles_rnumber my_spatial_low_limit_z = particles_rnumber(local_field_offset[IDXC_Z])*spatial_partition_width[IDXC_Z];
+ const particles_rnumber my_spatial_up_limit_z = particles_rnumber(local_field_offset[IDXC_Z]+local_field_dims[IDXC_Z])*spatial_partition_width[IDXC_Z];
// Create the particles system
using particles_system_type = particles_system<partsize_t, particles_rnumber, field_rnumber,
diff --git a/bfps/cpp/particles/particles_utils.hpp b/bfps/cpp/particles/particles_utils.hpp
index 146dc439..76371e64 100644
--- a/bfps/cpp/particles/particles_utils.hpp
+++ b/bfps/cpp/particles/particles_utils.hpp
@@ -19,16 +19,28 @@
#define AssertMpi(X) if(MPI_SUCCESS != (X)) { printf("MPI Error at line %d\n",__LINE__); fflush(stdout) ; throw std::runtime_error("Stop from from mpi erro"); }
#endif
-enum IDXS_3D {
- IDX_X = 0,
- IDX_Y = 1,
- IDX_Z = 2
+enum IDX_COMPONENT_3D {
+ IDXC_X = 0,
+ IDXC_Y = 1,
+ IDXC_Z = 2
};
-enum FIELD_IDXS_3D {
- FIELD_IDX_X = 2,
- FIELD_IDX_Y = 1,
- FIELD_IDX_Z = 0
+enum IDX_COMPONENT_DEL_3D {
+ IDXC_DX_X = 0,
+ IDXC_DX_Y = 1,
+ IDXC_DX_Z = 2,
+ IDXC_DY_X = 3,
+ IDXC_DY_Y = 4,
+ IDXC_DY_Z = 5,
+ IDXC_DZ_X = 6,
+ IDXC_DZ_Y = 7,
+ IDXC_DZ_Z = 8,
+};
+
+enum IDX_VARIABLE_3D {
+ IDXV_X = 2,
+ IDXV_Y = 1,
+ IDXV_Z = 0
};
namespace particles_utils {
@@ -123,7 +135,7 @@ inline void partition_extra_z(real_number* array, const partsize_t size, const i
if(nb_partitions == 2){
const partsize_t size_current = partition_extra<partsize_t, nb_values>(array, size,
[&](const real_number inval[]){
- return partitions_levels(inval[IDX_Z]) == 0;
+ return partitions_levels(inval[IDXC_Z]) == 0;
}, pdcswap);
partitions_size[0] = size_current;
partitions_size[1] = size-size_current;
@@ -152,7 +164,7 @@ inline void partition_extra_z(real_number* array, const partsize_t size, const i
const partsize_t size_current = partition_extra<partsize_t, nb_values>(&array[partitions_offset[current_part.first]*nb_values],
size_unpart,
[&](const real_number inval[]){
- return partitions_levels(inval[IDX_Z]) <= idx_middle;
+ return partitions_levels(inval[IDXC_Z]) <= idx_middle;
}, pdcswap, partitions_offset[current_part.first]);
partitions_offset[idx_middle+1] = size_current + partitions_offset[current_part.first];
--
GitLab
From bd28a9da8198144a62790a07898e651ee20587de Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 26 Sep 2018 13:51:35 +0200
Subject: [PATCH 245/342] fix typo
---
bfps/cpp/particles/p2p_distr_mpi.hpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bfps/cpp/particles/p2p_distr_mpi.hpp b/bfps/cpp/particles/p2p_distr_mpi.hpp
index 2758aecf..965f8ba8 100644
--- a/bfps/cpp/particles/p2p_distr_mpi.hpp
+++ b/bfps/cpp/particles/p2p_distr_mpi.hpp
@@ -202,7 +202,7 @@ public:
return limite;
}
- real_number apply_pbc(real_number pos, COMPONENT_3D dim) const{
+ real_number apply_pbc(real_number pos, IDX_COMPONENT_3D dim) const{
while( pos < spatial_box_offset[dim] ){
pos += spatial_box_width[dim];
}
--
GitLab
From aae823c1e1b4598116e19522f4ab98bf72d8a026 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 26 Sep 2018 14:15:00 +0200
Subject: [PATCH 246/342] injection rate should be read from file (it wasn't)
---
bfps/cpp/full_code/NSVE.cpp | 1 +
bfps/cpp/full_code/NSVEcomplex_particles.cpp | 2 +-
bfps/cpp/full_code/code_base.cpp | 2 +-
bfps/cpp/particles/particles_distr_mpi.hpp | 2 +-
bfps/cpp/vorticity_equation.cpp | 2 +-
5 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/bfps/cpp/full_code/NSVE.cpp b/bfps/cpp/full_code/NSVE.cpp
index ecec7db3..da26ad22 100644
--- a/bfps/cpp/full_code/NSVE.cpp
+++ b/bfps/cpp/full_code/NSVE.cpp
@@ -156,6 +156,7 @@ int NSVE<rnumber>::read_parameters(void)
this->fmode = hdf5_tools::read_value<int>(parameter_file, "parameters/fmode");
this->famplitude = hdf5_tools::read_value<double>(parameter_file, "parameters/famplitude");
this->friction_coefficient = hdf5_tools::read_value<double>(parameter_file, "parameters/friction_coefficient");
+ this->injection_rate = hdf5_tools::read_value<double>(parameter_file, "parameters/injection_rate");
this->fk0 = hdf5_tools::read_value<double>(parameter_file, "parameters/fk0");
this->fk1 = hdf5_tools::read_value<double>(parameter_file, "parameters/fk1");
this->energy = hdf5_tools::read_value<double>(parameter_file, "parameters/energy");
diff --git a/bfps/cpp/full_code/NSVEcomplex_particles.cpp b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
index 02a19931..487fe4bb 100644
--- a/bfps/cpp/full_code/NSVEcomplex_particles.cpp
+++ b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
@@ -24,7 +24,7 @@
-#define NDEBUG
+//#define NDEBUG
#include <string>
#include <cmath>
diff --git a/bfps/cpp/full_code/code_base.cpp b/bfps/cpp/full_code/code_base.cpp
index a6487c72..210f2c96 100644
--- a/bfps/cpp/full_code/code_base.cpp
+++ b/bfps/cpp/full_code/code_base.cpp
@@ -24,7 +24,7 @@
-#define NDEBUG
+//#define NDEBUG
#include "code_base.hpp"
#include "scope_timer.hpp"
diff --git a/bfps/cpp/particles/particles_distr_mpi.hpp b/bfps/cpp/particles/particles_distr_mpi.hpp
index 251119be..dbf15b6b 100644
--- a/bfps/cpp/particles/particles_distr_mpi.hpp
+++ b/bfps/cpp/particles/particles_distr_mpi.hpp
@@ -141,7 +141,7 @@ public:
const real_number particles_positions[],
real_number particles_current_rhs[],
const int interpolation_size){
- TIMEZONE("compute_distr");
+ TIMEZONE("particle_distr_mpi::compute_distr");
// Some processes might not be involved
if(nb_processes_involved <= my_rank){
diff --git a/bfps/cpp/vorticity_equation.cpp b/bfps/cpp/vorticity_equation.cpp
index ead9345a..320c51d8 100644
--- a/bfps/cpp/vorticity_equation.cpp
+++ b/bfps/cpp/vorticity_equation.cpp
@@ -24,7 +24,7 @@
-#define NDEBUG
+//#define NDEBUG
#include <limits>
#include <cassert>
--
GitLab
From b0fe4e2e8304cfe71f7f73532ca261a74e377aa9 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 26 Sep 2018 15:35:40 +0200
Subject: [PATCH 247/342] add NDEBUG
---
bfps/cpp/full_code/NSVEcomplex_particles.cpp | 2 +-
bfps/cpp/full_code/code_base.cpp | 2 +-
bfps/cpp/vorticity_equation.cpp | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/bfps/cpp/full_code/NSVEcomplex_particles.cpp b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
index 487fe4bb..02a19931 100644
--- a/bfps/cpp/full_code/NSVEcomplex_particles.cpp
+++ b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
@@ -24,7 +24,7 @@
-//#define NDEBUG
+#define NDEBUG
#include <string>
#include <cmath>
diff --git a/bfps/cpp/full_code/code_base.cpp b/bfps/cpp/full_code/code_base.cpp
index 210f2c96..a6487c72 100644
--- a/bfps/cpp/full_code/code_base.cpp
+++ b/bfps/cpp/full_code/code_base.cpp
@@ -24,7 +24,7 @@
-//#define NDEBUG
+#define NDEBUG
#include "code_base.hpp"
#include "scope_timer.hpp"
diff --git a/bfps/cpp/vorticity_equation.cpp b/bfps/cpp/vorticity_equation.cpp
index 320c51d8..ead9345a 100644
--- a/bfps/cpp/vorticity_equation.cpp
+++ b/bfps/cpp/vorticity_equation.cpp
@@ -24,7 +24,7 @@
-//#define NDEBUG
+#define NDEBUG
#include <limits>
#include <cassert>
--
GitLab
From 2e98c4b897609272531f655ef2f7352f40ed41a7 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 26 Sep 2018 16:36:42 +0200
Subject: [PATCH 248/342] add lambda parameter
---
bfps/DNS.py | 1 +
bfps/cpp/full_code/NSVEcomplex_particles.cpp | 3 +-
bfps/cpp/full_code/NSVEcomplex_particles.hpp | 3 +-
.../particles/particles_inner_computer.hpp | 34 +++++++++++++++++--
4 files changed, 37 insertions(+), 4 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index eb51862e..8e72593e 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -154,6 +154,7 @@ class DNS(_code):
self.NSVEp_extra_parameters['tracers0_enable_vorticity_omega'] = int(0)
self.NSVEp_extra_parameters['tracers0_cutoff'] = float(1)
self.NSVEp_extra_parameters['tracers0_inner_v0'] = float(1)
+ self.NSVEp_extra_parameters['tracers0_lambda'] = float(1)
#self.extra_parameters = {}
#for key in ['NSVE', 'NSVE_no_output', 'NSVEparticles', 'NSVEparticles_no_output', 'NSVEcomplex_particles']:
# self.extra_parameters[key] = {}
diff --git a/bfps/cpp/full_code/NSVEcomplex_particles.cpp b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
index 02a19931..b5882676 100644
--- a/bfps/cpp/full_code/NSVEcomplex_particles.cpp
+++ b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
@@ -43,7 +43,7 @@ int NSVEcomplex_particles<rnumber>::initialize(void)
p2p_computer<double, long long int> current_p2p_computer;
current_p2p_computer.setEnable(this->enable_p2p);
- particles_inner_computer<double, long long int> current_particles_inner_computer(inner_v0);
+ particles_inner_computer<double, long long int> current_particles_inner_computer(inner_v0, this->lambda);
current_particles_inner_computer.setEnable(enable_inner);
@@ -249,6 +249,7 @@ int NSVEcomplex_particles<rnumber>::read_parameters(void)
tval, this->enable_vorticity_omega);
this->cutoff = hdf5_tools::read_value<double>(parameter_file, "parameters/tracers0_cutoff");
this->inner_v0 = hdf5_tools::read_value<double>(parameter_file, "parameters/tracers0_inner_v0");
+ this->lambda = hdf5_tools::read_value<double>(parameter_file, "parameters/tracers0_lambda");
H5Fclose(parameter_file);
return EXIT_SUCCESS;
}
diff --git a/bfps/cpp/full_code/NSVEcomplex_particles.hpp b/bfps/cpp/full_code/NSVEcomplex_particles.hpp
index 2015ec5b..20a84a65 100644
--- a/bfps/cpp/full_code/NSVEcomplex_particles.hpp
+++ b/bfps/cpp/full_code/NSVEcomplex_particles.hpp
@@ -61,6 +61,7 @@ class NSVEcomplex_particles: public NSVE<rnumber>
double cutoff;
double inner_v0;
+ double lambda;
bool enable_p2p;
bool enable_inner;
bool enable_vorticity_omega;
@@ -80,7 +81,7 @@ class NSVEcomplex_particles: public NSVE<rnumber>
NSVE<rnumber>(
COMMUNICATOR,
simulation_name),
- cutoff(10), inner_v0(1), enable_p2p(true), enable_inner(true), enable_vorticity_omega(true){}
+ cutoff(10), inner_v0(1), lambda(1.0), enable_p2p(true), enable_inner(true), enable_vorticity_omega(true){}
~NSVEcomplex_particles(){}
int initialize(void);
diff --git a/bfps/cpp/particles/particles_inner_computer.hpp b/bfps/cpp/particles/particles_inner_computer.hpp
index 3e233370..59dbba64 100644
--- a/bfps/cpp/particles/particles_inner_computer.hpp
+++ b/bfps/cpp/particles/particles_inner_computer.hpp
@@ -9,10 +9,40 @@ template <class real_number, class partsize_t>
class particles_inner_computer{
bool isActive;
const real_number v0;
+ const real_number lambda;
+ const real_number lambda1;
+ const real_number lambda2;
+ const real_number lambda3;
public:
- explicit particles_inner_computer(const real_number inV0) : isActive(true), v0(inV0){
- }
+ explicit particles_inner_computer(const real_number inV0):
+ isActive(true),
+ v0(inV0),
+ lambda(0),
+ lambda1(0),
+ lambda2(0),
+ lambda3(0)
+ {}
+ explicit particles_inner_computer(const real_number inV0, const real_number inLambda):
+ isActive(true),
+ v0(inV0),
+ lambda(inLambda),
+ lambda1(0),
+ lambda2(0),
+ lambda3(0)
+ {}
+ explicit particles_inner_computer(
+ const real_number inV0,
+ const real_number inLambda1,
+ const real_number inLambda2,
+ const real_number inLambda3):
+ isActive(true),
+ v0(inV0),
+ lambda(0),
+ lambda1(inLambda1),
+ lambda2(inLambda2),
+ lambda3(inLambda3)
+ {}
template <int size_particle_positions, int size_particle_rhs>
void compute_interaction(
--
GitLab
From 9007bae33acf6e17bd51b29cfdbb5752701e098f Mon Sep 17 00:00:00 2001
From: Jose Agustin Arguedas Leiva <agustin.arguedas@ds.mpg.de>
Date: Wed, 26 Sep 2018 17:29:36 +0200
Subject: [PATCH 249/342] Add simple ellipsoid equation.
---
.../particles/particles_inner_computer.cpp | 25 +++++++++++++------
1 file changed, 18 insertions(+), 7 deletions(-)
diff --git a/bfps/cpp/particles/particles_inner_computer.cpp b/bfps/cpp/particles/particles_inner_computer.cpp
index f5967087..ad32e8ef 100644
--- a/bfps/cpp/particles/particles_inner_computer.cpp
+++ b/bfps/cpp/particles/particles_inner_computer.cpp
@@ -106,7 +106,7 @@ void particles_inner_computer<double, long long>::compute_interaction_with_extra
}
}
-template <>
+template <> //Work here
template <>
void particles_inner_computer<double, long long>::compute_interaction_with_extra<6,6,9>(
const long long nb_particles,
@@ -120,12 +120,23 @@ void particles_inner_computer<double, long long>::compute_interaction_with_extra
#pragma omp parallel for
for(long long idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
// Cross product vorticity/orientation
- rhs_part[idx_part*6 + 3+IDXC_X] += 0.5*(rhs_part_extra[idx_part*9 + IDXC_Y]*pos_part[idx_part*6 + 3+IDXC_Z] -
- rhs_part_extra[idx_part*9 + IDXC_Z]*pos_part[idx_part*6 + 3+IDXC_Y]);
- rhs_part[idx_part*6 + 3+IDXC_Y] += 0.5*(rhs_part_extra[idx_part*9 + IDXC_Z]*pos_part[idx_part*6 + 3+IDXC_X] -
- rhs_part_extra[idx_part*9 + IDXC_X]*pos_part[idx_part*6 + 3+IDXC_Z]);
- rhs_part[idx_part*6 + 3+IDXC_Z] += 0.5*(rhs_part_extra[idx_part*9 + IDXC_X]*pos_part[idx_part*6 + 3+IDXC_Y] -
- rhs_part_extra[idx_part*9 + IDXC_Y]*pos_part[idx_part*6 + 3+IDXC_X]);
+ rhs_part[idx_part*6 + 3+IDXC_X] += pos_part[idx_part*6 + 3+IDXC_Z])/(lambda**2+1.0)*
+ (lambda**2*rhs_part_extra[idx_part*9 + IDXC_DZ_X]-rhs_part_extra[idx_part*9 + IDXC_DX_Z])
+ +pos_part[idx_part*6 + 3+IDXC_Y])/(lambda**2+1.0)*
+ (lambda**2*rhs_part_extra[idx_part*9 + IDXC_DY_X]-rhs_part_extra[idx_part*9 + IDXC_DX_Y])
+ +pos_part[idx_part*6 + 3+IDXC_X])*(lambda**2-1.0)/(lambda**2+1.0)*rhs_part_extra[idx_part*9 + IDXC_DX_X];
+
+ rhs_part[idx_part*6 + 3+IDXC_Y] += pos_part[idx_part*6 + 3+IDXC_X])/(lambda**2+1.0)*
+ (lambda**2*rhs_part_extra[idx_part*9 + IDXC_DX_Y]-rhs_part_extra[idx_part*9 + IDXC_DY_X])
+ +pos_part[idx_part*6 + 3+IDXC_Z])/(lambda**2+1.0)*
+ (lambda**2*rhs_part_extra[idx_part*9 + IDXC_DZ_Y]-rhs_part_extra[idx_part*9 + IDXC_DY_Z])
+ +pos_part[idx_part*6 + 3+IDXC_Y])*(lambda**2-1.0)/(lambda**2+1.0)*rhs_part_extra[idx_part*9 + IDXC_DY_Y];
+
+ rhs_part[idx_part*6 + 3+IDXC_Z] += pos_part[idx_part*6 + 3+IDXC_Y])/(lambda**2+1.0)*
+ (lambda**2*rhs_part_extra[idx_part*9 + IDXC_DY_Z]-rhs_part_extra[idx_part*9 + IDXC_DZ_Y])
+ +pos_part[idx_part*6 + 3+IDXC_X])/(lambda**2+1.0)*
+ (lambda**2*rhs_part_extra[idx_part*9 + IDXC_DX_Z]-rhs_part_extra[idx_part*9 + IDXC_DZ_X])
+ +pos_part[idx_part*6 + 3+IDXC_Z])*(lambda**2-1.0)/(lambda**2+1.0)*rhs_part_extra[idx_part*9 + IDXC_DZ_Z];
}
}
--
GitLab
From ac6caacc4194c6ca8ab59d1baeffadad0120f2af Mon Sep 17 00:00:00 2001
From: Chichi Lalescu <chichilalescu@gmail.com>
Date: Wed, 26 Sep 2018 23:49:57 +0200
Subject: [PATCH 250/342] define alpha for ellipsoid equation
---
bfps/cpp/particles/particles_inner_computer.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bfps/cpp/particles/particles_inner_computer.cpp b/bfps/cpp/particles/particles_inner_computer.cpp
index ad32e8ef..088b6eb3 100644
--- a/bfps/cpp/particles/particles_inner_computer.cpp
+++ b/bfps/cpp/particles/particles_inner_computer.cpp
@@ -115,11 +115,11 @@ void particles_inner_computer<double, long long>::compute_interaction_with_extra
const double rhs_part_extra[]) const{
// call plain compute_interaction first
compute_interaction<6, 6>(nb_particles, pos_part, rhs_part);
+ const double alpha = (lambda*lambda - 1) / (lambda*lambda+1);
// now add vorticity term
#pragma omp parallel for
for(long long idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
- // Cross product vorticity/orientation
rhs_part[idx_part*6 + 3+IDXC_X] += pos_part[idx_part*6 + 3+IDXC_Z])/(lambda**2+1.0)*
(lambda**2*rhs_part_extra[idx_part*9 + IDXC_DZ_X]-rhs_part_extra[idx_part*9 + IDXC_DX_Z])
+pos_part[idx_part*6 + 3+IDXC_Y])/(lambda**2+1.0)*
--
GitLab
From a45f48a576af3ee6d8722b67c2dd16f1b81ebdf4 Mon Sep 17 00:00:00 2001
From: Jose Agustin Arguedas Leiva <agustin.arguedas@ds.mpg.de>
Date: Thu, 27 Sep 2018 09:08:35 +0200
Subject: [PATCH 251/342] Fixed syntax errors.
---
.../particles/particles_inner_computer.cpp | 22 ++++---------------
1 file changed, 4 insertions(+), 18 deletions(-)
diff --git a/bfps/cpp/particles/particles_inner_computer.cpp b/bfps/cpp/particles/particles_inner_computer.cpp
index 088b6eb3..5d7dd3f5 100644
--- a/bfps/cpp/particles/particles_inner_computer.cpp
+++ b/bfps/cpp/particles/particles_inner_computer.cpp
@@ -115,28 +115,14 @@ void particles_inner_computer<double, long long>::compute_interaction_with_extra
const double rhs_part_extra[]) const{
// call plain compute_interaction first
compute_interaction<6, 6>(nb_particles, pos_part, rhs_part);
- const double alpha = (lambda*lambda - 1) / (lambda*lambda+1);
+ const double ll2 = lambda*lambda;
// now add vorticity term
#pragma omp parallel for
for(long long idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
- rhs_part[idx_part*6 + 3+IDXC_X] += pos_part[idx_part*6 + 3+IDXC_Z])/(lambda**2+1.0)*
- (lambda**2*rhs_part_extra[idx_part*9 + IDXC_DZ_X]-rhs_part_extra[idx_part*9 + IDXC_DX_Z])
- +pos_part[idx_part*6 + 3+IDXC_Y])/(lambda**2+1.0)*
- (lambda**2*rhs_part_extra[idx_part*9 + IDXC_DY_X]-rhs_part_extra[idx_part*9 + IDXC_DX_Y])
- +pos_part[idx_part*6 + 3+IDXC_X])*(lambda**2-1.0)/(lambda**2+1.0)*rhs_part_extra[idx_part*9 + IDXC_DX_X];
-
- rhs_part[idx_part*6 + 3+IDXC_Y] += pos_part[idx_part*6 + 3+IDXC_X])/(lambda**2+1.0)*
- (lambda**2*rhs_part_extra[idx_part*9 + IDXC_DX_Y]-rhs_part_extra[idx_part*9 + IDXC_DY_X])
- +pos_part[idx_part*6 + 3+IDXC_Z])/(lambda**2+1.0)*
- (lambda**2*rhs_part_extra[idx_part*9 + IDXC_DZ_Y]-rhs_part_extra[idx_part*9 + IDXC_DY_Z])
- +pos_part[idx_part*6 + 3+IDXC_Y])*(lambda**2-1.0)/(lambda**2+1.0)*rhs_part_extra[idx_part*9 + IDXC_DY_Y];
-
- rhs_part[idx_part*6 + 3+IDXC_Z] += pos_part[idx_part*6 + 3+IDXC_Y])/(lambda**2+1.0)*
- (lambda**2*rhs_part_extra[idx_part*9 + IDXC_DY_Z]-rhs_part_extra[idx_part*9 + IDXC_DZ_Y])
- +pos_part[idx_part*6 + 3+IDXC_X])/(lambda**2+1.0)*
- (lambda**2*rhs_part_extra[idx_part*9 + IDXC_DX_Z]-rhs_part_extra[idx_part*9 + IDXC_DZ_X])
- +pos_part[idx_part*6 + 3+IDXC_Z])*(lambda**2-1.0)/(lambda**2+1.0)*rhs_part_extra[idx_part*9 + IDXC_DZ_Z];
+ rhs_part[idx_part*6 + 3+IDXC_X] += pos_part[idx_part*6 + 3+IDXC_Z]/(ll2+1.0)*(ll2*rhs_part_extra[idx_part*9 + IDXC_DZ_X]-rhs_part_extra[idx_part*9 + IDXC_DX_Z])+pos_part[idx_part*6 + 3+IDXC_Y]/(ll2+1.0)*(ll2*rhs_part_extra[idx_part*9 + IDXC_DY_X]-rhs_part_extra[idx_part*9 + IDXC_DX_Y])+pos_part[idx_part*6 + 3+IDXC_X]*(ll2-1.0)/(ll2+1.0)*rhs_part_extra[idx_part*9 + IDXC_DX_X];
+ rhs_part[idx_part*6 + 3+IDXC_Y] += pos_part[idx_part*6 + 3+IDXC_X]/(ll2+1.0)*(ll2*rhs_part_extra[idx_part*9 + IDXC_DX_Y]-rhs_part_extra[idx_part*9 + IDXC_DY_X])+pos_part[idx_part*6 + 3+IDXC_Z]/(ll2+1.0)*(ll2*rhs_part_extra[idx_part*9 + IDXC_DZ_Y]-rhs_part_extra[idx_part*9 + IDXC_DY_Z])+pos_part[idx_part*6 + 3+IDXC_Y]*(ll2-1.0)/(ll2+1.0)*rhs_part_extra[idx_part*9 + IDXC_DY_Y];
+ rhs_part[idx_part*6 + 3+IDXC_Z] += pos_part[idx_part*6 + 3+IDXC_Y]/(ll2+1.0)*(ll2*rhs_part_extra[idx_part*9 + IDXC_DY_Z]-rhs_part_extra[idx_part*9 + IDXC_DZ_Y])+pos_part[idx_part*6 + 3+IDXC_X]/(ll2+1.0)*(ll2*rhs_part_extra[idx_part*9 + IDXC_DX_Z]-rhs_part_extra[idx_part*9 + IDXC_DZ_X])+pos_part[idx_part*6 + 3+IDXC_Z]*(ll2-1.0)/(ll2+1.0)*rhs_part_extra[idx_part*9 + IDXC_DZ_Z];
}
}
--
GitLab
From bd4d9bb09e0ed585e0e474f838f23a1e60f1621b Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 27 Sep 2018 11:16:57 +0200
Subject: [PATCH 252/342] adapt code for solving ellipsoid equations
---
bfps/cpp/full_code/NSVEcomplex_particles.cpp | 21 ++++++----
.../particles/abstract_particles_system.hpp | 37 +++++++++++++-----
.../particles/particles_inner_computer.cpp | 17 +++++++--
bfps/cpp/particles/particles_system.hpp | 38 +++++++++++++++----
4 files changed, 84 insertions(+), 29 deletions(-)
diff --git a/bfps/cpp/full_code/NSVEcomplex_particles.cpp b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
index b5882676..c6b6d2ea 100644
--- a/bfps/cpp/full_code/NSVEcomplex_particles.cpp
+++ b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
@@ -91,17 +91,22 @@ int NSVEcomplex_particles<rnumber>::step(void)
{
TIMEZONE("NSVEcomplex_particles::step");
this->fs->compute_velocity(this->fs->cvorticity);
- this->fs->cvelocity->ift();
if(this->enable_vorticity_omega){
- *this->tmp_vec_field = this->fs->cvorticity->get_cdata();
- this->tmp_vec_field->ift();
- std::unique_ptr<double[]> sampled_vorticity(new double[3*this->ps->getLocalNbParticles()]);
- std::fill_n(sampled_vorticity.get(), 3*this->ps->getLocalNbParticles(), 0);
- this->ps->sample_compute_field(*this->tmp_vec_field, sampled_vorticity.get());
- DEBUG_MSG("sampled vorticity is %g\n", sampled_vorticity[0]);
- this->ps->completeLoopWithVorticity(this->dt, sampled_vorticity.get());
+ compute_gradient(
+ this->fs->kk,
+ this->fs->cvelocity,
+ this->nabla_u);
+ this->nabla_u->ift();
+ this->fs->cvelocity->ift(); // needed before completeloop
+ //std::unique_ptr<double[]> sampled_vorticity(new double[9*this->ps->getLocalNbParticles()]);
+ //std::fill_n(sampled_vorticity.get(), 9*this->ps->getLocalNbParticles(), 0);
+ //this->ps->sample_compute_field(*this->nabla_u, sampled_vorticity.get());
+ //*this->tmp_vec_field = this->fs->cvorticity->get_cdata();
+ //this->tmp_vec_field->ift();
+ this->ps->completeLoopWithExtraField(this->dt, *this->nabla_u);
}
else{
+ this->fs->cvelocity->ift();
this->ps->completeLoop(this->dt);
}
this->NSVE<rnumber>::step();
diff --git a/bfps/cpp/particles/abstract_particles_system.hpp b/bfps/cpp/particles/abstract_particles_system.hpp
index ffa89892..ee864b8c 100644
--- a/bfps/cpp/particles/abstract_particles_system.hpp
+++ b/bfps/cpp/particles/abstract_particles_system.hpp
@@ -24,7 +24,8 @@ public:
virtual void add_Lagrange_multipliers() = 0;
- virtual void compute_particles_inner(const real_number particle_extra_rhs[]) = 0;
+ virtual void compute_sphere_particles_inner(const real_number particle_extra_rhs[]) = 0;
+ virtual void compute_ellipsoid_particles_inner(const real_number particle_extra_rhs[]) = 0;
virtual void move(const real_number dt) = 0;
@@ -36,8 +37,13 @@ public:
virtual void completeLoop(const real_number dt) = 0;
- virtual void completeLoopWithVorticity(const real_number dt,
- const real_number particle_extra_rhs[]) = 0;
+ virtual void completeLoopWithVorticity(
+ const real_number dt,
+ const real_number sampled_vorticity[]) = 0;
+
+ virtual void completeLoopWithVelocityGradient(
+ const real_number dt,
+ const real_number sampled_velocity_gradient[]) = 0;
virtual const real_number* getParticlesState() const = 0;
@@ -71,13 +77,24 @@ public:
//- Not generic to enable sampling end
template <typename rnumber, field_backend be, field_components fc>
- void completeLoopWithVorticity(const real_number dt,
- const field<rnumber, be, fc>& in_field) {
- static_assert(fc == THREE, "only THREE is supported for now");
- std::unique_ptr<real_number[]> extra_rhs(new real_number[getLocalNbParticles()*3]());
- std::fill_n(extra_rhs.get(), 3*getLocalNbParticles(), 0);
- sample_compute_field(in_field, extra_rhs.get());
- completeLoopWithVorticity(dt, extra_rhs.get());
+ void completeLoopWithExtraField(
+ const real_number dt,
+ const field<rnumber, be, fc>& in_field) {
+ static_assert(fc == THREE || THREExTHREE, "only THREE or THREExTHREE is supported for now");
+ if (fc == THREE)
+ {
+ std::unique_ptr<real_number[]> extra_rhs(new real_number[getLocalNbParticles()*3]());
+ std::fill_n(extra_rhs.get(), 3*getLocalNbParticles(), 0);
+ sample_compute_field(in_field, extra_rhs.get());
+ completeLoopWithVorticity(dt, extra_rhs.get());
+ }
+ else if (fc == THREExTHREE)
+ {
+ std::unique_ptr<real_number[]> extra_rhs(new real_number[getLocalNbParticles()*9]());
+ std::fill_n(extra_rhs.get(), 9*getLocalNbParticles(), 0);
+ sample_compute_field(in_field, extra_rhs.get());
+ completeLoopWithVelocityGradient(dt, extra_rhs.get());
+ }
}
};
diff --git a/bfps/cpp/particles/particles_inner_computer.cpp b/bfps/cpp/particles/particles_inner_computer.cpp
index 5d7dd3f5..d4e24c2c 100644
--- a/bfps/cpp/particles/particles_inner_computer.cpp
+++ b/bfps/cpp/particles/particles_inner_computer.cpp
@@ -120,9 +120,20 @@ void particles_inner_computer<double, long long>::compute_interaction_with_extra
// now add vorticity term
#pragma omp parallel for
for(long long idx_part = 0 ; idx_part < nb_particles ; ++idx_part){
- rhs_part[idx_part*6 + 3+IDXC_X] += pos_part[idx_part*6 + 3+IDXC_Z]/(ll2+1.0)*(ll2*rhs_part_extra[idx_part*9 + IDXC_DZ_X]-rhs_part_extra[idx_part*9 + IDXC_DX_Z])+pos_part[idx_part*6 + 3+IDXC_Y]/(ll2+1.0)*(ll2*rhs_part_extra[idx_part*9 + IDXC_DY_X]-rhs_part_extra[idx_part*9 + IDXC_DX_Y])+pos_part[idx_part*6 + 3+IDXC_X]*(ll2-1.0)/(ll2+1.0)*rhs_part_extra[idx_part*9 + IDXC_DX_X];
- rhs_part[idx_part*6 + 3+IDXC_Y] += pos_part[idx_part*6 + 3+IDXC_X]/(ll2+1.0)*(ll2*rhs_part_extra[idx_part*9 + IDXC_DX_Y]-rhs_part_extra[idx_part*9 + IDXC_DY_X])+pos_part[idx_part*6 + 3+IDXC_Z]/(ll2+1.0)*(ll2*rhs_part_extra[idx_part*9 + IDXC_DZ_Y]-rhs_part_extra[idx_part*9 + IDXC_DY_Z])+pos_part[idx_part*6 + 3+IDXC_Y]*(ll2-1.0)/(ll2+1.0)*rhs_part_extra[idx_part*9 + IDXC_DY_Y];
- rhs_part[idx_part*6 + 3+IDXC_Z] += pos_part[idx_part*6 + 3+IDXC_Y]/(ll2+1.0)*(ll2*rhs_part_extra[idx_part*9 + IDXC_DY_Z]-rhs_part_extra[idx_part*9 + IDXC_DZ_Y])+pos_part[idx_part*6 + 3+IDXC_X]/(ll2+1.0)*(ll2*rhs_part_extra[idx_part*9 + IDXC_DX_Z]-rhs_part_extra[idx_part*9 + IDXC_DZ_X])+pos_part[idx_part*6 + 3+IDXC_Z]*(ll2-1.0)/(ll2+1.0)*rhs_part_extra[idx_part*9 + IDXC_DZ_Z];
+ long long idx_part6 = idx_part*6 + 3;
+ long long idx_part9 = idx_part*9;
+ rhs_part[idx_part6+IDXC_X] += (
+ pos_part[idx_part6+IDXC_Z]*(ll2*rhs_part_extra[idx_part9 + IDXC_DZ_X]-rhs_part_extra[idx_part9 + IDXC_DX_Z])
+ + pos_part[idx_part6+IDXC_Y]*(ll2*rhs_part_extra[idx_part9 + IDXC_DY_X]-rhs_part_extra[idx_part9 + IDXC_DX_Y])
+ + pos_part[idx_part6+IDXC_X]*(ll2-1)*rhs_part_extra[idx_part9 + IDXC_DX_X]) / (ll2+1);
+ rhs_part[idx_part6+IDXC_Y] += (
+ pos_part[idx_part6+IDXC_X]*(ll2*rhs_part_extra[idx_part9 + IDXC_DX_Y]-rhs_part_extra[idx_part9 + IDXC_DY_X])
+ + pos_part[idx_part6+IDXC_Z]*(ll2*rhs_part_extra[idx_part9 + IDXC_DZ_Y]-rhs_part_extra[idx_part9 + IDXC_DY_Z])
+ + pos_part[idx_part6+IDXC_Y]*(ll2-1)*rhs_part_extra[idx_part9 + IDXC_DY_Y]) / (ll2+1);
+ rhs_part[idx_part6+IDXC_Z] += (
+ pos_part[idx_part6+IDXC_Y]*(ll2*rhs_part_extra[idx_part9 + IDXC_DY_Z]-rhs_part_extra[idx_part9 + IDXC_DZ_Y])
+ + pos_part[idx_part6+IDXC_X]*(ll2*rhs_part_extra[idx_part9 + IDXC_DX_Z]-rhs_part_extra[idx_part9 + IDXC_DZ_X])
+ + pos_part[idx_part6+IDXC_Z]*(ll2-1)*rhs_part_extra[idx_part9 + IDXC_DZ_Z]) / (ll2+1);
}
}
diff --git a/bfps/cpp/particles/particles_system.hpp b/bfps/cpp/particles/particles_system.hpp
index 460383c3..ebc7e79a 100644
--- a/bfps/cpp/particles/particles_system.hpp
+++ b/bfps/cpp/particles/particles_system.hpp
@@ -179,12 +179,21 @@ public:
}
}
- void compute_particles_inner(const real_number particle_extra_rhs[]) final {
+ void compute_sphere_particles_inner(const real_number particle_extra_field[]) final {
if(computer_particules_inner.isEnable() == true){
- TIMEZONE("particles_system::compute_particles_inner");
+ TIMEZONE("particles_system::compute_sphere_particles_inner");
computer_particules_inner.template compute_interaction_with_extra<size_particle_positions, size_particle_rhs, 3>(
my_nb_particles, my_particles_positions.get(), my_particles_rhs.front().get(),
- particle_extra_rhs);
+ particle_extra_field);
+ }
+ }
+
+ void compute_ellipsoid_particles_inner(const real_number particle_extra_field[]) final {
+ if(computer_particules_inner.isEnable() == true){
+ TIMEZONE("particles_system::compute_ellipsoid_particles_inner");
+ computer_particules_inner.template compute_interaction_with_extra<size_particle_positions, size_particle_rhs, 9>(
+ my_nb_particles, my_particles_positions.get(), my_particles_rhs.front().get(),
+ particle_extra_field);
}
}
@@ -269,7 +278,6 @@ public:
compute();
compute_p2p();
compute_particles_inner();
- add_Lagrange_multipliers();
move(dt);
enforce_unit_orientation();
redistribute();
@@ -277,13 +285,27 @@ public:
shift_rhs_vectors();
}
- void completeLoopWithVorticity(const real_number dt,
- const real_number particle_extra_rhs[]) final {
+ void completeLoopWithVorticity(
+ const real_number dt,
+ const real_number particle_extra_field[]) final {
TIMEZONE("particles_system::completeLoopWithVorticity");
compute();
compute_p2p();
- compute_particles_inner(particle_extra_rhs);
- add_Lagrange_multipliers();
+ compute_sphere_particles_inner(particle_extra_field);
+ move(dt);
+ enforce_unit_orientation();
+ redistribute();
+ inc_step_idx();
+ shift_rhs_vectors();
+ }
+
+ void completeLoopWithVelocityGradient(
+ const real_number dt,
+ const real_number particle_extra_field[]) final {
+ TIMEZONE("particles_system::completeLoopWithVelocityGradient");
+ compute();
+ compute_p2p();
+ compute_ellipsoid_particles_inner(particle_extra_field);
move(dt);
enforce_unit_orientation();
redistribute();
--
GitLab
From c5435b261dc39ab6fa94f57ff6c68dfd74469d1c Mon Sep 17 00:00:00 2001
From: Cristian Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 27 Sep 2018 14:19:27 +0200
Subject: [PATCH 253/342] add explicit instantiation of template method
apparently some versions of compilers need this
---
bfps/cpp/particles/particles_inner_computer.cpp | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/bfps/cpp/particles/particles_inner_computer.cpp b/bfps/cpp/particles/particles_inner_computer.cpp
index d4e24c2c..deb85dee 100644
--- a/bfps/cpp/particles/particles_inner_computer.cpp
+++ b/bfps/cpp/particles/particles_inner_computer.cpp
@@ -160,3 +160,9 @@ void particles_inner_computer<double, long long>::enforce_unit_orientation<6>(
}
}
+template
+void particles_inner_computer<double, long long>::compute_interaction<6, 6>(
+ const long long nb_particles,
+ const double pos_part[],
+ double rhs_part[]) const;
+
--
GitLab
From 0292fbd03c38fbe455588176cf30554c632e8234 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 27 Sep 2018 14:41:36 +0200
Subject: [PATCH 254/342] add no-debug option for launcher
---
bfps/DNS.py | 7 ++++++-
bfps/_code.py | 11 ++++++++---
2 files changed, 14 insertions(+), 4 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 8e72593e..94081850 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -530,6 +530,10 @@ class DNS(_code):
metavar = 'NTHREADS_PER_PROCESS',
help = 'number of threads to use per MPI process',
default = 1)
+ parser.add_argument(
+ '--no-debug',
+ action = 'store_true',
+ dest = 'no_debug')
parser.add_argument(
'--no-submit',
action = 'store_true',
@@ -1051,6 +1055,7 @@ class DNS(_code):
njobs = opt.njobs,
hours = opt.minutes // 60,
minutes = opt.minutes % 60,
- no_submit = opt.no_submit)
+ no_submit = opt.no_submit,
+ no_debug = opt.no_debug)
return None
diff --git a/bfps/_code.py b/bfps/_code.py
index c29dda63..b3aa9ff9 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -194,7 +194,9 @@ class _code(_base):
outfile.write(self.main)
outfile.write(self.main_end)
return None
- def compile_code(self):
+ def compile_code(
+ self,
+ no_debug = False):
# compile code
if not os.path.isfile(os.path.join(bfps.header_dir, 'base.hpp')):
raise IOError('header not there:\n' +
@@ -206,6 +208,8 @@ class _code(_base):
command_strings = [bfps.install_info['compiler']]
command_strings += [self.name + '.cpp', '-o', self.name]
command_strings += bfps.install_info['extra_compile_args']
+ if no_debug:
+ command_strings += ['-DNDEBUG']
command_strings += ['-I' + idir for idir in bfps.install_info['include_dirs']]
command_strings.append('-I' + bfps.header_dir)
command_strings += ['-L' + ldir for ldir in bfps.install_info['library_dirs']]
@@ -237,7 +241,8 @@ class _code(_base):
hours = 0,
minutes = 10,
njobs = 1,
- no_submit = False):
+ no_submit = False,
+ no_debug = False):
self.read_parameters()
with h5py.File(os.path.join(self.work_dir, self.simname + '.h5'), 'r') as data_file:
iter0 = data_file['iteration'].value
@@ -249,7 +254,7 @@ class _code(_base):
need_to_compile = (datetime.fromtimestamp(os.path.getctime(os.path.join(self.work_dir, self.name))) <
bfps.install_info['install_date'])
if need_to_compile:
- assert self.compile_code() == 0
+ assert self.compile_code(no_debug = no_debug) == 0
if self.work_dir != os.path.realpath(os.getcwd()):
shutil.copy(self.name, self.work_dir)
if 'niter_todo' not in self.parameters.keys():
--
GitLab
From de8d7bd486b0b63fd192456a55f227314652fba8 Mon Sep 17 00:00:00 2001
From: Cristian Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 5 Oct 2018 12:50:17 +0200
Subject: [PATCH 255/342] bugfix --- fix second half of filtered field output
---
bfps/cpp/field.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/bfps/cpp/field.cpp b/bfps/cpp/field.cpp
index 87e18b67..452193c4 100644
--- a/bfps/cpp/field.cpp
+++ b/bfps/cpp/field.cpp
@@ -24,7 +24,7 @@
-#define NDEBUG
+//#define NDEBUG
#include <sys/stat.h>
#include <cmath>
@@ -769,13 +769,13 @@ int field<rnumber, be, fc>::write_filtered(
if (y0 <= hsize_t(this->clayout->sizes[0] - ny/2))
{
count[0] = y1 - (this->clayout->sizes[0] - ny/2);
- offset[0] = ny/2;
+ offset[0] = ny - (this->clayout->sizes[0] - y0);
memoffset[0] = this->clayout->subsizes[0] - count[0];
}
else
{
count[0] = this->clayout->subsizes[0];
- offset[0] = y0;
+ offset[0] = ny - (this->clayout->sizes[0] - y0);
memoffset[0] = 0;
}
}
--
GitLab
From 3a772ba445ba24e946c533c2e2b180c37dfda6da Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 5 Oct 2018 14:06:05 +0200
Subject: [PATCH 256/342] add back NDEBUG
---
bfps/cpp/field.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bfps/cpp/field.cpp b/bfps/cpp/field.cpp
index 452193c4..95a44f44 100644
--- a/bfps/cpp/field.cpp
+++ b/bfps/cpp/field.cpp
@@ -24,7 +24,7 @@
-//#define NDEBUG
+#define NDEBUG
#include <sys/stat.h>
#include <cmath>
--
GitLab
From e61407900a7636ddc12f68d8b5487738948547f4 Mon Sep 17 00:00:00 2001
From: Debarghya Banerjee <debarghya.banerjee@ds.mpg.de>
Date: Mon, 8 Oct 2018 14:55:25 +0200
Subject: [PATCH 257/342] M-filter implementation
---
bfps/cpp/kspace.cpp | 51 +++++++++++++++++++++++++++++++++++++++++++++
bfps/cpp/kspace.hpp | 7 +++++++
2 files changed, 58 insertions(+)
diff --git a/bfps/cpp/kspace.cpp b/bfps/cpp/kspace.cpp
index 3fb25000..8f593ea5 100644
--- a/bfps/cpp/kspace.cpp
+++ b/bfps/cpp/kspace.cpp
@@ -280,6 +280,43 @@ void kspace<be, dt>::ball_filter(
});
}
+/** \brief Filter a field using a M-filter to reproduce dissipation range.
+ *
+ * Filter's Fourier space expression:
+ * \f[
+ * \hat{\phi^M_\ell}(k) =
+ * \exp(-\frac{(5.0 k \ell)^a5}{2})
+ * \left( 1 + \frac{(k \eta/0.06)^3}{1 + (k \eta/ 0.07)^3} \right)^{1/2}
+ * \f]
+ */
+template <field_backend be,
+ kspace_dealias_type dt>
+template <typename rnumber,
+ field_components fc>
+void kspace<be, dt>::general_M_filter(
+ typename fftw_interface<rnumber>::complex *__restrict__ a,
+ const double ell,
+ const double a5)
+{
+ const double prefactor0 = 1.0;
+ this->CLOOP_K2(
+ [&](ptrdiff_t cindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex,
+ double k2){
+ if (k2 > 0)
+ {
+ double argument = sqrt(k2)*ell;
+ double prefactor = prefactor0;
+ for (unsigned int tcounter=0; tcounter<2*ncomp(fc); tcounter++)
+ ((rnumber*)a)[2*ncomp(fc)*cindex + tcounter] *= (
+ prefactor * (exp(-0.5*pow((5.0*argument),(a5))) * sqrt(1 + (pow((argument/0.06),3))/(1.0 + (pow((argument/0.07),3))))));
+ }
+ });
+}
+
+
/** \brief Filter a field using a Gaussian kernel.
*
* Filter's mathematical expression in Fourier space is as follows:
@@ -373,6 +410,13 @@ int kspace<be, dt>::filter(
a,
2*acos(0.)/wavenumber);
}
+ else if (filter_type == std::string("general_M"))
+ {
+ this->template general_M_filter<rnumber, fc>(
+ a,
+ 2*acos(0.)/wavenumber,
+ 0.9);
+ }
return EXIT_SUCCESS;
}
@@ -443,6 +487,13 @@ int kspace<be, dt>::filter_calibrated_ell(
a,
ell);
}
+ else if (filter_type == std::string("general_M"))
+ {
+ this->template general_M_filter<rnumber, fc>(
+ a,
+ ell,
+ 0.9);
+ }
return EXIT_SUCCESS;
}
diff --git a/bfps/cpp/kspace.hpp b/bfps/cpp/kspace.hpp
index c0bf2583..76059aa6 100644
--- a/bfps/cpp/kspace.hpp
+++ b/bfps/cpp/kspace.hpp
@@ -87,6 +87,13 @@ class kspace
typename fftw_interface<rnumber>::complex *__restrict__ a,
const double sigma);
+ template <typename rnumber,
+ field_components fc>
+ void general_M_filter(
+ typename fftw_interface<rnumber>::complex *__restrict__ a,
+ const double sigma,
+ const double a5);
+
template <typename rnumber,
field_components fc>
int filter(
--
GitLab
From b998d2ec91f734eff76c34481e655ab034f58307 Mon Sep 17 00:00:00 2001
From: Cristian Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 11 Oct 2018 11:16:54 +0200
Subject: [PATCH 258/342] add energy policy tag
---
bfps/_code.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/bfps/_code.py b/bfps/_code.py
index b3aa9ff9..cfddb6b3 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -423,6 +423,8 @@ class _code(_base):
nb_processes_per_node = int(nb_cpus_per_node // nb_threads_per_process)
first_node_tasks = int(nb_mpi_processes - (nb_nodes-1)*nb_processes_per_node)
+ script_file.write('# @ energy_policy_tag = {0}\n'.format(self.simname))
+ script_file.write('# @ minimize_time_to_solution = yes\n')
script_file.write('# @ resources = ConsumableCpus({})\n'.format(nb_threads_per_process))
script_file.write('# @ network.MPI = sn_all,not_shared,us\n')
script_file.write('# @ wall_clock_limit = {0}:{1:0>2d}:00\n'.format(hours, minutes))
@@ -528,6 +530,8 @@ class _code(_base):
script_file.write('# @ resources = ConsumableCpus({})\n'.format(nb_threads_per_process))
script_file.write('# @ network.MPI = sn_all,not_shared,us\n')
script_file.write('# @ wall_clock_limit = {0}:{1:0>2d}:00\n'.format(hours, minutes))
+ script_file.write('# @ energy_policy_tag = {0}\n'.format(self.simname))
+ script_file.write('# @ minimize_time_to_solution = yes\n')
assert type(self.host_info['environment']) != type(None)
script_file.write('# @ node = {0}\n'.format(nb_nodes))
script_file.write('# @ tasks_per_node = {0}\n'.format(nb_processes_per_node))
--
GitLab
From a46ad77047aad201894718e4767265baac2df15f Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 11 Oct 2018 13:04:50 +0200
Subject: [PATCH 259/342] bugfix --- fix dimensions for spectrum datasets
---
bfps/cpp/kspace.cpp | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/bfps/cpp/kspace.cpp b/bfps/cpp/kspace.cpp
index 5ceb2a80..c480d1b0 100644
--- a/bfps/cpp/kspace.cpp
+++ b/bfps/cpp/kspace.cpp
@@ -571,13 +571,13 @@ void kspace<be, dt>::cospectrum(
case THREExTHREE:
offset[4] = 0;
offset[5] = 0;
- count[4] = ncomp(fc);
- count[5] = ncomp(fc);
+ count[4] = 3;
+ count[5] = 3;
case THREE:
offset[2] = 0;
offset[3] = 0;
- count[2] = ncomp(fc);
- count[3] = ncomp(fc);
+ count[2] = 3;
+ count[3] = 3;
default:
offset[0] = toffset;
offset[1] = 0;
--
GitLab
From 45060516cc9aba5a8be73d29b4b43b1901927d23 Mon Sep 17 00:00:00 2001
From: Cristian Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 11 Oct 2018 13:37:03 +0200
Subject: [PATCH 260/342] fix energy policy tag thing
---
bfps/_code.py | 24 ++++++++++++++++++------
1 file changed, 18 insertions(+), 6 deletions(-)
diff --git a/bfps/_code.py b/bfps/_code.py
index cfddb6b3..297749f5 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -323,6 +323,14 @@ class _code(_base):
elif self.host_info['type'] == 'IBMLoadLeveler':
suffix = self.simname + '_{0}'.format(iter0)
job_script_name = 'run_' + suffix + '.sh'
+ energy_policy_tag = (
+ 'bfps'
+ + '_np{0}_ntpp{1}'.format(
+ nb_processes, nb_threads_per_process)
+ + '_Nx{0}_Ny{1}_Nz{2}'.format(
+ self.parameters['nx'], self.parameters['ny'], self.parameters['nz']))
+ if 'nparticles' in self.parameters.keys():
+ energy_policy_tag += '_nparticles{0}'.format(self.parameters['nparticles'])
if (njobs == 1):
self.write_IBMLoadLeveler_file_single_job(
file_name = os.path.join(self.work_dir, job_script_name),
@@ -333,7 +341,8 @@ class _code(_base):
out_file = out_file + '_' + suffix,
err_file = err_file + '_' + suffix,
nb_mpi_processes = nb_processes,
- nb_threads_per_process = nb_threads_per_process)
+ nb_threads_per_process = nb_threads_per_process,
+ energy_policy_tag = energy_policy_tag)
else:
self.write_IBMLoadLeveler_file_many_job(
file_name = os.path.join(self.work_dir, job_script_name),
@@ -345,7 +354,8 @@ class _code(_base):
err_file = err_file + '_' + suffix,
njobs = njobs,
nb_mpi_processes = nb_processes,
- nb_threads_per_process = nb_threads_per_process)
+ nb_threads_per_process = nb_threads_per_process,
+ energy_policy_tag = energy_policy_tag)
submit_atoms = ['llsubmit']
if not no_submit:
@@ -375,7 +385,8 @@ class _code(_base):
out_file = None,
err_file = None,
nb_mpi_processes = None,
- nb_threads_per_process = None):
+ nb_threads_per_process = None,
+ energy_policy_tag = 'bfps'):
script_file = open(file_name, 'w')
script_file.write('# @ shell=/bin/bash\n')
@@ -423,7 +434,7 @@ class _code(_base):
nb_processes_per_node = int(nb_cpus_per_node // nb_threads_per_process)
first_node_tasks = int(nb_mpi_processes - (nb_nodes-1)*nb_processes_per_node)
- script_file.write('# @ energy_policy_tag = {0}\n'.format(self.simname))
+ script_file.write('# @ energy_policy_tag = {0}\n'.format(energy_policy_tag))
script_file.write('# @ minimize_time_to_solution = yes\n')
script_file.write('# @ resources = ConsumableCpus({})\n'.format(nb_threads_per_process))
script_file.write('# @ network.MPI = sn_all,not_shared,us\n')
@@ -475,7 +486,8 @@ class _code(_base):
err_file = None,
njobs = 2,
nb_mpi_processes = None,
- nb_threads_per_process = None):
+ nb_threads_per_process = None,
+ energy_policy_tag = 'bfps'):
assert(type(self.host_info['environment']) != type(None))
script_file = open(file_name, 'w')
script_file.write('# @ shell=/bin/bash\n')
@@ -530,7 +542,7 @@ class _code(_base):
script_file.write('# @ resources = ConsumableCpus({})\n'.format(nb_threads_per_process))
script_file.write('# @ network.MPI = sn_all,not_shared,us\n')
script_file.write('# @ wall_clock_limit = {0}:{1:0>2d}:00\n'.format(hours, minutes))
- script_file.write('# @ energy_policy_tag = {0}\n'.format(self.simname))
+ script_file.write('# @ energy_policy_tag = {0}\n'.format(energy_policy_tag))
script_file.write('# @ minimize_time_to_solution = yes\n')
assert type(self.host_info['environment']) != type(None)
script_file.write('# @ node = {0}\n'.format(nb_nodes))
--
GitLab
From 8d23a02812a2775af4de50614dc7cddf974e7c4e Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 12 Oct 2018 14:45:24 +0200
Subject: [PATCH 261/342] add reference
---
bfps/DNS.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 94081850..660192c3 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -319,6 +319,9 @@ class DNS(_code):
return None
def compute_Reynolds_stress_invariants(
self):
+ """
+ see Choi and Lumley, JFM v436 p59 (2001)
+ """
Rij = self.statistics['R_ij(t)']
Rij /= (2*self.statistics['energy(t)'][:, None, None])
Rij[:, 0, 0] -= 1./3
--
GitLab
From c596a65e2a7214d4e58b899c334ac0c289f8aa13 Mon Sep 17 00:00:00 2001
From: Debarghya Banerjee <debarghya.banerjee@ds.mpg.de>
Date: Mon, 15 Oct 2018 10:51:07 +0200
Subject: [PATCH 262/342] e different default a5 value
---
bfps/cpp/kspace.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/bfps/cpp/kspace.cpp b/bfps/cpp/kspace.cpp
index 3bc94990..f0e7185a 100644
--- a/bfps/cpp/kspace.cpp
+++ b/bfps/cpp/kspace.cpp
@@ -420,7 +420,7 @@ int kspace<be, dt>::filter(
this->template general_M_filter<rnumber, fc>(
a,
2*acos(0.)/wavenumber,
- 0.9);
+ 0.7);
}
return EXIT_SUCCESS;
}
@@ -497,7 +497,7 @@ int kspace<be, dt>::filter_calibrated_ell(
this->template general_M_filter<rnumber, fc>(
a,
ell,
- 0.9);
+ 0.7);
}
return EXIT_SUCCESS;
}
--
GitLab
From ba9e1927e7bfd0fac61723347ea291fec87708da Mon Sep 17 00:00:00 2001
From: Debarghya Banerjee <debarghya.banerjee@ds.mpg.de>
Date: Mon, 22 Oct 2018 16:01:53 +0200
Subject: [PATCH 263/342] M-filter changes needs fixing for large ell
---
bfps/cpp/kspace.cpp | 16 +++++++---------
bfps/cpp/kspace.hpp | 3 +--
2 files changed, 8 insertions(+), 11 deletions(-)
diff --git a/bfps/cpp/kspace.cpp b/bfps/cpp/kspace.cpp
index f0e7185a..a7b59bc2 100644
--- a/bfps/cpp/kspace.cpp
+++ b/bfps/cpp/kspace.cpp
@@ -290,8 +290,8 @@ void kspace<be, dt>::ball_filter(
* Filter's Fourier space expression:
* \f[
* \hat{\phi^M_\ell}(k) =
- * \exp(-\frac{(5.0 k \ell)^a5}{2})
- * \left( 1 + \frac{(k \eta/0.06)^3}{1 + (k \eta/ 0.07)^3} \right)^{1/2}
+ * \exp(-\frac{(3.54 k \ell)^(122*(\ell)^(0.0836))}{2})
+ * \left( 1 + \frac{(k \eta/0.0636)^3.44}{1 + (k \eta/ 0.0621)^3.44} \right)^{1/2}
* \f]
*/
template <field_backend be,
@@ -300,8 +300,7 @@ template <typename rnumber,
field_components fc>
void kspace<be, dt>::general_M_filter(
typename fftw_interface<rnumber>::complex *__restrict__ a,
- const double ell,
- const double a5)
+ const double ell)
{
const double prefactor0 = 1.0;
this->CLOOP_K2(
@@ -316,7 +315,8 @@ void kspace<be, dt>::general_M_filter(
double prefactor = prefactor0;
for (unsigned int tcounter=0; tcounter<2*ncomp(fc); tcounter++)
((rnumber*)a)[2*ncomp(fc)*cindex + tcounter] *= (
- prefactor * (exp(-0.5*pow((5.0*argument),(a5))) * sqrt(1 + (pow((argument/0.06),3))/(1.0 + (pow((argument/0.07),3))))));
+ prefactor * (exp(-0.5*pow((3.54*argument),(122*(pow(ell,0.836))))) * sqrt(1.0 + (pow((argument/0.0636),3.44))/(1.0 + (pow((argument/0.0621),3.44))))));
+// prefactor * (exp(-0.5*pow((3.54*argument),(122*pow(ell,0.836)))) * sqrt(1.0 + (pow((argument/0.0636),3.44))/(1.0 + (pow((argument/0.0621),3.44))))));
}
});
}
@@ -419,8 +419,7 @@ int kspace<be, dt>::filter(
{
this->template general_M_filter<rnumber, fc>(
a,
- 2*acos(0.)/wavenumber,
- 0.7);
+ 2*acos(0.)/wavenumber);
}
return EXIT_SUCCESS;
}
@@ -496,8 +495,7 @@ int kspace<be, dt>::filter_calibrated_ell(
{
this->template general_M_filter<rnumber, fc>(
a,
- ell,
- 0.7);
+ ell);
}
return EXIT_SUCCESS;
}
diff --git a/bfps/cpp/kspace.hpp b/bfps/cpp/kspace.hpp
index 76059aa6..71a09998 100644
--- a/bfps/cpp/kspace.hpp
+++ b/bfps/cpp/kspace.hpp
@@ -91,8 +91,7 @@ class kspace
field_components fc>
void general_M_filter(
typename fftw_interface<rnumber>::complex *__restrict__ a,
- const double sigma,
- const double a5);
+ const double sigma);
template <typename rnumber,
field_components fc>
--
GitLab
From f1b48459e1cae9b00865aac5801bf8f741a8d724 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 23 Oct 2018 13:07:03 +0200
Subject: [PATCH 264/342] add misc test of overflow avoidance
---
tests/misc/pow_overflow.cpp | 30 ++++++++++++++++++++++++++++++
1 file changed, 30 insertions(+)
create mode 100644 tests/misc/pow_overflow.cpp
diff --git a/tests/misc/pow_overflow.cpp b/tests/misc/pow_overflow.cpp
new file mode 100644
index 00000000..48cc8aae
--- /dev/null
+++ b/tests/misc/pow_overflow.cpp
@@ -0,0 +1,30 @@
+#include <cfenv>
+#include <cmath>
+#include <iostream>
+#include <limits>
+
+int main()
+{
+ feenableexcept(FE_ALL_EXCEPT);
+ double p0 = 3.54;
+ double p4 = 122;
+ double p5 = 0.836;
+ double ell = 1.0;
+ double result = 0.;
+ double argument = 0.;
+
+ for (int k = 0; k<128; k++)
+ {
+ argument = p0*k*ell;
+ // double exponent = p4*pow(ell, p5);
+ // //if (exponent*log(argument) <2*std::numeric_limits<double>::min())
+ // // result = 0.;
+ // //else
+ // //{
+ // // double result0 = pow(p0*argument, p4*pow(ell, p5));
+ // // result = exp(-0.5*result0);
+ // //}
+ }
+ std::cout << argument << std::endl;
+ return 0;
+}
--
GitLab
From d3e8dc284b95a374782aa93d64237b04c5f910f2 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 25 Oct 2018 17:35:02 +0200
Subject: [PATCH 265/342] bugfix: sample velocity correctly when NSVE stats
were not called
---
bfps/cpp/full_code/NSVEparticles.cpp | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/bfps/cpp/full_code/NSVEparticles.cpp b/bfps/cpp/full_code/NSVEparticles.cpp
index bcb2f435..05069308 100644
--- a/bfps/cpp/full_code/NSVEparticles.cpp
+++ b/bfps/cpp/full_code/NSVEparticles.cpp
@@ -122,6 +122,13 @@ int NSVEparticles<rnumber>::do_stats()
/// sample velocity
std::fill_n(pdata.get(), 3*this->ps->getLocalNbParticles(), 0);
+ if (!(this->iteration % this->niter_stat == 0))
+ {
+ // we need to compute velocity field manually, because it didn't happen in NSVE::do_stats()
+ this->fs->compute_velocity(fs->cvorticity);
+ *this->tmp_vec_field = this->fs->cvelocity->get_cdata();
+ this->tmp_vec_field->ift();
+ }
this->ps->sample_compute_field(*this->tmp_vec_field, pdata.get());
this->particles_sample_writer_mpi->template save_dataset<3>(
"tracers0",
--
GitLab
From 4f907a450659664c61022592a7d2476acc4e1ab2 Mon Sep 17 00:00:00 2001
From: Cristian Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Sat, 27 Oct 2018 10:46:04 +0200
Subject: [PATCH 266/342] bugfix --- use "this" to access fluid solver
---
bfps/cpp/full_code/NSVEparticles.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bfps/cpp/full_code/NSVEparticles.cpp b/bfps/cpp/full_code/NSVEparticles.cpp
index 05069308..277e6095 100644
--- a/bfps/cpp/full_code/NSVEparticles.cpp
+++ b/bfps/cpp/full_code/NSVEparticles.cpp
@@ -125,7 +125,7 @@ int NSVEparticles<rnumber>::do_stats()
if (!(this->iteration % this->niter_stat == 0))
{
// we need to compute velocity field manually, because it didn't happen in NSVE::do_stats()
- this->fs->compute_velocity(fs->cvorticity);
+ this->fs->compute_velocity(this->fs->cvorticity);
*this->tmp_vec_field = this->fs->cvelocity->get_cdata();
this->tmp_vec_field->ift();
}
--
GitLab
From 96308ee91b81247f238f998ed8cd734fd6cd4ccc Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 2 Nov 2018 22:20:04 +0100
Subject: [PATCH 267/342] add fine sampling option for particles
---
bfps/DNS.py | 2 ++
bfps/cpp/full_code/NSVEparticles.cpp | 9 ++++++++-
bfps/cpp/full_code/NSVEparticles.hpp | 2 ++
3 files changed, 12 insertions(+), 1 deletion(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 660192c3..50f62ef7 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -145,6 +145,8 @@ class DNS(_code):
# parameters specific to particle version
self.NSVEp_extra_parameters = {}
self.NSVEp_extra_parameters['niter_part'] = int(1)
+ self.NSVEp_extra_parameters['niter_part_fine_period'] = int(10)
+ self.NSVEp_extra_parameters['niter_part_fine_duration'] = int(0)
self.NSVEp_extra_parameters['nparticles'] = int(10)
self.NSVEp_extra_parameters['tracers0_integration_steps'] = int(4)
self.NSVEp_extra_parameters['tracers0_neighbours'] = int(1)
diff --git a/bfps/cpp/full_code/NSVEparticles.cpp b/bfps/cpp/full_code/NSVEparticles.cpp
index 277e6095..90184d8b 100644
--- a/bfps/cpp/full_code/NSVEparticles.cpp
+++ b/bfps/cpp/full_code/NSVEparticles.cpp
@@ -99,7 +99,12 @@ int NSVEparticles<rnumber>::do_stats()
this->NSVE<rnumber>::do_stats();
- if (!(this->iteration % this->niter_part == 0))
+ /// either one of two conditions suffices to compute statistics:
+ /// 1) current iteration is a multiple of niter_part
+ /// 2) we are within niter_part_fine_duration/2 of a multiple of niter_part_fine_period
+ if (!(this->iteration % this->niter_part == 0 ||
+ ((this->iteration + this->niter_part_fine_duration/2) % this->niter_part_fine_period <=
+ this->niter_part_fine_duration)))
return EXIT_SUCCESS;
// allocate temporary data array
@@ -167,6 +172,8 @@ int NSVEparticles<rnumber>::read_parameters(void)
this->NSVE<rnumber>::read_parameters();
hid_t parameter_file = H5Fopen((this->simname + ".h5").c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
this->niter_part = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_part");
+ this->niter_part_fine_period = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_part_fine_period");
+ this->niter_part_fine_duration = hdf5_tools::read_value<int>(parameter_file, "parameters/niter_part_fine_duration");
this->nparticles = hdf5_tools::read_value<int>(parameter_file, "parameters/nparticles");
this->tracers0_integration_steps = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_integration_steps");
this->tracers0_neighbours = hdf5_tools::read_value<int>(parameter_file, "parameters/tracers0_neighbours");
diff --git a/bfps/cpp/full_code/NSVEparticles.hpp b/bfps/cpp/full_code/NSVEparticles.hpp
index 1d73e129..8b70ead9 100644
--- a/bfps/cpp/full_code/NSVEparticles.hpp
+++ b/bfps/cpp/full_code/NSVEparticles.hpp
@@ -51,6 +51,8 @@ class NSVEparticles: public NSVE<rnumber>
/* parameters that are read in read_parameters */
int niter_part;
+ int niter_part_fine_period;
+ int niter_part_fine_duration;
int nparticles;
int tracers0_integration_steps;
int tracers0_neighbours;
--
GitLab
From 24d796ad374fc1e2593c79518cf3a5827f851600 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 24 Dec 2018 15:19:10 +0100
Subject: [PATCH 268/342] only create tracer initial conditions if they don't
exist
---
bfps/DNS.py | 33 +++++++++++++++++----------------
1 file changed, 17 insertions(+), 16 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 50f62ef7..f360db8b 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -496,22 +496,23 @@ class DNS(_code):
ncomponents = 6
with h5py.File(self.get_checkpoint_0_fname(), 'a') as ofile:
s = 0
- ofile.create_group('tracers{0}'.format(s))
- ofile.create_group('tracers{0}/rhs'.format(s))
- ofile.create_group('tracers{0}/state'.format(s))
- ofile['tracers{0}/rhs'.format(s)].create_dataset(
- '0',
- shape = (
- (self.parameters['tracers{0}_integration_steps'.format(s)],) +
- pbase_shape +
- (ncomponents,)),
- dtype = np.float)
- ofile['tracers{0}/state'.format(s)].create_dataset(
- '0',
- shape = (
- pbase_shape +
- (ncomponents,)),
- dtype = np.float)
+ if not 'tracers{0}'.format(s) in ofile.keys():
+ ofile.create_group('tracers{0}'.format(s))
+ ofile.create_group('tracers{0}/rhs'.format(s))
+ ofile.create_group('tracers{0}/state'.format(s))
+ ofile['tracers{0}/rhs'.format(s)].create_dataset(
+ '0',
+ shape = (
+ (self.parameters['tracers{0}_integration_steps'.format(s)],) +
+ pbase_shape +
+ (ncomponents,)),
+ dtype = np.float)
+ ofile['tracers{0}/state'.format(s)].create_dataset(
+ '0',
+ shape = (
+ pbase_shape +
+ (ncomponents,)),
+ dtype = np.float)
return None
def job_parser_arguments(
self,
--
GitLab
From d61299324c8eceabdd0ca7e754ff7dd687104650 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Sat, 29 Dec 2018 07:56:38 +0100
Subject: [PATCH 269/342] call release() method of std::unique_ptr
---
bfps/cpp/full_code/NSVEparticles.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bfps/cpp/full_code/NSVEparticles.cpp b/bfps/cpp/full_code/NSVEparticles.cpp
index 90184d8b..37b1b833 100644
--- a/bfps/cpp/full_code/NSVEparticles.cpp
+++ b/bfps/cpp/full_code/NSVEparticles.cpp
@@ -160,7 +160,7 @@ int NSVEparticles<rnumber>::do_stats()
// deallocate temporary data array
// TODO: is it required/safe to call the release method here?
- //pdata.release();
+ pdata.release();
return EXIT_SUCCESS;
}
--
GitLab
From 88e38a1f320eaa2d2ea22b7ce6db2678c8031ca7 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Sat, 29 Dec 2018 11:15:18 +0100
Subject: [PATCH 270/342] release particle sample array for complex particles
---
bfps/cpp/full_code/NSVEcomplex_particles.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bfps/cpp/full_code/NSVEcomplex_particles.cpp b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
index c6b6d2ea..43de34aa 100644
--- a/bfps/cpp/full_code/NSVEcomplex_particles.cpp
+++ b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
@@ -230,7 +230,7 @@ int NSVEcomplex_particles<rnumber>::do_stats()
// deallocate temporary data array
// TODO: is it required/safe to call the release method here?
- //pdata.release();
+ pdata.release();
return EXIT_SUCCESS;
}
--
GitLab
From 1b3875006de178fb0a7cb9c71e0d756280a6fb89 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Sun, 30 Dec 2018 11:07:27 +0100
Subject: [PATCH 271/342] fix typo
---
bfps/cpp/full_code/NSVEcomplex_particles.cpp | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/bfps/cpp/full_code/NSVEcomplex_particles.cpp b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
index 43de34aa..c7bc9424 100644
--- a/bfps/cpp/full_code/NSVEcomplex_particles.cpp
+++ b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
@@ -230,7 +230,9 @@ int NSVEcomplex_particles<rnumber>::do_stats()
// deallocate temporary data array
// TODO: is it required/safe to call the release method here?
- pdata.release();
+ pdata0.release();
+ pdata1.release();
+ pdata2.release();
return EXIT_SUCCESS;
}
--
GitLab
From 9bdbcc6ed182c1319f0db6073003e38291a8220e Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 15 Jan 2019 13:17:29 +0100
Subject: [PATCH 272/342] fix generation of _fields.h5 file
---
bfps/PP.py | 25 ++++++++++---------------
1 file changed, 10 insertions(+), 15 deletions(-)
diff --git a/bfps/PP.py b/bfps/PP.py
index 5716a7fe..77bf9d6c 100644
--- a/bfps/PP.py
+++ b/bfps/PP.py
@@ -33,6 +33,7 @@ import h5py
import math
import numpy as np
import warnings
+import glob
import bfps
from ._code import _code
@@ -804,21 +805,15 @@ class PP(_code):
with h5py.File(os.path.join(self.work_dir, self.simname + '_fields.h5'), 'a') as ff:
ff.require_group('vorticity')
ff.require_group('vorticity/complex')
- checkpoint = (iter0 // niter_out) // cppf
- while True:
- cpf_name = os.path.join(
- self.work_dir,
- self.simname + '_checkpoint_{0}.h5'.format(checkpoint))
- if os.path.exists(cpf_name):
- cpf = h5py.File(cpf_name, 'r')
- for iter_name in cpf['vorticity/complex'].keys():
- if iter_name not in ff['vorticity/complex'].keys():
- ff['vorticity/complex/' + iter_name] = h5py.ExternalLink(
- cpf_name,
- 'vorticity/complex/' + iter_name)
- checkpoint += 1
- else:
- break
+ checkpoint_file_list = glob.glob(self.simname + '_checkpoint_*.h5')
+ for cpf_name in checkpoint_file_list:
+ cpf = h5py.File(cpf_name, 'r')
+ for iter_name in cpf['vorticity/complex'].keys():
+ if iter_name not in ff['vorticity/complex'].keys():
+ ff['vorticity/complex/' + iter_name] = h5py.ExternalLink(
+ cpf_name,
+ 'vorticity/complex/' + iter_name)
+ cpf.close()
return None
def launch_jobs(
self,
--
GitLab
From d1f23313a408fdf663e699a0a5ffb9cfc8455dae Mon Sep 17 00:00:00 2001
From: Debarghya Banerjee <debarghya.banerjee@ds.mpg.de>
Date: Tue, 15 Jan 2019 16:09:08 +0100
Subject: [PATCH 273/342] Update filter parameters
---
bfps/cpp/kspace.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bfps/cpp/kspace.cpp b/bfps/cpp/kspace.cpp
index a7b59bc2..89006cdd 100644
--- a/bfps/cpp/kspace.cpp
+++ b/bfps/cpp/kspace.cpp
@@ -315,7 +315,7 @@ void kspace<be, dt>::general_M_filter(
double prefactor = prefactor0;
for (unsigned int tcounter=0; tcounter<2*ncomp(fc); tcounter++)
((rnumber*)a)[2*ncomp(fc)*cindex + tcounter] *= (
- prefactor * (exp(-0.5*pow((3.54*argument),(122*(pow(ell,0.836))))) * sqrt(1.0 + (pow((argument/0.0636),3.44))/(1.0 + (pow((argument/0.0621),3.44))))));
+ prefactor * (exp(-0.5*pow((3.25*argument),(16.98*(pow(ell,0.5))))) * sqrt(1.0 + (pow((argument/0.06),3.76))/(1.0 + (pow((argument/0.06),3.76))))));
// prefactor * (exp(-0.5*pow((3.54*argument),(122*pow(ell,0.836)))) * sqrt(1.0 + (pow((argument/0.0636),3.44))/(1.0 + (pow((argument/0.0621),3.44))))));
}
});
--
GitLab
From 853ee86b3563476fb79ed9c3e8143755b25c5f6e Mon Sep 17 00:00:00 2001
From: Debarghya Banerjee <debarghya.banerjee@ds.mpg.de>
Date: Thu, 17 Jan 2019 09:23:44 +0100
Subject: [PATCH 274/342] kspace.cpp general_M filter
---
bfps/cpp/kspace.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/bfps/cpp/kspace.cpp b/bfps/cpp/kspace.cpp
index a7b59bc2..83cfeedb 100644
--- a/bfps/cpp/kspace.cpp
+++ b/bfps/cpp/kspace.cpp
@@ -315,8 +315,9 @@ void kspace<be, dt>::general_M_filter(
double prefactor = prefactor0;
for (unsigned int tcounter=0; tcounter<2*ncomp(fc); tcounter++)
((rnumber*)a)[2*ncomp(fc)*cindex + tcounter] *= (
- prefactor * (exp(-0.5*pow((3.54*argument),(122*(pow(ell,0.836))))) * sqrt(1.0 + (pow((argument/0.0636),3.44))/(1.0 + (pow((argument/0.0621),3.44))))));
+// prefactor * (exp(-0.5*pow((5.00113370*argument),(120.000074*(pow(ell,0.954544955))))) * sqrt(1.0 + (pow((argument/0.0444952819),2.99975517))/(1.0 + (pow((argument/0.0456538014),2.99975517))))));
// prefactor * (exp(-0.5*pow((3.54*argument),(122*pow(ell,0.836)))) * sqrt(1.0 + (pow((argument/0.0636),3.44))/(1.0 + (pow((argument/0.0621),3.44))))));
+ prefactor * (exp(-0.5*pow((3.25*argument),(16.98*(pow(ell,0.5))))) * sqrt(1.0 + (pow((argument/0.06),3.76))/(1.0 + (pow((argument/0.06),3.76))))));
}
});
}
--
GitLab
From 98e4b4a1e9dc34f5a239aacf0de46926b2aefd25 Mon Sep 17 00:00:00 2001
From: Debarghya Banerjee <debarghya.banerjee@ds.mpg.de>
Date: Thu, 24 Jan 2019 13:25:03 +0100
Subject: [PATCH 275/342] filter parameters modified
---
bfps/cpp/kspace.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/bfps/cpp/kspace.cpp b/bfps/cpp/kspace.cpp
index 87eb3a79..5314f9b1 100644
--- a/bfps/cpp/kspace.cpp
+++ b/bfps/cpp/kspace.cpp
@@ -315,8 +315,7 @@ void kspace<be, dt>::general_M_filter(
double prefactor = prefactor0;
for (unsigned int tcounter=0; tcounter<2*ncomp(fc); tcounter++)
((rnumber*)a)[2*ncomp(fc)*cindex + tcounter] *= (
-// prefactor * (exp(-0.5*pow((3.54*argument),(122*pow(ell,0.836)))) * sqrt(1.0 + (pow((argument/0.0636),3.44))/(1.0 + (pow((argument/0.0621),3.44))))));
- prefactor * (exp(-0.5*pow((3.25*argument),(16.98*(pow(ell,0.5))))) * sqrt(1.0 + (pow((argument/0.06),3.76))/(1.0 + (pow((argument/0.06),3.76))))));
+ prefactor * (exp(-0.5*pow((2.9*argument),(68.0*(pow(ell,0.74))))) * sqrt(1.0 + (pow((argument/0.06),3.8))/(1.0 + (pow((argument/0.057),3.8))))));
}
});
}
--
GitLab
From 540181bccbb239e118b2575e10f331a7b3e61fe8 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 30 Jan 2019 09:01:44 +0100
Subject: [PATCH 276/342] update documentation
---
documentation/_static/overview.rst | 17 ++++++++++-------
1 file changed, 10 insertions(+), 7 deletions(-)
diff --git a/documentation/_static/overview.rst b/documentation/_static/overview.rst
index afe7a753..26c924ba 100644
--- a/documentation/_static/overview.rst
+++ b/documentation/_static/overview.rst
@@ -184,16 +184,17 @@ available, called ``bfps``, that you can execute.
Just executing it will run a small test DNS on a real space grid of size
:math:`32 \times 32 \times 32`, in the current
folder, with the simulation name ``test``.
-So, open a console, and type ``bfps NavierStokes``:
+So, open a console, and type ``bfps DNS NSVE``:
.. code:: bash
# depending on how curious you are, you may have a look at the
# options first:
bfps --help
- bfps NavierStokes --help
+ bfps DNS --help
+ bfps DNS NS --help
# or you may just run it:
- bfps NavierStokes
+ bfps DNS NSVE
The simulation itself should not take more than a few seconds, since
this is just a :math:`32^3` simulation run for 8 iterations.
@@ -205,9 +206,9 @@ the following:
.. code:: python
import numpy as np
- from bfps import NavierStokes
+ from bfps import DNS
- c = NavierStokes(
+ c = DNS(
work_dir = '/location/of/simulation/data',
simname = 'simulation_name_goes_here')
c.compute_statistics()
@@ -223,7 +224,7 @@ the following:
data_file['iteration'].value*c.parameters['dt'] / c.statistics['Tint'],
data_file['iteration'].value*c.parameters['dt'] / c.statistics['tauK']))
-:func:`compute_statistics <bfps.NavierStokes.NavierStokes.compute_statistics>`
+:func:`compute_statistics <bfps.DNS.DNS.compute_statistics>`
will read the data
file generated by the DNS, compute a bunch of basic statistics, for
example the Taylor scale Reynolds number :math:`R_\lambda` that we're
@@ -233,7 +234,7 @@ What happens is that the DNS will have generated an ``HDF5`` file
containing a bunch of specific datasets (spectra, moments of real space
representations, etc).
The function
-:func:`compute_statistics <bfps.NavierStokes.NavierStokes.compute_statistics>`
+:func:`compute_statistics <bfps.DNS.DNS.compute_statistics>`
performs simple postprocessing that may however be expensive, therefore
it also saves some data into a ``<simname>_postprocess.h5`` file, and
then it also performs some time averages, yielding the ``statistics``
@@ -242,6 +243,8 @@ dictionary that is used in the above code.
Behind the scenes
-----------------
+TODO FIXME obsolete documentation
+
In brief the following takes place:
1. An instance ``c`` of
--
GitLab
From d2f10788ce2ab77343d5328d4993934d2e7e4eed Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 1 Feb 2019 23:43:25 +0100
Subject: [PATCH 277/342] fix tutorial command
---
documentation/_static/overview.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/documentation/_static/overview.rst b/documentation/_static/overview.rst
index 26c924ba..58af5653 100644
--- a/documentation/_static/overview.rst
+++ b/documentation/_static/overview.rst
@@ -192,7 +192,7 @@ So, open a console, and type ``bfps DNS NSVE``:
# options first:
bfps --help
bfps DNS --help
- bfps DNS NS --help
+ bfps DNS NSVE --help
# or you may just run it:
bfps DNS NSVE
--
GitLab
From c8211acccf967cf5a308d7593cdeb78f87033c49 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 1 Feb 2019 23:43:34 +0100
Subject: [PATCH 278/342] fix comparison
---
bfps/cpp/particles/abstract_particles_system.hpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bfps/cpp/particles/abstract_particles_system.hpp b/bfps/cpp/particles/abstract_particles_system.hpp
index ee864b8c..5d49c8f4 100644
--- a/bfps/cpp/particles/abstract_particles_system.hpp
+++ b/bfps/cpp/particles/abstract_particles_system.hpp
@@ -80,7 +80,7 @@ public:
void completeLoopWithExtraField(
const real_number dt,
const field<rnumber, be, fc>& in_field) {
- static_assert(fc == THREE || THREExTHREE, "only THREE or THREExTHREE is supported for now");
+ static_assert((fc == THREE) || (fc == THREExTHREE), "only THREE or THREExTHREE is supported for now");
if (fc == THREE)
{
std::unique_ptr<real_number[]> extra_rhs(new real_number[getLocalNbParticles()*3]());
--
GitLab
From 072613189e028c229ae8c154bebe16dd6165b75f Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Sun, 3 Feb 2019 11:29:16 +0100
Subject: [PATCH 279/342] clean up field initial condition code
---
bfps/DNS.py | 42 +++++++++++++++++-------------------------
1 file changed, 17 insertions(+), 25 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index f360db8b..c1835062 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -1001,8 +1001,23 @@ class DNS(_code):
particle_initial_condition = None):
if not os.path.exists(os.path.join(self.work_dir, self.simname + '.h5')):
# take care of fields' initial condition
+ # first, check if initial field exists
+ need_field = False
if not os.path.exists(self.get_checkpoint_0_fname()):
- f = h5py.File(self.get_checkpoint_0_fname(), 'w')
+ need_field = True
+ else:
+ f = h5py.File(self.get_checkpoint_0_fname(), 'r')
+ try:
+ dset = f['vorticity/complex/0']
+ need_field = (dset.shape == (self.parameters['ny'],
+ self.parameters['nz'],
+ self.parameters['nx']//2+1,
+ 3))
+ except:
+ need_field = True
+ f.close()
+ if need_field:
+ f = h5py.File(self.get_checkpoint_0_fname(), 'a')
if len(opt.src_simname) > 0:
source_cp = 0
src_file = 'not_a_file'
@@ -1027,32 +1042,9 @@ class DNS(_code):
amplitude = 0.05)
f['vorticity/complex/{0}'.format(0)] = data
f.close()
- ## take care of particles' initial condition
- #if self.dns_type in ['NSVEparticles', 'NSVEparticles_no_output']:
- # if opt.pclouds > 1:
- # np.random.seed(opt.particle_rand_seed)
- # if opt.pcloud_type == 'random-cube':
- # particle_initial_condition = (
- # np.random.random((opt.pclouds, 1, 3))*2*np.pi +
- # np.random.random((1, self.parameters['nparticles'], 3))*opt.particle_cloud_size)
- # elif opt.pcloud_type == 'regular-cube':
- # onedarray = np.linspace(
- # -opt.particle_cloud_size/2,
- # opt.particle_cloud_size/2,
- # self.parameters['nparticles'])
- # particle_initial_condition = np.zeros(
- # (opt.pclouds,
- # self.parameters['nparticles'],
- # self.parameters['nparticles'],
- # self.parameters['nparticles'], 3),
- # dtype = np.float64)
- # particle_initial_condition[:] = \
- # np.random.random((opt.pclouds, 1, 1, 1, 3))*2*np.pi
- # particle_initial_condition[..., 0] += onedarray[None, None, None, :]
- # particle_initial_condition[..., 1] += onedarray[None, None, :, None]
- # particle_initial_condition[..., 2] += onedarray[None, :, None, None]
self.write_par(
particle_ic = None)
+ # take care of particles' initial condition
if self.dns_type in ['NSVEparticles', 'NSVEcomplex_particles', 'NSVEparticles_no_output', 'NSVEp_extra_sampling']:
self.generate_particle_data(opt = opt)
self.run(
--
GitLab
From a2b474e05ae5444dc11567f78b548effc136b259 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 4 Feb 2019 09:44:55 +0100
Subject: [PATCH 280/342] checkpoint
---
bfps/DNS.py | 107 +++++++++++++++---------------
bfps/test/test_particle_clouds.py | 66 ++++++++++++++++++
2 files changed, 120 insertions(+), 53 deletions(-)
create mode 100644 bfps/test/test_particle_clouds.py
diff --git a/bfps/DNS.py b/bfps/DNS.py
index c1835062..3aa47fd4 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -430,9 +430,7 @@ class DNS(_code):
return None
def write_par(
self,
- iter0 = 0,
- particle_ic = None,
- particles_off = False):
+ iter0 = 0):
assert (self.parameters['niter_todo'] % self.parameters['niter_stat'] == 0)
assert (self.parameters['niter_todo'] % self.parameters['niter_out'] == 0)
assert (self.parameters['niter_out'] % self.parameters['niter_stat'] == 0)
@@ -479,7 +477,7 @@ class DNS(_code):
4),
dtype = np.int64)
ofile['checkpoint'] = int(0)
- if (self.dns_type in ['NSVE', 'NSVE_no_output']) or particles_off:
+ if (self.dns_type in ['NSVE', 'NSVE_no_output']):
return None
if type(particle_ic) == type(None):
@@ -995,58 +993,61 @@ class DNS(_code):
particle_file.create_group('tracers0/pressure_gradient')
particle_file.create_group('tracers0/pressure_Hessian')
return None
- def launch_jobs(
+ def generate_initial_condition(
self,
- opt = None,
- particle_initial_condition = None):
- if not os.path.exists(os.path.join(self.work_dir, self.simname + '.h5')):
- # take care of fields' initial condition
- # first, check if initial field exists
- need_field = False
- if not os.path.exists(self.get_checkpoint_0_fname()):
+ opt = None):
+ # take care of fields' initial condition
+ # first, check if initial field exists
+ need_field = False
+ if not os.path.exists(self.get_checkpoint_0_fname()):
+ need_field = True
+ else:
+ f = h5py.File(self.get_checkpoint_0_fname(), 'r')
+ try:
+ dset = f['vorticity/complex/0']
+ need_field = (dset.shape == (self.parameters['ny'],
+ self.parameters['nz'],
+ self.parameters['nx']//2+1,
+ 3))
+ except:
need_field = True
+ f.close()
+ if need_field:
+ f = h5py.File(self.get_checkpoint_0_fname(), 'a')
+ if len(opt.src_simname) > 0:
+ source_cp = 0
+ src_file = 'not_a_file'
+ while True:
+ src_file = os.path.join(
+ os.path.realpath(opt.src_work_dir),
+ opt.src_simname + '_checkpoint_{0}.h5'.format(source_cp))
+ f0 = h5py.File(src_file, 'r')
+ if '{0}'.format(opt.src_iteration) in f0['vorticity/complex'].keys():
+ f0.close()
+ break
+ source_cp += 1
+ self.copy_complex_field(
+ src_file,
+ 'vorticity/complex/{0}'.format(opt.src_iteration),
+ f,
+ 'vorticity/complex/{0}'.format(0))
else:
- f = h5py.File(self.get_checkpoint_0_fname(), 'r')
- try:
- dset = f['vorticity/complex/0']
- need_field = (dset.shape == (self.parameters['ny'],
- self.parameters['nz'],
- self.parameters['nx']//2+1,
- 3))
- except:
- need_field = True
- f.close()
- if need_field:
- f = h5py.File(self.get_checkpoint_0_fname(), 'a')
- if len(opt.src_simname) > 0:
- source_cp = 0
- src_file = 'not_a_file'
- while True:
- src_file = os.path.join(
- os.path.realpath(opt.src_work_dir),
- opt.src_simname + '_checkpoint_{0}.h5'.format(source_cp))
- f0 = h5py.File(src_file, 'r')
- if '{0}'.format(opt.src_iteration) in f0['vorticity/complex'].keys():
- f0.close()
- break
- source_cp += 1
- self.copy_complex_field(
- src_file,
- 'vorticity/complex/{0}'.format(opt.src_iteration),
- f,
- 'vorticity/complex/{0}'.format(0))
- else:
- data = self.generate_vector_field(
- write_to_file = False,
- spectra_slope = 2.0,
- amplitude = 0.05)
- f['vorticity/complex/{0}'.format(0)] = data
- f.close()
- self.write_par(
- particle_ic = None)
- # take care of particles' initial condition
- if self.dns_type in ['NSVEparticles', 'NSVEcomplex_particles', 'NSVEparticles_no_output', 'NSVEp_extra_sampling']:
- self.generate_particle_data(opt = opt)
+ data = self.generate_vector_field(
+ write_to_file = False,
+ spectra_slope = 2.0,
+ amplitude = 0.05)
+ f['vorticity/complex/{0}'.format(0)] = data
+ f.close()
+ # now take care of particles' initial condition
+ if self.dns_type in ['NSVEparticles', 'NSVEcomplex_particles', 'NSVEparticles_no_output', 'NSVEp_extra_sampling']:
+ self.generate_particle_data(opt = opt)
+ return None
+ def launch_jobs(
+ self,
+ opt = None):
+ if not os.path.exists(self.get_data_file_name()):
+ self.generate_initial_condition()
+ self.write_par()
self.run(
nb_processes = opt.nb_processes,
nb_threads_per_process = opt.nb_threads_per_process,
diff --git a/bfps/test/test_particle_clouds.py b/bfps/test/test_particle_clouds.py
new file mode 100644
index 00000000..1a890495
--- /dev/null
+++ b/bfps/test/test_particle_clouds.py
@@ -0,0 +1,66 @@
+#! /usr/bin/env python
+#######################################################################
+# #
+# Copyright 2019 Max Planck Institute #
+# for Dynamics and Self-Organization #
+# #
+# This file is part of bfps. #
+# #
+# bfps is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# bfps is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with bfps. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+#######################################################################
+
+
+
+import os
+import numpy as np
+import h5py
+import sys
+
+import bfps
+from bfps import DNS
+
+def main():
+ nclouds = 4
+ nparticles_per_cloud = 3
+ nparticles = nclouds*nparticles_per_cloud
+ niterations = 32
+ c = DNS()
+ ic_file = h5py.File(c.get_checkpoint_0_fname(), 'a')
+ ic_file['tracers0/state/0'] = np.random.random((nclouds, nparticles_per_cloud, 3))
+ ic_file['tracers0/rhs/0'] = np.zeros((2, nclouds, nparticles_per_cloud, 3))
+ ic_file.close()
+ c.launch(
+ ['NSVEparticles',
+ '-n', '32',
+ '--src-simname', 'B32p1e4',
+ '--forcing_type', 'linear',
+ '--src-wd', bfps.lib_dir + '/test',
+ '--src-iteration', '0',
+ '--np', '4',
+ '--ntpp', '1',
+ '--fftw_plan_rigor', 'FFTW_PATIENT',
+ '--niter_todo', '{0}'.format(niterations),
+ '--niter_out', '{0}'.format(niterations),
+ '--niter_stat', '1',
+ '--nparticles', '{0}'.format(nparticles),
+ '--tracers0_integration_steps', '2',
+ '--wd', './'])
+ return None
+
+if __name__ == '__main__':
+ main()
+
--
GitLab
From 1c9cf2cf5f7aaa0c4185ccee294f5c435131986a Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 4 Feb 2019 13:27:34 +0100
Subject: [PATCH 281/342] reading arbitrary shaped particle IC works
---
bfps/DNS.py | 110 +++++++++-----------
bfps/cpp/full_code/NSVEparticles.cpp | 2 +-
bfps/cpp/particles/particles_input_hdf5.hpp | 18 ++--
3 files changed, 60 insertions(+), 70 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 3aa47fd4..dd75854d 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -479,38 +479,6 @@ class DNS(_code):
ofile['checkpoint'] = int(0)
if (self.dns_type in ['NSVE', 'NSVE_no_output']):
return None
-
- if type(particle_ic) == type(None):
- pbase_shape = (self.parameters['nparticles'],)
- number_of_particles = self.parameters['nparticles']
- else:
- pbase_shape = particle_ic.shape[:-1]
- assert(particle_ic.shape[-1] == 3)
- number_of_particles = 1
- for val in pbase_shape[1:]:
- number_of_particles *= val
- ncomponents = 3
- if self.dns_type in ['NSVEcomplex_particles']:
- ncomponents = 6
- with h5py.File(self.get_checkpoint_0_fname(), 'a') as ofile:
- s = 0
- if not 'tracers{0}'.format(s) in ofile.keys():
- ofile.create_group('tracers{0}'.format(s))
- ofile.create_group('tracers{0}/rhs'.format(s))
- ofile.create_group('tracers{0}/state'.format(s))
- ofile['tracers{0}/rhs'.format(s)].create_dataset(
- '0',
- shape = (
- (self.parameters['tracers{0}_integration_steps'.format(s)],) +
- pbase_shape +
- (ncomponents,)),
- dtype = np.float)
- ofile['tracers{0}/state'.format(s)].create_dataset(
- '0',
- shape = (
- pbase_shape +
- (ncomponents,)),
- dtype = np.float)
return None
def job_parser_arguments(
self,
@@ -817,34 +785,48 @@ class DNS(_code):
self,
rseed = None,
species = 0):
- with h5py.File(self.get_checkpoint_0_fname(), 'a') as data_file:
- dset = data_file[
- 'tracers{0}/state/0'.format(species)]
- if not type(rseed) == type(None):
- np.random.seed(rseed)
- nn = self.parameters['nparticles']
- cc = int(0)
- batch_size = int(1e6)
- def get_random_phases(npoints):
- return np.random.random(
- (npoints, 3))*2*np.pi
- def get_random_versors(npoints):
- bla = np.random.normal(
- size = (npoints, 3))
- bla /= np.sum(bla**2, axis = 1)[:, None]**.5
- return bla
- while nn > 0:
- if nn > batch_size:
- dset[cc*batch_size:(cc+1)*batch_size, :3] = get_random_phases(batch_size)
- if dset.shape[1] == 6:
- dset[cc*batch_size:(cc+1)*batch_size, 3:] = get_random_versors(batch_size)
- nn -= batch_size
- else:
- dset[cc*batch_size:cc*batch_size+nn, :3] = get_random_phases(nn)
- if dset.shape[1] == 6:
- dset[cc*batch_size:cc*batch_size+nn, 3:] = get_random_versors(nn)
- nn = 0
- cc += 1
+ try:
+ ncomponents = 3
+ if self.dns_type in ['NSVEcomplex_particles']:
+ ncomponents = 6
+ with h5py.File(self.get_checkpoint_0_fname(), 'a') as data_file:
+ nn = self.parameters['nparticles']
+ data_file['tracers{0}/rhs'.format(species)].create_dataset(
+ '0',
+ shape = (
+ (self.parameters['tracers{0}_integration_steps'.format(species)],) +
+ (nn, ncomponents,)),
+ dtype = np.float)
+ dset = data_file['tracers{0}/state'.format(s)].create_dataset(
+ '0',
+ shape = (nn, ncomponents,),
+ dtype = np.float)
+ if not type(rseed) == type(None):
+ np.random.seed(rseed)
+ cc = int(0)
+ batch_size = int(1e6)
+ def get_random_phases(npoints):
+ return np.random.random(
+ (npoints, 3))*2*np.pi
+ def get_random_versors(npoints):
+ bla = np.random.normal(
+ size = (npoints, 3))
+ bla /= np.sum(bla**2, axis = 1)[:, None]**.5
+ return bla
+ while nn > 0:
+ if nn > batch_size:
+ dset[cc*batch_size:(cc+1)*batch_size, :3] = get_random_phases(batch_size)
+ if dset.shape[1] == 6:
+ dset[cc*batch_size:(cc+1)*batch_size, 3:] = get_random_versors(batch_size)
+ nn -= batch_size
+ else:
+ dset[cc*batch_size:cc*batch_size+nn, :3] = get_random_phases(nn)
+ if dset.shape[1] == 6:
+ dset[cc*batch_size:cc*batch_size+nn, 3:] = get_random_versors(nn)
+ nn = 0
+ cc += 1
+ except Exception as e:
+ print(e)
return None
def generate_vector_field(
self,
@@ -976,6 +958,12 @@ class DNS(_code):
self,
opt = None):
if self.parameters['nparticles'] > 0:
+ with h5py.File(self.get_checkpoint_0_fname(), 'a') as ofile:
+ s = 0
+ if not 'tracers{0}'.format(s) in ofile.keys():
+ ofile.create_group('tracers{0}'.format(s))
+ ofile.create_group('tracers{0}/rhs'.format(s))
+ ofile.create_group('tracers{0}/state'.format(s))
self.generate_tracer_state(
species = 0,
rseed = opt.particle_rand_seed)
@@ -1046,7 +1034,7 @@ class DNS(_code):
self,
opt = None):
if not os.path.exists(self.get_data_file_name()):
- self.generate_initial_condition()
+ self.generate_initial_condition(opt = opt)
self.write_par()
self.run(
nb_processes = opt.nb_processes,
diff --git a/bfps/cpp/full_code/NSVEparticles.cpp b/bfps/cpp/full_code/NSVEparticles.cpp
index 90184d8b..72a1a85a 100644
--- a/bfps/cpp/full_code/NSVEparticles.cpp
+++ b/bfps/cpp/full_code/NSVEparticles.cpp
@@ -2,7 +2,7 @@
-#define NDEBUG
+//#define NDEBUG
#include <string>
#include <cmath>
diff --git a/bfps/cpp/particles/particles_input_hdf5.hpp b/bfps/cpp/particles/particles_input_hdf5.hpp
index 5231872d..cd08b602 100644
--- a/bfps/cpp/particles/particles_input_hdf5.hpp
+++ b/bfps/cpp/particles/particles_input_hdf5.hpp
@@ -158,7 +158,8 @@ public:
hid_t dset = H5Dopen(particle_file, inDatanameState.c_str(), H5P_DEFAULT);
assert(dset >= 0);
- hid_t rspace = H5Dget_space(dset);
+ hsize_t file_space_dims[2] = {nb_total_particles, size_particle_positions};
+ hid_t rspace = H5Screate_simple(2, file_space_dims, NULL);
assert(rspace >= 0);
hsize_t offset[2] = {load_splitter.getMyOffset(), 0};
@@ -184,11 +185,11 @@ public:
TIMEZONE("rhs-read");
hid_t dset = H5Dopen(particle_file, inDatanameRhs.c_str(), H5P_DEFAULT);
assert(dset >= 0);
+ hsize_t file_space_dims[3] = {nb_rhs, nb_total_particles, size_particle_rhs};
+ hid_t rspace = H5Screate_simple(3, file_space_dims, NULL);
+ assert(rspace >= 0);
for(hsize_t idx_rhs = 0 ; idx_rhs < nb_rhs ; ++idx_rhs){
- hid_t rspace = H5Dget_space(dset);
- assert(rspace >= 0);
-
if(load_splitter.getMySize()){
split_particles_rhs[idx_rhs].reset(new real_number[load_splitter.getMySize()*size_particle_rhs]);
}
@@ -203,16 +204,17 @@ public:
NULL, mem_dims, NULL);
variable_used_only_in_assert(rethdf);
assert(rethdf >= 0);
+ //DEBUG_MSG("");
rethdf = H5Dread(dset, type_id, mspace, rspace, H5P_DEFAULT, split_particles_rhs[idx_rhs].get());
assert(rethdf >= 0);
rethdf = H5Sclose(mspace);
assert(rethdf >= 0);
-
- rethdf = H5Sclose(rspace);
- assert(rethdf >= 0);
}
- int rethdf = H5Dclose(dset);
+
+ int rethdf = H5Sclose(rspace);
+ assert(rethdf >= 0);
+ rethdf = H5Dclose(dset);
variable_used_only_in_assert(rethdf);
assert(rethdf >= 0);
}
--
GitLab
From ad82946b809185028e842b7353fee786a1a20479 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 4 Feb 2019 13:36:49 +0100
Subject: [PATCH 282/342] rename variable
---
bfps/cpp/particles/particles_input_hdf5.hpp | 66 ++++++++++++++-------
1 file changed, 46 insertions(+), 20 deletions(-)
diff --git a/bfps/cpp/particles/particles_input_hdf5.hpp b/bfps/cpp/particles/particles_input_hdf5.hpp
index cd08b602..e32b4355 100644
--- a/bfps/cpp/particles/particles_input_hdf5.hpp
+++ b/bfps/cpp/particles/particles_input_hdf5.hpp
@@ -22,7 +22,7 @@ class particles_input_hdf5 : public abstract_particles_input<partsize_t, real_nu
int my_rank;
int nb_processes;
- hsize_t nb_total_particles;
+ hsize_t total_number_of_particles;
hsize_t nb_rhs;
partsize_t nb_particles_for_me;
@@ -30,8 +30,10 @@ class particles_input_hdf5 : public abstract_particles_input<partsize_t, real_nu
std::unique_ptr<partsize_t[]> my_particles_indexes;
std::vector<std::unique_ptr<real_number[]>> my_particles_rhs;
- static std::vector<real_number> BuildLimitsAllProcesses(MPI_Comm mpi_comm,
- const real_number my_spatial_low_limit, const real_number my_spatial_up_limit){
+ static std::vector<real_number> BuildLimitsAllProcesses(
+ MPI_Comm mpi_comm,
+ const real_number my_spatial_low_limit,
+ const real_number my_spatial_up_limit){
int my_rank;
int nb_processes;
@@ -41,8 +43,15 @@ class particles_input_hdf5 : public abstract_particles_input<partsize_t, real_nu
std::vector<real_number> spatial_limit_per_proc(nb_processes*2);
real_number intervalToSend[2] = {my_spatial_low_limit, my_spatial_up_limit};
- AssertMpi(MPI_Allgather(intervalToSend, 2, particles_utils::GetMpiType(real_number()),
- spatial_limit_per_proc.data(), 2, particles_utils::GetMpiType(real_number()), mpi_comm));
+ AssertMpi(
+ MPI_Allgather(
+ intervalToSend,
+ 2,
+ particles_utils::GetMpiType(real_number()),
+ spatial_limit_per_proc.data(),
+ 2,
+ particles_utils::GetMpiType(real_number()),
+ mpi_comm));
for(int idx_proc = 0; idx_proc < nb_processes-1 ; ++idx_proc){
assert(spatial_limit_per_proc[idx_proc*2] <= spatial_limit_per_proc[idx_proc*2+1]);
@@ -56,18 +65,35 @@ class particles_input_hdf5 : public abstract_particles_input<partsize_t, real_nu
}
public:
- particles_input_hdf5(const MPI_Comm in_mpi_comm,const std::string& inFilename,
- const std::string& inDatanameState, const std::string& inDatanameRhs,
- const real_number my_spatial_low_limit, const real_number my_spatial_up_limit)
- : particles_input_hdf5(in_mpi_comm, inFilename, inDatanameState, inDatanameRhs,
- BuildLimitsAllProcesses(in_mpi_comm, my_spatial_low_limit, my_spatial_up_limit)){
+ particles_input_hdf5(
+ const MPI_Comm in_mpi_comm,
+ const std::string& inFilename,
+ const std::string& inDatanameState,
+ const std::string& inDatanameRhs,
+ const real_number my_spatial_low_limit,
+ const real_number my_spatial_up_limit)
+ : particles_input_hdf5(
+ in_mpi_comm,
+ inFilename,
+ inDatanameState,
+ inDatanameRhs,
+ BuildLimitsAllProcesses(
+ in_mpi_comm,
+ my_spatial_low_limit,
+ my_spatial_up_limit)){
}
- particles_input_hdf5(const MPI_Comm in_mpi_comm,const std::string& inFilename,
- const std::string& inDatanameState, const std::string& inDatanameRhs,
- const std::vector<real_number>& in_spatial_limit_per_proc)
+ particles_input_hdf5(
+ const MPI_Comm in_mpi_comm,
+ const std::string& inFilename,
+ const std::string& inDatanameState,
+ const std::string& inDatanameRhs,
+ const std::vector<real_number>& in_spatial_limit_per_proc)
: filename(inFilename),
- mpi_comm(in_mpi_comm), my_rank(-1), nb_processes(-1), nb_total_particles(0),
+ mpi_comm(in_mpi_comm),
+ my_rank(-1),
+ nb_processes(-1),
+ total_number_of_particles(0),
nb_particles_for_me(0){
TIMEZONE("particles_input_hdf5");
@@ -104,9 +130,9 @@ public:
// Last value is the position dim of the particles
assert(state_dim_array.back() == size_particle_positions);
- nb_total_particles = 1;
+ total_number_of_particles = 1;
for (size_t idx_dim = 0; idx_dim < state_dim_array.size()-1; ++idx_dim){
- nb_total_particles *= state_dim_array[idx_dim];
+ total_number_of_particles *= state_dim_array[idx_dim];
}
hdfret = H5Sclose(dspace);
@@ -141,7 +167,7 @@ public:
assert(hdfret >= 0);
}
- particles_utils::IntervalSplitter<hsize_t> load_splitter(nb_total_particles, nb_processes, my_rank);
+ particles_utils::IntervalSplitter<hsize_t> load_splitter(total_number_of_particles, nb_processes, my_rank);
static_assert(std::is_same<real_number, double>::value
|| std::is_same<real_number, float>::value, "real_number must be double or float");
@@ -158,7 +184,7 @@ public:
hid_t dset = H5Dopen(particle_file, inDatanameState.c_str(), H5P_DEFAULT);
assert(dset >= 0);
- hsize_t file_space_dims[2] = {nb_total_particles, size_particle_positions};
+ hsize_t file_space_dims[2] = {total_number_of_particles, size_particle_positions};
hid_t rspace = H5Screate_simple(2, file_space_dims, NULL);
assert(rspace >= 0);
@@ -185,7 +211,7 @@ public:
TIMEZONE("rhs-read");
hid_t dset = H5Dopen(particle_file, inDatanameRhs.c_str(), H5P_DEFAULT);
assert(dset >= 0);
- hsize_t file_space_dims[3] = {nb_rhs, nb_total_particles, size_particle_rhs};
+ hsize_t file_space_dims[3] = {nb_rhs, total_number_of_particles, size_particle_rhs};
hid_t rspace = H5Screate_simple(3, file_space_dims, NULL);
assert(rspace >= 0);
@@ -304,7 +330,7 @@ public:
}
partsize_t getTotalNbParticles() final{
- return partsize_t(nb_total_particles);
+ return partsize_t(total_number_of_particles);
}
partsize_t getLocalNbParticles() final{
--
GitLab
From 684e5426849259c906870357ddb7cb980f74bede Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 4 Feb 2019 16:42:44 +0100
Subject: [PATCH 283/342] all checkpoints use the same shape of particle data
---
bfps/cpp/full_code/NSVEparticles.cpp | 1 +
.../particles/abstract_particles_system.hpp | 3 ++
bfps/cpp/particles/particles_input_hdf5.hpp | 8 ++++
bfps/cpp/particles/particles_output_hdf5.hpp | 42 +++++++++++++------
bfps/cpp/particles/particles_system.hpp | 12 ++++++
.../particles/particles_system_builder.hpp | 3 ++
6 files changed, 57 insertions(+), 12 deletions(-)
diff --git a/bfps/cpp/full_code/NSVEparticles.cpp b/bfps/cpp/full_code/NSVEparticles.cpp
index 72a1a85a..a0bd1c61 100644
--- a/bfps/cpp/full_code/NSVEparticles.cpp
+++ b/bfps/cpp/full_code/NSVEparticles.cpp
@@ -39,6 +39,7 @@ int NSVEparticles<rnumber>::initialize(void)
"tracers0",
nparticles,
tracers0_integration_steps);
+ this->particles_output_writer_mpi->setParticleFileLayout(this->ps->getParticleFileLayout());
this->particles_sample_writer_mpi = new particles_output_sampling_hdf5<
long long int, double, 3>(
MPI_COMM_WORLD,
diff --git a/bfps/cpp/particles/abstract_particles_system.hpp b/bfps/cpp/particles/abstract_particles_system.hpp
index 5d49c8f4..67c46855 100644
--- a/bfps/cpp/particles/abstract_particles_system.hpp
+++ b/bfps/cpp/particles/abstract_particles_system.hpp
@@ -96,6 +96,9 @@ public:
completeLoopWithVelocityGradient(dt, extra_rhs.get());
}
}
+
+ virtual int setParticleFileLayout(std::vector<hsize_t>) = 0;
+ virtual std::vector<hsize_t> getParticleFileLayout() = 0;
};
#endif
diff --git a/bfps/cpp/particles/particles_input_hdf5.hpp b/bfps/cpp/particles/particles_input_hdf5.hpp
index e32b4355..5201744d 100644
--- a/bfps/cpp/particles/particles_input_hdf5.hpp
+++ b/bfps/cpp/particles/particles_input_hdf5.hpp
@@ -25,6 +25,7 @@ class particles_input_hdf5 : public abstract_particles_input<partsize_t, real_nu
hsize_t total_number_of_particles;
hsize_t nb_rhs;
partsize_t nb_particles_for_me;
+ std::vector<hsize_t> particle_file_layout; // to hold the shape of initial condition array
std::unique_ptr<real_number[]> my_particles_positions;
std::unique_ptr<partsize_t[]> my_particles_indexes;
@@ -130,9 +131,12 @@ public:
// Last value is the position dim of the particles
assert(state_dim_array.back() == size_particle_positions);
+ // compute total number of particles, store initial condition array shape
total_number_of_particles = 1;
+ particle_file_layout.resize(state_dim_array.size()-1);
for (size_t idx_dim = 0; idx_dim < state_dim_array.size()-1; ++idx_dim){
total_number_of_particles *= state_dim_array[idx_dim];
+ particle_file_layout[idx_dim] = state_dim_array[idx_dim];
}
hdfret = H5Sclose(dspace);
@@ -355,6 +359,10 @@ public:
assert(my_particles_indexes != nullptr || nb_particles_for_me == 0);
return std::move(my_particles_indexes);
}
+
+ std::vector<hsize_t> getParticleFileLayout(){
+ return std::move(this->particle_file_layout);
+ }
};
#endif
diff --git a/bfps/cpp/particles/particles_output_hdf5.hpp b/bfps/cpp/particles/particles_output_hdf5.hpp
index 0098ba54..e62dfc3a 100644
--- a/bfps/cpp/particles/particles_output_hdf5.hpp
+++ b/bfps/cpp/particles/particles_output_hdf5.hpp
@@ -22,6 +22,7 @@ class particles_output_hdf5 : public abstract_particles_output<partsize_t,
hid_t file_id;
const partsize_t total_nb_particles;
+ std::vector<hsize_t> particle_file_layout; // to hold the shape of initial condition array
hid_t dset_id_state;
hid_t dset_id_rhs;
@@ -204,12 +205,9 @@ public:
}
{
- assert(total_nb_particles >= 0);
- assert(size_particle_positions >= 0);
- const hsize_t datacount[2] = {
- hsize_t(total_nb_particles),
- hsize_t(size_particle_positions)};
- hid_t dataspace = H5Screate_simple(2, datacount, NULL);
+ std::vector<hsize_t> datacount = std::vector<hsize_t>(this->particle_file_layout);
+ datacount.push_back(size_particle_positions);
+ hid_t dataspace = H5Screate_simple(datacount.size(), &datacount.front(), NULL);
assert(dataspace >= 0);
hid_t dataset_id = H5Dcreate( dset_id_state,
@@ -228,7 +226,12 @@ public:
hid_t memspace = H5Screate_simple(2, count, NULL);
assert(memspace >= 0);
- hid_t filespace = H5Dget_space(dataset_id);
+ assert(total_nb_particles >= 0);
+ assert(size_particle_positions >= 0);
+ const hsize_t file_count[2] = {hsize_t(total_nb_particles), size_particle_positions};
+ hid_t filespace = H5Screate_simple(2, file_count, NULL);
+ assert(filespace >= 0);
+
int rethdf = H5Sselect_hyperslab(
filespace,
H5S_SELECT_SET,
@@ -257,10 +260,10 @@ public:
}
{
assert(size_particle_rhs >= 0);
- const hsize_t datacount[3] = {hsize_t(Parent::getNbRhs()),
- hsize_t(total_nb_particles),
- hsize_t(size_particle_rhs)};
- hid_t dataspace = H5Screate_simple(3, datacount, NULL);
+ std::vector<hsize_t> datacount = std::vector<hsize_t>(this->particle_file_layout);
+ datacount.insert(datacount.begin(), hsize_t(Parent::getNbRhs()));
+ datacount.push_back(size_particle_positions);
+ hid_t dataspace = H5Screate_simple(datacount.size(), &datacount.front(), NULL);
assert(dataspace >= 0);
hid_t dataset_id = H5Dcreate( dset_id_rhs,
@@ -285,8 +288,12 @@ public:
hid_t memspace = H5Screate_simple(3, count, NULL);
assert(memspace >= 0);
- hid_t filespace = H5Dget_space(dataset_id);
+ assert(total_nb_particles >= 0);
+ assert(size_particle_positions >= 0);
+ const hsize_t file_count[3] = {hsize_t(Parent::getNbRhs()), hsize_t(total_nb_particles), size_particle_positions};
+ hid_t filespace = H5Screate_simple(3, file_count, NULL);
assert(filespace >= 0);
+
int rethdf = H5Sselect_hyperslab(
filespace,
H5S_SELECT_SET,
@@ -322,6 +329,17 @@ public:
assert(rethdf >= 0);
}
}
+
+ int setParticleFileLayout(std::vector<hsize_t> input_layout){
+ this->particle_file_layout.resize(input_layout.size());
+ for (unsigned int i=0; i<this->particle_file_layout.size(); i++)
+ this->particle_file_layout[i] = input_layout[i];
+ return EXIT_SUCCESS;
+ }
+
+ std::vector<hsize_t> getParticleFileLayout(void){
+ return std::move(this->particle_file_layout);
+ }
};
#endif//PARTICLES_OUTPUT_HDF5_HPP
diff --git a/bfps/cpp/particles/particles_system.hpp b/bfps/cpp/particles/particles_system.hpp
index ebc7e79a..201a57a1 100644
--- a/bfps/cpp/particles/particles_system.hpp
+++ b/bfps/cpp/particles/particles_system.hpp
@@ -48,6 +48,7 @@ class particles_system : public abstract_particles_system<partsize_t, real_numbe
partsize_t my_nb_particles;
const partsize_t total_nb_particles;
std::vector<std::unique_ptr<real_number[]>> my_particles_rhs;
+ std::vector<hsize_t> particle_file_layout;
int step_idx;
@@ -351,6 +352,17 @@ public:
return int(my_particles_rhs.size());
}
+ int setParticleFileLayout(std::vector<hsize_t> input_layout) final{
+ this->particle_file_layout.resize(input_layout.size());
+ for (unsigned int i=0; i<this->particle_file_layout.size(); i++)
+ this->particle_file_layout[i] = input_layout[i];
+ return EXIT_SUCCESS;
+ }
+
+ std::vector<hsize_t> getParticleFileLayout(void) final{
+ return std::move(this->particle_file_layout);
+ }
+
void checkNan() const { // TODO remove
for(partsize_t idx_part = 0 ; idx_part < my_nb_particles ; ++idx_part){ // TODO remove me
assert(std::isnan(my_particles_positions[idx_part*size_particle_positions+IDXC_X]) == false);
diff --git a/bfps/cpp/particles/particles_system_builder.hpp b/bfps/cpp/particles/particles_system_builder.hpp
index a9a140ac..916ab4bf 100644
--- a/bfps/cpp/particles/particles_system_builder.hpp
+++ b/bfps/cpp/particles/particles_system_builder.hpp
@@ -243,6 +243,9 @@ struct particles_system_build_container {
assert(part_sys->getNbRhs() == nsteps);
+ // store particle file layout
+ part_sys->setParticleFileLayout(generator.getParticleFileLayout());
+
// Return the created particles system
return std::unique_ptr<abstract_particles_system<partsize_t, particles_rnumber>>(part_sys);
}
--
GitLab
From 54d91695f0abe1c31b4b7527e027fb698465ea81 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 4 Feb 2019 17:17:52 +0100
Subject: [PATCH 284/342] sampling preserves initial particle array shape
---
bfps/cpp/full_code/NSVEparticles.cpp | 1 +
bfps/cpp/particles/particles_input_hdf5.hpp | 3 +--
bfps/cpp/particles/particles_output_hdf5.hpp | 2 +-
.../particles_output_sampling_hdf5.hpp | 21 +++++++++++++++----
bfps/cpp/particles/particles_system.hpp | 2 +-
5 files changed, 21 insertions(+), 8 deletions(-)
diff --git a/bfps/cpp/full_code/NSVEparticles.cpp b/bfps/cpp/full_code/NSVEparticles.cpp
index a0bd1c61..edc0f9a6 100644
--- a/bfps/cpp/full_code/NSVEparticles.cpp
+++ b/bfps/cpp/full_code/NSVEparticles.cpp
@@ -47,6 +47,7 @@ int NSVEparticles<rnumber>::initialize(void)
(this->simname + "_particles.h5"),
"tracers0",
"position/0");
+ this->particles_sample_writer_mpi->setParticleFileLayout(this->ps->getParticleFileLayout());
return EXIT_SUCCESS;
}
diff --git a/bfps/cpp/particles/particles_input_hdf5.hpp b/bfps/cpp/particles/particles_input_hdf5.hpp
index 5201744d..e10377bf 100644
--- a/bfps/cpp/particles/particles_input_hdf5.hpp
+++ b/bfps/cpp/particles/particles_input_hdf5.hpp
@@ -234,7 +234,6 @@ public:
NULL, mem_dims, NULL);
variable_used_only_in_assert(rethdf);
assert(rethdf >= 0);
- //DEBUG_MSG("");
rethdf = H5Dread(dset, type_id, mspace, rspace, H5P_DEFAULT, split_particles_rhs[idx_rhs].get());
assert(rethdf >= 0);
@@ -361,7 +360,7 @@ public:
}
std::vector<hsize_t> getParticleFileLayout(){
- return std::move(this->particle_file_layout);
+ return std::vector<hsize_t>(this->particle_file_layout);
}
};
diff --git a/bfps/cpp/particles/particles_output_hdf5.hpp b/bfps/cpp/particles/particles_output_hdf5.hpp
index e62dfc3a..d7c987ee 100644
--- a/bfps/cpp/particles/particles_output_hdf5.hpp
+++ b/bfps/cpp/particles/particles_output_hdf5.hpp
@@ -338,7 +338,7 @@ public:
}
std::vector<hsize_t> getParticleFileLayout(void){
- return std::move(this->particle_file_layout);
+ return std::vector<hsize_t>(this->particle_file_layout);
}
};
diff --git a/bfps/cpp/particles/particles_output_sampling_hdf5.hpp b/bfps/cpp/particles/particles_output_sampling_hdf5.hpp
index 22dafaed..fd79a49f 100644
--- a/bfps/cpp/particles/particles_output_sampling_hdf5.hpp
+++ b/bfps/cpp/particles/particles_output_sampling_hdf5.hpp
@@ -19,6 +19,7 @@ class particles_output_sampling_hdf5 : public abstract_particles_output<
hid_t file_id, pgroup_id;
std::string dataset_name;
+ std::vector<hsize_t> particle_file_layout; // to hold the shape of initial condition array
const bool use_collective_io;
public:
@@ -191,9 +192,9 @@ public:
}
{
assert(size_particle_rhs >= 0);
- const hsize_t datacount[2] = {hsize_t(Parent::getTotalNbParticles()),
- hsize_t(size_particle_rhs)};
- hid_t dataspace = H5Screate_simple(2, datacount, NULL);
+ std::vector<hsize_t> datacount = std::vector<hsize_t>(this->particle_file_layout);
+ datacount.push_back(size_particle_positions);
+ hid_t dataspace = H5Screate_simple(datacount.size(), &datacount.front(), NULL);
assert(dataspace >= 0);
hid_t dataset_id = H5Dcreate( pgroup_id,
@@ -215,7 +216,8 @@ public:
hid_t memspace = H5Screate_simple(2, count, NULL);
assert(memspace >= 0);
- hid_t filespace = H5Dget_space(dataset_id);
+ const hsize_t file_count[2] = {hsize_t(Parent::getTotalNbParticles()), size_particle_positions};
+ hid_t filespace = H5Screate_simple(2, file_count, NULL);
assert(filespace >= 0);
int rethdf = H5Sselect_hyperslab(
filespace,
@@ -250,6 +252,17 @@ public:
assert(rethdf >= 0);
}
}
+
+ int setParticleFileLayout(std::vector<hsize_t> input_layout){
+ this->particle_file_layout.resize(input_layout.size());
+ for (unsigned int i=0; i<this->particle_file_layout.size(); i++)
+ this->particle_file_layout[i] = input_layout[i];
+ return EXIT_SUCCESS;
+ }
+
+ std::vector<hsize_t> getParticleFileLayout(void){
+ return std::vector<hsize_t>(this->particle_file_layout);
+ }
};
#endif
diff --git a/bfps/cpp/particles/particles_system.hpp b/bfps/cpp/particles/particles_system.hpp
index 201a57a1..db651904 100644
--- a/bfps/cpp/particles/particles_system.hpp
+++ b/bfps/cpp/particles/particles_system.hpp
@@ -360,7 +360,7 @@ public:
}
std::vector<hsize_t> getParticleFileLayout(void) final{
- return std::move(this->particle_file_layout);
+ return std::vector<hsize_t>(this->particle_file_layout);
}
void checkNan() const { // TODO remove
--
GitLab
From 96e9436e92b3be3942f0363d6c0e1c9dfbbdeecc Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 4 Feb 2019 23:08:36 +0100
Subject: [PATCH 285/342] finalize test of particle cloud code
---
bfps/DNS.py | 12 ++++-----
bfps/__main__.py | 2 +-
bfps/test/test_bfps_NSVEparticles.py | 25 +++++++++++++++++++
bfps/test/test_particle_clouds.py | 37 ++++++++++++++++++++++++----
4 files changed, 63 insertions(+), 13 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index dd75854d..ffcaea49 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -791,13 +791,17 @@ class DNS(_code):
ncomponents = 6
with h5py.File(self.get_checkpoint_0_fname(), 'a') as data_file:
nn = self.parameters['nparticles']
+ if not 'tracers{0}'.format(species) in data_file.keys():
+ data_file.create_group('tracers{0}'.format(species))
+ data_file.create_group('tracers{0}/rhs'.format(species))
+ data_file.create_group('tracers{0}/state'.format(species))
data_file['tracers{0}/rhs'.format(species)].create_dataset(
'0',
shape = (
(self.parameters['tracers{0}_integration_steps'.format(species)],) +
(nn, ncomponents,)),
dtype = np.float)
- dset = data_file['tracers{0}/state'.format(s)].create_dataset(
+ dset = data_file['tracers{0}/state'.format(species)].create_dataset(
'0',
shape = (nn, ncomponents,),
dtype = np.float)
@@ -958,12 +962,6 @@ class DNS(_code):
self,
opt = None):
if self.parameters['nparticles'] > 0:
- with h5py.File(self.get_checkpoint_0_fname(), 'a') as ofile:
- s = 0
- if not 'tracers{0}'.format(s) in ofile.keys():
- ofile.create_group('tracers{0}'.format(s))
- ofile.create_group('tracers{0}/rhs'.format(s))
- ofile.create_group('tracers{0}/state'.format(s))
self.generate_tracer_state(
species = 0,
rseed = opt.particle_rand_seed)
diff --git a/bfps/__main__.py b/bfps/__main__.py
index 16a7cf7d..cf269edb 100644
--- a/bfps/__main__.py
+++ b/bfps/__main__.py
@@ -33,7 +33,7 @@ from .PP import PP
from .TEST import TEST
def main():
- parser = argparse.ArgumentParser(prog = 'bfps')
+ parser = argparse.ArgumentParser(prog = 'bfps', conflict_handler = 'resolve')
parser.add_argument(
'-v', '--version',
action = 'version',
diff --git a/bfps/test/test_bfps_NSVEparticles.py b/bfps/test/test_bfps_NSVEparticles.py
index f914ad7d..fe1e7875 100644
--- a/bfps/test/test_bfps_NSVEparticles.py
+++ b/bfps/test/test_bfps_NSVEparticles.py
@@ -1,4 +1,29 @@
#! /usr/bin/env python
+#######################################################################
+# #
+# Copyright 2019 Max Planck Institute #
+# for Dynamics and Self-Organization #
+# #
+# This file is part of bfps. #
+# #
+# bfps is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# bfps is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with bfps. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+#######################################################################
+
+
import os
import numpy as np
diff --git a/bfps/test/test_particle_clouds.py b/bfps/test/test_particle_clouds.py
index 1a890495..5d204539 100644
--- a/bfps/test/test_particle_clouds.py
+++ b/bfps/test/test_particle_clouds.py
@@ -33,15 +33,22 @@ import sys
import bfps
from bfps import DNS
+
def main():
- nclouds = 4
- nparticles_per_cloud = 3
+ nclouds = 10
+ nparticles_per_cloud = 1000
nparticles = nclouds*nparticles_per_cloud
niterations = 32
c = DNS()
+ c.dns_type = 'NSVEparticles'
+ c.parameters['nparticles'] = nparticles
+ c.parameters['tracers1_integration_steps'] = 4
+ c.generate_tracer_state(rseed = 2, species = 1)
+ del c.parameters['nparticles']
+ del c.parameters['tracers1_integration_steps']
ic_file = h5py.File(c.get_checkpoint_0_fname(), 'a')
- ic_file['tracers0/state/0'] = np.random.random((nclouds, nparticles_per_cloud, 3))
- ic_file['tracers0/rhs/0'] = np.zeros((2, nclouds, nparticles_per_cloud, 3))
+ ic_file['tracers0/state/0'] = ic_file['tracers1/state/0'].value.reshape(nclouds, nparticles_per_cloud, 3)
+ ic_file['tracers0/rhs/0'] = ic_file['tracers1/rhs/0'].value.reshape(4, nclouds, nparticles_per_cloud, 3)
ic_file.close()
c.launch(
['NSVEparticles',
@@ -57,8 +64,28 @@ def main():
'--niter_out', '{0}'.format(niterations),
'--niter_stat', '1',
'--nparticles', '{0}'.format(nparticles),
- '--tracers0_integration_steps', '2',
+ '--njobs', '2',
'--wd', './'])
+ f0 = h5py.File(
+ os.path.join(
+ os.path.join(bfps.lib_dir, 'test'),
+ 'B32p1e4_checkpoint_0.h5'),
+ 'r')
+ f1 = h5py.File(c.get_checkpoint_0_fname(), 'r')
+ for iteration in [0, 32, 64]:
+ field0 = f0['vorticity/complex/{0}'.format(iteration)].value
+ field1 = f1['vorticity/complex/{0}'.format(iteration)].value
+ field_error = np.max(np.abs(field0 - field1))
+ x0 = f0['tracers0/state/{0}'.format(iteration)].value
+ x1 = f1['tracers0/state/{0}'.format(iteration)].value.reshape(x0.shape)
+ traj_error = np.max(np.abs(x0 - x1))
+ y0 = f0['tracers0/rhs/{0}'.format(iteration)].value
+ y1 = f1['tracers0/rhs/{0}'.format(iteration)].value.reshape(y0.shape)
+ rhs_error = np.max(np.abs(y0 - y1))
+ assert(field_error < 1e-5)
+ assert(traj_error < 1e-5)
+ assert(rhs_error < 1e-5)
+ print('SUCCESS! Basic test passed.')
return None
if __name__ == '__main__':
--
GitLab
From 3e488108b6eae019c03ca40670c003a98d626c0f Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 5 Feb 2019 15:48:02 +0100
Subject: [PATCH 286/342] fix sampling file space error
---
bfps/DNS.py | 15 +++++----
bfps/cpp/full_code/NSVEparticles.cpp | 32 ++++++++++++++++---
bfps/cpp/particles/particles_output_hdf5.hpp | 4 +++
.../particles_output_sampling_hdf5.hpp | 2 +-
4 files changed, 41 insertions(+), 12 deletions(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index ffcaea49..b72527a8 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -784,10 +784,15 @@ class DNS(_code):
def generate_tracer_state(
self,
rseed = None,
- species = 0):
+ species = 0,
+ integration_steps = None,
+ ncomponents = 3):
try:
- ncomponents = 3
- if self.dns_type in ['NSVEcomplex_particles']:
+ if type(integration_steps) == type(None):
+ integration_steps = self.NSVEp_extra_parameters['tracers0_integration_steps']
+ if 'tracers{0}_integration_steps'.format(species) in self.parameters.keys():
+ integration_steps = self.parameters['tracers{0}_integration_steps'.format(species)]
+ if self.dns_type == 'NSVEcomplex_particles' and species == 0:
ncomponents = 6
with h5py.File(self.get_checkpoint_0_fname(), 'a') as data_file:
nn = self.parameters['nparticles']
@@ -797,9 +802,7 @@ class DNS(_code):
data_file.create_group('tracers{0}/state'.format(species))
data_file['tracers{0}/rhs'.format(species)].create_dataset(
'0',
- shape = (
- (self.parameters['tracers{0}_integration_steps'.format(species)],) +
- (nn, ncomponents,)),
+ shape = (integration_steps, nn, ncomponents,),
dtype = np.float)
dset = data_file['tracers{0}/state'.format(species)].create_dataset(
'0',
diff --git a/bfps/cpp/full_code/NSVEparticles.cpp b/bfps/cpp/full_code/NSVEparticles.cpp
index edc0f9a6..e7471640 100644
--- a/bfps/cpp/full_code/NSVEparticles.cpp
+++ b/bfps/cpp/full_code/NSVEparticles.cpp
@@ -1,8 +1,30 @@
-
-
-
-
-//#define NDEBUG
+/**********************************************************************
+* *
+* Copyright 2019 Max Planck Institute *
+* for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+**********************************************************************/
+
+
+
+#define NDEBUG
#include <string>
#include <cmath>
diff --git a/bfps/cpp/particles/particles_output_hdf5.hpp b/bfps/cpp/particles/particles_output_hdf5.hpp
index d7c987ee..8f2b45ed 100644
--- a/bfps/cpp/particles/particles_output_hdf5.hpp
+++ b/bfps/cpp/particles/particles_output_hdf5.hpp
@@ -242,6 +242,7 @@ public:
variable_used_only_in_assert(rethdf);
assert(rethdf >= 0);
+ DEBUG_MSG("before writing state\n");
herr_t status = H5Dwrite(
dataset_id,
type_id,
@@ -257,6 +258,7 @@ public:
assert(rethdf >= 0);
rethdf = H5Sclose(filespace);
assert(rethdf >= 0);
+ DEBUG_MSG("after writing state\n");
}
{
assert(size_particle_rhs >= 0);
@@ -304,6 +306,7 @@ public:
variable_used_only_in_assert(rethdf);
assert(rethdf >= 0);
+ DEBUG_MSG("before writing rhs %d\n", idx_rhs);
herr_t status = H5Dwrite(
dataset_id,
type_id,
@@ -317,6 +320,7 @@ public:
assert(rethdf >= 0);
rethdf = H5Sclose(memspace);
assert(rethdf >= 0);
+ DEBUG_MSG("after writing rhs %d\n", idx_rhs);
}
int rethdf = H5Dclose(dataset_id);
variable_used_only_in_assert(rethdf);
diff --git a/bfps/cpp/particles/particles_output_sampling_hdf5.hpp b/bfps/cpp/particles/particles_output_sampling_hdf5.hpp
index fd79a49f..3693a587 100644
--- a/bfps/cpp/particles/particles_output_sampling_hdf5.hpp
+++ b/bfps/cpp/particles/particles_output_sampling_hdf5.hpp
@@ -216,7 +216,7 @@ public:
hid_t memspace = H5Screate_simple(2, count, NULL);
assert(memspace >= 0);
- const hsize_t file_count[2] = {hsize_t(Parent::getTotalNbParticles()), size_particle_positions};
+ const hsize_t file_count[2] = {hsize_t(Parent::getTotalNbParticles()), size_particle_rhs};
hid_t filespace = H5Screate_simple(2, file_count, NULL);
assert(filespace >= 0);
int rethdf = H5Sselect_hyperslab(
--
GitLab
From c9f276b53c0318f3950b6dd1dd02932aa83ba697 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 5 Feb 2019 16:12:34 +0100
Subject: [PATCH 287/342] remove debug messages
---
bfps/cpp/particles/particles_output_hdf5.hpp | 4 ----
1 file changed, 4 deletions(-)
diff --git a/bfps/cpp/particles/particles_output_hdf5.hpp b/bfps/cpp/particles/particles_output_hdf5.hpp
index 8f2b45ed..d7c987ee 100644
--- a/bfps/cpp/particles/particles_output_hdf5.hpp
+++ b/bfps/cpp/particles/particles_output_hdf5.hpp
@@ -242,7 +242,6 @@ public:
variable_used_only_in_assert(rethdf);
assert(rethdf >= 0);
- DEBUG_MSG("before writing state\n");
herr_t status = H5Dwrite(
dataset_id,
type_id,
@@ -258,7 +257,6 @@ public:
assert(rethdf >= 0);
rethdf = H5Sclose(filespace);
assert(rethdf >= 0);
- DEBUG_MSG("after writing state\n");
}
{
assert(size_particle_rhs >= 0);
@@ -306,7 +304,6 @@ public:
variable_used_only_in_assert(rethdf);
assert(rethdf >= 0);
- DEBUG_MSG("before writing rhs %d\n", idx_rhs);
herr_t status = H5Dwrite(
dataset_id,
type_id,
@@ -320,7 +317,6 @@ public:
assert(rethdf >= 0);
rethdf = H5Sclose(memspace);
assert(rethdf >= 0);
- DEBUG_MSG("after writing rhs %d\n", idx_rhs);
}
int rethdf = H5Dclose(dataset_id);
variable_used_only_in_assert(rethdf);
--
GitLab
From b928c7ec36c7f1336103aee5c938ed84b48265de Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 13 Feb 2019 22:48:49 +0100
Subject: [PATCH 288/342] fix call to write_par
---
bfps/DNS.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index b72527a8..fdaa0f63 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -1036,7 +1036,7 @@ class DNS(_code):
opt = None):
if not os.path.exists(self.get_data_file_name()):
self.generate_initial_condition(opt = opt)
- self.write_par()
+ self.write_par()
self.run(
nb_processes = opt.nb_processes,
nb_threads_per_process = opt.nb_threads_per_process,
--
GitLab
From 9f5be7bf77182bf15b614a093dcc83d357d250ea Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 15 Feb 2019 09:16:01 +0100
Subject: [PATCH 289/342] tweak plain fftw test
---
tests/misc/makefile | 1 +
tests/misc/test_fftw.c | 14 ++++++++++----
2 files changed, 11 insertions(+), 4 deletions(-)
diff --git a/tests/misc/makefile b/tests/misc/makefile
index c8f5f788..d44b9f04 100644
--- a/tests/misc/makefile
+++ b/tests/misc/makefile
@@ -1,5 +1,6 @@
test_fftw: test_fftw.c
mpicc \
+ -DFFTW_PLAN_RIGOR=FFTW_ESTIMATE \
-I/stuff/ext_installs/include \
-fopenmp \
test_fftw.c \
diff --git a/tests/misc/test_fftw.c b/tests/misc/test_fftw.c
index 6da0099c..af9fef7b 100644
--- a/tests/misc/test_fftw.c
+++ b/tests/misc/test_fftw.c
@@ -5,6 +5,12 @@
#include <assert.h>
#include <math.h>
+#ifndef FFTW_PLAN_RIGOR
+
+#define FFTW_PLAN_RIGOR FFTW_ESTIMATE
+
+#endif
+
//#define NO_FFTWOMP
#define NX 36
@@ -104,7 +110,7 @@ int main(
cdata,
data,
MPI_COMM_WORLD,
- FFTW_ESTIMATE | FFTW_MPI_TRANSPOSED_IN);
+ FFTW_PLAN_RIGOR | FFTW_MPI_TRANSPOSED_IN);
r2c_plan = fftwf_mpi_plan_many_dft_r2c(
3, nfftw, 3,
@@ -112,7 +118,7 @@ int main(
data,
cdata,
MPI_COMM_WORLD,
- FFTW_ESTIMATE | FFTW_MPI_TRANSPOSED_OUT);
+ FFTW_PLAN_RIGOR | FFTW_MPI_TRANSPOSED_OUT);
kx = (double*)malloc(sizeof(double)*(nx/2+1));
ky = (double*)malloc(sizeof(double)*local_n1);
@@ -124,8 +130,7 @@ int main(
if (jy + local_1_start <= ny/2)
ky[jy] = dky*(jy + local_1_start);
else
- ky[jy] = dky*((jy + local_1_start) - ny);
- }
+ ky[jy] = dky*((jy + local_1_start) - ny); }
for (jz = 0; jz < nz; jz++)
{
if (jz <= nz/2)
@@ -305,6 +310,7 @@ int main(
//L2norm1 = sqrt(L2norm1 / (nx*ny*nz));
//L2norm2 = sqrt(L2norm2 / (nx*ny*nz));
+ printf("FFTW_PLAN_RIGOR=%d\n", FFTW_PLAN_RIGOR);
printf("L2normk = %g, L2norm1 = %g, relative error = %g\n",
L2normk, L2norm1, fabs(L2normk - L2norm1) / (L2normk));
--
GitLab
From 61df2487283ed7a75bca2568db0021f339125a56 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 15 Feb 2019 10:06:45 +0100
Subject: [PATCH 290/342] fix documentation
---
README.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/README.rst b/README.rst
index 5418d711..efafb817 100644
--- a/README.rst
+++ b/README.rst
@@ -130,7 +130,7 @@ enough).
./configure --prefix=PREFIX --enable-single --enable-sse --enable-mpi --enable-openmp --enable-threads
make
make install
- ./configure --prefix=PREFIX --enable-sse --enable-sse2 --enable-mpi --enable-openmp --enable-threads
+ ./configure --prefix=PREFIX --enable-sse2 --enable-mpi --enable-openmp --enable-threads
make
make install
--
GitLab
From 551203bf039b21602469fecb4b3b267aa59f9a55 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 15 Feb 2019 10:30:52 +0100
Subject: [PATCH 291/342] fix possible spectrum memory aliasing issue
---
bfps/cpp/field.cpp | 1 -
bfps/cpp/kspace.cpp | 80 ++++++++++++++++++++++++++++++++++++++++++++-
bfps/cpp/kspace.hpp | 8 +++++
3 files changed, 87 insertions(+), 2 deletions(-)
diff --git a/bfps/cpp/field.cpp b/bfps/cpp/field.cpp
index 95a44f44..9113b138 100644
--- a/bfps/cpp/field.cpp
+++ b/bfps/cpp/field.cpp
@@ -1387,7 +1387,6 @@ void field<rnumber, be, fc>::compute_stats(
// what follows gave me a headache until I found this link:
// http://stackoverflow.com/questions/8256636/expected-primary-expression-error-on-template-method-using
kk->template cospectrum<rnumber, fc>(
- (typename fftw_interface<rnumber>::complex*)this->data,
(typename fftw_interface<rnumber>::complex*)this->data,
group,
dset_name + "_" + dset_name,
diff --git a/bfps/cpp/kspace.cpp b/bfps/cpp/kspace.cpp
index c480d1b0..a156b061 100644
--- a/bfps/cpp/kspace.cpp
+++ b/bfps/cpp/kspace.cpp
@@ -525,7 +525,7 @@ void kspace<be, dt>::cospectrum(
const std::string dset_name,
const hsize_t toffset)
{
- TIMEZONE("field::cospectrum");
+ TIMEZONE("field::cospectrum2");
shared_array<double> spec_local_thread(this->nshells*ncomp(fc)*ncomp(fc),[&](double* spec_local){
std::fill_n(spec_local, this->nshells*ncomp(fc)*ncomp(fc), 0);
});
@@ -593,6 +593,84 @@ void kspace<be, dt>::cospectrum(
}
}
+template <field_backend be,
+ kspace_dealias_type dt>
+template <typename rnumber,
+ field_components fc>
+void kspace<be, dt>::cospectrum(
+ const rnumber(* __restrict a)[2],
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset)
+{
+ TIMEZONE("field::cospectrum1");
+ shared_array<double> spec_local_thread(this->nshells*ncomp(fc)*ncomp(fc),[&](double* spec_local){
+ std::fill_n(spec_local, this->nshells*ncomp(fc)*ncomp(fc), 0);
+ });
+
+ this->CLOOP_K2_NXMODES(
+ [&](ptrdiff_t cindex,
+ ptrdiff_t xindex,
+ ptrdiff_t yindex,
+ ptrdiff_t zindex,
+ double k2,
+ int nxmodes){
+ if (k2 <= this->kM2)
+ {
+ double* spec_local = spec_local_thread.getMine();
+ int tmp_int = int(sqrt(k2) / this->dk)*ncomp(fc)*ncomp(fc);
+ for (hsize_t i=0; i<ncomp(fc); i++)
+ for (hsize_t j=0; j<ncomp(fc); j++){
+ spec_local[tmp_int + i*ncomp(fc)+j] += nxmodes * (
+ (a[ncomp(fc)*cindex + i][0] * a[ncomp(fc)*cindex + j][0]) +
+ (a[ncomp(fc)*cindex + i][1] * a[ncomp(fc)*cindex + j][1]));
+ }
+ }
+ });
+
+ spec_local_thread.mergeParallel();
+
+ std::vector<double> spec;
+ spec.resize(this->nshells*ncomp(fc)*ncomp(fc), 0);
+ MPI_Allreduce(
+ spec_local_thread.getMasterData(),
+ &spec.front(),
+ spec.size(),
+ MPI_DOUBLE, MPI_SUM, this->layout->comm);
+ if (this->layout->myrank == 0)
+ {
+ hid_t dset, wspace, mspace;
+ hsize_t count[(ndim(fc)-2)*2], offset[(ndim(fc)-2)*2], dims[(ndim(fc)-2)*2];
+ dset = H5Dopen(group, ("spectra/" + dset_name).c_str(), H5P_DEFAULT);
+ wspace = H5Dget_space(dset);
+ H5Sget_simple_extent_dims(wspace, dims, NULL);
+ switch (fc)
+ {
+ case THREExTHREE:
+ offset[4] = 0;
+ offset[5] = 0;
+ count[4] = 3;
+ count[5] = 3;
+ case THREE:
+ offset[2] = 0;
+ offset[3] = 0;
+ count[2] = 3;
+ count[3] = 3;
+ default:
+ offset[0] = toffset;
+ offset[1] = 0;
+ count[0] = 1;
+ count[1] = this->nshells;
+ }
+ mspace = H5Screate_simple((ndim(fc)-2)*2, count, NULL);
+ H5Sselect_hyperslab(wspace, H5S_SELECT_SET, offset, NULL, count, NULL);
+ H5Dwrite(dset, H5T_NATIVE_DOUBLE, mspace, wspace, H5P_DEFAULT, &spec.front());
+ H5Sclose(wspace);
+ H5Sclose(mspace);
+ H5Dclose(dset);
+ }
+}
+
template <field_backend be,
kspace_dealias_type dt>
template <typename rnumber,
diff --git a/bfps/cpp/kspace.hpp b/bfps/cpp/kspace.hpp
index c0bf2583..94582da8 100644
--- a/bfps/cpp/kspace.hpp
+++ b/bfps/cpp/kspace.hpp
@@ -114,6 +114,14 @@ class kspace
const std::string dset_name,
const hsize_t toffset);
+ template <typename rnumber,
+ field_components fc>
+ void cospectrum(
+ const rnumber(* __restrict__ a)[2],
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset);
+
template <typename rnumber,
field_components fc>
double L2norm(
--
GitLab
From 5603b077dd16945f56c3ace896370104ae1d1880 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 15 Feb 2019 10:35:25 +0100
Subject: [PATCH 292/342] make explicit cast
compiler was warning that something is reduced from int to hsize_t.
---
bfps/cpp/particles/particles_output_sampling_hdf5.hpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bfps/cpp/particles/particles_output_sampling_hdf5.hpp b/bfps/cpp/particles/particles_output_sampling_hdf5.hpp
index 3693a587..ff3782b5 100644
--- a/bfps/cpp/particles/particles_output_sampling_hdf5.hpp
+++ b/bfps/cpp/particles/particles_output_sampling_hdf5.hpp
@@ -216,7 +216,7 @@ public:
hid_t memspace = H5Screate_simple(2, count, NULL);
assert(memspace >= 0);
- const hsize_t file_count[2] = {hsize_t(Parent::getTotalNbParticles()), size_particle_rhs};
+ const hsize_t file_count[2] = {hsize_t(Parent::getTotalNbParticles()), hsize_t(size_particle_rhs)};
hid_t filespace = H5Screate_simple(2, file_count, NULL);
assert(filespace >= 0);
int rethdf = H5Sselect_hyperslab(
--
GitLab
From 23a27b746de095d6cbb40ae220987d285f65c6bf Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 15 Feb 2019 11:13:23 +0100
Subject: [PATCH 293/342] add template definition/instance thing
---
bfps/cpp/kspace.cpp | 62 +++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 62 insertions(+)
diff --git a/bfps/cpp/kspace.cpp b/bfps/cpp/kspace.cpp
index a156b061..04d5dcd8 100644
--- a/bfps/cpp/kspace.cpp
+++ b/bfps/cpp/kspace.cpp
@@ -915,6 +915,68 @@ template void kspace<FFTW, SMOOTH>::cospectrum<double, THREExTHREE>(
const std::string dset_name,
const hsize_t toffset);
+template void kspace<FFTW, TWO_THIRDS>::cospectrum<float, ONE>(
+ const typename fftw_interface<float>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset);
+template void kspace<FFTW, TWO_THIRDS>::cospectrum<float, THREE>(
+ const typename fftw_interface<float>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset);
+template void kspace<FFTW, TWO_THIRDS>::cospectrum<float, THREExTHREE>(
+ const typename fftw_interface<float>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset);
+template void kspace<FFTW, TWO_THIRDS>::cospectrum<double, ONE>(
+ const typename fftw_interface<double>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset);
+template void kspace<FFTW, TWO_THIRDS>::cospectrum<double, THREE>(
+ const typename fftw_interface<double>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset);
+template void kspace<FFTW, TWO_THIRDS>::cospectrum<double, THREExTHREE>(
+ const typename fftw_interface<double>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset);
+
+template void kspace<FFTW, SMOOTH>::cospectrum<float, ONE>(
+ const typename fftw_interface<float>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset);
+template void kspace<FFTW, SMOOTH>::cospectrum<float, THREE>(
+ const typename fftw_interface<float>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset);
+template void kspace<FFTW, SMOOTH>::cospectrum<float, THREExTHREE>(
+ const typename fftw_interface<float>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset);
+template void kspace<FFTW, SMOOTH>::cospectrum<double, ONE>(
+ const typename fftw_interface<double>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset);
+template void kspace<FFTW, SMOOTH>::cospectrum<double, THREE>(
+ const typename fftw_interface<double>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset);
+template void kspace<FFTW, SMOOTH>::cospectrum<double, THREExTHREE>(
+ const typename fftw_interface<double>::complex *__restrict__ a,
+ const hid_t group,
+ const std::string dset_name,
+ const hsize_t toffset);
+
template double kspace<FFTW, TWO_THIRDS>::L2norm<float, ONE>(
const typename fftw_interface<float>::complex *__restrict__ a);
template double kspace<FFTW, TWO_THIRDS>::L2norm<float, THREE>(
--
GitLab
From 1f14b12c18da4b88e07f3704feefa66711369fe8 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 15 Feb 2019 11:13:45 +0100
Subject: [PATCH 294/342] don't include deprecated header
---
bfps/cpp/fftw_tools.hpp | 1 -
1 file changed, 1 deletion(-)
diff --git a/bfps/cpp/fftw_tools.hpp b/bfps/cpp/fftw_tools.hpp
index e32500fd..52d80f25 100644
--- a/bfps/cpp/fftw_tools.hpp
+++ b/bfps/cpp/fftw_tools.hpp
@@ -26,7 +26,6 @@
#include <mpi.h>
#include <fftw3-mpi.h>
-#include "field_descriptor.hpp"
#ifndef FFTW_TOOLS
--
GitLab
From 7cd44280b927abc4f18c37a7b1c2b0c149750bed Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 15 Feb 2019 15:52:38 +0100
Subject: [PATCH 295/342] add required header
---
bfps/cpp/fftw_tools.hpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/bfps/cpp/fftw_tools.hpp b/bfps/cpp/fftw_tools.hpp
index 52d80f25..b41cd2a4 100644
--- a/bfps/cpp/fftw_tools.hpp
+++ b/bfps/cpp/fftw_tools.hpp
@@ -26,6 +26,7 @@
#include <mpi.h>
#include <fftw3-mpi.h>
+#include <map>
#ifndef FFTW_TOOLS
--
GitLab
From 79e0f520507dd16bcce0c212c5aabeac01a5e33a Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 15 Feb 2019 21:13:12 +0100
Subject: [PATCH 296/342] add ability to handle mpich pc installations
i.e. different mpirun syntax
---
bfps/_code.py | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/bfps/_code.py b/bfps/_code.py
index 297749f5..f997d651 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -262,10 +262,16 @@ class _code(_base):
current_dir = os.getcwd()
os.chdir(self.work_dir)
os.chdir(current_dir)
+ if not 'MPI' in self.host_info.keys():
+ self.host_info['MPI'] = 'openmpi'
+ if self.host_info['MPI'] == 'openmpi':
+ mpirun_environment_set = 'x'
+ else:
+ mpirun_environment_set = 'env'
command_atoms = ['mpirun',
'-np',
'{0}'.format(nb_processes),
- '-x',
+ '-' + mpirun_environment_set,
'OMP_NUM_THREADS={0}'.format(nb_threads_per_process),
'./' + self.name,
self.simname]
--
GitLab
From 05c44ecaaaecc46451a96022a9aac2850a3f5e75 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Sun, 17 Feb 2019 13:28:10 +0100
Subject: [PATCH 297/342] partial fix
when using unique_ptr for arrays, they are cleaned up by calling
`delete[] bla.release()`.
At least, cppreference says that the release method
"returns a pointer to the managed object and releases the ownership ".
https://en.cppreference.com/w/cpp/memory/unique_ptr
---
bfps/cpp/full_code/NSVEcomplex_particles.cpp | 7 +++----
bfps/cpp/full_code/NSVEparticles.cpp | 7 +++----
bfps/cpp/full_code/test_interpolation.cpp | 6 ++++--
bfps/cpp/hdf5_tools.cpp | 3 +++
bfps/cpp/particles/abstract_particles_output.hpp | 12 ++++++------
bfps/cpp/particles/p2p_distr_mpi.hpp | 6 +++---
bfps/cpp/particles/particles_distr_mpi.hpp | 10 +++++-----
bfps/cpp/particles/particles_input_hdf5.hpp | 4 ++--
8 files changed, 29 insertions(+), 26 deletions(-)
diff --git a/bfps/cpp/full_code/NSVEcomplex_particles.cpp b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
index c7bc9424..3bd27102 100644
--- a/bfps/cpp/full_code/NSVEcomplex_particles.cpp
+++ b/bfps/cpp/full_code/NSVEcomplex_particles.cpp
@@ -229,10 +229,9 @@ int NSVEcomplex_particles<rnumber>::do_stats()
this->ps->get_step_idx()-1);
// deallocate temporary data array
- // TODO: is it required/safe to call the release method here?
- pdata0.release();
- pdata1.release();
- pdata2.release();
+ delete[] pdata0.release();
+ delete[] pdata1.release();
+ delete[] pdata2.release();
return EXIT_SUCCESS;
}
diff --git a/bfps/cpp/full_code/NSVEparticles.cpp b/bfps/cpp/full_code/NSVEparticles.cpp
index c8918996..9b8743cd 100644
--- a/bfps/cpp/full_code/NSVEparticles.cpp
+++ b/bfps/cpp/full_code/NSVEparticles.cpp
@@ -105,7 +105,7 @@ int NSVEparticles<rnumber>::finalize(void)
{
TIMEZONE("NSVEparticles::finalize");
delete this->pressure;
- this->ps.release();
+ delete this->ps.release();
delete this->particles_output_writer_mpi;
delete this->particles_sample_writer_mpi;
this->NSVE<rnumber>::finalize();
@@ -134,7 +134,7 @@ int NSVEparticles<rnumber>::do_stats()
// allocate temporary data array
std::unique_ptr<double[]> pdata(new double[3*this->ps->getLocalNbParticles()]);
- // copy position data
+ /// copy position data
/// sample position
std::copy(this->ps->getParticlesState(),
@@ -183,8 +183,7 @@ int NSVEparticles<rnumber>::do_stats()
this->ps->get_step_idx()-1);
// deallocate temporary data array
- // TODO: is it required/safe to call the release method here?
- pdata.release();
+ delete[] pdata.release();
return EXIT_SUCCESS;
}
diff --git a/bfps/cpp/full_code/test_interpolation.cpp b/bfps/cpp/full_code/test_interpolation.cpp
index 2acd3c27..c3103fb4 100644
--- a/bfps/cpp/full_code/test_interpolation.cpp
+++ b/bfps/cpp/full_code/test_interpolation.cpp
@@ -92,7 +92,7 @@ int test_interpolation<rnumber>::finalize(void)
delete this->nabla_u;
delete this->velocity;
delete this->vorticity;
- this->ps.release();
+ delete this->ps.release();
delete this->kk;
delete particles_output_writer_mpi;
delete particles_sample_writer_mpi;
@@ -198,7 +198,9 @@ int test_interpolation<rnumber>::do_work()
this->ps->getLocalNbParticles(),
this->ps->get_step_idx()-1);
- // no need to deallocate because we used "unique_ptr"
+ // deallocate temporary arrays
+ delete[] p3data.release();
+ delete[] p9data.release();
return EXIT_SUCCESS;
}
diff --git a/bfps/cpp/hdf5_tools.cpp b/bfps/cpp/hdf5_tools.cpp
index 25acaf21..5a3aef39 100644
--- a/bfps/cpp/hdf5_tools.cpp
+++ b/bfps/cpp/hdf5_tools.cpp
@@ -213,6 +213,9 @@ std::string hdf5_tools::read_string(
hid_t dset = H5Dopen(group, dset_name.c_str(), H5P_DEFAULT);
hid_t space = H5Dget_space(dset);
hid_t memtype = H5Dget_type(dset);
+ // fsanitize complains unless I have a static array here
+ // but that doesn't actually work (data is read incorrectly).
+ // this is caught by bfps.test_NSVEparticles
char *string_data = (char*)malloc(256);
H5Dread(dset, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &string_data);
std::string std_string_data = std::string(string_data);
diff --git a/bfps/cpp/particles/abstract_particles_output.hpp b/bfps/cpp/particles/abstract_particles_output.hpp
index 4fc344d3..a457689a 100644
--- a/bfps/cpp/particles/abstract_particles_output.hpp
+++ b/bfps/cpp/particles/abstract_particles_output.hpp
@@ -137,15 +137,15 @@ public:
}
void releaseMemory(){
- buffer_indexes_send.release();
- buffer_particles_positions_send.release();
+ delete[] buffer_indexes_send.release();
+ delete[] buffer_particles_positions_send.release();
size_buffers_send = 0;
- buffer_indexes_recv.release();
- buffer_particles_positions_recv.release();
+ delete[] buffer_indexes_recv.release();
+ delete[] buffer_particles_positions_recv.release();
size_buffers_recv = 0;
for(int idx_rhs = 0 ; idx_rhs < nb_rhs ; ++idx_rhs){
- buffer_particles_rhs_send[idx_rhs].release();
- buffer_particles_rhs_recv[idx_rhs].release();
+ delete[] buffer_particles_rhs_send[idx_rhs].release();
+ delete[] buffer_particles_rhs_recv[idx_rhs].release();
}
buffers_size_particle_rhs_send = 0;
buffers_size_particle_rhs_recv = 0;
diff --git a/bfps/cpp/particles/p2p_distr_mpi.hpp b/bfps/cpp/particles/p2p_distr_mpi.hpp
index 965f8ba8..9b8a5b13 100644
--- a/bfps/cpp/particles/p2p_distr_mpi.hpp
+++ b/bfps/cpp/particles/p2p_distr_mpi.hpp
@@ -625,7 +625,7 @@ public:
AssertMpi(MPI_Isend(descriptor.results.get(), int(NbParticlesToReceive*size_particle_rhs),
particles_utils::GetMpiType(real_number()), destProc, TAG_RESULT_PARTICLES,
current_com, &mpiRequests.back()));
- descriptor.toCompute.release();
+ delete[] descriptor.toCompute.release();
}
//////////////////////////////////////////////////////////////////////
/// Release memory that was sent back
@@ -634,7 +634,7 @@ public:
NeighborDescriptor& descriptor = neigDescriptors[releasedAction.second];
assert(descriptor.results != nullptr);
assert(descriptor.isRecv);
- descriptor.results.release();
+ delete[] descriptor.results.release();
}
//////////////////////////////////////////////////////////////////////
/// Merge
@@ -646,7 +646,7 @@ public:
assert(descriptor.toRecvAndMerge != nullptr);
in_computer.template reduce_particles_rhs<size_particle_rhs>(&particles_current_rhs[particles_offset_layers[my_nb_cell_levels-descriptor.nbLevelsToExchange]*size_particle_rhs],
descriptor.toRecvAndMerge.get(), descriptor.nbParticlesToExchange);
- descriptor.toRecvAndMerge.release();
+ delete[] descriptor.toRecvAndMerge.release();
}
}
}
diff --git a/bfps/cpp/particles/particles_distr_mpi.hpp b/bfps/cpp/particles/particles_distr_mpi.hpp
index cec734e1..8a2b77ca 100644
--- a/bfps/cpp/particles/particles_distr_mpi.hpp
+++ b/bfps/cpp/particles/particles_distr_mpi.hpp
@@ -418,7 +418,7 @@ public:
if(releasedAction.first == RELEASE_BUFFER_PARTICLES){
NeighborDescriptor& descriptor = neigDescriptors[releasedAction.second];
assert(descriptor.toCompute != nullptr);
- descriptor.toCompute.release();
+ delete[] descriptor.toCompute.release();
}
//////////////////////////////////////////////////////////////////////
/// Merge
@@ -430,14 +430,14 @@ public:
TIMEZONE("reduce");
assert(descriptor.toRecvAndMerge != nullptr);
in_computer.template reduce_particles_rhs<size_particle_rhs>(&particles_current_rhs[0], descriptor.toRecvAndMerge.get(), descriptor.nbParticlesToSend);
- descriptor.toRecvAndMerge.release();
+ delete[] descriptor.toRecvAndMerge.release();
}
else {
TIMEZONE("reduce");
assert(descriptor.toRecvAndMerge != nullptr);
in_computer.template reduce_particles_rhs<size_particle_rhs>(&particles_current_rhs[(current_offset_particles_for_partition[current_partition_size]-descriptor.nbParticlesToSend)*size_particle_rhs],
descriptor.toRecvAndMerge.get(), descriptor.nbParticlesToSend);
- descriptor.toRecvAndMerge.release();
+ delete[] descriptor.toRecvAndMerge.release();
}
}
}
@@ -475,14 +475,14 @@ public:
TIMEZONE("reduce_later");
assert(descriptor.toRecvAndMerge != nullptr);
in_computer.template reduce_particles_rhs<size_particle_rhs>(&particles_current_rhs[0], descriptor.toRecvAndMerge.get(), descriptor.nbParticlesToSend);
- descriptor.toRecvAndMerge.release();
+ delete[] descriptor.toRecvAndMerge.release();
}
else {
TIMEZONE("reduce_later");
assert(descriptor.toRecvAndMerge != nullptr);
in_computer.template reduce_particles_rhs<size_particle_rhs>(&particles_current_rhs[(current_offset_particles_for_partition[current_partition_size]-descriptor.nbParticlesToSend)*size_particle_rhs],
descriptor.toRecvAndMerge.get(), descriptor.nbParticlesToSend);
- descriptor.toRecvAndMerge.release();
+ delete[] descriptor.toRecvAndMerge.release();
}
}
}
diff --git a/bfps/cpp/particles/particles_input_hdf5.hpp b/bfps/cpp/particles/particles_input_hdf5.hpp
index e10377bf..33406314 100644
--- a/bfps/cpp/particles/particles_input_hdf5.hpp
+++ b/bfps/cpp/particles/particles_input_hdf5.hpp
@@ -302,13 +302,13 @@ public:
my_particles_positions.reset(new real_number[exchanger.getTotalToRecv()*size_particle_positions]);
}
exchanger.alltoallv<real_number>(split_particles_positions.get(), my_particles_positions.get(), size_particle_positions);
- split_particles_positions.release();
+ delete[] split_particles_positions.release();
if(nb_particles_for_me){
my_particles_indexes.reset(new partsize_t[exchanger.getTotalToRecv()]);
}
exchanger.alltoallv<partsize_t>(split_particles_indexes.get(), my_particles_indexes.get());
- split_particles_indexes.release();
+ delete[] split_particles_indexes.release();
my_particles_rhs.resize(nb_rhs);
for(int idx_rhs = 0 ; idx_rhs < int(nb_rhs) ; ++idx_rhs){
--
GitLab
From 4b19af4ef8fc5bc4b7912f54c028cd99e4602288 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 21 Feb 2019 16:01:28 +0100
Subject: [PATCH 298/342] fix documentation
---
README.rst | 3 +++
1 file changed, 3 insertions(+)
diff --git a/README.rst b/README.rst
index efafb817..d5d0c8ba 100644
--- a/README.rst
+++ b/README.rst
@@ -127,6 +127,7 @@ enough).
optimisation flags for your own computer:
.. code:: bash
+
./configure --prefix=PREFIX --enable-single --enable-sse --enable-mpi --enable-openmp --enable-threads
make
make install
@@ -139,6 +140,7 @@ enough).
We are using parallel I/O, therefore we use the plain C interface of HDF5:
.. code:: bash
+
./configure --prefix=PREFIX --enable-parallel
make
make install
@@ -150,6 +152,7 @@ enough).
4. Clone bfps repository.
.. code:: bash
+
git clone git@gitlab.mpcdf.mpg.de:clalescu/bfps.git
5. Tweak host_information.py and machine_settings.py for your user and your machine and place under ~/.config/bfps.
--
GitLab
From 16c64a36cb1327f13801f9ed09cab5ddcfbe38af Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 22 Feb 2019 06:24:06 +0100
Subject: [PATCH 299/342] update README
---
README.rst | 17 ++++++++++++++---
1 file changed, 14 insertions(+), 3 deletions(-)
diff --git a/README.rst b/README.rst
index d5d0c8ba..71d4f44f 100644
--- a/README.rst
+++ b/README.rst
@@ -76,6 +76,18 @@ needed.
python setup.py compile_library
python setup.py install
+For `machine_settings.py`, please keep in mind to turn on optimizations
+for production environments.
+In particular, for clusters of unknown architecture it helps to log into
+individual nodes and run the following command:
+
+.. code:: bash
+ gcc -march=native -Q --help=target
+
+The results can be used to then compile on the frontend node(s) without
+using `-march=native` (since the frontend node may have different
+architecture).
+
-------------
Documentation
-------------
@@ -105,9 +117,8 @@ Comments
* Code is only known to work with HDF5 1.8.x.
-* Code is used mainly with Python 3.4 and 3.5.
- In principle it should be easy to maintain compatibility with Python
- 2.7.x, but as of `bfps 1.8` this is no longer a main concern.
+* Code is used mainly with Python 3.5 and later, and it is not tested at
+ all with Python 2.x
-------------------------------
Installation with prerequisites
--
GitLab
From 20c6ce712caed8558a8f2b3912fde643a1d88d5e Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 1 Mar 2019 16:45:11 +0100
Subject: [PATCH 300/342] fix readme
---
README.rst | 1 +
1 file changed, 1 insertion(+)
diff --git a/README.rst b/README.rst
index 71d4f44f..e4f6d110 100644
--- a/README.rst
+++ b/README.rst
@@ -82,6 +82,7 @@ In particular, for clusters of unknown architecture it helps to log into
individual nodes and run the following command:
.. code:: bash
+
gcc -march=native -Q --help=target
The results can be used to then compile on the frontend node(s) without
--
GitLab
From 2836c5b6a2af365b2f99ad61e17db8ed80205455 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Sun, 3 Mar 2019 11:08:21 +0100
Subject: [PATCH 301/342] don't run any CI scripts
---
tests/ci-scripts/test.sh | 84 ++++++++++++++++++++--------------------
1 file changed, 43 insertions(+), 41 deletions(-)
diff --git a/tests/ci-scripts/test.sh b/tests/ci-scripts/test.sh
index e2adc661..bb6eaa85 100644
--- a/tests/ci-scripts/test.sh
+++ b/tests/ci-scripts/test.sh
@@ -5,45 +5,47 @@ set -x
# stops when fails
set -e
-# Init
-export destdir=$(pwd)"/ci-installdir"
-export pythonbin=/home/ubuntu/anaconda3/bin/python3
-export bfpspythonpath=$destdir/lib/python3.6/site-packages/
-export PYTHONPATH=:$bfpspythonpath$PYTHONPATH
-export PATH=$destdir/bin/:/home/ubuntu/hdf5/install/bin/:$PATH
-export LD_LIBRARY_PATH=/home/ubuntu/hdf5/install/lib/:/home/ubuntu/fftw/install/lib/
-
-echo "destdir = $destdir"
-echo "pythonbin = $pythonbin"
-echo "bfpspythonpath = $bfpspythonpath"
-
-# Remove possible previous installation
-if [[ -d $destdir ]] ; then
- rm -rf $destdir ;
-fi
-
-# Create install path
-if [[ ! -d $bfpspythonpath ]] ; then
- mkdir -p $bfpspythonpath ;
-fi
-
-# Build
-$pythonbin setup.py compile_library --timing-output 1
-# Install
-$pythonbin setup.py install --prefix=$destdir
-
-# Test
-ls $destdir
-ls $destdir/bin/
-
-$pythonbin $destdir/bin/bfps.test_fftw
-
-$pythonbin $destdir/bin/bfps.test_Parseval
-
-$pythonbin $destdir/bin/bfps.test_NSVEparticles
-
-# Clean
-if [[ -d $destdir ]] ; then
- rm -rf $destdir ;
-fi
+echo "please check VM before turning tests back on"
+
+## Init
+#export destdir=$(pwd)"/ci-installdir"
+#export pythonbin=/home/ubuntu/anaconda3/bin/python3
+#export bfpspythonpath=$destdir/lib/python3.6/site-packages/
+#export PYTHONPATH=:$bfpspythonpath$PYTHONPATH
+#export PATH=$destdir/bin/:/home/ubuntu/hdf5/install/bin/:$PATH
+#export LD_LIBRARY_PATH=/home/ubuntu/hdf5/install/lib/:/home/ubuntu/fftw/install/lib/
+#
+#echo "destdir = $destdir"
+#echo "pythonbin = $pythonbin"
+#echo "bfpspythonpath = $bfpspythonpath"
+#
+## Remove possible previous installation
+#if [[ -d $destdir ]] ; then
+# rm -rf $destdir ;
+#fi
+#
+## Create install path
+#if [[ ! -d $bfpspythonpath ]] ; then
+# mkdir -p $bfpspythonpath ;
+#fi
+#
+## Build
+#$pythonbin setup.py compile_library --timing-output 1
+## Install
+#$pythonbin setup.py install --prefix=$destdir
+#
+## Test
+#ls $destdir
+#ls $destdir/bin/
+#
+#$pythonbin $destdir/bin/bfps.test_fftw
+#
+#$pythonbin $destdir/bin/bfps.test_Parseval
+#
+#$pythonbin $destdir/bin/bfps.test_NSVEparticles
+#
+## Clean
+#if [[ -d $destdir ]] ; then
+# rm -rf $destdir ;
+#fi
--
GitLab
From 438f76dc43d2dc7e6fb844f64395b1c4d1e17b27 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Sun, 3 Mar 2019 11:08:35 +0100
Subject: [PATCH 302/342] add more general checkpoint file name method
---
bfps/DNS.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index fdaa0f63..9dd9f182 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -781,6 +781,10 @@ class DNS(_code):
return os.path.join(
self.work_dir,
self.simname + '_checkpoint_0.h5')
+ def get_checkpoint_fname(self, checkpoint = 0):
+ return os.path.join(
+ self.work_dir,
+ self.simname + '_checkpoint_{0}.h5'.format(checkpoint))
def generate_tracer_state(
self,
rseed = None,
--
GitLab
From 7ee015a8b053a70def3e66a194df730bef3c9f80 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 4 Mar 2019 09:48:40 +0100
Subject: [PATCH 303/342] update checkpoint file name method
---
bfps/DNS.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 9dd9f182..e0d83caf 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -781,7 +781,8 @@ class DNS(_code):
return os.path.join(
self.work_dir,
self.simname + '_checkpoint_0.h5')
- def get_checkpoint_fname(self, checkpoint = 0):
+ def get_checkpoint_fname(self, iteration = 0):
+ checkpoint = iteration // self.parameters['checkpoints_per_file']
return os.path.join(
self.work_dir,
self.simname + '_checkpoint_{0}.h5'.format(checkpoint))
--
GitLab
From abe32518f78b744a68bacbe3ad921393a2739e16 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 6 Mar 2019 10:46:18 +0100
Subject: [PATCH 304/342] testing cmake setup
---
CMakeLists.txt | 38 ++++++++++++++++++++++++++++++++++++++
1 file changed, 38 insertions(+)
create mode 100644 CMakeLists.txt
diff --git a/CMakeLists.txt b/CMakeLists.txt
new file mode 100644
index 00000000..14bea37e
--- /dev/null
+++ b/CMakeLists.txt
@@ -0,0 +1,38 @@
+cmake_minimum_required(VERSION 2.8.11)
+
+project(BFPS)
+include_directories(
+ ${PROJECT_SOURCE_DIR}/bfps/cpp
+ ${PROJECT_SOURCE_DIR}/bfps/cpp/particles
+ ${PROJECT_SOURCE_DIR}/bfps/cpp/full_code
+ )
+
+file(GLOB all_SRCS
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/*.hpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/*.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/particles/*.hpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/particles/*.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/*.hpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/*.cpp"
+ )
+
+find_package(MPI REQUIRED)
+include_directories(${MPI_INCLUDE_PATH})
+set(CMAKE_CXX_COMPILE_FLAGS "${CMAKE_CXX_COMPILE_FLAGS} -Wall -std=c++11 -fopenmp")
+set(CMAKE_CXX_COMPILE_FLAGS ${CMAKE_CXX_COMPILE_FLAGS} ${MPI_COMPILE_FLAGS})
+set(CMAKE_CXX_LINK_FLAGS ${CMAKE_CXX_LINK_FLAGS} ${MPI_LINK_FLAGS})
+
+add_library(bfps
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/field.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/kspace.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/vorticity_equation.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/NSVE.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/NSVEparticles.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/NSVEcomplex_particles.cpp"
+ )
+
+target_link_libraries(bfps ${MPI_LIBRARIES})
+set_target_properties(bfps PROPERTIES
+ COMPILE_FLAGS "${MPI_COMPILE_FLAGS}")
+set_target_properties(bfps PROPERTIES
+ LINK_FLAGS "${MPI_LINK_FLAGS}")
--
GitLab
From e6de0c98746e0d21aff710ac2824dc480493969c Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 6 Mar 2019 22:39:44 +0100
Subject: [PATCH 305/342] link all cpp into static library
---
CMakeLists.txt | 44 ++++++++++++++++++++++++++++++++++----------
1 file changed, 34 insertions(+), 10 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 14bea37e..494b11b4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -7,15 +7,6 @@ include_directories(
${PROJECT_SOURCE_DIR}/bfps/cpp/full_code
)
-file(GLOB all_SRCS
- "${PROJECT_SOURCE_DIR}/bfps/cpp/*.hpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/*.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/particles/*.hpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/particles/*.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/*.hpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/*.cpp"
- )
-
find_package(MPI REQUIRED)
include_directories(${MPI_INCLUDE_PATH})
set(CMAKE_CXX_COMPILE_FLAGS "${CMAKE_CXX_COMPILE_FLAGS} -Wall -std=c++11 -fopenmp")
@@ -23,12 +14,44 @@ set(CMAKE_CXX_COMPILE_FLAGS ${CMAKE_CXX_COMPILE_FLAGS} ${MPI_COMPILE_FLAGS})
set(CMAKE_CXX_LINK_FLAGS ${CMAKE_CXX_LINK_FLAGS} ${MPI_LINK_FLAGS})
add_library(bfps
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/code_base.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/direct_numerical_simulation.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/NSVE.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/joint_acc_vel_stats.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/test.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/filter_test.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/field_test.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/symmetrize_test.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/field_output_test.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/get_rfields.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/field_single_to_double.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/resize.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/NSVE_field_stats.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/native_binary_to_hdf5.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/postprocess.cpp"
"${PROJECT_SOURCE_DIR}/bfps/cpp/field.cpp"
"${PROJECT_SOURCE_DIR}/bfps/cpp/kspace.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/field_layout.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/hdf5_tools.cpp"
"${PROJECT_SOURCE_DIR}/bfps/cpp/vorticity_equation.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/NSVE.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/field_binary_IO.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/spline_n1.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/spline_n2.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/spline_n3.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/spline_n4.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/spline_n5.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/spline_n6.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/spline_n7.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/spline_n8.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/spline_n9.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/spline_n10.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/Lagrange_polys.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/scope_timer.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/test_interpolation.cpp"
"${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/NSVEparticles.cpp"
"${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/NSVEcomplex_particles.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/NSVEp_extra_sampling.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/particles/particles_inner_computer.cpp"
)
target_link_libraries(bfps ${MPI_LIBRARIES})
@@ -36,3 +59,4 @@ set_target_properties(bfps PROPERTIES
COMPILE_FLAGS "${MPI_COMPILE_FLAGS}")
set_target_properties(bfps PROPERTIES
LINK_FLAGS "${MPI_LINK_FLAGS}")
+
--
GitLab
From 20fc592b58e99371195d23181cac22fd44359e71 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 7 Mar 2019 09:12:04 +0100
Subject: [PATCH 306/342] update to find HDF5 as well as MPI
---
CMakeLists.txt | 38 +++++++++++++++++++++++++++++++-------
1 file changed, 31 insertions(+), 7 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 494b11b4..91b7fc32 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -8,10 +8,26 @@ include_directories(
)
find_package(MPI REQUIRED)
-include_directories(${MPI_INCLUDE_PATH})
-set(CMAKE_CXX_COMPILE_FLAGS "${CMAKE_CXX_COMPILE_FLAGS} -Wall -std=c++11 -fopenmp")
-set(CMAKE_CXX_COMPILE_FLAGS ${CMAKE_CXX_COMPILE_FLAGS} ${MPI_COMPILE_FLAGS})
-set(CMAKE_CXX_LINK_FLAGS ${CMAKE_CXX_LINK_FLAGS} ${MPI_LINK_FLAGS})
+#find_package(FFTW REQUIRED)
+find_package(HDF5 REQUIRED)
+include_directories(
+ ${MPI_INCLUDE_PATH}
+ ${HDF5_INCLUDE_PATH}
+ )
+
+set(CMAKE_CXX_COMPILE_FLAGS
+ ${CMAKE_CXX_COMPILE_FLAGS}
+ "-Wall -std=c++11 -fopenmp"
+ )
+set(CMAKE_CXX_COMPILE_FLAGS
+ ${CMAKE_CXX_COMPILE_FLAGS}
+ ${MPI_COMPILE_FLAGS}
+ )
+set(CMAKE_CXX_LINK_FLAGS
+ ${MPI_LINK_FLAGS}
+ ${HDF5_LINK_FLAGS}
+ ${CMAKE_CXX_LINK_FLAGS}
+ )
add_library(bfps
"${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/code_base.cpp"
@@ -54,9 +70,17 @@ add_library(bfps
"${PROJECT_SOURCE_DIR}/bfps/cpp/particles/particles_inner_computer.cpp"
)
-target_link_libraries(bfps ${MPI_LIBRARIES})
+target_link_libraries(
+ bfps
+ ${MPI_LIBRARIES}
+ ${HDF5_LIBRARIES}
+ )
set_target_properties(bfps PROPERTIES
- COMPILE_FLAGS "${MPI_COMPILE_FLAGS}")
+ COMPILE_FLAGS
+ "${MPI_COMPILE_FLAGS}"
+ )
set_target_properties(bfps PROPERTIES
- LINK_FLAGS "${MPI_LINK_FLAGS}")
+ LINK_FLAGS
+ "${MPI_LINK_FLAGS} ${HDF5_LINK_FLAGS}"
+ )
--
GitLab
From dde655ecc2817015de2fa12ae1108f9b3d50f751 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Thu, 7 Mar 2019 22:22:14 +0100
Subject: [PATCH 307/342] split CMakeLists.txt
the plan is to use the generic `CMakeLists_package.txt` for calling
`cmake` from within python scripts. Otherwise the python script would
write something like `CMakeLists.txt`, but with an executable instead of
a library.
---
CMakeLists.txt | 22 +-------------------
CMakeLists_packages.txt | 46 +++++++++++++++++++++++++++++++++++++++++
2 files changed, 47 insertions(+), 21 deletions(-)
create mode 100644 CMakeLists_packages.txt
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 91b7fc32..e77521f2 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -7,27 +7,7 @@ include_directories(
${PROJECT_SOURCE_DIR}/bfps/cpp/full_code
)
-find_package(MPI REQUIRED)
-#find_package(FFTW REQUIRED)
-find_package(HDF5 REQUIRED)
-include_directories(
- ${MPI_INCLUDE_PATH}
- ${HDF5_INCLUDE_PATH}
- )
-
-set(CMAKE_CXX_COMPILE_FLAGS
- ${CMAKE_CXX_COMPILE_FLAGS}
- "-Wall -std=c++11 -fopenmp"
- )
-set(CMAKE_CXX_COMPILE_FLAGS
- ${CMAKE_CXX_COMPILE_FLAGS}
- ${MPI_COMPILE_FLAGS}
- )
-set(CMAKE_CXX_LINK_FLAGS
- ${MPI_LINK_FLAGS}
- ${HDF5_LINK_FLAGS}
- ${CMAKE_CXX_LINK_FLAGS}
- )
+include(${PROJECT_SOURCE_DIR}/CMakeLists_packages.txt)
add_library(bfps
"${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/code_base.cpp"
diff --git a/CMakeLists_packages.txt b/CMakeLists_packages.txt
new file mode 100644
index 00000000..e08621bb
--- /dev/null
+++ b/CMakeLists_packages.txt
@@ -0,0 +1,46 @@
+find_package(MPI REQUIRED)
+find_package(HDF5 REQUIRED)
+
+###############################################################################
+# manual settings for FFTW. ugly.
+set(FFTW_INCLUDE_FLAGS
+ "${FFTW_BASE}/include"
+ )
+set(FFTW_LINK_FLAGS
+ "${FFTW_OPENMP_LIB} ${FFTW_LIB}"
+ )
+###############################################################################
+
+###############################################################################
+# manual settings for SZIP. ugly.
+# needed for non-gcc environments.
+if(SZIP_LIB)
+ set(SZIP_LINK_FLAGS
+ "${SZIP_LIB} -lz")
+else()
+ set(SZIP_LINK_FLAGS
+ "")
+endif()
+###############################################################################
+
+include_directories(
+ ${MPI_INCLUDE_PATH}
+ ${HDF5_INCLUDE_PATH}
+ ${FFTW_INCLUDE_PATH}
+ )
+
+set(CMAKE_CXX_COMPILE_FLAGS
+ ${CMAKE_CXX_COMPILE_FLAGS}
+ "-Wall -std=c++11 -fopenmp"
+ )
+set(CMAKE_CXX_COMPILE_FLAGS
+ ${CMAKE_CXX_COMPILE_FLAGS}
+ ${MPI_COMPILE_FLAGS}
+ )
+set(CMAKE_CXX_LINK_FLAGS
+ ${MPI_LINK_FLAGS}
+ ${HDF5_LINK_FLAGS}
+ ${FFTW_LINK_FLAGS}
+ ${CMAKE_CXX_LINK_FLAGS}
+ )
+
--
GitLab
From 293ac29d76d28474e4344301f0e476ea4e068959 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 8 Mar 2019 11:32:06 +0100
Subject: [PATCH 308/342] make cmake compilation work on desktop
---
CMakeLists.txt | 4 ++--
CMakeLists_packages.txt | 32 ++++++++++++++++++++++++++------
2 files changed, 28 insertions(+), 8 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e77521f2..1a8f6793 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -57,10 +57,10 @@ target_link_libraries(
)
set_target_properties(bfps PROPERTIES
COMPILE_FLAGS
- "${MPI_COMPILE_FLAGS}"
+ "${CMAKE_CXX_COMPILE_FLAGS} ${HDF5_COMPILE_FLAGS} ${MPI_COMPILE_FLAGS}"
)
set_target_properties(bfps PROPERTIES
LINK_FLAGS
- "${MPI_LINK_FLAGS} ${HDF5_LINK_FLAGS}"
+ "${MPI_LINK_FLAGS} ${HDF5_LINK_FLAGS} ${FFTW_LINK_FLAGS}"
)
diff --git a/CMakeLists_packages.txt b/CMakeLists_packages.txt
index e08621bb..97bf9220 100644
--- a/CMakeLists_packages.txt
+++ b/CMakeLists_packages.txt
@@ -1,22 +1,41 @@
+# vim: syntax=cmake
+
+cmake_policy(SET CMP0074 NEW)
find_package(MPI REQUIRED)
find_package(HDF5 REQUIRED)
###############################################################################
-# manual settings for FFTW. ugly.
+# ugly manual setting for HDF5
+if(NOT DEFINED HDF5_INCLUDE_PATH)
+ set(HDF5_INCLUDE_PATH "$ENV{HDF5_ROOT}/include")
+endif()
+###############################################################################
+
+###############################################################################
+# ugly manual settings for FFTW.
+if (NOT DEFINED ENV{FFTW_BASE})
+ message(FATAL_ERROR "Required FFTW_BASE environment variable is not defined.")
+endif()
set(FFTW_INCLUDE_FLAGS
- "${FFTW_BASE}/include"
+ "$ENV{FFTW_BASE}/include"
)
+if (NOT DEFINED ENV{FFTW_LIB})
+ message(FATAL_ERROR "Required FFTW_LIB environment variable is not defined.")
+endif()
+if (NOT DEFINED ENV{FFTW_OPENMP_LIB})
+ message(FATAL_ERROR "Required FFTW_OPENMP_LIB environment variable is not defined.")
+endif()
set(FFTW_LINK_FLAGS
- "${FFTW_OPENMP_LIB} ${FFTW_LIB}"
+ "$ENV{FFTW_OPENMP_LIB} $ENV{FFTW_LIB}"
)
###############################################################################
###############################################################################
-# manual settings for SZIP. ugly.
+# ugly manual settings for SZIP.
# needed for non-gcc environments.
-if(SZIP_LIB)
+if(DEFINED env{SZIP_LIB})
set(SZIP_LINK_FLAGS
- "${SZIP_LIB} -lz")
+ "$env{SZIP_LIB} -lz")
else()
set(SZIP_LINK_FLAGS
"")
@@ -35,6 +54,7 @@ set(CMAKE_CXX_COMPILE_FLAGS
)
set(CMAKE_CXX_COMPILE_FLAGS
${CMAKE_CXX_COMPILE_FLAGS}
+ ${HDF5_COMPILE_FLAGS}
${MPI_COMPILE_FLAGS}
)
set(CMAKE_CXX_LINK_FLAGS
--
GitLab
From b443b20383d58366e19a81c9e9cbbda644d91d2a Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 8 Mar 2019 12:43:26 +0100
Subject: [PATCH 309/342] don't set policy, since it demands very new version
of cmake
---
CMakeLists_packages.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/CMakeLists_packages.txt b/CMakeLists_packages.txt
index 97bf9220..7b745a6d 100644
--- a/CMakeLists_packages.txt
+++ b/CMakeLists_packages.txt
@@ -1,6 +1,6 @@
# vim: syntax=cmake
-cmake_policy(SET CMP0074 NEW)
+#cmake_policy(SET CMP0074 NEW)
find_package(MPI REQUIRED)
find_package(HDF5 REQUIRED)
--
GitLab
From 813cfbec05a459061640b61d6f62315bb763fd91 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 8 Mar 2019 13:25:14 +0100
Subject: [PATCH 310/342] [broken] setup.py now uses cmake instead of gcc
package is broken, when I try `run_all_tests.sh" I get a linking
problem:
compiling code with command
g++ symmetrize_test-single-v2.21.2.post31+gb443b20.dirty.cpp -o symmetrize_test-single-v2.21.2.post31+gb443b20.dirty -Wall -Wfatal-errors -O2 -g -mtune=native -march=native -ffast-math -std=c++11 -fopenmp -gdwarf-2 -rdynamic -fPIC -I/usr/lib/openmpi/include -I/scratch.local/chichi/installs/include -I/scratch.local/chichi/installs/py3/lib/python3.5/site-packages/bfps-2.21.2.post31+gb443b20.dirty-py3.5.egg/bfps/cpp -L/usr/lib/openmpi/lib -L/scratch.local/chichi/installs/lib -L/scratch.local/chichi/installs/lib64 -Wl,-rpath=/usr/lib/openmpi/lib -Wl,-rpath=/scratch.local/chichi/installs/lib -Wl,-rpath=/scratch.local/chichi/installs/lib64 -L/scratch.local/chichi/installs/py3/lib/python3.5/site-packages/bfps-2.21.2.post31+gb443b20.dirty-py3.5.egg/bfps -Wl,-rpath=/scratch.local/chichi/installs/py3/lib/python3.5/site-packages/bfps-2.21.2.post31+gb443b20.dirty-py3.5.egg/bfps -lbfps -lhdf5 -lfftw3_omp -lfftw3f_omp -lfftw3_mpi -lfftw3 -lfftw3f_mpi -lfftw3f -lmpi_cxx -lmpi -fopenmp
/scratch.local/chichi/installs/py3/lib/python3.5/site-packages/bfps-2.21.2.post31+gb443b20.dirty-py3.5.egg/bfps/libbfps.a(symmetrize_test.cpp.o): In function `symmetrize_test<float>::do_work()':
symmetrize_test.cpp:(.text._ZN15symmetrize_testIfE7do_workEv[_ZN15symmetrize_testIfE7do_workEv]+0x42): undefined reference to `fftw_planner_string_to_flag[abi:cxx11]'
symmetrize_test.cpp:(.text._ZN15symmetrize_testIfE7do_workEv[_ZN15symmetrize_testIfE7do_workEv]+0xc8): undefined reference to `fftw_planner_string_to_flag[abi:cxx11]'
/scratch.local/chichi/installs/py3/lib/python3.5/site-packages/bfps-2.21.2.post31+gb443b20.dirty-py3.5.egg/bfps/libbfps.a(symmetrize_test.cpp.o): In function `symmetrize_test<double>::do_work()':
symmetrize_test.cpp:(.text._ZN15symmetrize_testIdE7do_workEv[_ZN15symmetrize_testIdE7do_workEv]+0x42): undefined reference to `fftw_planner_string_to_flag[abi:cxx11]'
symmetrize_test.cpp:(.text._ZN15symmetrize_testIdE7do_workEv[_ZN15symmetrize_testIdE7do_workEv]+0xc8): undefined reference to `fftw_planner_string_to_flag[abi:cxx11]'
collect2: error: ld returned 1 exit status
---
CMakeLists_packages.txt | 5 ++++
setup.py | 61 ++++++++++++++++++++++-------------------
2 files changed, 38 insertions(+), 28 deletions(-)
diff --git a/CMakeLists_packages.txt b/CMakeLists_packages.txt
index 7b745a6d..17a1099c 100644
--- a/CMakeLists_packages.txt
+++ b/CMakeLists_packages.txt
@@ -1,7 +1,11 @@
# vim: syntax=cmake
#cmake_policy(SET CMP0074 NEW)
+
+set(MPI_STATIC ON)
find_package(MPI REQUIRED)
+
+set(HDF5_STATIC ON)
find_package(HDF5 REQUIRED)
###############################################################################
@@ -13,6 +17,7 @@ endif()
###############################################################################
# ugly manual settings for FFTW.
+set(FFTW_STATIC ON)
if (NOT DEFINED ENV{FFTW_BASE})
message(FATAL_ERROR "Required FFTW_BASE environment variable is not defined.")
endif()
diff --git a/setup.py b/setup.py
index 34f63082..23f9c266 100644
--- a/setup.py
+++ b/setup.py
@@ -145,7 +145,6 @@ particle_headers = [
'cpp/particles/particles_field_computer.hpp',
'cpp/particles/particles_generic_interp.hpp',
'cpp/particles/particles_inner_computer_empty.hpp',
- #'cpp/particles/particles_inner_computer.hpp',
'cpp/particles/particles_input_hdf5.hpp',
'cpp/particles/particles_output_hdf5.hpp',
'cpp/particles/particles_output_mpiio.hpp',
@@ -231,33 +230,39 @@ class CompileLibCommand(distutils.cmd.Command):
eca += ['-DUSE_FFTWESTIMATE']
if self.disable_fftw_omp:
eca += ['-DNO_FFTWOMP']
- for fname in src_file_list:
- ifile = 'bfps/cpp/' + fname + '.cpp'
- ofile = 'obj/' + fname + '.o'
- if not os.path.exists(ofile):
- need_to_compile_file = True
- else:
- need_to_compile_file = False
- if not need_to_compile:
- latest = libtime
- dependency_list = get_file_dependency_list(fname)
- for depname in dependency_list.split()[1:]:
- latest = max(latest,
- datetime.datetime.fromtimestamp(os.path.getctime(depname)))
- need_to_compile_file = (latest > libtime)
- if need_to_compile_file:
- command_strings = [compiler, '-c']
- command_strings += ['bfps/cpp/' + fname + '.cpp']
- command_strings += ['-o', 'obj/' + fname + '.o']
- command_strings += eca
- command_strings += ['-I' + idir for idir in include_dirs]
- command_strings.append('-Ibfps/cpp/')
- print(' '.join(command_strings))
- subprocess.check_call(command_strings)
- command_strings = ['ar', 'rvs', 'bfps/libbfps.a']
- command_strings += ['obj/' + fname + '.o' for fname in src_file_list]
- print(' '.join(command_strings))
- subprocess.check_call(command_strings)
+ os.makedirs('cmake_build_dir', exist_ok = True)
+ os.chdir('cmake_build_dir')
+ subprocess.check_call(['cmake', '..'])
+ subprocess.check_call(['make', '-j4'])
+ os.chdir('..')
+ subprocess.check_call(['cp', 'cmake_build_dir/libbfps.a', 'bfps/'])
+ #for fname in src_file_list:
+ # ifile = 'bfps/cpp/' + fname + '.cpp'
+ # ofile = 'obj/' + fname + '.o'
+ # if not os.path.exists(ofile):
+ # need_to_compile_file = True
+ # else:
+ # need_to_compile_file = False
+ # if not need_to_compile:
+ # latest = libtime
+ # dependency_list = get_file_dependency_list(fname)
+ # for depname in dependency_list.split()[1:]:
+ # latest = max(latest,
+ # datetime.datetime.fromtimestamp(os.path.getctime(depname)))
+ # need_to_compile_file = (latest > libtime)
+ # if need_to_compile_file:
+ # command_strings = [compiler, '-c']
+ # command_strings += ['bfps/cpp/' + fname + '.cpp']
+ # command_strings += ['-o', 'obj/' + fname + '.o']
+ # command_strings += eca
+ # command_strings += ['-I' + idir for idir in include_dirs]
+ # command_strings.append('-Ibfps/cpp/')
+ # print(' '.join(command_strings))
+ # subprocess.check_call(command_strings)
+ #command_strings = ['ar', 'rvs', 'bfps/libbfps.a']
+ #command_strings += ['obj/' + fname + '.o' for fname in src_file_list]
+ #print(' '.join(command_strings))
+ #subprocess.check_call(command_strings)
### save compiling information
pickle.dump(
--
GitLab
From da6ad6f1a21dabab2a5b97a34a80ba95ba9a12b7 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 8 Mar 2019 13:43:14 +0100
Subject: [PATCH 311/342] add required source file to list
---
CMakeLists.txt | 1 +
1 file changed, 1 insertion(+)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1a8f6793..4b8da6e1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -29,6 +29,7 @@ add_library(bfps
"${PROJECT_SOURCE_DIR}/bfps/cpp/kspace.cpp"
"${PROJECT_SOURCE_DIR}/bfps/cpp/field_layout.cpp"
"${PROJECT_SOURCE_DIR}/bfps/cpp/hdf5_tools.cpp"
+ "${PROJECT_SOURCE_DIR}/bfps/cpp/fftw_tools.cpp"
"${PROJECT_SOURCE_DIR}/bfps/cpp/vorticity_equation.cpp"
"${PROJECT_SOURCE_DIR}/bfps/cpp/field_binary_IO.cpp"
"${PROJECT_SOURCE_DIR}/bfps/cpp/spline_n1.cpp"
--
GitLab
From 811274b34543fbd5301ef7b058eef36282701525 Mon Sep 17 00:00:00 2001
From: Berenger Bramas <Berenger.Bramas@inria.fr>
Date: Fri, 8 Mar 2019 17:03:36 +0100
Subject: [PATCH 312/342] WIP to have the cmake working perfectly
---
CMakeLists.txt | 106 ++++++++++------------
cmake/BFPSConfig.cmake.in | 50 +++++++++++
cmake/BFPSUtils.cmake | 184 ++++++++++++++++++++++++++++++++++++++
cmake/LoadFFTW.cmake | 17 ++++
cmake/LoadHDF5.cmake | 17 ++++
cmake/LoadSZIP.cmake | 17 ++++
6 files changed, 334 insertions(+), 57 deletions(-)
create mode 100644 cmake/BFPSConfig.cmake.in
create mode 100644 cmake/BFPSUtils.cmake
create mode 100644 cmake/LoadFFTW.cmake
create mode 100644 cmake/LoadHDF5.cmake
create mode 100644 cmake/LoadSZIP.cmake
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4b8da6e1..1185a98b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,67 +1,59 @@
cmake_minimum_required(VERSION 2.8.11)
-project(BFPS)
+project(BFPS CXX)
+
+#####################################################################################
+## Dependencies
+
+set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH})
+include(BFPSUtils)
+
+BFPSMustAdd(FFTW)
+BFPSMustAdd(HDF5)
+BFPSMustAdd(SZIP)
+
+message(STATUS "Include dirs from dependencies: ${FFTW_INCLUDE_DIRS} ${HDF5_INCLUDE_DIRS} ${SZIP_INCLUDE_DIRS}")
+include_directories(${FFTW_INCLUDE_DIRS} ${HDF5_INCLUDE_DIRS} ${SZIP_INCLUDE_DIRS})
+
+set(CMAKE_CXX_COMPILE_FLAGS ${CMAKE_CXX_COMPILE_FLAGS} "-Wall -std=c++11 -fopenmp")
+
+#####################################################################################
+## Build the lib
+
include_directories(
${PROJECT_SOURCE_DIR}/bfps/cpp
${PROJECT_SOURCE_DIR}/bfps/cpp/particles
${PROJECT_SOURCE_DIR}/bfps/cpp/full_code
)
-include(${PROJECT_SOURCE_DIR}/CMakeLists_packages.txt)
-
-add_library(bfps
- "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/code_base.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/direct_numerical_simulation.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/NSVE.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/joint_acc_vel_stats.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/test.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/filter_test.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/field_test.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/symmetrize_test.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/field_output_test.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/get_rfields.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/field_single_to_double.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/resize.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/NSVE_field_stats.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/native_binary_to_hdf5.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/postprocess.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/field.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/kspace.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/field_layout.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/hdf5_tools.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/fftw_tools.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/vorticity_equation.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/field_binary_IO.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/spline_n1.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/spline_n2.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/spline_n3.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/spline_n4.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/spline_n5.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/spline_n6.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/spline_n7.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/spline_n8.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/spline_n9.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/spline_n10.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/Lagrange_polys.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/scope_timer.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/test_interpolation.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/NSVEparticles.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/NSVEcomplex_particles.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/full_code/NSVEp_extra_sampling.cpp"
- "${PROJECT_SOURCE_DIR}/bfps/cpp/particles/particles_inner_computer.cpp"
- )
+file(GLOB_RECURSE cpp_for_lib ${PROJECT_SOURCE_DIR}/bfps/*.cpp)
+file(GLOB_RECURSE hpp_for_lib ${PROJECT_SOURCE_DIR}/bfps/*.hpp)
+LIST(APPEND source_files ${cpp_for_lib} ${hpp_for_lib} )
-target_link_libraries(
- bfps
- ${MPI_LIBRARIES}
- ${HDF5_LIBRARIES}
- )
-set_target_properties(bfps PROPERTIES
- COMPILE_FLAGS
- "${CMAKE_CXX_COMPILE_FLAGS} ${HDF5_COMPILE_FLAGS} ${MPI_COMPILE_FLAGS}"
- )
-set_target_properties(bfps PROPERTIES
- LINK_FLAGS
- "${MPI_LINK_FLAGS} ${HDF5_LINK_FLAGS} ${FFTW_LINK_FLAGS}"
- )
+add_library(bfps ${source_files})
+
+target_link_libraries(bfps ${FFTW_LIBRARIES} ${HDF5_LIBRARIES} ${SZIP_LIBRARIES})
+# set_target_properties(bfps PROPERTIES
+# COMPILE_FLAGS
+# "${CMAKE_CXX_COMPILE_FLAGS} ${HDF5_COMPILE_FLAGS} ${MPI_COMPILE_FLAGS}"
+# )
+# set_target_properties(bfps PROPERTIES
+# LINK_FLAGS
+# "${FFTW_LIBRARIES} ${HDF5_LIBRARIES} ${SZIP_LIBRARIES"
+# )
+
+#####################################################################################
+##Â Export the configuration
+
+export(TARGETS bfps FILE "${PROJECT_BINARY_DIR}/BFPSLibraryDepends.cmake")
+
+# Create a BFPSConfig.cmake file for the use from the build tree
+set(BFPS_INCLUDE_DIRS
+ "${PROJECT_SOURCE_DIR}/include"
+ "${PROJECT_BINARY_DIR}")
+set(BFPS_LIB_DIR "${PROJECT_BINARY_DIR}/BFPS")
+set(BFPS_CMAKE_DIR "${PROJECT_BINARY_DIR}")
+configure_file(${PROJECT_SOURCE_DIR}/cmake/BFPSConfig.cmake.in ${PROJECT_BINARY_DIR}/BFPSConfig.cmake @ONLY)
+
+install(FILES "${PROJECT_BINARY_DIR}/InstallFiles/BFPSConfig.cmake" DESTINATION "${BFPS_CMAKE_DIR}")
diff --git a/cmake/BFPSConfig.cmake.in b/cmake/BFPSConfig.cmake.in
new file mode 100644
index 00000000..1d3e4d37
--- /dev/null
+++ b/cmake/BFPSConfig.cmake.in
@@ -0,0 +1,50 @@
+#-----------------------------------------------------------------------------
+#
+# BFPSConfig.cmake - BFPS CMake configuration file for external projects.
+#
+# This file is configured by BFPS and used by the BFPS.cmake module
+# to load BFPS's settings for an external project.
+#
+@BFPS_CONFIG_INSTALL_ONLY@
+
+#
+SET(BFPS_VERSION "@BFPS_VERSION@")
+
+#
+SET(HAVE_BFPS TRUE)
+SET(BFPS_PREFIX "@CMAKE_INSTALL_PREFIX@")
+SET(BFPS_INCLUDE_DIR "@CMAKE_INSTALL_PREFIX@/include")
+SET(BFPS_LIBRARIES_DIR "@CMAKE_INSTALL_PREFIX@/lib")
+SET(BFPS_LIBRARIES_ADD "@BFPS_LIBRARIES@")
+SET(BFPS_LIBRARY_NAME "@BFPS_LIB@")
+SET(BFPS_LINK_DIRECTORIES "@BFPS_LINK_DIRECTORIES@")
+
+#
+SET(BFPS_SOURCE_DIR "@BFPS_SOURCE_DIR@")
+
+#
+SET(BFPS_BUILD_TYPE "@CMAKE_BUILD_TYPE@")
+
+# Flag
+set(BFPS_DEFINITIONS @BFPS_DEFINITIONS@)
+
+SET(BFPS_USE_FFTW "@BFPS_USE_FFTW@")
+SET(BFPS_USE_HDF5 "@BFPS_USE_HDF5@")
+SET(BFPS_USE_SZIP "@BFPS_USE_SZIP@")
+
+SET(BFPS_DOC_TAGS "@CMAKE_BINARY_DIR@/Doc/BFPS.tag")
+
+# Tell the user project where to find our headers and libraries
+set(BFPS_INCLUDE_DIRS "@BFPS_INCLUDE_DIRS@")
+set(BFPS_LIBRARY_DIRS "@BFPS_LIB_DIR@")
+
+# List of compilation flags -DTOTO to export
+set(BFPS_DEFINITIONS @BFPS_DEFINITIONS@)
+
+# Our library dependencies (contains definitions for IMPORTED targets)
+include("@BFPS_CMAKE_DIR@/BFPSLibraryDepends.cmake")
+
+# These are IMPORTED targets created by BFPSLibraryDepends.cmake
+set(BFPS_LIBRARIES bfps)
+
+
diff --git a/cmake/BFPSUtils.cmake b/cmake/BFPSUtils.cmake
new file mode 100644
index 00000000..ad50e395
--- /dev/null
+++ b/cmake/BFPSUtils.cmake
@@ -0,0 +1,184 @@
+###########################################################################################
+# BFPSAddLibrary : check if a package exists (its lib, headers and its dependencies).
+# Here we do not test to find some functions we simply try to find the files.
+# If environment variables ${MYPACK}_LIB} or ${MYPACK}_INC are set, they are used
+# to give some hints to the find process.
+# The variables set are:
+# ${MYPACK}_LIBRARY_DIRS : one or more dir to libs.
+# ${MYPACK}_INCLUDE_DIRS : one or more dir to headers.
+# ${MYPACK}_LIBRARIES : one or more libs.
+# ${MYPACK}_FOUND : Set to true if the libs has been found completly (with its dependencies).
+# ${MYPACK}_DEPENDENCIES : list of dependencies.
+# ${MYPACK}_FOUND_DEPENDENDCIES : TRUE if dependencies are all TRUE.
+###########################################################################################
+macro(BFPSFindLib MYPACK MYLIBS MYHEADERS MYDEPS)
+ # If it has not been already proceed
+ if(NOT DEFINED ${MYPACK}_FOUND)
+ # Inform user about the research
+ MESSAGE( STATUS "Try to find ${MYPACK}..." )
+
+ # Find include files
+ if(${MYHEADERS})
+ find_path(${MYPACK}_INCLUDE_DIRS NAMES ${MYHEADERS} HINTS ${${MYPACK}_INCLUDE_DIR} ${${MYPACK}_INCLUDE_DIRS} ${${MYPACK}_INCLUDE_PATH} ENV ${MYPACK}_INCLUDE_DIR ${MYPACK}_INCLUDE_PATH ${MYPACK}_INC "${MYPACK}_HOME" "${MYPACK}_ROOT" DOC "${MYPACK} include directory")
+ else()
+ set(${MYPACK}_INCLUDE_DIRS "")
+ endif()
+
+ if((NOT ${MYPACK}_INCLUDE_DIRS) AND (${MYHEADERS}) )
+ MESSAGE( STATUS " [${MYPACK}] Include not found!" )
+ set(${MYPACK}_FOUND FALSE)
+ else()
+ set(${MYPACK}_FOUND TRUE)
+
+ # Find libs
+ set(${MYPACK}_LIBRARY_HINT_DIRS ${${MYPACK}_LIBRARY_DIRS})
+ set(${MYPACK}_LIBRARY_DIRS "")
+ foreach (lib_ ${MYLIBS})
+ MESSAGE( STATUS " [${MYPACK}] Try to find ${lib_}..." )
+ # Ask for this lib
+ find_library(${MYPACK}_LIBRARY_${lib_}_PATH NAMES ${lib_} HINTS ${${MYPACK}_LIBRARY_DIR} ${${MYPACK}_LIBRARY_HINT_DIRS} ${${MYPACK}_LIB_PATH} ${${MYPACK}_LIBS_PATH} ENV ${MYPACK}_LIB_DIR ${MYPACK}_LIB_PATH ${MYPACK}_LIB "${MYPACK}_HOME" "${MYPACK}_ROOT" DOC "${MYPACK} lib directory")
+ # Stop if not found
+ if(NOT ${MYPACK}_LIBRARY_${lib_}_PATH)
+ MESSAGE( STATUS " [${MYPACK}] ${lib_} Failed!" )
+ set(${MYPACK}_LIBRARY_DIRS NOTFOUND)
+ set(${MYPACK}_FOUND FALSE)
+ break()
+ endif()
+ get_filename_component(lib_dir_ ${${MYPACK}_LIBRARY_${lib_}_PATH} PATH)
+ list(APPEND ${MYPACK}_LIBRARY_DIRS ${lib_dir_})
+ list(REMOVE_DUPLICATES ${MYPACK}_LIBRARY_DIRS)
+ endforeach()
+
+ # Check dependencies
+ set(${MYPACK}_DEPENDENCIES ${MYDEPS})
+
+ # If all files exist we set to TRUE
+ if(${MYPACK}_FOUND AND (${MYPACK}_LIBRARY_DIRS OR (NOT MYLIBS)) AND (${MYPACK}_INCLUDE_DIRS OR (NOT MYHEADERS)) )
+ set(${MYPACK}_FOUND_DEPENDENDCIES TRUE)
+
+ MESSAGE( STATUS " [${MYPACK}] Check deps: ${${MYPACK}_DEPENDENCIES}" )
+ foreach (dep_ ${${MYPACK}_DEPENDENCIES})
+ # Be sure it has been config
+ if(NOT DEFINED ${dep_}_FOUND)
+ MESSAGE( STATUS " [${MYPACK}] Try to find dep: ${dep_}..." )
+ BFPSCheckIfFound(${dep_})
+ else()
+ MESSAGE( STATUS " [${MYPACK}] Dep ${dep_} has been found already..." )
+ endif()
+ # If not found stop here
+ if(NOT ${dep_}_FOUND)
+ set(${MYPACK}_FOUND_DEPENDENDCIES FALSE)
+ set(${MYPACK}_FOUND FALSE)
+ break()
+ endif()
+ endforeach()
+ endif()
+ endif()
+
+ # If found and dependencies are found set to true
+ if(${MYPACK}_FOUND AND ${MYPACK}_FOUND_DEPENDENDCIES)
+ set(${MYPACK}_LIBRARIES ${MYLIBS})
+ set(${MYPACK}_FOUND TRUE)
+ else()
+ # Else make clean results
+ set(${${MYPACK}_LIBRARY_DIRS} NOTFOUND)
+ set(${${MYPACK}_INCLUDE_DIRS} NOTFOUND)
+ set(${MYPACK}_LIBRARIES)
+ set(${MYPACK}_FOUND FALSE)
+ endif()
+
+ # Print results
+ MESSAGE( STATUS "${MYPACK}_FOUND = ${${MYPACK}_FOUND}" )
+ endif()
+endmacro(BFPSFindLib MYPACK MYLIBS MYHEADERS)
+
+###########################################################################################
+# BFPSAddLibrary : try to add a package (its lib, headers and its dependencies).
+# The variables set are:
+# ${PROJECT_NAME}_USE_${MYPACK} : Set to true if package and dependencies have been found)
+# link_directories and include_directories are called
+# ${PROJECT_NAME}_LIBRARIES is modified in order to contains the package libs
+###########################################################################################
+macro(BFPSAddLibrary MYPACK)
+ if(NOT DEFINED ${PROJECT_NAME}_USE_${MYPACK})
+ # Check it has been found
+ if(NOT ${MYPACK}_FOUND)
+ MESSAGE( FATAL_ERROR "Try to load ${MYPACK} but ${MYPACK}_FOUND has value ${${MYPACK}_FOUND}" )
+ endif()
+ # Check dependencies have been found
+ foreach (dep_ ${${MYPACK}_DEPENDENCIES})
+ if(NOT ${dep_}_FOUND)
+ MESSAGE( FATAL_ERROR "Try to load ${MYPACK} but ${dep_} has value ${${dep_}_FOUND}" )
+ endif()
+ endforeach()
+
+ # Set use variable
+ set(${PROJECT_NAME}_USE_${MYPACK} TRUE)
+
+ # Add lib directory
+ if( ${MYPACK}_LIBRARY_DIRS )
+ link_directories(${${MYPACK}_LIBRARY_DIRS})
+ list(APPEND ${PROJECT_NAME}_LINK_DIRECTORIES ${${MYPACK}_LIBRARY_DIRS})
+ endif()
+ # Add inc directory
+ if( ${MYPACK}_INCLUDE_DIRS )
+ include_directories(${${MYPACK}_INCLUDE_DIRS})
+ endif()
+ endif()
+ if( ${PROJECT_NAME}_USE_${MYPACK} )
+ # Add libs to variables
+ if( ${MYPACK}_LIBRARIES )
+ list(APPEND ${PROJECT_NAME}_LIBRARIES ${${MYPACK}_LIBRARIES})
+ endif()
+
+ # Add dependencies
+ foreach (dep_ ${${MYPACK}_DEPENDENCIES})
+ BFPSAddLibrary( ${dep_} )
+ set(${PROJECT_NAME}_${MYPACK}_USE_${dep_} TRUE)
+ endforeach()
+ endif()
+endmacro(BFPSAddLibrary MYPACK)
+
+###########################################################################################
+# BFPSCheckIfFound : check if a package exists (its lib, headers and its dependencies).
+# The variables set by BFPSFindLib are returned.
+###########################################################################################
+macro(BFPSCheckIfFound PACKTOADD)
+ include(Load${PACKTOADD} RESULT_VARIABLE ${PACKTOADD}_LOADFILE_EXISTS)
+ if(NOT ${PACKTOADD}_LOADFILE_EXISTS)
+ MESSAGE( FATAL_ERROR "Try to load ${PACKTOADD} but Load${PACKTOADD}.cmake cannot be found." )
+ endif()
+
+ BFPSFindLib("${PACKTOADD}" "${${PACKTOADD}_CONFIG_LIBS}" "${${PACKTOADD}_CONFIG_HEADERS}" "${${PACKTOADD}_CONFIG_DEPS}")
+endmacro(BFPSCheckIfFound PACKTOADD)
+
+###########################################################################################
+# BFPSAddIfFound : adds a package if it is found.
+# The variables set by BFPSCheckIfFound and BFPSAddLibrary are returned.
+###########################################################################################
+macro(BFPSAddIfFound PACKTOADD)
+ BFPSCheckIfFound(${PACKTOADD})
+ if( ${PACKTOADD}_FOUND )
+ BFPSAddLibrary(${PACKTOADD})
+ endif()
+endmacro(BFPSAddIfFound PACKTOADD)
+
+###########################################################################################
+# BFPSMustAdd : adds a package (it must succed or a fatal error will be launched).
+# The variables set by BFPSAddIfFound are returned.
+###########################################################################################
+macro(BFPSMustAdd PACKTOADD)
+ BFPSAddIfFound(${PACKTOADD})
+ if(NOT ${PACKTOADD}_FOUND)
+ MESSAGE( FATAL_ERROR "Try to load ${PACKTOADD} but ${PACKTOADD}_FOUND has value ${${PACKTOADD}_FOUND}." )
+ endif()
+endmacro(BFPSMustAdd PACKTOADD)
+###########################################################################################
+# BFPSSetIfndef : define a variable if it is not already defined
+###########################################################################################
+macro(BFPSSetIfndef VarToSet PotentialValue)
+if(NOT DEFINED ${VarToSet})
+ set(${VarToSet} ${PotentialValue})
+ MESSAGE( STATUS "${VarToSet} = ${${VarToSet}}" )
+endif()
+endmacro(BFPSSetIfndef VarToSet PotentialValue)
diff --git a/cmake/LoadFFTW.cmake b/cmake/LoadFFTW.cmake
new file mode 100644
index 00000000..32e8c1aa
--- /dev/null
+++ b/cmake/LoadFFTW.cmake
@@ -0,0 +1,17 @@
+# Should define all the configure variables for a given Package.
+# MYPACK => The package name
+# ${MYPACK}_CONFIG_LIBS => The needed libraries
+# ${MYPACK}_CONFIG_HEADERS => The needed headers
+# ${MYPACK}_CONFIG_DEPS => A list of dependencies (in the Package name format)
+
+# This file should not be edited, but should be changed by setting
+# set( BFPS_ENABLE_MYPACK VALUE CACHE FILEPATH "" FORCE)
+# in the initialCache corresponding
+
+include(BFPSUtils)
+
+set(MYPACK FFTW)
+BFPSSetIfndef(${MYPACK}_CONFIG_LIBS "fftw3")
+BFPSSetIfndef(${MYPACK}_CONFIG_HEADERS "fftw3.h")
+
+
diff --git a/cmake/LoadHDF5.cmake b/cmake/LoadHDF5.cmake
new file mode 100644
index 00000000..1b930af7
--- /dev/null
+++ b/cmake/LoadHDF5.cmake
@@ -0,0 +1,17 @@
+# Should define all the configure variables for a given Package.
+# MYPACK => The package name
+# ${MYPACK}_CONFIG_LIBS => The needed libraries
+# ${MYPACK}_CONFIG_HEADERS => The needed headers
+# ${MYPACK}_CONFIG_DEPS => A list of dependencies (in the Package name format)
+
+# This file should not be edited, but should be changed by setting
+# set( BFPS_ENABLE_MYPACK VALUE CACHE FILEPATH "" FORCE)
+# in the initialCache corresponding
+
+include(BFPSUtils)
+
+set(MYPACK HDF5)
+BFPSSetIfndef(${MYPACK}_CONFIG_LIBS "hdf5")
+BFPSSetIfndef(${MYPACK}_CONFIG_HEADERS "hdf5.h")
+BFPSSetIfndef(${MYPACK}_CONFIG_DEPS "")
+
diff --git a/cmake/LoadSZIP.cmake b/cmake/LoadSZIP.cmake
new file mode 100644
index 00000000..2cf052a1
--- /dev/null
+++ b/cmake/LoadSZIP.cmake
@@ -0,0 +1,17 @@
+# Should define all the configure variables for a given Package.
+# MYPACK => The package name
+# ${MYPACK}_CONFIG_LIBS => The needed libraries
+# ${MYPACK}_CONFIG_HEADERS => The needed headers
+# ${MYPACK}_CONFIG_DEPS => A list of dependencies (in the Package name format)
+
+# This file should not be edited, but should be changed by setting
+# set( BFPS_ENABLE_MYPACK VALUE CACHE FILEPATH "" FORCE)
+# in the initialCache corresponding
+
+include(BFPSUtils)
+
+set(MYPACK SZIP)
+BFPSSetIfndef(${MYPACK}_CONFIG_LIBS "z")
+BFPSSetIfndef(${MYPACK}_CONFIG_HEADERS "")
+BFPSSetIfndef(${MYPACK}_CONFIG_DEPS "")
+
--
GitLab
From 5cfd9e1b16dfb2d7e65b0aa4120574a2b4603e3f Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 8 Mar 2019 17:30:58 +0100
Subject: [PATCH 313/342] fix mpi include path
---
CMakeLists.txt | 29 +++++++++++++++++------------
1 file changed, 17 insertions(+), 12 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1185a98b..efb931c7 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -12,15 +12,20 @@ BFPSMustAdd(FFTW)
BFPSMustAdd(HDF5)
BFPSMustAdd(SZIP)
-message(STATUS "Include dirs from dependencies: ${FFTW_INCLUDE_DIRS} ${HDF5_INCLUDE_DIRS} ${SZIP_INCLUDE_DIRS}")
-include_directories(${FFTW_INCLUDE_DIRS} ${HDF5_INCLUDE_DIRS} ${SZIP_INCLUDE_DIRS})
+set(MPI_STATIC ON)
+find_package(MPI REQUIRED)
-set(CMAKE_CXX_COMPILE_FLAGS ${CMAKE_CXX_COMPILE_FLAGS} "-Wall -std=c++11 -fopenmp")
+set(CMAKE_CXX_COMPILE_FLAGS ${CMAKE_CXX_COMPILE_FLAGS} ${MPI_COMPILE_FLAGS} "-Wall -std=c++11 -fopenmp")
#####################################################################################
## Build the lib
+message(STATUS "Include dirs from dependencies: ${FFTW_INCLUDE_DIRS} ${HDF5_INCLUDE_DIRS} ${SZIP_INCLUDE_DIRS} ${MPI_INCLUDE_PATH}")
include_directories(
+ ${FFTW_INCLUDE_DIRS}
+ ${HDF5_INCLUDE_DIRS}
+ ${SZIP_INCLUDE_DIRS}
+ ${MPI_INCLUDE_PATH}
${PROJECT_SOURCE_DIR}/bfps/cpp
${PROJECT_SOURCE_DIR}/bfps/cpp/particles
${PROJECT_SOURCE_DIR}/bfps/cpp/full_code
@@ -32,15 +37,15 @@ LIST(APPEND source_files ${cpp_for_lib} ${hpp_for_lib} )
add_library(bfps ${source_files})
-target_link_libraries(bfps ${FFTW_LIBRARIES} ${HDF5_LIBRARIES} ${SZIP_LIBRARIES})
-# set_target_properties(bfps PROPERTIES
-# COMPILE_FLAGS
-# "${CMAKE_CXX_COMPILE_FLAGS} ${HDF5_COMPILE_FLAGS} ${MPI_COMPILE_FLAGS}"
-# )
-# set_target_properties(bfps PROPERTIES
-# LINK_FLAGS
-# "${FFTW_LIBRARIES} ${HDF5_LIBRARIES} ${SZIP_LIBRARIES"
-# )
+target_link_libraries(bfps ${FFTW_LIBRARIES} ${HDF5_LIBRARIES} ${SZIP_LIBRARIES} ${MPI_LIBRARIES})
+set_target_properties(bfps PROPERTIES
+ COMPILE_FLAGS
+ "${CMAKE_CXX_COMPILE_FLAGS} ${HDF5_COMPILE_FLAGS} ${MPI_COMPILE_FLAGS}"
+ )
+set_target_properties(bfps PROPERTIES
+ LINK_FLAGS
+ "${FFTW_LIBRARIES} ${HDF5_LIBRARIES} ${SZIP_LIBRARIES}"
+ )
#####################################################################################
##Â Export the configuration
--
GitLab
From 2e75ad060828fe28f5e4369af9439096cbe62852 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 8 Mar 2019 21:25:23 +0100
Subject: [PATCH 314/342] clean up source tree
old files can't be compiled, and new cmake config wants to compile
everything.
successfuly compiled library on my laptop, but installation fails.
---
CMakeLists.txt | 5 +-
bfps/cpp/field_descriptor.cpp | 543 ----------------------------------
bfps/cpp/field_descriptor.hpp | 114 -------
bfps/cpp/particles.cpp | 254 ----------------
bfps/cpp/particles.hpp | 99 -------
bfps/cpp/particles_base.cpp | 424 --------------------------
bfps/cpp/particles_base.hpp | 136 ---------
7 files changed, 4 insertions(+), 1571 deletions(-)
delete mode 100644 bfps/cpp/field_descriptor.cpp
delete mode 100644 bfps/cpp/field_descriptor.hpp
delete mode 100644 bfps/cpp/particles.cpp
delete mode 100644 bfps/cpp/particles.hpp
delete mode 100644 bfps/cpp/particles_base.cpp
delete mode 100644 bfps/cpp/particles_base.hpp
diff --git a/CMakeLists.txt b/CMakeLists.txt
index efb931c7..f102e4d5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -10,7 +10,10 @@ include(BFPSUtils)
BFPSMustAdd(FFTW)
BFPSMustAdd(HDF5)
-BFPSMustAdd(SZIP)
+
+if(DEFINED env{SZIP_LIB})
+ BFPSMustAdd(SZIP)
+endif()
set(MPI_STATIC ON)
find_package(MPI REQUIRED)
diff --git a/bfps/cpp/field_descriptor.cpp b/bfps/cpp/field_descriptor.cpp
deleted file mode 100644
index cb7da995..00000000
--- a/bfps/cpp/field_descriptor.cpp
+++ /dev/null
@@ -1,543 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#define NDEBUG
-
-#include <stdlib.h>
-#include <algorithm>
-#include <iostream>
-#include "base.hpp"
-#include "field_descriptor.hpp"
-#include "fftw_interface.hpp"
-#include "scope_timer.hpp"
-
-/*****************************************************************************/
-/* macro for specializations to numeric types compatible with FFTW */
-
-
-template <class rnumber>
-field_descriptor<rnumber>::field_descriptor(
- int ndims,
- int *n,
- MPI_Datatype element_type,
- MPI_Comm COMM_TO_USE)
-{
- TIMEZONE("field_descriptor");
- DEBUG_MSG("entered field_descriptor::field_descriptor\n");
- this->comm = COMM_TO_USE;
- MPI_Comm_rank(this->comm, &this->myrank);
- MPI_Comm_size(this->comm, &this->nprocs);
- this->ndims = ndims;
- this->sizes = new int[ndims];
- this->subsizes = new int[ndims];
- this->starts = new int[ndims];
- int tsizes [ndims];
- int tsubsizes[ndims];
- int tstarts [ndims];
- std::vector<ptrdiff_t> nfftw;
- nfftw.resize(ndims);
- ptrdiff_t local_n0, local_0_start;
- for (int i = 0; i < this->ndims; i++)
- nfftw[i] = n[i];
- this->local_size = fftw_interface<rnumber>::mpi_local_size_many(
- this->ndims,
- &nfftw.front(),
- 1,
- FFTW_MPI_DEFAULT_BLOCK,
- this->comm,
- &local_n0,
- &local_0_start);
- this->sizes[0] = n[0];
- this->subsizes[0] = (int)local_n0;
- this->starts[0] = (int)local_0_start;
- DEBUG_MSG_WAIT(
- this->comm,
- "first subsizes[0] = %d %d %d\n",
- this->subsizes[0],
- tsubsizes[0],
- (int)local_n0);
- tsizes[0] = n[0];
- tsubsizes[0] = (int)local_n0;
- tstarts[0] = (int)local_0_start;
- DEBUG_MSG_WAIT(
- this->comm,
- "second subsizes[0] = %d %d %d\n",
- this->subsizes[0],
- tsubsizes[0],
- (int)local_n0);
- this->mpi_dtype = element_type;
- this->slice_size = 1;
- this->full_size = this->sizes[0];
- for (int i = 1; i < this->ndims; i++)
- {
- this->sizes[i] = n[i];
- this->subsizes[i] = n[i];
- this->starts[i] = 0;
- this->slice_size *= this->subsizes[i];
- this->full_size *= this->sizes[i];
- tsizes[i] = this->sizes[i];
- tsubsizes[i] = this->subsizes[i];
- tstarts[i] = this->starts[i];
- }
- tsizes[ndims-1] *= sizeof(rnumber);
- tsubsizes[ndims-1] *= sizeof(rnumber);
- tstarts[ndims-1] *= sizeof(rnumber);
- if (this->mpi_dtype == mpi_real_type<rnumber>::complex())
- {
- tsizes[ndims-1] *= 2;
- tsubsizes[ndims-1] *= 2;
- tstarts[ndims-1] *= 2;
- }
- int local_zero_array[this->nprocs], zero_array[this->nprocs];
- for (int i=0; i<this->nprocs; i++)
- local_zero_array[i] = 0;
- local_zero_array[this->myrank] = (this->subsizes[0] == 0) ? 1 : 0;
- MPI_Allreduce(
- local_zero_array,
- zero_array,
- this->nprocs,
- MPI_INT,
- MPI_SUM,
- this->comm);
- int no_of_excluded_ranks = 0;
- for (int i = 0; i<this->nprocs; i++)
- no_of_excluded_ranks += zero_array[i];
- DEBUG_MSG_WAIT(
- this->comm,
- "subsizes[0] = %d %d\n",
- this->subsizes[0],
- tsubsizes[0]);
- if (no_of_excluded_ranks == 0)
- {
- this->io_comm = this->comm;
- this->io_nprocs = this->nprocs;
- this->io_myrank = this->myrank;
- }
- else
- {
- int excluded_rank[no_of_excluded_ranks];
- for (int i=0, j=0; i<this->nprocs; i++)
- if (zero_array[i])
- {
- excluded_rank[j] = i;
- j++;
- }
- MPI_Group tgroup0, tgroup;
- MPI_Comm_group(this->comm, &tgroup0);
- MPI_Group_excl(tgroup0, no_of_excluded_ranks, excluded_rank, &tgroup);
- MPI_Comm_create(this->comm, tgroup, &this->io_comm);
- MPI_Group_free(&tgroup0);
- MPI_Group_free(&tgroup);
- if (this->subsizes[0] > 0)
- {
- MPI_Comm_rank(this->io_comm, &this->io_myrank);
- MPI_Comm_size(this->io_comm, &this->io_nprocs);
- }
- else
- {
- this->io_myrank = MPI_PROC_NULL;
- this->io_nprocs = -1;
- }
- }
- DEBUG_MSG_WAIT(
- this->comm,
- "inside field_descriptor constructor, about to call "
- "MPI_Type_create_subarray "
- "%d %d %d\n",
- this->sizes[0],
- this->subsizes[0],
- this->starts[0]);
- for (int i=0; i<this->ndims; i++)
- DEBUG_MSG_WAIT(
- this->comm,
- "tsizes "
- "%d %d %d\n",
- tsizes[i],
- tsubsizes[i],
- tstarts[i]);
- if (this->subsizes[0] > 0)
- {
- DEBUG_MSG("creating subarray\n");
- MPI_Type_create_subarray(
- ndims,
- tsizes,
- tsubsizes,
- tstarts,
- MPI_ORDER_C,
- MPI_UNSIGNED_CHAR,
- &this->mpi_array_dtype);
- MPI_Type_commit(&this->mpi_array_dtype);
- }
- this->rank = new int[this->sizes[0]];
- int *local_rank = new int[this->sizes[0]];
- std::fill_n(local_rank, this->sizes[0], 0);
- for (int i = 0; i < this->sizes[0]; i++)
- if (i >= this->starts[0] && i < this->starts[0] + this->subsizes[0])
- local_rank[i] = this->myrank;
- MPI_Allreduce(
- local_rank,
- this->rank,
- this->sizes[0],
- MPI_INT,
- MPI_SUM,
- this->comm);
- delete[] local_rank;
- this->all_start0 = new int[this->nprocs];
- int *local_start0 = new int[this->nprocs];
- std::fill_n(local_start0, this->nprocs, 0);
- for (int i = 0; i < this->nprocs; i++)
- if (this->myrank == i)
- local_start0[i] = this->starts[0];
- MPI_Allreduce(
- local_start0,
- this->all_start0,
- this->nprocs,
- MPI_INT,
- MPI_SUM,
- this->comm);
- delete[] local_start0;
- this->all_size0 = new int[this->nprocs];
- int *local_size0 = new int[this->nprocs];
- std::fill_n(local_size0, this->nprocs, 0);
- for (int i = 0; i < this->nprocs; i++)
- if (this->myrank == i)
- local_size0[i] = this->subsizes[0];
- MPI_Allreduce(
- local_size0,
- this->all_size0,
- this->nprocs,
- MPI_INT,
- MPI_SUM,
- this->comm);
- delete[] local_size0;
- DEBUG_MSG("exiting field_descriptor constructor\n");
-}
-
-template <class rnumber>
-int field_descriptor<rnumber>::read(
- const char *fname,
- void *buffer)
-{
- TIMEZONE("field_descriptor::read");
- DEBUG_MSG("entered field_descriptor::read\n");
- char representation[] = "native";
- if (this->subsizes[0] > 0)
- {
- MPI_Info info;
- MPI_Info_create(&info);
- MPI_File f;
- ptrdiff_t read_size = this->local_size*sizeof(rnumber);
- DEBUG_MSG("read size is %ld\n", read_size);
- char ffname[200];
- if (this->mpi_dtype == mpi_real_type<rnumber>::complex())
- read_size *= 2;
- DEBUG_MSG("read size is %ld\n", read_size);
- sprintf(ffname, "%s", fname);
-
- MPI_File_open(
- this->io_comm,
- ffname,
- MPI_MODE_RDONLY,
- info,
- &f);
- DEBUG_MSG("opened file\n");
- MPI_File_set_view(
- f,
- 0,
- MPI_UNSIGNED_CHAR,
- this->mpi_array_dtype,
- representation,
- info);
- DEBUG_MSG("view is set\n");
- MPI_File_read_all(
- f,
- buffer,
- read_size,
- MPI_UNSIGNED_CHAR,
- MPI_STATUS_IGNORE);
- DEBUG_MSG("info is read\n");
- MPI_File_close(&f);
- }
- DEBUG_MSG("finished with field_descriptor::read\n");
- return EXIT_SUCCESS;
-}
-
-template <class rnumber>
-int field_descriptor<rnumber>::write(
- const char *fname,
- void *buffer)
-{
- TIMEZONE("field_descriptor::write");
- char representation[] = "native";
- if (this->subsizes[0] > 0)
- {
- MPI_Info info;
- MPI_Info_create(&info);
- MPI_File f;
- ptrdiff_t read_size = this->local_size*sizeof(rnumber);
- char ffname[200];
- if (this->mpi_dtype == mpi_real_type<rnumber>::complex())
- read_size *= 2;
- sprintf(ffname, "%s", fname);
-
- MPI_File_open(
- this->io_comm,
- ffname,
- MPI_MODE_CREATE | MPI_MODE_WRONLY,
- info,
- &f);
- MPI_File_set_view(
- f,
- 0,
- MPI_UNSIGNED_CHAR,
- this->mpi_array_dtype,
- representation,
- info);
- MPI_File_write_all(
- f,
- buffer,
- read_size,
- MPI_UNSIGNED_CHAR,
- MPI_STATUS_IGNORE);
- MPI_File_close(&f);
- }
-
- return EXIT_SUCCESS;
-}
-
-template <class rnumber>
-int field_descriptor<rnumber>::transpose(
- rnumber *input,
- rnumber *output)
-{
- TIMEZONE("field_descriptor::transpose");
- /* IMPORTANT NOTE:
- for 3D transposition, the input data is messed up */
- typename fftw_interface<rnumber>::plan tplan;
- if (this->ndims == 3)
- {
- /* transpose the two local dimensions 1 and 2 */
- rnumber *atmp;
- atmp = fftw_interface<rnumber>::alloc_real(this->slice_size);
- for (int k = 0; k < this->subsizes[0]; k++)
- {
- /* put transposed slice in atmp */
- for (int j = 0; j < this->sizes[1]; j++)
- for (int i = 0; i < this->sizes[2]; i++)
- atmp[i*this->sizes[1] + j] =
- input[(k*this->sizes[1] + j)*this->sizes[2] + i];
- /* copy back transposed slice */
- std::copy(
- atmp,
- atmp + this->slice_size,
- input + k*this->slice_size);
- }
- fftw_interface<rnumber>::free(atmp);
- }
- tplan = fftw_interface<rnumber>::mpi_plan_transpose(
- this->sizes[0], this->slice_size,
- input, output,
- this->comm,
- DEFAULT_FFTW_FLAG);
- fftw_interface<rnumber>::execute(tplan);
- fftw_interface<rnumber>::destroy_plan(tplan);
- return EXIT_SUCCESS;
-}
-
-template <class rnumber>
-int field_descriptor<rnumber>::transpose(
- typename fftw_interface<rnumber>::complex *input,
- typename fftw_interface<rnumber>::complex *output)
-{
- TIMEZONE("field_descriptor::transpose2");
- switch (this->ndims)
- {
- case 2:
- /* do a global transpose over the 2 dimensions */
- if (output == NULL)
- {
- std::cerr << "bad arguments for transpose.\n" << std::endl;
- return EXIT_FAILURE;
- }
- typename fftw_interface<rnumber>::plan tplan;
- tplan = fftw_interface<rnumber>::mpi_plan_many_transpose(
- this->sizes[0], this->sizes[1], 2,
- FFTW_MPI_DEFAULT_BLOCK,
- FFTW_MPI_DEFAULT_BLOCK,
- (rnumber*)input, (rnumber*)output,
- this->comm,
- DEFAULT_FFTW_FLAG);
- fftw_interface<rnumber>::execute(tplan);
- fftw_interface<rnumber>::destroy_plan(tplan);
- break;
- case 3:
- /* transpose the two local dimensions 1 and 2 */
- typename fftw_interface<rnumber>::complex *atmp;
- atmp = fftw_interface<rnumber>::alloc_complex(this->slice_size);
- for (int k = 0; k < this->subsizes[0]; k++)
- {
- /* put transposed slice in atmp */
- for (int j = 0; j < this->sizes[1]; j++)
- for (int i = 0; i < this->sizes[2]; i++)
- {
- atmp[i*this->sizes[1] + j][0] =
- input[(k*this->sizes[1] + j)*this->sizes[2] + i][0];
- atmp[i*this->sizes[1] + j][1] =
- input[(k*this->sizes[1] + j)*this->sizes[2] + i][1];
- }
- /* copy back transposed slice */
- std::copy(
- (rnumber*)(atmp),
- (rnumber*)(atmp + this->slice_size),
- (rnumber*)(input + k*this->slice_size));
- }
- fftw_interface<rnumber>::free(atmp);
- break;
- default:
- return EXIT_FAILURE;
- break;
- }
- return EXIT_SUCCESS;
-}
-
-template <class rnumber>
-int field_descriptor<rnumber>::interleave(
- rnumber *a,
- int dim)
-{
- TIMEZONE("field_descriptor::interleav");
- /* the following is copied from
- * http://agentzlerich.blogspot.com/2010/01/using-fftw-for-in-place-matrix.html
- * */
- typename fftw_interface<rnumber>::iodim howmany_dims[2];
- howmany_dims[0].n = dim;
- howmany_dims[0].is = this->local_size;
- howmany_dims[0].os = 1;
- howmany_dims[1].n = this->local_size;
- howmany_dims[1].is = 1;
- howmany_dims[1].os = dim;
- const int howmany_rank = sizeof(howmany_dims)/sizeof(howmany_dims[0]);
-
- typename fftw_interface<rnumber>::plan tmp = fftw_interface<rnumber>::plan_guru_r2r(
- /*rank*/0,
- /*dims*/nullptr,
- howmany_rank,
- howmany_dims,
- a,
- a,
- /*kind*/nullptr,
- DEFAULT_FFTW_FLAG);
- fftw_interface<rnumber>::execute(tmp);
- fftw_interface<rnumber>::destroy_plan(tmp);
- return EXIT_SUCCESS;
-}
-
-template <class rnumber>
-int field_descriptor<rnumber>::interleave(
- typename fftw_interface<rnumber>::complex *a,
- int dim)
-{
- TIMEZONE("field_descriptor::interleave2");
- typename fftw_interface<rnumber>::iodim howmany_dims[2];
- howmany_dims[0].n = dim;
- howmany_dims[0].is = this->local_size;
- howmany_dims[0].os = 1;
- howmany_dims[1].n = this->local_size;
- howmany_dims[1].is = 1;
- howmany_dims[1].os = dim;
- const int howmany_rank = sizeof(howmany_dims)/sizeof(howmany_dims[0]);
-
- typename fftw_interface<rnumber>::plan tmp = fftw_interface<rnumber>::plan_guru_dft(
- /*rank*/0,
- /*dims*/nullptr,
- howmany_rank,
- howmany_dims,
- a,
- a,
- +1,
- DEFAULT_FFTW_FLAG);
- fftw_interface<rnumber>::execute(tmp);
- fftw_interface<rnumber>::destroy_plan(tmp);
- return EXIT_SUCCESS;
-}
-
-template <class rnumber>
-field_descriptor<rnumber>* field_descriptor<rnumber>::get_transpose()
-{
- TIMEZONE("field_descriptor::get_transpose");
- int n[this->ndims];
- for (int i=0; i<this->ndims; i++)
- n[i] = this->sizes[this->ndims - i - 1];
- return new field_descriptor<rnumber>(this->ndims, n, this->mpi_dtype, this->comm);
-}
-
-/*****************************************************************************/
-/*****************************************************************************/
-
-
-
-/*****************************************************************************/
-/* destructor looks the same for both float and double */
-template <class rnumber>
-field_descriptor<rnumber>::~field_descriptor()
-{
- DEBUG_MSG_WAIT(
- MPI_COMM_WORLD,
- this->io_comm == MPI_COMM_NULL ? "null\n" : "not null\n");
- DEBUG_MSG_WAIT(
- MPI_COMM_WORLD,
- "subsizes[0] = %d \n", this->subsizes[0]);
- if (this->subsizes[0] > 0)
- {
- DEBUG_MSG_WAIT(
- this->io_comm,
- "deallocating mpi_array_dtype\n");
- MPI_Type_free(&this->mpi_array_dtype);
- }
- if (this->nprocs != this->io_nprocs && this->io_myrank != MPI_PROC_NULL)
- {
- DEBUG_MSG_WAIT(
- this->io_comm,
- "freeing io_comm\n");
- MPI_Comm_free(&this->io_comm);
- }
- delete[] this->sizes;
- delete[] this->subsizes;
- delete[] this->starts;
- delete[] this->rank;
- delete[] this->all_start0;
- delete[] this->all_size0;
-}
-/*****************************************************************************/
-
-
-
-/*****************************************************************************/
-/* finally, force generation of code */
-template class field_descriptor<float>;
-template class field_descriptor<double>;
-/*****************************************************************************/
-
diff --git a/bfps/cpp/field_descriptor.hpp b/bfps/cpp/field_descriptor.hpp
deleted file mode 100644
index 2fb491bc..00000000
--- a/bfps/cpp/field_descriptor.hpp
+++ /dev/null
@@ -1,114 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#include <mpi.h>
-#include <fftw3-mpi.h>
-#include "fftw_interface.hpp"
-
-#ifndef FIELD_DESCRIPTOR
-
-#define FIELD_DESCRIPTOR
-
-extern int myrank, nprocs;
-
-template <class rnumber>
-class field_descriptor
-{
- private:
- typedef rnumber cnumber[2];
- public:
-
- /* data */
- int *sizes;
- int *subsizes;
- int *starts;
- int ndims;
- int *rank;
- int *all_start0;
- int *all_size0;
- ptrdiff_t slice_size, local_size, full_size;
- MPI_Datatype mpi_array_dtype, mpi_dtype;
- int myrank, nprocs, io_myrank, io_nprocs;
- MPI_Comm comm, io_comm;
-
-
- /* methods */
- field_descriptor(
- int ndims,
- int *n,
- MPI_Datatype element_type,
- MPI_Comm COMM_TO_USE);
- ~field_descriptor();
-
- /* io is performed using MPI_File stuff, and our
- * own mpi_array_dtype that was defined in the constructor.
- * */
- int read(
- const char *fname,
- void *buffer);
- int write(
- const char *fname,
- void *buffer);
-
- /* a function that generates the transposed descriptor.
- * don't forget to delete the result once you're done with it.
- * the transposed descriptor is useful for io operations.
- * */
- field_descriptor<rnumber> *get_transpose();
-
- /* we don't actually need the transposed descriptor to perform
- * the transpose operation: we only need the in/out fields.
- * */
- int transpose(
- rnumber *input,
- rnumber *output);
- int transpose(
- typename fftw_interface<rnumber>::complex *input,
- typename fftw_interface<rnumber>::complex *output = NULL);
-
- int interleave(
- rnumber *input,
- int dim);
- int interleave(
- typename fftw_interface<rnumber>::complex *input,
- int dim);
-};
-
-
-inline float btle(const float be)
- {
- float le;
- char *befloat = (char *) & be;
- char *lefloat = (char *) & le;
- lefloat[0] = befloat[3];
- lefloat[1] = befloat[2];
- lefloat[2] = befloat[1];
- lefloat[3] = befloat[0];
- return le;
- }
-
-#endif//FIELD_DESCRIPTOR
-
diff --git a/bfps/cpp/particles.cpp b/bfps/cpp/particles.cpp
deleted file mode 100644
index cdaf157c..00000000
--- a/bfps/cpp/particles.cpp
+++ /dev/null
@@ -1,254 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#define NDEBUG
-
-#include <cmath>
-#include <cassert>
-#include <cstring>
-#include <string>
-#include <sstream>
-
-#include "base.hpp"
-#include "particles.hpp"
-#include "fftw_tools.hpp"
-
-
-extern int myrank, nprocs;
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-particles<particle_type, rnumber, interp_neighbours>::particles(
- const char *NAME,
- const hid_t data_file_id,
- interpolator_base<rnumber, interp_neighbours> *VEL,
- const int TRAJ_SKIP,
- const int INTEGRATION_STEPS) : particles_io_base<particle_type>(
- NAME,
- TRAJ_SKIP,
- data_file_id,
- VEL->descriptor->comm)
-{
- assert((INTEGRATION_STEPS <= 6) &&
- (INTEGRATION_STEPS >= 1));
- this->vel = VEL;
- this->integration_steps = INTEGRATION_STEPS;
- this->array_size = this->nparticles * state_dimension(particle_type);
- this->state = new double[this->array_size];
- std::fill_n(this->state, this->array_size, 0.0);
- for (int i=0; i < this->integration_steps; i++)
- {
- this->rhs[i] = new double[this->array_size];
- std::fill_n(this->rhs[i], this->array_size, 0.0);
- }
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-particles<particle_type, rnumber, interp_neighbours>::~particles()
-{
- delete[] this->state;
- for (int i=0; i < this->integration_steps; i++)
- {
- delete[] this->rhs[i];
- }
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void particles<particle_type, rnumber, interp_neighbours>::get_rhs(double *x, double *y)
-{
- switch(particle_type)
- {
- case VELOCITY_TRACER:
- this->vel->sample(this->nparticles, state_dimension(particle_type), x, y);
- break;
- }
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void particles<particle_type, rnumber, interp_neighbours>::roll_rhs()
-{
- for (int i=this->integration_steps-2; i>=0; i--)
- std::copy(this->rhs[i],
- this->rhs[i] + this->array_size,
- this->rhs[i+1]);
-}
-
-
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void particles<particle_type, rnumber, interp_neighbours>::AdamsBashforth(
- const int nsteps)
-{
- ptrdiff_t ii;
- this->get_rhs(this->state, this->rhs[0]);
- switch(nsteps)
- {
- case 1:
- for (unsigned int p=0; p<this->nparticles; p++)
- for (unsigned int i=0; i<state_dimension(particle_type); i++)
- {
- ii = p*state_dimension(particle_type)+i;
- this->state[ii] += this->dt*this->rhs[0][ii];
- }
- break;
- case 2:
- for (unsigned int p=0; p<this->nparticles; p++)
- for (unsigned int i=0; i<state_dimension(particle_type); i++)
- {
- ii = p*state_dimension(particle_type)+i;
- this->state[ii] += this->dt*(3*this->rhs[0][ii]
- - this->rhs[1][ii])/2;
- }
- break;
- case 3:
- for (unsigned int p=0; p<this->nparticles; p++)
- for (unsigned int i=0; i<state_dimension(particle_type); i++)
- {
- ii = p*state_dimension(particle_type)+i;
- this->state[ii] += this->dt*(23*this->rhs[0][ii]
- - 16*this->rhs[1][ii]
- + 5*this->rhs[2][ii])/12;
- }
- break;
- case 4:
- for (unsigned int p=0; p<this->nparticles; p++)
- for (unsigned int i=0; i<state_dimension(particle_type); i++)
- {
- ii = p*state_dimension(particle_type)+i;
- this->state[ii] += this->dt*(55*this->rhs[0][ii]
- - 59*this->rhs[1][ii]
- + 37*this->rhs[2][ii]
- - 9*this->rhs[3][ii])/24;
- }
- break;
- case 5:
- for (unsigned int p=0; p<this->nparticles; p++)
- for (unsigned int i=0; i<state_dimension(particle_type); i++)
- {
- ii = p*state_dimension(particle_type)+i;
- this->state[ii] += this->dt*(1901*this->rhs[0][ii]
- - 2774*this->rhs[1][ii]
- + 2616*this->rhs[2][ii]
- - 1274*this->rhs[3][ii]
- + 251*this->rhs[4][ii])/720;
- }
- break;
- case 6:
- for (unsigned int p=0; p<this->nparticles; p++)
- for (unsigned int i=0; i<state_dimension(particle_type); i++)
- {
- ii = p*state_dimension(particle_type)+i;
- this->state[ii] += this->dt*(4277*this->rhs[0][ii]
- - 7923*this->rhs[1][ii]
- + 9982*this->rhs[2][ii]
- - 7298*this->rhs[3][ii]
- + 2877*this->rhs[4][ii]
- - 475*this->rhs[5][ii])/1440;
- }
- break;
- }
- this->roll_rhs();
-}
-
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void particles<particle_type, rnumber, interp_neighbours>::step()
-{
- this->AdamsBashforth((this->iteration < this->integration_steps) ?
- this->iteration+1 :
- this->integration_steps);
- this->iteration++;
-}
-
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void particles<particle_type, rnumber, interp_neighbours>::read()
-{
- if (this->myrank == 0)
- for (unsigned int cindex=0; cindex<this->get_number_of_chunks(); cindex++)
- {
- this->read_state_chunk(cindex, this->state+cindex*this->chunk_size*state_dimension(particle_type));
- if (this->iteration > 0)
- for (int i=0; i<this->integration_steps; i++)
- this->read_rhs_chunk(cindex, i, this->rhs[i]+cindex*this->chunk_size*state_dimension(particle_type));
- }
- MPI_Bcast(
- this->state,
- this->array_size,
- MPI_DOUBLE,
- 0,
- this->comm);
- if (this->iteration > 0)
- for (int i = 0; i<this->integration_steps; i++)
- MPI_Bcast(
- this->rhs[i],
- this->array_size,
- MPI_DOUBLE,
- 0,
- this->comm);
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void particles<particle_type, rnumber, interp_neighbours>::write(
- const bool write_rhs)
-{
- if (this->myrank == 0)
- for (unsigned int cindex=0; cindex<this->get_number_of_chunks(); cindex++)
- {
- this->write_state_chunk(cindex, this->state+cindex*this->chunk_size*state_dimension(particle_type));
- if (write_rhs)
- for (int i=0; i<this->integration_steps; i++)
- this->write_rhs_chunk(cindex, i, this->rhs[i]+cindex*this->chunk_size*state_dimension(particle_type));
- }
-}
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-void particles<particle_type, rnumber, interp_neighbours>::sample(
- interpolator_base<rnumber, interp_neighbours> *field,
- const char *dset_name)
-{
- double *y = new double[this->nparticles*3];
- field->sample(this->nparticles, state_dimension(particle_type), this->state, y);
- if (this->myrank == 0)
- for (unsigned int cindex=0; cindex<this->get_number_of_chunks(); cindex++)
- this->write_point3D_chunk(dset_name, cindex, y+cindex*this->chunk_size*3);
- delete[] y;
-}
-
-
-/*****************************************************************************/
-template class particles<VELOCITY_TRACER, float, 1>;
-template class particles<VELOCITY_TRACER, float, 2>;
-template class particles<VELOCITY_TRACER, float, 3>;
-template class particles<VELOCITY_TRACER, float, 4>;
-template class particles<VELOCITY_TRACER, float, 5>;
-template class particles<VELOCITY_TRACER, float, 6>;
-template class particles<VELOCITY_TRACER, double, 1>;
-template class particles<VELOCITY_TRACER, double, 2>;
-template class particles<VELOCITY_TRACER, double, 3>;
-template class particles<VELOCITY_TRACER, double, 4>;
-template class particles<VELOCITY_TRACER, double, 5>;
-template class particles<VELOCITY_TRACER, double, 6>;
-/*****************************************************************************/
diff --git a/bfps/cpp/particles.hpp b/bfps/cpp/particles.hpp
deleted file mode 100644
index 03daf3e3..00000000
--- a/bfps/cpp/particles.hpp
+++ /dev/null
@@ -1,99 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <iostream>
-#include <hdf5.h>
-#include "base.hpp"
-#include "particles_base.hpp"
-#include "fluid_solver_base.hpp"
-#include "interpolator_base.hpp"
-
-#ifndef PARTICLES
-
-#define PARTICLES
-
-template <particle_types particle_type, class rnumber, int interp_neighbours>
-class particles: public particles_io_base<particle_type>
-{
- private:
- double *state;
- double *rhs[6];
-
- public:
- int array_size;
- int integration_steps;
- interpolator_base<rnumber, interp_neighbours> *vel;
-
- /* simulation parameters */
- double dt;
-
- /* methods */
-
- /* constructor and destructor.
- * allocate and deallocate:
- * this->state
- * this->rhs
- * */
- particles(
- const char *NAME,
- const hid_t data_file_id,
- interpolator_base<rnumber, interp_neighbours> *FIELD,
- const int TRAJ_SKIP,
- const int INTEGRATION_STEPS = 2);
- ~particles();
-
- void sample(
- interpolator_base<rnumber, interp_neighbours> *field,
- const char *dset_name);
-
- inline void sample(
- interpolator_base<rnumber, interp_neighbours> *field,
- double *y)
- {
- field->sample(this->nparticles, state_dimension(particle_type), this->state, y);
- }
-
- void get_rhs(
- double *__restrict__ x,
- double *__restrict__ rhs);
-
- /* input/output */
- void read();
- void write(
- const char *dset_name,
- const double *data);
- void write(const bool write_rhs = true);
-
- /* solvers */
- void step();
- void roll_rhs();
- void AdamsBashforth(const int nsteps);
-};
-
-#endif//PARTICLES
-
diff --git a/bfps/cpp/particles_base.cpp b/bfps/cpp/particles_base.cpp
deleted file mode 100644
index 14104884..00000000
--- a/bfps/cpp/particles_base.cpp
+++ /dev/null
@@ -1,424 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#define NDEBUG
-
-#include <algorithm>
-#include <cassert>
-#include "particles_base.hpp"
-#include "scope_timer.hpp"
-
-template <particle_types particle_type>
-single_particle_state<particle_type>::single_particle_state()
-{
- std::fill_n(this->data, state_dimension(particle_type), 0);
-}
-
-template <particle_types particle_type>
-single_particle_state<particle_type>::single_particle_state(
- const single_particle_state<particle_type> &src)
-{
- std::copy(
- src.data,
- src.data + state_dimension(particle_type),
- this->data);
-}
-
-template <particle_types particle_type>
-single_particle_state<particle_type>::single_particle_state(
- const double *src)
-{
- std::copy(
- src,
- src + state_dimension(particle_type),
- this->data);
-}
-
-template <particle_types particle_type>
-single_particle_state<particle_type>::~single_particle_state()
-{
-}
-
-template <particle_types particle_type>
-single_particle_state<particle_type> &single_particle_state<particle_type>::operator=(
- const single_particle_state &src)
-{
- std::copy(
- src.data,
- src.data + state_dimension(particle_type),
- this->data);
- return *this;
-}
-
-template <particle_types particle_type>
-single_particle_state<particle_type> &single_particle_state<particle_type>::operator=(
- const double *src)
-{
- std::copy(
- src,
- src + state_dimension(particle_type),
- this->data);
- return *this;
-}
-
-int get_chunk_offsets(
- std::vector<hsize_t> data_dims,
- std::vector<hsize_t> chnk_dims,
- std::vector<std::vector<hsize_t>> &co)
-{
- TIMEZONE("get_chunk_offsets");
- std::vector<hsize_t> nchunks(data_dims);
- int total_number_of_chunks = 1;
- for (unsigned i=0; i<nchunks.size(); i++)
- {
- DEBUG_MSG("get_chunk_offset nchunks[%d] = %ld, chnk_dims[%d] = %ld\n",
- i, nchunks[i], i, chnk_dims[i]);
- nchunks[i] = data_dims[i] / chnk_dims[i];
- total_number_of_chunks *= nchunks[i];
- }
- co.resize(total_number_of_chunks);
- DEBUG_MSG("total number of chunks is %d\n", total_number_of_chunks);
- for (int cindex=0; cindex < total_number_of_chunks; cindex++)
- {
- int cc = cindex;
- for (unsigned i=0; i<nchunks.size(); i++)
- {
- int ii = nchunks.size()-1-i;
- co[cindex].resize(nchunks.size());
- co[cindex][ii] = cc % nchunks[ii];
- cc = (cc - co[cindex][ii]) / nchunks[ii];
- co[cindex][ii] *= chnk_dims[ii];
- }
- }
- return EXIT_SUCCESS;
-}
-
-template <particle_types particle_type>
-particles_io_base<particle_type>::particles_io_base(
- const char *NAME,
- const int TRAJ_SKIP,
- const hid_t data_file_id,
- MPI_Comm COMM)
-{
- TIMEZONE("particles_io_base::particles_io_base");
- this->name = std::string(NAME);
- this->traj_skip = TRAJ_SKIP;
- this->comm = COMM;
- MPI_Comm_rank(COMM, &this->myrank);
- MPI_Comm_size(COMM, &this->nprocs);
-
- if (this->myrank == 0)
- {
- hid_t dset, prop_list, dspace;
- this->hdf5_group_id = H5Gopen(data_file_id, this->name.c_str(), H5P_DEFAULT);
- dset = H5Dopen(this->hdf5_group_id, "state", H5P_DEFAULT);
- dspace = H5Dget_space(dset);
- this->hdf5_state_dims.resize(H5Sget_simple_extent_ndims(dspace));
- H5Sget_simple_extent_dims(dspace, &this->hdf5_state_dims.front(), NULL);
- assert(this->hdf5_state_dims[this->hdf5_state_dims.size()-1] == state_dimension(particle_type));
- this->nparticles = 1;
- for (unsigned int i=1; i<this->hdf5_state_dims.size()-1; i++)
- this->nparticles *= this->hdf5_state_dims[i];
- prop_list = H5Dget_create_plist(dset);
- this->hdf5_state_chunks.resize(this->hdf5_state_dims.size());
- H5Pget_chunk(prop_list, this->hdf5_state_dims.size(), &this->hdf5_state_chunks.front());
- H5Pclose(prop_list);
- H5Sclose(dspace);
- H5Dclose(dset);
- this->chunk_size = 1;
- for (unsigned int i=1; i<this->hdf5_state_dims.size()-1; i++)
- this->chunk_size *= this->hdf5_state_chunks[i];
- dset = H5Dopen(this->hdf5_group_id, "rhs", H5P_DEFAULT);
- dspace = H5Dget_space(dset);
- this->hdf5_rhs_dims.resize(H5Sget_simple_extent_ndims(dspace));
- H5Sget_simple_extent_dims(dspace, &this->hdf5_rhs_dims.front(), NULL);
- prop_list = H5Dget_create_plist(dset);
- this->hdf5_rhs_chunks.resize(this->hdf5_rhs_dims.size());
- H5Pget_chunk(prop_list, this->hdf5_rhs_dims.size(), &this->hdf5_rhs_chunks.front());
- H5Pclose(prop_list);
- H5Sclose(dspace);
- H5Dclose(dset);
- }
- DEBUG_MSG("hello, rank 0 just read particle thingie\n");
-
- int tmp;
- tmp = this->hdf5_state_dims.size();
- MPI_Bcast(
- &tmp,
- 1,
- MPI_INTEGER,
- 0,
- this->comm);
- if (this->myrank != 0)
- {
- this->hdf5_state_dims.resize(tmp);
- this->hdf5_state_chunks.resize(tmp);
- }
- DEBUG_MSG("successfully resized state_dims and state_chunks\n");
- MPI_Bcast(
- &this->hdf5_state_dims.front(),
- this->hdf5_state_dims.size(),
- // hsize_t is in fact unsigned long long. Will this ever change...?
- MPI_UNSIGNED_LONG_LONG,
- 0,
- this->comm);
- MPI_Bcast(
- &this->hdf5_state_chunks.front(),
- this->hdf5_state_chunks.size(),
- MPI_UNSIGNED_LONG_LONG,
- 0,
- this->comm);
- DEBUG_MSG("successfully broadcasted state_dims and state_chunks\n");
- for (unsigned i=0; i<this->hdf5_state_chunks.size(); i++)
- DEBUG_MSG(
- "hdf5_state_dims[%d] = %ld, hdf5_state_chunks[%d] = %ld\n",
- i, this->hdf5_state_dims[i],
- i, this->hdf5_state_chunks[i]
- );
- std::vector<hsize_t> tdims(this->hdf5_state_dims), tchnk(this->hdf5_state_chunks);
- tdims.erase(tdims.begin()+0);
- tchnk.erase(tchnk.begin()+0);
- tdims.erase(tdims.end()-1);
- tchnk.erase(tchnk.end()-1);
- DEBUG_MSG("before get_chunk_offsets\n");
- get_chunk_offsets(tdims, tchnk, this->chunk_offsets);
- DEBUG_MSG("after get_chunk_offsets\n");
- MPI_Bcast(
- &this->chunk_size,
- 1,
- MPI_UNSIGNED,
- 0,
- this->comm);
- MPI_Bcast(
- &this->nparticles,
- 1,
- MPI_UNSIGNED,
- 0,
- this->comm);
- DEBUG_MSG("nparticles = %d, chunk_size = %d\n",
- this->nparticles,
- this->chunk_size);
- DEBUG_MSG("exiting particles_io_base constructor\n");
-}
-
-template <particle_types particle_type>
-particles_io_base<particle_type>::~particles_io_base()
-{
- if(this->myrank == 0)
- H5Gclose(this->hdf5_group_id);
-}
-
-template <particle_types particle_type>
-void particles_io_base<particle_type>::read_state_chunk(
- const int cindex,
- double *data)
-{
- TIMEZONE("particles_io_base::read_state_chunk");
- DEBUG_MSG("entered read_state_chunk\n");
- hid_t dset = H5Dopen(this->hdf5_group_id, "state", H5P_DEFAULT);
- hid_t rspace = H5Dget_space(dset);
- std::vector<hsize_t> mem_dims(this->hdf5_state_chunks);
- mem_dims[0] = 1;
- hid_t mspace = H5Screate_simple(
- this->hdf5_state_dims.size(),
- &mem_dims.front(),
- NULL);
- hsize_t *offset = new hsize_t[this->hdf5_state_dims.size()];
- offset[0] = this->iteration / this->traj_skip;
- for (unsigned int i=1; i<this->hdf5_state_dims.size()-1; i++)
- offset[i] = this->chunk_offsets[cindex][i-1];
- offset[this->hdf5_state_dims.size()-1] = 0;
- H5Sselect_hyperslab(
- rspace,
- H5S_SELECT_SET,
- offset,
- NULL,
- &mem_dims.front(),
- NULL);
- H5Dread(dset, H5T_NATIVE_DOUBLE, mspace, rspace, H5P_DEFAULT, data);
- H5Sclose(mspace);
- H5Sclose(rspace);
- H5Dclose(dset);
- delete[] offset;
- DEBUG_MSG("exiting read_state_chunk\n");
-}
-
-template <particle_types particle_type>
-void particles_io_base<particle_type>::write_state_chunk(
- const int cindex,
- const double *data)
-{
- TIMEZONE("particles_io_base::write_state_chunk");
- hid_t dset = H5Dopen(this->hdf5_group_id, "state", H5P_DEFAULT);
- hid_t rspace = H5Dget_space(dset);
- std::vector<hsize_t> mem_dims(this->hdf5_state_chunks);
- mem_dims[0] = 1;
- hid_t mspace = H5Screate_simple(
- this->hdf5_state_dims.size(),
- &mem_dims.front(),
- NULL);
- hsize_t *offset = new hsize_t[this->hdf5_state_dims.size()];
- offset[0] = this->iteration / this->traj_skip;
- for (unsigned int i=1; i<this->hdf5_state_dims.size()-1; i++)
- offset[i] = this->chunk_offsets[cindex][i-1];
- offset[this->hdf5_state_dims.size()-1] = 0;
- H5Sselect_hyperslab(
- rspace,
- H5S_SELECT_SET,
- offset,
- NULL,
- &mem_dims.front(),
- NULL);
- H5Dwrite(dset, H5T_NATIVE_DOUBLE, mspace, rspace, H5P_DEFAULT, data);
- H5Sclose(mspace);
- H5Sclose(rspace);
- H5Dclose(dset);
- delete[] offset;
-}
-
-template <particle_types particle_type>
-void particles_io_base<particle_type>::read_rhs_chunk(
- const int cindex,
- const int rhsindex,
- double *data)
-{
- TIMEZONE("particles_io_base::read_rhs_chunk");
- //DEBUG_MSG("entered read_rhs_chunk\n");
- hid_t dset = H5Dopen(this->hdf5_group_id, "rhs", H5P_DEFAULT);
- hid_t rspace = H5Dget_space(dset);
- std::vector<hsize_t> mem_dims(this->hdf5_rhs_chunks);
- mem_dims[0] = 1;
- mem_dims[1] = 1;
- hid_t mspace = H5Screate_simple(
- this->hdf5_rhs_dims.size(),
- &mem_dims.front(),
- NULL);
- hsize_t *offset = new hsize_t[this->hdf5_rhs_dims.size()];
- offset[0] = this->hdf5_rhs_dims[0]-2;
- offset[1] = rhsindex;
- for (unsigned int i=2; i<this->hdf5_rhs_dims.size()-1; i++)
- offset[i] = this->chunk_offsets[cindex][i-2];
- offset[this->hdf5_rhs_dims.size()-1] = 0;
- //for (int i=0; i<this->hdf5_rhs_dims.size(); i++)
- // DEBUG_MSG("rhs dim %d: size=%d chunk=%d offset=%d\n",
- // i, this->hdf5_rhs_dims[i], this->hdf5_rhs_chunks[i], offset[i]);
- H5Sselect_hyperslab(
- rspace,
- H5S_SELECT_SET,
- offset,
- NULL,
- &mem_dims.front(),
- NULL);
- //DEBUG_MSG("selected hyperslab\n");
- H5Dread(dset, H5T_NATIVE_DOUBLE, mspace, rspace, H5P_DEFAULT, data);
- //DEBUG_MSG("data has been read\n");
- H5Sclose(mspace);
- H5Sclose(rspace);
- H5Dclose(dset);
- delete[] offset;
- //DEBUG_MSG("exiting read_rhs_chunk\n");
-}
-
-template <particle_types particle_type>
-void particles_io_base<particle_type>::write_rhs_chunk(
- const int cindex,
- const int rhsindex,
- const double *data)
-{
- TIMEZONE("particles_io_base::write_rhs_chunk");
- hid_t dset = H5Dopen(this->hdf5_group_id, "rhs", H5P_DEFAULT);
- hid_t rspace = H5Dget_space(dset);
- std::vector<hsize_t> mem_dims(this->hdf5_rhs_chunks);
- mem_dims[0] = 1;
- mem_dims[1] = 1;
- hid_t mspace = H5Screate_simple(
- this->hdf5_rhs_dims.size(),
- &mem_dims.front(),
- NULL);
- hsize_t *offset = new hsize_t[this->hdf5_rhs_dims.size()];
- offset[0] = this->hdf5_rhs_dims[0]-1;
- offset[1] = rhsindex;
- for (unsigned int i=2; i<this->hdf5_rhs_dims.size()-1; i++)
- offset[i] = this->chunk_offsets[cindex][i-2];
- offset[this->hdf5_rhs_dims.size()-1] = 0;
- DEBUG_MSG("rhs write offsets are %d %d %d %d\n",
- offset[0], offset[1], offset[2], offset[3]);
- H5Sselect_hyperslab(
- rspace,
- H5S_SELECT_SET,
- offset,
- NULL,
- &mem_dims.front(),
- NULL);
- H5Dwrite(dset, H5T_NATIVE_DOUBLE, mspace, rspace, H5P_DEFAULT, data);
- H5Sclose(mspace);
- H5Sclose(rspace);
- H5Dclose(dset);
- delete[] offset;
-}
-
-template <particle_types particle_type>
-void particles_io_base<particle_type>::write_point3D_chunk(
- const std::string dset_name,
- const int cindex,
- const double *data)
-{
- TIMEZONE("particles_io_base::write_point3D_chunk");
- hid_t dset = H5Dopen(this->hdf5_group_id, dset_name.c_str(), H5P_DEFAULT);
- hid_t rspace = H5Dget_space(dset);
- std::vector<hsize_t> mem_dims(this->hdf5_state_chunks);
- mem_dims[0] = 1;
- mem_dims[mem_dims.size()-1] = 3;
- hid_t mspace = H5Screate_simple(
- this->hdf5_state_dims.size(),
- &mem_dims.front(),
- NULL);
- hsize_t *offset = new hsize_t[this->hdf5_state_dims.size()];
- offset[0] = this->iteration / this->traj_skip;
- for (unsigned int i=1; i<this->hdf5_state_dims.size()-1; i++)
- offset[i] = this->chunk_offsets[cindex][i-1];
- offset[this->hdf5_state_dims.size()-1] = 0;
- H5Sselect_hyperslab(
- rspace,
- H5S_SELECT_SET,
- offset,
- NULL,
- &mem_dims.front(),
- NULL);
- H5Dwrite(dset, H5T_NATIVE_DOUBLE, mspace, rspace, H5P_DEFAULT, data);
- H5Sclose(mspace);
- H5Sclose(rspace);
- H5Dclose(dset);
- delete[] offset;
-}
-
-/*****************************************************************************/
-template class single_particle_state<POINT3D>;
-template class single_particle_state<VELOCITY_TRACER>;
-
-template class particles_io_base<VELOCITY_TRACER>;
-/*****************************************************************************/
-
diff --git a/bfps/cpp/particles_base.hpp b/bfps/cpp/particles_base.hpp
deleted file mode 100644
index 8afd5d43..00000000
--- a/bfps/cpp/particles_base.hpp
+++ /dev/null
@@ -1,136 +0,0 @@
-/**********************************************************************
-* *
-* Copyright 2015 Max Planck Institute *
-* for Dynamics and Self-Organization *
-* *
-* This file is part of bfps. *
-* *
-* bfps is free software: you can redistribute it and/or modify *
-* it under the terms of the GNU General Public License as published *
-* by the Free Software Foundation, either version 3 of the License, *
-* or (at your option) any later version. *
-* *
-* bfps is distributed in the hope that it will be useful, *
-* but WITHOUT ANY WARRANTY; without even the implied warranty of *
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
-* GNU General Public License for more details. *
-* *
-* You should have received a copy of the GNU General Public License *
-* along with bfps. If not, see <http://www.gnu.org/licenses/> *
-* *
-* Contact: Cristian.Lalescu@ds.mpg.de *
-* *
-**********************************************************************/
-
-
-
-#include <vector>
-#include <hdf5.h>
-#include <unordered_map>
-#include "interpolator_base.hpp"
-
-#ifndef PARTICLES_BASE
-
-#define PARTICLES_BASE
-
-/* particle types */
-enum particle_types {POINT3D, VELOCITY_TRACER};
-
-/* space dimension */
-constexpr unsigned int state_dimension(particle_types particle_type)
-{
- return ((particle_type == POINT3D) ? 3 : (
- (particle_type == VELOCITY_TRACER) ? 3 :
- 3));
-}
-
-/* 1 particle state type */
-
-template <particle_types particle_type>
-class single_particle_state
-{
- public:
- double data[state_dimension(particle_type)];
-
- single_particle_state();
- single_particle_state(const single_particle_state &src);
- single_particle_state(const double *src);
- ~single_particle_state();
-
- single_particle_state<particle_type> &operator=(const single_particle_state &src);
- single_particle_state<particle_type> &operator=(const double *src);
-
- inline double &operator[](const int i)
- {
- return this->data[i];
- }
-};
-
-std::vector<std::vector<hsize_t>> get_chunk_offsets(
- std::vector<hsize_t> data_dims,
- std::vector<hsize_t> chnk_dims);
-
-template <particle_types particle_type>
-class particles_io_base
-{
- protected:
- int myrank, nprocs;
- MPI_Comm comm;
-
- unsigned int nparticles;
-
- std::string name;
- unsigned int chunk_size;
- int traj_skip;
-
- hid_t hdf5_group_id;
- std::vector<hsize_t> hdf5_state_dims, hdf5_state_chunks;
- std::vector<hsize_t> hdf5_rhs_dims, hdf5_rhs_chunks;
-
- std::vector<std::vector<hsize_t>> chunk_offsets;
-
- particles_io_base(
- const char *NAME,
- const int TRAJ_SKIP,
- const hid_t data_file_id,
- MPI_Comm COMM);
- virtual ~particles_io_base();
-
- void read_state_chunk(
- const int cindex,
- double *__restrict__ data);
- void write_state_chunk(
- const int cindex,
- const double *data);
- void read_rhs_chunk(
- const int cindex,
- const int rhsindex,
- double *__restrict__ data);
- void write_rhs_chunk(
- const int cindex,
- const int rhsindex,
- const double *data);
-
- void write_point3D_chunk(
- const std::string dset_name,
- const int cindex,
- const double *data);
-
- public:
- int iteration;
-
- inline const char *get_name()
- {
- return this->name.c_str();
- }
- inline const unsigned int get_number_of_chunks()
- {
- return this->chunk_offsets.size();
- }
- inline const unsigned int get_number_of_rhs_chunks();
- virtual void read() = 0;
- virtual void write(const bool write_rhs = true) = 0;
-};
-
-#endif//PARTICLES_BASE
-
--
GitLab
From 4f1e39a7f7c43b2026fadfaf8972dc3ff66dccec Mon Sep 17 00:00:00 2001
From: Berenger Bramas <Berenger.Bramas@inria.fr>
Date: Mon, 11 Mar 2019 11:41:39 +0100
Subject: [PATCH 315/342] Update cmake -- remove previous system and use find
instead (including find fftw from morse) -- export config for external reuse
-- update README to give more information
---
CMakeLists.txt | 127 ++-
README.rst | 47 +-
cmake/BFPSConfig.cmake.in | 37 +-
cmake/BFPSUtils.cmake | 184 ----
cmake/LoadFFTW.cmake | 17 -
cmake/LoadHDF5.cmake | 17 -
cmake/LoadSZIP.cmake | 17 -
cmake/morse/FindCommon.cmake | 47 +
cmake/morse/FindFFTW.cmake | 832 ++++++++++++++++++
cmake/morse/FindHeadersAndLibs.cmake | 94 ++
cmake/morse/FindInit.cmake | 45 +
.../FindPkgconfigLibrariesAbsolutePath.cmake | 99 +++
cmake/morse/LICENCE.txt | 42 +
cmake/morse/LibrariesAbsolutePath.cmake | 70 ++
cmake/morse/MorseInit.cmake | 67 ++
cmake/morse/PrintFindStatus.cmake | 207 +++++
16 files changed, 1643 insertions(+), 306 deletions(-)
delete mode 100644 cmake/BFPSUtils.cmake
delete mode 100644 cmake/LoadFFTW.cmake
delete mode 100644 cmake/LoadHDF5.cmake
delete mode 100644 cmake/LoadSZIP.cmake
create mode 100644 cmake/morse/FindCommon.cmake
create mode 100644 cmake/morse/FindFFTW.cmake
create mode 100644 cmake/morse/FindHeadersAndLibs.cmake
create mode 100644 cmake/morse/FindInit.cmake
create mode 100644 cmake/morse/FindPkgconfigLibrariesAbsolutePath.cmake
create mode 100644 cmake/morse/LICENCE.txt
create mode 100644 cmake/morse/LibrariesAbsolutePath.cmake
create mode 100644 cmake/morse/MorseInit.cmake
create mode 100644 cmake/morse/PrintFindStatus.cmake
diff --git a/CMakeLists.txt b/CMakeLists.txt
index f102e4d5..b7d65d0d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,34 +1,97 @@
-cmake_minimum_required(VERSION 2.8.11)
+cmake_minimum_required(VERSION 3.10)
-project(BFPS CXX)
+project(BFPS C CXX)
+
+
+set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/morse ${CMAKE_MODULE_PATH})
+set(BFPS_LIBS "")
+
+#####################################################################################
+## MPI
+
+set(MPI_STATIC ON)
+find_package(MPI REQUIRED)
+
+set(CMAKE_CXX_COMPILE_FLAGS "${CMAKE_CXX_COMPILE_FLAGS} ${MPI_CXX_COMPILE_OPTIONS}")
+set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${MPI_CXX_LINK_FLAGS}")
+include_directories(${MPI_CXX_INCLUDE_DIRS})
+add_definitions(${MPI_CXX_COMPILE_DEFINITIONS})
+list(APPEND BFPS_LIBS "${MPI_CXX_LIBRARIES}")
+
+#####################################################################################
+## CXX Standard
+
+set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+# set(CMAKE_CXX_EXTENSIONS OFF)
#####################################################################################
-## Dependencies
+## OpenMP
+
+find_package(OpenMP REQUIRED)
-set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH})
-include(BFPSUtils)
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}")
+list(APPEND BFPS_LIBS "${OpenMP_CXX_LIB_NAMES}")
+
+#####################################################################################
+## Extra flags
-BFPSMustAdd(FFTW)
-BFPSMustAdd(HDF5)
+set(CMAKE_CXX_COMPILE_FLAGS "${CMAKE_CXX_COMPILE_FLAGS} -Wall")
-if(DEFINED env{SZIP_LIB})
- BFPSMustAdd(SZIP)
+#####################################################################################
+## HDF5
+
+if(NOT DEFINED ENV{HDF5_ROOT})
+ message(WARNING "The environment variable HDF5_ROOT is undefined, this might cause trouble in finding the HDF5")
endif()
-set(MPI_STATIC ON)
-find_package(MPI REQUIRED)
+set(HDF5_PREFER_PARALLEL TRUE)
+find_package(HDF5 REQUIRED)
-set(CMAKE_CXX_COMPILE_FLAGS ${CMAKE_CXX_COMPILE_FLAGS} ${MPI_COMPILE_FLAGS} "-Wall -std=c++11 -fopenmp")
+message(STATUS "HDF5_C_INCLUDE_DIRS ${HDF5_C_INCLUDE_DIRS}")
+
+include_directories(${HDF5_C_INCLUDE_DIRS})
+add_definitions(${HDF5_C_DEFINITIONS})
+list(APPEND BFPS_LIBS "${HDF5_C_LIBRARIES}")
+
+option(BFPS_HDF5_USE_SZIP "Set to on to also link against SZIP" OFF)
+
+if(BFPS_HDF5_USE_SZIP)
+ option(BFPS_HDF5_SZIP_LIB_PATH "Additional lib path for SZIP" "")
+ if(BFPS_HDF5_SZIP_LIB_PATH)
+ link_directories(${BFPS_HDF5_SZIP_LIB_PATH})
+ endif()
+ list(APPEND BFPS_LIBS "z")
+endif()
+
+#####################################################################################
+## FFTW
+
+if(NOT DEFINED ENV{FFTW_DIR})
+ message(WARNING "The environment variable FFTW_DIR is undefined, this might cause trouble in finding the FFTW")
+endif()
+
+find_package(FFTW REQUIRED OMP)
+
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FFTW_CFLAGS_OTHER}")
+set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${FFTW_LDFLAGS_OTHER}")
+list(APPEND BFPS_LIBS "${FFTW_LIBRARIES}")
+include_directories(${FFTW_INCLUDE_DIRS})
+link_directories(${FFTW_LIBRARY_DIRS})
+
+find_package(FFTW REQUIRED OMP SIMPLE)
+
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FFTW_CFLAGS_OTHER}")
+set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${FFTW_LDFLAGS_OTHER}")
+list(APPEND BFPS_LIBS "${FFTW_LIBRARIES}")
+include_directories(${FFTW_INCLUDE_DIRS})
+link_directories(${FFTW_LIBRARY_DIRS})
#####################################################################################
## Build the lib
-message(STATUS "Include dirs from dependencies: ${FFTW_INCLUDE_DIRS} ${HDF5_INCLUDE_DIRS} ${SZIP_INCLUDE_DIRS} ${MPI_INCLUDE_PATH}")
include_directories(
- ${FFTW_INCLUDE_DIRS}
- ${HDF5_INCLUDE_DIRS}
- ${SZIP_INCLUDE_DIRS}
- ${MPI_INCLUDE_PATH}
${PROJECT_SOURCE_DIR}/bfps/cpp
${PROJECT_SOURCE_DIR}/bfps/cpp/particles
${PROJECT_SOURCE_DIR}/bfps/cpp/full_code
@@ -36,32 +99,24 @@ include_directories(
file(GLOB_RECURSE cpp_for_lib ${PROJECT_SOURCE_DIR}/bfps/*.cpp)
file(GLOB_RECURSE hpp_for_lib ${PROJECT_SOURCE_DIR}/bfps/*.hpp)
-LIST(APPEND source_files ${cpp_for_lib} ${hpp_for_lib} )
+LIST(APPEND source_files ${cpp_for_lib})
add_library(bfps ${source_files})
-target_link_libraries(bfps ${FFTW_LIBRARIES} ${HDF5_LIBRARIES} ${SZIP_LIBRARIES} ${MPI_LIBRARIES})
-set_target_properties(bfps PROPERTIES
- COMPILE_FLAGS
- "${CMAKE_CXX_COMPILE_FLAGS} ${HDF5_COMPILE_FLAGS} ${MPI_COMPILE_FLAGS}"
- )
-set_target_properties(bfps PROPERTIES
- LINK_FLAGS
- "${FFTW_LIBRARIES} ${HDF5_LIBRARIES} ${SZIP_LIBRARIES}"
- )
+target_link_libraries(bfps ${BFPS_LIBS})
+set_target_properties(bfps PROPERTIES PUBLIC_HEADER "${hpp_for_lib}")
+
+install(TARGETS bfps EXPORT BFPS_EXPORT DESTINATION lib/ PUBLIC_HEADER DESTINATION include/bfps )
#####################################################################################
##Â Export the configuration
-export(TARGETS bfps FILE "${PROJECT_BINARY_DIR}/BFPSLibraryDepends.cmake")
-
-# Create a BFPSConfig.cmake file for the use from the build tree
-set(BFPS_INCLUDE_DIRS
- "${PROJECT_SOURCE_DIR}/include"
- "${PROJECT_BINARY_DIR}")
-set(BFPS_LIB_DIR "${PROJECT_BINARY_DIR}/BFPS")
-set(BFPS_CMAKE_DIR "${PROJECT_BINARY_DIR}")
+get_property(ALL_INCLUDE_DIRS DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES)
+get_property(ALL_LINK_DIRS DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY LINK_DIRECTORIES)
configure_file(${PROJECT_SOURCE_DIR}/cmake/BFPSConfig.cmake.in ${PROJECT_BINARY_DIR}/BFPSConfig.cmake @ONLY)
-install(FILES "${PROJECT_BINARY_DIR}/InstallFiles/BFPSConfig.cmake" DESTINATION "${BFPS_CMAKE_DIR}")
+install(FILES "${PROJECT_BINARY_DIR}/BFPSConfig.cmake" DESTINATION lib/)
+export(TARGETS bfps FILE "${PROJECT_BINARY_DIR}/BFPSLibraryDepends.cmake")
+install(EXPORT BFPS_EXPORT DESTINATION lib/)
+
diff --git a/README.rst b/README.rst
index e4f6d110..b562ba94 100644
--- a/README.rst
+++ b/README.rst
@@ -136,19 +136,23 @@ enough).
2. Download, compile, install FFTW (latest version 3.x from http://www.fftw.org/).
Execute the following commands in order, feel free to customize
- optimisation flags for your own computer:
+ optimisation flags for your own computer (see http://www.fftw.org/fftw3_doc/Installation-on-Unix.html):
.. code:: bash
- ./configure --prefix=PREFIX --enable-single --enable-sse --enable-mpi --enable-openmp --enable-threads
+ ./configure --prefix=PREFIX --enable-float --enable-sse --enable-sse2 --enable-avx --enable-avx2 --enable-avx-128-fma --enable-avx512 --enable-mpi --enable-openmp --enable-threads
make
make install
- ./configure --prefix=PREFIX --enable-sse2 --enable-mpi --enable-openmp --enable-threads
+ ./configure --prefix=PREFIX --enable-sse --enable-sse2 --enable-avx --enable-avx2 --enable-avx-128-fma --enable-avx512 --enable-mpi --enable-openmp --enable-threads
make
make install
+ BFPS will try to find HDF5 using the FindFFTW from the Morse project.
+ If the package is installed in a non standard location, it is recommanded
+ to setup the environment variables: FFTW_DIR (or FFTW_INCDIR and FFTW_LIBDIR)
+
3. Download, compile, install HDF5 (version 1.8.x, currently available
- at https://support.hdfgroup.org/HDF5/release/obtainsrc518.html.
+ at https://portal.hdfgroup.org/display/support/HDF5+1.8.20#files).
We are using parallel I/O, therefore we use the plain C interface of HDF5:
.. code:: bash
@@ -156,6 +160,10 @@ enough).
./configure --prefix=PREFIX --enable-parallel
make
make install
+
+ BFPS will try to find HDF5 using the regular FindHDF5.
+ Therefore, if the package is installed in a non standard location, it is recommanded
+ to setup the environment variable: HDF5_ROOT
3. This step may be ommited.
I recommend the creation of a virtual python3 environment (also under PREFIX) that will be used for installing bfps and dependencies.
@@ -167,14 +175,31 @@ enough).
git clone git@gitlab.mpcdf.mpg.de:clalescu/bfps.git
-5. Tweak host_information.py and machine_settings.py for your user and your machine and place under ~/.config/bfps.
-
-6. Activate virtual environment.
-
-7. Go into bfps repository, execute
+5. Go into bfps repository, execute
.. code:: bash
- python setup.py compile_library
- python setup.py install
+ mkdir build
+ cd build
+ cmake ..
+ # possibly : cmake .. -DCMAKE_INSTALL_PREFIX=INSTALL_DIR
+ make ..
+ # to get a verbose compilation process, use VERBOSE=1 make
+ # make install
+
+6. Using BFPS from an external project. Along with all the BFPS files
+ (lib and headers) can be found different cmake files
+ that contains all the information related to the compilation of BFPS.
+ It is possible to load this file in another CMake based application
+ to know the dependencies and paths.
+
+ For instance, the installation will create these files:
+
+.. code:: bash
+ -- Installing: install/lib/BFPSConfig.cmake
+ -- Installing: install/lib/BFPS_EXPORT.cmake
+ -- Installing: install/lib/BFPS_EXPORT-noconfig.cmake
+
+ In case an information is not provided, it is necessary to update
+ the cmake input config file: bfps/cmake/BFPSConfig.cmake.in
diff --git a/cmake/BFPSConfig.cmake.in b/cmake/BFPSConfig.cmake.in
index 1d3e4d37..80660897 100644
--- a/cmake/BFPSConfig.cmake.in
+++ b/cmake/BFPSConfig.cmake.in
@@ -15,9 +15,13 @@ SET(HAVE_BFPS TRUE)
SET(BFPS_PREFIX "@CMAKE_INSTALL_PREFIX@")
SET(BFPS_INCLUDE_DIR "@CMAKE_INSTALL_PREFIX@/include")
SET(BFPS_LIBRARIES_DIR "@CMAKE_INSTALL_PREFIX@/lib")
-SET(BFPS_LIBRARIES_ADD "@BFPS_LIBRARIES@")
-SET(BFPS_LIBRARY_NAME "@BFPS_LIB@")
-SET(BFPS_LINK_DIRECTORIES "@BFPS_LINK_DIRECTORIES@")
+
+SET(BFPS_LINK_DIRECTORIES "@ALL_LINK_DIRS@")
+SET(BFPS_INCLUDE_DIRECTORIES "@ALL_INCLUDE_DIRS@")
+
+SET(BFPS_CXX_COMPILE_FLAGS "@CMAKE_CXX_COMPILE_FLAGS@")
+SET(BFPS_EXE_LINKER_FLAGS "@CMAKE_EXE_LINKER_FLAGS@")
+set(BFPS_DEFINITIONS @COMPILE_DEFINITIONS@)
#
SET(BFPS_SOURCE_DIR "@BFPS_SOURCE_DIR@")
@@ -25,26 +29,11 @@ SET(BFPS_SOURCE_DIR "@BFPS_SOURCE_DIR@")
#
SET(BFPS_BUILD_TYPE "@CMAKE_BUILD_TYPE@")
-# Flag
-set(BFPS_DEFINITIONS @BFPS_DEFINITIONS@)
-
-SET(BFPS_USE_FFTW "@BFPS_USE_FFTW@")
-SET(BFPS_USE_HDF5 "@BFPS_USE_HDF5@")
-SET(BFPS_USE_SZIP "@BFPS_USE_SZIP@")
-
-SET(BFPS_DOC_TAGS "@CMAKE_BINARY_DIR@/Doc/BFPS.tag")
-
-# Tell the user project where to find our headers and libraries
-set(BFPS_INCLUDE_DIRS "@BFPS_INCLUDE_DIRS@")
-set(BFPS_LIBRARY_DIRS "@BFPS_LIB_DIR@")
-
-# List of compilation flags -DTOTO to export
-set(BFPS_DEFINITIONS @BFPS_DEFINITIONS@)
-
-# Our library dependencies (contains definitions for IMPORTED targets)
-include("@BFPS_CMAKE_DIR@/BFPSLibraryDepends.cmake")
-
-# These are IMPORTED targets created by BFPSLibraryDepends.cmake
-set(BFPS_LIBRARIES bfps)
+#
+SET(BFPS_HDF5_USE_SZIP "@BFPS_HDF5_USE_SZIP@")
+SET(BFPS_HDF5_SZIP_LIB_PATH "@BFPS_HDF5_SZIP_LIB_PATH@")
+#
+set(BFPS_SRC_INCLUDE_DIRS "@BFPS_INCLUDE_DIRS@")
+set(BFPS_BUILD_LIBRARY_DIRS "@BFPS_LIB_DIR@")
diff --git a/cmake/BFPSUtils.cmake b/cmake/BFPSUtils.cmake
deleted file mode 100644
index ad50e395..00000000
--- a/cmake/BFPSUtils.cmake
+++ /dev/null
@@ -1,184 +0,0 @@
-###########################################################################################
-# BFPSAddLibrary : check if a package exists (its lib, headers and its dependencies).
-# Here we do not test to find some functions we simply try to find the files.
-# If environment variables ${MYPACK}_LIB} or ${MYPACK}_INC are set, they are used
-# to give some hints to the find process.
-# The variables set are:
-# ${MYPACK}_LIBRARY_DIRS : one or more dir to libs.
-# ${MYPACK}_INCLUDE_DIRS : one or more dir to headers.
-# ${MYPACK}_LIBRARIES : one or more libs.
-# ${MYPACK}_FOUND : Set to true if the libs has been found completly (with its dependencies).
-# ${MYPACK}_DEPENDENCIES : list of dependencies.
-# ${MYPACK}_FOUND_DEPENDENDCIES : TRUE if dependencies are all TRUE.
-###########################################################################################
-macro(BFPSFindLib MYPACK MYLIBS MYHEADERS MYDEPS)
- # If it has not been already proceed
- if(NOT DEFINED ${MYPACK}_FOUND)
- # Inform user about the research
- MESSAGE( STATUS "Try to find ${MYPACK}..." )
-
- # Find include files
- if(${MYHEADERS})
- find_path(${MYPACK}_INCLUDE_DIRS NAMES ${MYHEADERS} HINTS ${${MYPACK}_INCLUDE_DIR} ${${MYPACK}_INCLUDE_DIRS} ${${MYPACK}_INCLUDE_PATH} ENV ${MYPACK}_INCLUDE_DIR ${MYPACK}_INCLUDE_PATH ${MYPACK}_INC "${MYPACK}_HOME" "${MYPACK}_ROOT" DOC "${MYPACK} include directory")
- else()
- set(${MYPACK}_INCLUDE_DIRS "")
- endif()
-
- if((NOT ${MYPACK}_INCLUDE_DIRS) AND (${MYHEADERS}) )
- MESSAGE( STATUS " [${MYPACK}] Include not found!" )
- set(${MYPACK}_FOUND FALSE)
- else()
- set(${MYPACK}_FOUND TRUE)
-
- # Find libs
- set(${MYPACK}_LIBRARY_HINT_DIRS ${${MYPACK}_LIBRARY_DIRS})
- set(${MYPACK}_LIBRARY_DIRS "")
- foreach (lib_ ${MYLIBS})
- MESSAGE( STATUS " [${MYPACK}] Try to find ${lib_}..." )
- # Ask for this lib
- find_library(${MYPACK}_LIBRARY_${lib_}_PATH NAMES ${lib_} HINTS ${${MYPACK}_LIBRARY_DIR} ${${MYPACK}_LIBRARY_HINT_DIRS} ${${MYPACK}_LIB_PATH} ${${MYPACK}_LIBS_PATH} ENV ${MYPACK}_LIB_DIR ${MYPACK}_LIB_PATH ${MYPACK}_LIB "${MYPACK}_HOME" "${MYPACK}_ROOT" DOC "${MYPACK} lib directory")
- # Stop if not found
- if(NOT ${MYPACK}_LIBRARY_${lib_}_PATH)
- MESSAGE( STATUS " [${MYPACK}] ${lib_} Failed!" )
- set(${MYPACK}_LIBRARY_DIRS NOTFOUND)
- set(${MYPACK}_FOUND FALSE)
- break()
- endif()
- get_filename_component(lib_dir_ ${${MYPACK}_LIBRARY_${lib_}_PATH} PATH)
- list(APPEND ${MYPACK}_LIBRARY_DIRS ${lib_dir_})
- list(REMOVE_DUPLICATES ${MYPACK}_LIBRARY_DIRS)
- endforeach()
-
- # Check dependencies
- set(${MYPACK}_DEPENDENCIES ${MYDEPS})
-
- # If all files exist we set to TRUE
- if(${MYPACK}_FOUND AND (${MYPACK}_LIBRARY_DIRS OR (NOT MYLIBS)) AND (${MYPACK}_INCLUDE_DIRS OR (NOT MYHEADERS)) )
- set(${MYPACK}_FOUND_DEPENDENDCIES TRUE)
-
- MESSAGE( STATUS " [${MYPACK}] Check deps: ${${MYPACK}_DEPENDENCIES}" )
- foreach (dep_ ${${MYPACK}_DEPENDENCIES})
- # Be sure it has been config
- if(NOT DEFINED ${dep_}_FOUND)
- MESSAGE( STATUS " [${MYPACK}] Try to find dep: ${dep_}..." )
- BFPSCheckIfFound(${dep_})
- else()
- MESSAGE( STATUS " [${MYPACK}] Dep ${dep_} has been found already..." )
- endif()
- # If not found stop here
- if(NOT ${dep_}_FOUND)
- set(${MYPACK}_FOUND_DEPENDENDCIES FALSE)
- set(${MYPACK}_FOUND FALSE)
- break()
- endif()
- endforeach()
- endif()
- endif()
-
- # If found and dependencies are found set to true
- if(${MYPACK}_FOUND AND ${MYPACK}_FOUND_DEPENDENDCIES)
- set(${MYPACK}_LIBRARIES ${MYLIBS})
- set(${MYPACK}_FOUND TRUE)
- else()
- # Else make clean results
- set(${${MYPACK}_LIBRARY_DIRS} NOTFOUND)
- set(${${MYPACK}_INCLUDE_DIRS} NOTFOUND)
- set(${MYPACK}_LIBRARIES)
- set(${MYPACK}_FOUND FALSE)
- endif()
-
- # Print results
- MESSAGE( STATUS "${MYPACK}_FOUND = ${${MYPACK}_FOUND}" )
- endif()
-endmacro(BFPSFindLib MYPACK MYLIBS MYHEADERS)
-
-###########################################################################################
-# BFPSAddLibrary : try to add a package (its lib, headers and its dependencies).
-# The variables set are:
-# ${PROJECT_NAME}_USE_${MYPACK} : Set to true if package and dependencies have been found)
-# link_directories and include_directories are called
-# ${PROJECT_NAME}_LIBRARIES is modified in order to contains the package libs
-###########################################################################################
-macro(BFPSAddLibrary MYPACK)
- if(NOT DEFINED ${PROJECT_NAME}_USE_${MYPACK})
- # Check it has been found
- if(NOT ${MYPACK}_FOUND)
- MESSAGE( FATAL_ERROR "Try to load ${MYPACK} but ${MYPACK}_FOUND has value ${${MYPACK}_FOUND}" )
- endif()
- # Check dependencies have been found
- foreach (dep_ ${${MYPACK}_DEPENDENCIES})
- if(NOT ${dep_}_FOUND)
- MESSAGE( FATAL_ERROR "Try to load ${MYPACK} but ${dep_} has value ${${dep_}_FOUND}" )
- endif()
- endforeach()
-
- # Set use variable
- set(${PROJECT_NAME}_USE_${MYPACK} TRUE)
-
- # Add lib directory
- if( ${MYPACK}_LIBRARY_DIRS )
- link_directories(${${MYPACK}_LIBRARY_DIRS})
- list(APPEND ${PROJECT_NAME}_LINK_DIRECTORIES ${${MYPACK}_LIBRARY_DIRS})
- endif()
- # Add inc directory
- if( ${MYPACK}_INCLUDE_DIRS )
- include_directories(${${MYPACK}_INCLUDE_DIRS})
- endif()
- endif()
- if( ${PROJECT_NAME}_USE_${MYPACK} )
- # Add libs to variables
- if( ${MYPACK}_LIBRARIES )
- list(APPEND ${PROJECT_NAME}_LIBRARIES ${${MYPACK}_LIBRARIES})
- endif()
-
- # Add dependencies
- foreach (dep_ ${${MYPACK}_DEPENDENCIES})
- BFPSAddLibrary( ${dep_} )
- set(${PROJECT_NAME}_${MYPACK}_USE_${dep_} TRUE)
- endforeach()
- endif()
-endmacro(BFPSAddLibrary MYPACK)
-
-###########################################################################################
-# BFPSCheckIfFound : check if a package exists (its lib, headers and its dependencies).
-# The variables set by BFPSFindLib are returned.
-###########################################################################################
-macro(BFPSCheckIfFound PACKTOADD)
- include(Load${PACKTOADD} RESULT_VARIABLE ${PACKTOADD}_LOADFILE_EXISTS)
- if(NOT ${PACKTOADD}_LOADFILE_EXISTS)
- MESSAGE( FATAL_ERROR "Try to load ${PACKTOADD} but Load${PACKTOADD}.cmake cannot be found." )
- endif()
-
- BFPSFindLib("${PACKTOADD}" "${${PACKTOADD}_CONFIG_LIBS}" "${${PACKTOADD}_CONFIG_HEADERS}" "${${PACKTOADD}_CONFIG_DEPS}")
-endmacro(BFPSCheckIfFound PACKTOADD)
-
-###########################################################################################
-# BFPSAddIfFound : adds a package if it is found.
-# The variables set by BFPSCheckIfFound and BFPSAddLibrary are returned.
-###########################################################################################
-macro(BFPSAddIfFound PACKTOADD)
- BFPSCheckIfFound(${PACKTOADD})
- if( ${PACKTOADD}_FOUND )
- BFPSAddLibrary(${PACKTOADD})
- endif()
-endmacro(BFPSAddIfFound PACKTOADD)
-
-###########################################################################################
-# BFPSMustAdd : adds a package (it must succed or a fatal error will be launched).
-# The variables set by BFPSAddIfFound are returned.
-###########################################################################################
-macro(BFPSMustAdd PACKTOADD)
- BFPSAddIfFound(${PACKTOADD})
- if(NOT ${PACKTOADD}_FOUND)
- MESSAGE( FATAL_ERROR "Try to load ${PACKTOADD} but ${PACKTOADD}_FOUND has value ${${PACKTOADD}_FOUND}." )
- endif()
-endmacro(BFPSMustAdd PACKTOADD)
-###########################################################################################
-# BFPSSetIfndef : define a variable if it is not already defined
-###########################################################################################
-macro(BFPSSetIfndef VarToSet PotentialValue)
-if(NOT DEFINED ${VarToSet})
- set(${VarToSet} ${PotentialValue})
- MESSAGE( STATUS "${VarToSet} = ${${VarToSet}}" )
-endif()
-endmacro(BFPSSetIfndef VarToSet PotentialValue)
diff --git a/cmake/LoadFFTW.cmake b/cmake/LoadFFTW.cmake
deleted file mode 100644
index 32e8c1aa..00000000
--- a/cmake/LoadFFTW.cmake
+++ /dev/null
@@ -1,17 +0,0 @@
-# Should define all the configure variables for a given Package.
-# MYPACK => The package name
-# ${MYPACK}_CONFIG_LIBS => The needed libraries
-# ${MYPACK}_CONFIG_HEADERS => The needed headers
-# ${MYPACK}_CONFIG_DEPS => A list of dependencies (in the Package name format)
-
-# This file should not be edited, but should be changed by setting
-# set( BFPS_ENABLE_MYPACK VALUE CACHE FILEPATH "" FORCE)
-# in the initialCache corresponding
-
-include(BFPSUtils)
-
-set(MYPACK FFTW)
-BFPSSetIfndef(${MYPACK}_CONFIG_LIBS "fftw3")
-BFPSSetIfndef(${MYPACK}_CONFIG_HEADERS "fftw3.h")
-
-
diff --git a/cmake/LoadHDF5.cmake b/cmake/LoadHDF5.cmake
deleted file mode 100644
index 1b930af7..00000000
--- a/cmake/LoadHDF5.cmake
+++ /dev/null
@@ -1,17 +0,0 @@
-# Should define all the configure variables for a given Package.
-# MYPACK => The package name
-# ${MYPACK}_CONFIG_LIBS => The needed libraries
-# ${MYPACK}_CONFIG_HEADERS => The needed headers
-# ${MYPACK}_CONFIG_DEPS => A list of dependencies (in the Package name format)
-
-# This file should not be edited, but should be changed by setting
-# set( BFPS_ENABLE_MYPACK VALUE CACHE FILEPATH "" FORCE)
-# in the initialCache corresponding
-
-include(BFPSUtils)
-
-set(MYPACK HDF5)
-BFPSSetIfndef(${MYPACK}_CONFIG_LIBS "hdf5")
-BFPSSetIfndef(${MYPACK}_CONFIG_HEADERS "hdf5.h")
-BFPSSetIfndef(${MYPACK}_CONFIG_DEPS "")
-
diff --git a/cmake/LoadSZIP.cmake b/cmake/LoadSZIP.cmake
deleted file mode 100644
index 2cf052a1..00000000
--- a/cmake/LoadSZIP.cmake
+++ /dev/null
@@ -1,17 +0,0 @@
-# Should define all the configure variables for a given Package.
-# MYPACK => The package name
-# ${MYPACK}_CONFIG_LIBS => The needed libraries
-# ${MYPACK}_CONFIG_HEADERS => The needed headers
-# ${MYPACK}_CONFIG_DEPS => A list of dependencies (in the Package name format)
-
-# This file should not be edited, but should be changed by setting
-# set( BFPS_ENABLE_MYPACK VALUE CACHE FILEPATH "" FORCE)
-# in the initialCache corresponding
-
-include(BFPSUtils)
-
-set(MYPACK SZIP)
-BFPSSetIfndef(${MYPACK}_CONFIG_LIBS "z")
-BFPSSetIfndef(${MYPACK}_CONFIG_HEADERS "")
-BFPSSetIfndef(${MYPACK}_CONFIG_DEPS "")
-
diff --git a/cmake/morse/FindCommon.cmake b/cmake/morse/FindCommon.cmake
new file mode 100644
index 00000000..95d8c1f5
--- /dev/null
+++ b/cmake/morse/FindCommon.cmake
@@ -0,0 +1,47 @@
+###
+#
+# @copyright (c) 2018 Inria. All rights reserved.
+#
+###
+#
+# @file FindCommon.cmake
+#
+# @project MORSE
+# MORSE is a software package provided by:
+# Inria Bordeaux - Sud-Ouest,
+# Univ. of Tennessee,
+# King Abdullah Univesity of Science and Technology
+# Univ. of California Berkeley,
+# Univ. of Colorado Denver.
+#
+# @version 1.0.0
+# @author Florent Pruvost
+# @date 13-04-2018
+#
+###
+
+# clean these variables before using them in CMAKE_REQUIRED_* variables in
+# check_function_exists
+macro(finds_remove_duplicates)
+ if (REQUIRED_DEFINITIONS)
+ list(REMOVE_DUPLICATES REQUIRED_DEFINITIONS)
+ endif()
+ if (REQUIRED_INCDIRS)
+ list(REMOVE_DUPLICATES REQUIRED_INCDIRS)
+ endif()
+ if (REQUIRED_FLAGS)
+ list(REMOVE_DUPLICATES REQUIRED_FLAGS)
+ endif()
+ if (REQUIRED_LDFLAGS)
+ list(REMOVE_DUPLICATES REQUIRED_LDFLAGS)
+ endif()
+ if (REQUIRED_LIBS)
+ list(REVERSE REQUIRED_LIBS)
+ list(REMOVE_DUPLICATES REQUIRED_LIBS)
+ list(REVERSE REQUIRED_LIBS)
+ endif()
+endmacro()
+
+##
+## @end file FindCommon
+##
diff --git a/cmake/morse/FindFFTW.cmake b/cmake/morse/FindFFTW.cmake
new file mode 100644
index 00000000..37450bae
--- /dev/null
+++ b/cmake/morse/FindFFTW.cmake
@@ -0,0 +1,832 @@
+###
+#
+# @copyright (c) 2009-2014 The University of Tennessee and The University
+# of Tennessee Research Foundation.
+# All rights reserved.
+# @copyright (c) 2012-2018 Inria. All rights reserved.
+# @copyright (c) 2012-2014 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
+#
+###
+#
+# - Find FFTW Version 3 include dirs and libraries
+# Default configuration will find the real double precision fftw library version
+# without THREADS|OMP.
+# Use this module by invoking find_package with the form:
+# find_package(FFTW
+# [REQUIRED] # Fail with error if fftw is not found
+# [COMPONENTS MKL]
+#
+# COMPONENTS can be some of the following:
+# - MKL: to detect the FFTW from Intel MKL
+# - ESSL: to detect the FFTW from IBM ESSL
+# - THREADS: to detect the Threads version of FFTW
+# - OMP: to detect the OpenMP version of FFTW
+# - SIMPLE: to detect the FFTW simple precision fftw3f
+# - LONG: to detect the FFTW long double precision fftw3l
+# - QUAD: to detect the FFTW quadruple precision fftw3q
+#
+# This module finds headers and fftw library.
+# Results are reported in variables:
+# FFTW_FOUND - True if headers and requested libraries were found
+# FFTW_CFLAGS_OTHER - fftw compiler flags without headers paths
+# FFTW_LDFLAGS_OTHER - fftw linker flags without libraries
+# FFTW_INCLUDE_DIRS - fftw include directories
+# FFTW_LIBRARY_DIRS - fftw link directories
+# FFTW_LIBRARIES - fftw libraries to be linked (absolute path)
+# FFTW_CFLAGS_OTHER_DEP - fftw + dependencies compiler flags without headers paths
+# FFTW_LDFLAGS_OTHER_DEP - fftw + dependencies linker flags without libraries
+# FFTW_INCLUDE_DIRS_DEP - fftw + dependencies include directories
+# FFTW_LIBRARY_DIRS_DEP - fftw + dependencies link directories
+# FFTW_LIBRARIES_DEP - fftw + dependencies libraries
+#
+# FFTW_FOUND_WITH_PKGCONFIG - True if found with pkg-config
+# if found with pkg-config the following variables are set
+# <PREFIX> = FFTW3F or FFTW3 or FFTW3L or FFTW3Q
+# <XPREFIX> = <PREFIX> for common case
+# <XPREFIX> = <PREFIX>_STATIC for static linking
+# <XPREFIX>_FOUND ... set to 1 if module(s) exist
+# <XPREFIX>_LIBRARIES ... only the libraries (w/o the '-l')
+# <XPREFIX>_LIBRARY_DIRS ... the paths of the libraries (w/o the '-L')
+# <XPREFIX>_LDFLAGS ... all required linker flags
+# <XPREFIX>_LDFLAGS_OTHER ... all other linker flags
+# <XPREFIX>_INCLUDE_DIRS ... the '-I' preprocessor flags (w/o the '-I')
+# <XPREFIX>_CFLAGS ... all required cflags
+# <XPREFIX>_CFLAGS_OTHER ... the other compiler flags
+#
+# The user can give specific paths where to find the libraries adding cmake
+# options at configure (ex: cmake path/to/project -DFFTW_DIR=path/to/fftw):
+# FFTW_DIR - Where to find the base directory of fftw
+# FFTW_INCDIR - Where to find the header files
+# FFTW_LIBDIR - Where to find the library files
+# The module can also look for the following environment variables if paths
+# are not given as cmake variable: FFTW_DIR, FFTW_INCDIR, FFTW_LIBDIR
+# For MKL case and if no paths are given as hints, we will try to use the MKLROOT
+# environment variable
+
+#=============================================================================
+# Copyright 2012-2018 Inria
+# Copyright 2012-2013 Emmanuel Agullo
+# Copyright 2012-2013 Mathieu Faverge
+# Copyright 2012 Cedric Castagnede
+# Copyright 2013-2018 Florent Pruvost
+#
+# Distributed under the OSI-approved BSD License (the "License");
+# see accompanying file MORSE-Copyright.txt for details.
+#
+# This software is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the License for more information.
+#=============================================================================
+# (To distribute this file outside of Morse, substitute the full
+# License text for the above reference.)
+
+# Common macros to use in finds
+include(FindInit)
+
+if (NOT FFTW_FOUND)
+ set(FFTW_DIR "" CACHE PATH "Installation directory of FFTW library given by user")
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "A cache variable, namely FFTW_DIR, has been set to specify the install directory of FFTW")
+ endif()
+endif()
+
+# Set the version to find
+set(FFTW_LOOK_FOR_MKL OFF)
+set(FFTW_LOOK_FOR_ESSL OFF)
+set(FFTW_LOOK_FOR_THREADS OFF)
+set(FFTW_LOOK_FOR_OMP OFF)
+set(FFTW_LOOK_FOR_FFTW_SIMPLE OFF)
+set(FFTW_LOOK_FOR_FFTW_LONG OFF)
+set(FFTW_LOOK_FOR_FFTW_QUAD OFF)
+
+if( FFTW_FIND_COMPONENTS )
+ foreach( component ${FFTW_FIND_COMPONENTS} )
+ if (${component} STREQUAL "THREADS")
+ # means we look for the Threads version of FFTW
+ set(FFTW_LOOK_FOR_THREADS ON)
+ endif()
+ if (${component} STREQUAL "OMP")
+ # means we look for the OpenMP version of FFTW
+ set(FFTW_LOOK_FOR_OMP ON)
+ endif()
+ if (${component} STREQUAL "SIMPLE")
+ # means we look for FFTW simple precision (fftw3f)
+ set(FFTW_LOOK_FOR_FFTW_SIMPLE ON)
+ set(FFTW_LOOK_FOR_FFTW_LONG OFF)
+ set(FFTW_LOOK_FOR_FFTW_QUAD OFF)
+ endif()
+ if (${component} STREQUAL "LONG")
+ # means we look for FFTW long double precision (fftw3l)
+ set(FFTW_LOOK_FOR_FFTW_SIMPLE OFF)
+ set(FFTW_LOOK_FOR_FFTW_LONG ON)
+ set(FFTW_LOOK_FOR_FFTW_QUAD OFF)
+ endif()
+ if (${component} STREQUAL "QUAD")
+ # means we look for FFTW quad precision (fftw3q)
+ set(FFTW_LOOK_FOR_FFTW_SIMPLE OFF)
+ set(FFTW_LOOK_FOR_FFTW_LONG OFF)
+ set(FFTW_LOOK_FOR_FFTW_QUAD ON)
+ endif()
+ if (${component} STREQUAL "MKL")
+ # means we look for the Intel MKL version of FFTW
+ set(FFTW_LOOK_FOR_MKL ON)
+ if (FFTW_LOOK_FOR_FFTW_LONG)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(WARNING "Looking for FFTW -- long precision functions do not exist in MKL FFTW")
+ endif()
+ set(FFTW_LOOK_FOR_FFTW_LONG OFF)
+ endif()
+ if (FFTW_LOOK_FOR_FFTW_QUAD)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(WARNING "Looking for FFTW -- quadruple functions do not exist in MKL FFTW")
+ endif()
+ set(FFTW_LOOK_FOR_FFTW_QUAD OFF)
+ endif()
+ endif()
+ if (${component} STREQUAL "ESSL")
+ # means we look for the Intel MKL version of FFTW
+ set(FFTW_LOOK_FOR_ESSL ON)
+ if (FFTW_LOOK_FOR_FFTW_LONG)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(WARNING "Looking for FFTW -- long precision functions do not exist in FFTW_ESSL")
+ endif()
+ set(FFTW_LOOK_FOR_FFTW_LONG OFF)
+ endif()
+ if (FFTW_LOOK_FOR_FFTW_QUAD)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(WARNING "Looking for FFTW -- quadruple functions do not exist in FFTW_ESSL")
+ endif()
+ set(FFTW_LOOK_FOR_FFTW_QUAD OFF)
+ endif()
+ if (FFTW_LOOK_FOR_OMP)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(WARNING "Looking for FFTW -- FFTW_ESSL does not use OpenMP")
+ endif()
+ set(FFTW_LOOK_FOR_OMP OFF)
+ endif()
+ endif()
+ endforeach()
+endif()
+
+if (FFTW_LOOK_FOR_THREADS)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "FFTW looks for threads")
+ endif()
+ if (FFTW_FIND_REQUIRED AND FFTW_FIND_REQUIRED_THREADS)
+ find_package(Threads REQUIRED)
+ else()
+ find_package(Threads)
+ endif()
+endif()
+
+if (FFTW_LOOK_FOR_OMP)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "FFTW looks for openmp")
+ endif()
+ if (FFTW_FIND_REQUIRED AND FFTW_FIND_REQUIRED_OMP)
+ find_package(OpenMP REQUIRED)
+ else()
+ find_package(OpenMP)
+ endif()
+endif()
+
+if (FFTW_LOOK_FOR_MKL)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "FFTW looks for threads and Intel MKL")
+ endif()
+ if (FFTW_LOOK_FOR_THREADS)
+ set(BLA_VENDOR "Intel10_64lp")
+ else()
+ set(BLA_VENDOR "Intel10_64lp_seq")
+ endif()
+ if (FFTW_FIND_REQUIRED AND FFTW_FIND_REQUIRED_MKL)
+ find_package(Threads REQUIRED)
+ find_package(BLAS REQUIRED)
+ else()
+ find_package(Threads)
+ find_package(BLAS)
+ endif()
+endif()
+
+if (FFTW_LOOK_FOR_ESSL)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "FFTW looks for IBM ESSL")
+ endif()
+ if (FFTW_LOOK_FOR_THREADS)
+ set(BLA_VENDOR "IBMESSLMT")
+ else()
+ set(BLA_VENDOR "IBMESSL")
+ endif()
+ if (FFTW_FIND_REQUIRED AND FFTW_FIND_REQUIRED_ESSL)
+ find_package(BLAS REQUIRED)
+ else()
+ find_package(BLAS)
+ endif()
+endif()
+
+
+if( THREADS_FOUND )
+ libraries_absolute_path(CMAKE_THREAD_LIBS_INIT "")
+endif ()
+
+set(ENV_FFTW_DIR "$ENV{FFTW_DIR}")
+set(ENV_FFTW_INCDIR "$ENV{FFTW_INCDIR}")
+set(ENV_FFTW_LIBDIR "$ENV{FFTW_LIBDIR}")
+set(FFTW_GIVEN_BY_USER "FALSE")
+if ( FFTW_DIR OR ( FFTW_INCDIR AND FFTW_LIBDIR) OR ENV_FFTW_DIR OR (ENV_FFTW_INCDIR AND ENV_FFTW_LIBDIR) )
+ set(FFTW_GIVEN_BY_USER "TRUE")
+endif()
+
+
+# Optionally use pkg-config to detect include/library dirs (if pkg-config is available)
+# -------------------------------------------------------------------------------------
+if (NOT FFTW_LOOK_FOR_MKL AND NOT FFTW_LOOK_FOR_ESSL)
+ include(FindPkgConfig)
+ find_package(PkgConfig QUIET)
+ if( PKG_CONFIG_EXECUTABLE AND NOT FFTW_GIVEN_BY_USER )
+
+ set(FFTW_INCLUDE_DIRS)
+ set(FFTW_LIBRARY_DIRS)
+ set(FFTW_LIBRARIES)
+
+ if(FFTW_LOOK_FOR_FFTW_SIMPLE)
+ pkg_search_module(FFTW3F fftw3f)
+ pkg_search_module(FFTW3 fftw3)
+ if (FFTW3F_FOUND)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "Looking for FFTW3F - found using PkgConfig")
+ endif()
+ if (FFTW3F_LIBRARIES)
+ find_pkgconfig_libraries_absolute_path(FFTW3F)
+ list(APPEND FFTW_LIBRARIES "${FFTW3F_LIBRARIES}")
+ endif()
+ if(FFTW3F_INCLUDE_DIRS)
+ list(APPEND FFTW_INCLUDE_DIRS "${FFTW3F_INCLUDE_DIRS}")
+ else()
+ if (NOT FFTW_FIND_QUIETLY)
+ message(WARNING "FFTW3F_INCLUDE_DIRS is empty using PkgConfig."
+ "Perhaps the path to fftw3f headers is already present in your"
+ "CPATH/C(PLUS)_INCLUDE_PATH environment variables.")
+ endif()
+ endif()
+ if(FFTW3F_LIBRARY_DIRS)
+ list(APPEND FFTW_LIBRARY_DIRS "${FFTW3F_LIBRARY_DIRS}")
+ endif()
+ else(FFTW3F_FOUND)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "Looking for FFTW3F - not found using PkgConfig."
+ "\n Perhaps you should add the directory containing fftw3f.pc to"
+ "\n the PKG_CONFIG_PATH environment variable.")
+ endif()
+ endif(FFTW3F_FOUND)
+ elseif(FFTW_LOOK_FOR_FFTW_LONG)
+ pkg_search_module(FFTW3L fftw3l)
+ pkg_search_module(FFTW3 fftw3)
+ if (FFTW3L_FOUND)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "Looking for FFTW3L - found using PkgConfig")
+ endif()
+ if (FFTW3L_LIBRARIES)
+ find_pkgconfig_libraries_absolute_path(FFTW3L)
+ list(APPEND FFTW_LIBRARIES "${FFTW3L_LIBRARIES}")
+ endif()
+ if(FFTW3L_INCLUDE_DIRS)
+ list(APPEND FFTW_INCLUDE_DIRS "${FFTW3L_INCLUDE_DIRS}")
+ else()
+ if (NOT FFTW_FIND_QUIETLY)
+ message(WARNING "FFTW3L_INCLUDE_DIRS is empty using PkgConfig."
+ "Perhaps the path to fftw3l headers is already present in your"
+ "CPATH/C(PLUS)_INCLUDE_PATH environment variables.")
+ endif()
+ endif()
+ if(FFTW3L_LIBRARY_DIRS)
+ list(APPEND FFTW_LIBRARY_DIRS "${FFTW3L_LIBRARY_DIRS}")
+ endif()
+ else(FFTW3L_FOUND)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "Looking for FFTW3L - not found using PkgConfig."
+ "\n Perhaps you should add the directory containing fftw3l.pc to"
+ "\n the PKG_CONFIG_PATH environment variable.")
+ endif()
+ endif(FFTW3L_FOUND)
+ elseif(FFTW_LOOK_FOR_FFTW_QUAD)
+ pkg_search_module(FFTW3Q fftw3q)
+ pkg_search_module(FFTW3 fftw3)
+ if (FFTW3Q_FOUND)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "Looking for FFTW3Q - found using PkgConfig")
+ endif()
+ if (FFTW3Q_LIBRARIES)
+ find_pkgconfig_libraries_absolute_path(FFTW3Q)
+ list(APPEND FFTW_LIBRARIES "${FFTW3Q_LIBRARIES}")
+ endif()
+ if(FFTW3Q_INCLUDE_DIRS)
+ list(APPEND FFTW_INCLUDE_DIRS "${FFTW3Q_INCLUDE_DIRS}")
+ else()
+ if (NOT FFTW_FIND_QUIETLY)
+ message(WARNING "FFTW3Q_INCLUDE_DIRS is empty using PkgConfig."
+ "Perhaps the path to fftw3q headers is already present in your"
+ "CPATH/C(PLUS)_INCLUDE_PATH environment variables.")
+ endif()
+ endif()
+ if(FFTW3Q_LIBRARY_DIRS)
+ list(APPEND FFTW_LIBRARY_DIRS "${FFTW3Q_LIBRARY_DIRS}")
+ endif()
+ else(FFTW3Q_FOUND)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "Looking for FFTW3Q - not found using PkgConfig."
+ "\n Perhaps you should add the directory containing fftw3q.pc to"
+ "\n the PKG_CONFIG_PATH environment variable.")
+ endif()
+ endif(FFTW3Q_FOUND)
+ else()
+ pkg_search_module(FFTW3 fftw3)
+ if (FFTW3_FOUND AND FFTW3_LIBRARIES)
+ find_pkgconfig_libraries_absolute_path(FFTW3)
+ endif()
+ endif()
+ if (FFTW3_FOUND)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "Looking for FFTW3 - found using PkgConfig")
+ endif()
+ if (FFTW3_LIBRARIES)
+ find_pkgconfig_libraries_absolute_path(FFTW3)
+ list(APPEND FFTW_LIBRARIES "${FFTW3_LIBRARIES}")
+ endif()
+ if(FFTW3_INCLUDE_DIRS)
+ list(APPEND FFTW_INCLUDE_DIRS "${FFTW3_INCLUDE_DIRS}")
+ else()
+ if (NOT FFTW_FIND_QUIETLY)
+ message(WARNING "FFTW3_INCLUDE_DIRS is empty using PkgConfig."
+ "Perhaps the path to fftw3 headers is already present in your"
+ "CPATH/C(PLUS)_INCLUDE_PATH environment variables.")
+ endif()
+ endif()
+ if(FFTW3_LIBRARY_DIRS)
+ list(APPEND FFTW_LIBRARY_DIRS "${FFTW3_LIBRARY_DIRS}")
+ endif()
+ else(FFTW3_FOUND)
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "Looking for FFTW3 - not found using PkgConfig."
+ "\n Perhaps you should add the directory containing fftw3.pc to"
+ "\n the PKG_CONFIG_PATH environment variable.")
+ endif()
+ endif(FFTW3_FOUND)
+
+ if (FFTW_FOUND AND FFTW_LIBRARIES)
+ set(FFTW_FOUND_WITH_PKGCONFIG "TRUE")
+ else()
+ set(FFTW_FOUND_WITH_PKGCONFIG "FALSE")
+ endif()
+
+ endif( PKG_CONFIG_EXECUTABLE AND NOT FFTW_GIVEN_BY_USER )
+
+endif(NOT FFTW_LOOK_FOR_MKL AND NOT FFTW_LOOK_FOR_ESSL)
+
+if( (NOT PKG_CONFIG_EXECUTABLE) OR
+ (PKG_CONFIG_EXECUTABLE AND NOT FFTW_FOUND) OR
+ FFTW_GIVEN_BY_USER OR
+ FFTW_LOOK_FOR_MKL OR
+ FFTW_LOOK_FOR_ESSL
+ )
+
+ # Looking for include
+ # -------------------
+
+ # Add system include paths to search include
+ # ------------------------------------------
+ unset(_inc_env)
+ set(ENV_MKLROOT "$ENV{MKLROOT}")
+ set(ENV_FFTW_DIR "$ENV{FFTW_DIR}")
+ set(ENV_FFTW_INCDIR "$ENV{FFTW_INCDIR}")
+ if(ENV_FFTW_INCDIR)
+ list(APPEND _inc_env "${ENV_FFTW_INCDIR}")
+ elseif(ENV_FFTW_DIR)
+ list(APPEND _inc_env "${ENV_FFTW_DIR}")
+ list(APPEND _inc_env "${ENV_FFTW_DIR}/include")
+ list(APPEND _inc_env "${ENV_FFTW_DIR}/include/fftw")
+ else()
+ if (ENV_MKLROOT)
+ list(APPEND _inc_env "${ENV_MKLROOT}/include/fftw")
+ endif()
+ # system variables
+ if(WIN32)
+ string(REPLACE ":" ";" _path_env "$ENV{INCLUDE}")
+ list(APPEND _inc_env "${_path_env}")
+ else()
+ string(REPLACE ":" ";" _path_env "$ENV{INCLUDE}")
+ list(APPEND _inc_env "${_path_env}")
+ string(REPLACE ":" ";" _path_env "$ENV{C_INCLUDE_PATH}")
+ list(APPEND _inc_env "${_path_env}")
+ string(REPLACE ":" ";" _path_env "$ENV{CPATH}")
+ list(APPEND _inc_env "${_path_env}")
+ string(REPLACE ":" ";" _path_env "$ENV{INCLUDE_PATH}")
+ list(APPEND _inc_env "${_path_env}")
+ endif()
+ endif()
+ list(APPEND _inc_env "${CMAKE_C_IMPLICIT_INCLUDE_DIRECTORIES}")
+ list(REMOVE_DUPLICATES _inc_env)
+
+ # set paths where to look for
+ set(PATH_TO_LOOK_FOR "${_inc_env}")
+
+ if (FFTW_LOOK_FOR_ESSL)
+ set(FFTW3_HEADER_TO_FIND "fftw3_essl.h")
+ else()
+ set(FFTW3_HEADER_TO_FIND "fftw3.h")
+ endif()
+
+ # Try to find the fftw header in the given paths
+ # -------------------------------------------------
+ # call cmake macro to find the header path
+ if(FFTW_INCDIR)
+ set(FFTW_${FFTW3_HEADER_TO_FIND}_DIRS "FFTW_${FFTW3_HEADER_TO_FIND}_DIRS-NOTFOUND")
+ find_path(FFTW_${FFTW3_HEADER_TO_FIND}_DIRS
+ NAMES ${FFTW3_HEADER_TO_FIND}
+ HINTS ${FFTW_INCDIR})
+ else()
+ if(FFTW_DIR)
+ set(FFTW_${FFTW3_HEADER_TO_FIND}_DIRS "FFTW_${FFTW3_HEADER_TO_FIND}_DIRS-NOTFOUND")
+ find_path(FFTW_${FFTW3_HEADER_TO_FIND}_DIRS
+ NAMES ${FFTW3_HEADER_TO_FIND}
+ HINTS ${FFTW_DIR}
+ PATH_SUFFIXES "include" "include/fftw")
+ else()
+ set(FFTW_${FFTW3_HEADER_TO_FIND}_DIRS "FFTW_${FFTW3_HEADER_TO_FIND}_DIRS-NOTFOUND")
+ find_path(FFTW_${FFTW3_HEADER_TO_FIND}_DIRS
+ NAMES ${FFTW3_HEADER_TO_FIND}
+ HINTS ${PATH_TO_LOOK_FOR}
+ PATH_SUFFIXES "fftw")
+ endif()
+ endif()
+ mark_as_advanced(FFTW_${FFTW3_HEADER_TO_FIND}_DIRS)
+
+ # Add path to cmake variable
+ # ------------------------------------
+ if (FFTW_${FFTW3_HEADER_TO_FIND}_DIRS)
+ set(FFTW_INCLUDE_DIRS "${FFTW_${FFTW3_HEADER_TO_FIND}_DIRS}")
+ else ()
+ set(FFTW_INCLUDE_DIRS "FFTW_INCLUDE_DIRS-NOTFOUND")
+ if(NOT FFTW_FIND_QUIETLY)
+ message(STATUS "Looking for FFTW -- ${FFTW3_HEADER_TO_FIND} not found")
+ endif()
+ endif ()
+
+
+ # Looking for lib
+ # ---------------
+
+ # Add system library paths to search lib
+ # --------------------------------------
+ unset(_lib_env)
+ set(ENV_FFTW_LIBDIR "$ENV{FFTW_LIBDIR}")
+ if(ENV_FFTW_LIBDIR)
+ list(APPEND _lib_env "${ENV_FFTW_LIBDIR}")
+ elseif(ENV_FFTW_DIR)
+ list(APPEND _lib_env "${ENV_FFTW_DIR}")
+ list(APPEND _lib_env "${ENV_FFTW_DIR}/lib")
+ if("${CMAKE_SIZEOF_VOID_P}" EQUAL "8")
+ list(APPEND _lib_env "${ENV_FFTW_DIR}/lib64")
+ list(APPEND _lib_env "${ENV_FFTW_DIR}/lib/intel64")
+ else()
+ list(APPEND _lib_env "${ENV_FFTW_DIR}/lib32")
+ list(APPEND _lib_env "${ENV_FFTW_DIR}/lib/ia32")
+ endif()
+ else()
+ if (ENV_MKLROOT)
+ list(APPEND _lib_env "${ENV_MKLROOT}/lib")
+ if("${CMAKE_SIZEOF_VOID_P}" EQUAL "8")
+ list(APPEND _lib_env "${ENV_MKLROOT}/lib64")
+ list(APPEND _lib_env "${ENV_MKLROOT}/lib/intel64")
+ else()
+ list(APPEND _lib_env "${ENV_MKLROOT}/lib32")
+ list(APPEND _lib_env "${ENV_MKLROOT}/lib/ia32")
+ endif()
+ endif()
+ list(APPEND _lib_env "$ENV{LIBRARY_PATH}")
+ if(WIN32)
+ string(REPLACE ":" ";" _lib_env2 "$ENV{LIB}")
+ elseif(APPLE)
+ string(REPLACE ":" ";" _lib_env2 "$ENV{DYLD_LIBRARY_PATH}")
+ else()
+ string(REPLACE ":" ";" _lib_env2 "$ENV{LD_LIBRARY_PATH}")
+ endif()
+ list(APPEND _lib_env "${_lib_env2}")
+ list(APPEND _lib_env "${CMAKE_C_IMPLICIT_LINK_DIRECTORIES}")
+ endif()
+ list(REMOVE_DUPLICATES _lib_env)
+
+ # set paths where to look for
+ set(PATH_TO_LOOK_FOR "${_lib_env}")
+
+ if(FFTW_LOOK_FOR_FFTW_SIMPLE)
+ set(FFTW_PREC "f")
+ set(FFTW_PREC_TESTFUNC "s")
+ elseif(FFTW_LOOK_FOR_FFTW_LONG)
+ set(FFTW_PREC "l")
+ set(FFTW_PREC_TESTFUNC "l")
+ elseif(FFTW_LOOK_FOR_FFTW_QUAD)
+ set(FFTW_PREC "q")
+ set(FFTW_PREC_TESTFUNC "q")
+ else()
+ set(FFTW_PREC "")
+ set(FFTW_PREC_TESTFUNC "d")
+ endif()
+
+ set(FFTW_LIBRARIES "")
+ set(FFTW_LIBRARY_DIRS "")
+
+ if(NOT FFTW_LOOK_FOR_MKL)
+
+ if (FFTW_LOOK_FOR_THREADS)
+ set(FFTW_libs_to_find "fftw3${FFTW_PREC}_threads;fftw3${FFTW_PREC};fftw3")
+ elseif (FFTW_LOOK_FOR_OMP)
+ set(FFTW_libs_to_find "fftw3${FFTW_PREC}_omp;fftw3${FFTW_PREC};fftw3")
+ else()
+ set(FFTW_libs_to_find "fftw3${FFTW_PREC};fftw3")
+ endif()
+ if (FFTW_LOOK_FOR_FFTW_QUAD)
+ if (NOT FFTW_LOOK_FOR_MKL AND NOT FFTW_LOOK_FOR_ESSL)
+ list(APPEND FFTW_libs_to_find "quadmath")
+ endif()
+ endif()
+
+ if (FFTW_LOOK_FOR_ESSL)
+ set(FFTW_libs_to_find "fftw3_essl")
+ endif()
+
+ # Try to find the fftw lib in the given paths
+ # ----------------------------------------------
+
+ # call cmake macro to find the lib path
+ if(FFTW_LIBDIR)
+ foreach(fftw_lib ${FFTW_libs_to_find})
+ set(FFTW_${fftw_lib}_LIBRARY "FFTW_${fftw_lib}_LIBRARY-NOTFOUND")
+ find_library(FFTW_${fftw_lib}_LIBRARY
+ NAMES ${fftw_lib}
+ HINTS ${FFTW_LIBDIR})
+ endforeach()
+ else()
+ if(FFTW_DIR)
+ foreach(fftw_lib ${FFTW_libs_to_find})
+ set(FFTW_${fftw_lib}_LIBRARY "FFTW_${fftw_lib}_LIBRARY-NOTFOUND")
+ find_library(FFTW_${fftw_lib}_LIBRARY
+ NAMES ${fftw_lib}
+ HINTS ${FFTW_DIR}
+ PATH_SUFFIXES lib lib32 lib64)
+ endforeach()
+ else()
+ foreach(fftw_lib ${FFTW_libs_to_find})
+ set(FFTW_${fftw_lib}_LIBRARY "FFTW_${fftw_lib}_LIBRARY-NOTFOUND")
+ find_library(FFTW_${fftw_lib}_LIBRARY
+ NAMES ${fftw_lib}
+ HINTS ${PATH_TO_LOOK_FOR})
+ endforeach()
+ endif()
+ endif()
+
+ # If found, add path to cmake variable
+ # ------------------------------------
+ foreach(fftw_lib ${FFTW_libs_to_find})
+
+ if (FFTW_${fftw_lib}_LIBRARY)
+ get_filename_component(${fftw_lib}_lib_path "${FFTW_${fftw_lib}_LIBRARY}" PATH)
+ # set cmake variables
+ list(APPEND FFTW_LIBRARIES "${FFTW_${fftw_lib}_LIBRARY}")
+ list(APPEND FFTW_LIBRARY_DIRS "${${fftw_lib}_lib_path}")
+ else ()
+ list(APPEND FFTW_LIBRARIES "${FFTW_${fftw_lib}_LIBRARY}")
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "Looking for FFTW -- lib ${fftw_lib} not found")
+ endif()
+ endif ()
+ mark_as_advanced(FFTW_${fftw_lib}_LIBRARY)
+
+ endforeach()
+
+ # check if one lib is NOTFOUND
+ foreach(lib ${FFTW_LIBRARIES})
+ if (NOT lib)
+ set(FFTW_LIBRARIES "FFTW_LIBRARIES-NOTFOUND")
+ endif()
+ endforeach()
+
+ endif(NOT FFTW_LOOK_FOR_MKL)
+
+ if (FFTW_LOOK_FOR_MKL OR FFTW_LOOK_FOR_ESSL)
+
+ # FFTW relies on blas libs
+ if (FFTW_LOOK_FOR_THREADS)
+ if (FFTW_LOOK_FOR_MKL)
+ if (BLAS_LIBRARIES_PAR)
+ list(APPEND FFTW_LIBRARIES "${BLAS_LIBRARIES_PAR}")
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "Multithreaded FFTW has been found: ${FFTW_LIBRARIES}")
+ endif()
+ else()
+ if (NOT FFTW_FIND_QUIETLY)
+ if (FFTW_FIND_REQUIRED AND FFTW_FIND_REQUIRED_MKL)
+ message(FATAL_ERROR "FFTW is required but not found.")
+ else()
+ message(STATUS "Multithreaded FFTW not found.")
+ endif()
+ endif()
+ endif(BLAS_LIBRARIES_PAR)
+ elseif (FFTW_LOOK_FOR_ESSL)
+ if (FFTW_LIBRARIES AND BLAS_LIBRARIES_PAR)
+ list(APPEND FFTW_LIBRARIES "${BLAS_LIBRARIES_PAR}")
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "Multithreaded FFTW has been found: ${FFTW_LIBRARIES}")
+ endif()
+ else()
+ if (NOT FFTW_FIND_QUIETLY)
+ if (FFTW_FIND_REQUIRED AND FFTW_FIND_REQUIRED_MKL)
+ message(FATAL_ERROR "FFTW is required but not found.")
+ else()
+ message(STATUS "Multithreaded FFTW not found.")
+ endif()
+ endif()
+ endif(FFTW_LIBRARIES AND BLAS_LIBRARIES_PAR)
+ endif()
+ else(FFTW_LOOK_FOR_THREADS)
+ if (FFTW_LOOK_FOR_MKL)
+ if (BLAS_LIBRARIES_SEQ)
+ list(APPEND FFTW_LIBRARIES "${BLAS_LIBRARIES_SEQ}")
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "FFTW has been found: ${FFTW_LIBRARIES}")
+ endif()
+ else()
+ if (NOT FFTW_FIND_QUIETLY)
+ if (FFTW_FIND_REQUIRED AND FFTW_FIND_REQUIRED_MKL)
+ message(FATAL_ERROR "FFTW is required but not found.")
+ else()
+ message(STATUS "FFTW not found.")
+ endif()
+ endif()
+ endif(BLAS_LIBRARIES_SEQ)
+ elseif (FFTW_LOOK_FOR_ESSL)
+ if (FFTW_LIBRARIES AND BLAS_LIBRARIES_SEQ)
+ list(APPEND FFTW_LIBRARIES "${BLAS_LIBRARIES_SEQ}")
+ if (NOT FFTW_FIND_QUIETLY)
+ message(STATUS "FFTW has been found: ${FFTW_LIBRARIES}")
+ endif()
+ else()
+ if (NOT FFTW_FIND_QUIETLY)
+ if (FFTW_FIND_REQUIRED AND FFTW_FIND_REQUIRED_MKL)
+ message(FATAL_ERROR "FFTW is required but not found.")
+ else()
+ message(STATUS "FFTW not found.")
+ endif()
+ endif()
+ endif(FFTW_LIBRARIES AND BLAS_LIBRARIES_SEQ)
+ endif()
+ endif(FFTW_LOOK_FOR_THREADS)
+
+ if (BLAS_LIBRARY_DIRS)
+ list(APPEND FFTW_LIBRARY_DIRS "${BLAS_LIBRARY_DIRS}")
+ else()
+ if (NOT FFTW_FIND_QUIETLY)
+ message(WARNING "FFTW_LIBRARY_DIRS may not be complete because BLAS_LIBRARY_DIRS is empty.")
+ endif()
+ endif()
+
+ endif(FFTW_LOOK_FOR_MKL OR FFTW_LOOK_FOR_ESSL)
+
+ list(REMOVE_DUPLICATES FFTW_INCLUDE_DIRS)
+ list(REMOVE_DUPLICATES FFTW_LIBRARY_DIRS)
+
+ # check if one lib is NOTFOUND
+ foreach(lib ${FFTW_LIBRARIES})
+ if (NOT lib)
+ set(FFTW_LIBRARIES "FFTW_LIBRARIES-NOTFOUND")
+ endif()
+ endforeach()
+
+endif( (NOT PKG_CONFIG_EXECUTABLE) OR
+ (PKG_CONFIG_EXECUTABLE AND NOT FFTW_FOUND) OR
+ FFTW_GIVEN_BY_USER OR
+ FFTW_LOOK_FOR_MKL OR
+ FFTW_LOOK_FOR_ESSL
+ )
+
+# check a function to validate the find
+if(FFTW_LIBRARIES)
+
+ set(REQUIRED_FLAGS)
+ set(REQUIRED_LDFLAGS)
+ set(REQUIRED_INCDIRS)
+ set(REQUIRED_LIBDIRS)
+ set(REQUIRED_LIBS)
+
+ # FFTW
+ if (FFTW_INCLUDE_DIRS)
+ set(REQUIRED_INCDIRS "${FFTW_INCLUDE_DIRS}")
+ endif()
+ if (FFTW_CFLAGS_OTHER)
+ set(REQUIRED_FLAGS "${FFTW_CFLAGS_OTHER}")
+ endif()
+ if (FFTW_LDFLAGS_OTHER)
+ set(REQUIRED_LDFLAGS "${FFTW_LDFLAGS_OTHER}")
+ endif()
+ if (FFTW_LIBRARY_DIRS)
+ set(REQUIRED_LIBDIRS "${FFTW_LIBRARY_DIRS}")
+ endif()
+ set(REQUIRED_LIBS "${FFTW_LIBRARIES}")
+ # THREADS
+ if (FFTW_LOOK_FOR_THREADS)
+ list(APPEND REQUIRED_LIBS "${CMAKE_THREAD_LIBS_INIT}")
+ endif()
+ # OMP
+ if(FFTW_LOOK_FOR_OMP)
+ list(APPEND REQUIRED_FLAGS "${OPENMP_C_FLAGS}")
+ endif()
+ # MKL
+ if(FFTW_LOOK_FOR_MKL)
+ list(APPEND REQUIRED_LIBS "${CMAKE_THREAD_LIBS_INIT}")
+ if (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND CMAKE_SYSTEM_NAME STREQUAL "Linux")
+ list(APPEND REQUIRED_LDFLAGS "-Wl,--no-as-needed")
+ endif()
+ endif()
+ # m
+ find_library(M_LIBRARY NAMES m)
+ mark_as_advanced(M_LIBRARY)
+ if(M_LIBRARY)
+ list(APPEND REQUIRED_LIBS "-lm")
+ endif()
+
+ # set required libraries for link
+ set(CMAKE_REQUIRED_INCLUDES "${REQUIRED_INCDIRS}")
+ if (REQUIRED_FLAGS)
+ set(REQUIRED_FLAGS_COPY "${REQUIRED_FLAGS}")
+ set(REQUIRED_FLAGS)
+ set(REQUIRED_DEFINITIONS)
+ foreach(_flag ${REQUIRED_FLAGS_COPY})
+ if (_flag MATCHES "^-D")
+ list(APPEND REQUIRED_DEFINITIONS "${_flag}")
+ endif()
+ string(REGEX REPLACE "^-D.*" "" _flag "${_flag}")
+ list(APPEND REQUIRED_FLAGS "${_flag}")
+ endforeach()
+ endif()
+ finds_remove_duplicates()
+ set(CMAKE_REQUIRED_DEFINITIONS "${REQUIRED_DEFINITIONS}")
+ set(CMAKE_REQUIRED_FLAGS "${REQUIRED_FLAGS}")
+ set(CMAKE_REQUIRED_LIBRARIES)
+ list(APPEND CMAKE_REQUIRED_LIBRARIES "${REQUIRED_LDFLAGS}")
+ list(APPEND CMAKE_REQUIRED_LIBRARIES "${REQUIRED_LIBS}")
+ list(APPEND CMAKE_REQUIRED_FLAGS "${REQUIRED_FLAGS}")
+ string(REGEX REPLACE "^ -" "-" CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES}")
+
+ # test link
+ unset(FFTW_WORKS CACHE)
+ include(CheckFunctionExists)
+ if (FFTW_LOOK_FOR_ESSL)
+ check_function_exists(${FFTW_PREC_TESTFUNC}fftw_execute FFTW_WORKS)
+ else()
+ check_function_exists(${FFTW_PREC_TESTFUNC}fftw_execute_ FFTW_WORKS)
+ endif()
+ mark_as_advanced(FFTW_WORKS)
+
+ if(FFTW_WORKS)
+ # save link with dependencies
+ set(FFTW_LIBRARIES_DEP "${REQUIRED_LIBS}")
+ set(FFTW_LIBRARY_DIRS_DEP "${REQUIRED_LIBDIRS}")
+ set(FFTW_INCLUDE_DIRS_DEP "${REQUIRED_INCDIRS}")
+ set(FFTW_CFLAGS_OTHER_DEP "${REQUIRED_FLAGS}")
+ set(FFTW_LDFLAGS_OTHER_DEP "${REQUIRED_LDFLAGS}")
+ else()
+ if(NOT FFTW_FIND_QUIETLY)
+ message(STATUS "Looking for FFTW : test of ${FFTW_PREC_TESTFUNC}fftw_execute_ with fftw library fails")
+ message(STATUS "CMAKE_REQUIRED_LIBRARIES: ${CMAKE_REQUIRED_LIBRARIES}")
+ message(STATUS "CMAKE_REQUIRED_INCLUDES: ${CMAKE_REQUIRED_INCLUDES}")
+ message(STATUS "CMAKE_REQUIRED_FLAGS: ${CMAKE_REQUIRED_FLAGS}")
+ message(STATUS "Check in CMakeFiles/CMakeError.log to figure out why it fails")
+ endif()
+ endif()
+ set(CMAKE_REQUIRED_INCLUDES)
+ set(CMAKE_REQUIRED_FLAGS)
+ set(CMAKE_REQUIRED_LIBRARIES)
+endif(FFTW_LIBRARIES)
+
+if (FFTW_LIBRARIES)
+ list(GET FFTW_LIBRARIES 0 first_lib)
+ get_filename_component(first_lib_path "${first_lib}" PATH)
+ if (NOT FFTW_LIBRARY_DIRS)
+ set(FFTW_LIBRARY_DIRS "${first_lib_path}")
+ endif()
+ if (${first_lib_path} MATCHES "(/lib(32|64)?$)|(/lib/intel64$|/lib/ia32$)")
+ string(REGEX REPLACE "(/lib(32|64)?$)|(/lib/intel64$|/lib/ia32$)" "" not_cached_dir "${first_lib_path}")
+ set(FFTW_DIR_FOUND "${not_cached_dir}" CACHE PATH "Installation directory of FFTW library" FORCE)
+ else()
+ set(FFTW_DIR_FOUND "${first_lib_path}" CACHE PATH "Installation directory of FFTW library" FORCE)
+ endif()
+endif()
+mark_as_advanced(FFTW_DIR)
+mark_as_advanced(FFTW_DIR_FOUND)
+
+# check that FFTW has been found
+# -------------------------------
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(FFTW DEFAULT_MSG
+ FFTW_LIBRARIES
+ FFTW_WORKS)
diff --git a/cmake/morse/FindHeadersAndLibs.cmake b/cmake/morse/FindHeadersAndLibs.cmake
new file mode 100644
index 00000000..64144bdb
--- /dev/null
+++ b/cmake/morse/FindHeadersAndLibs.cmake
@@ -0,0 +1,94 @@
+###
+#
+# @copyright (c) 2009-2014 The University of Tennessee and The University
+# of Tennessee Research Foundation.
+# All rights reserved.
+# @copyright (c) 2012-2014 Inria. All rights reserved.
+# @copyright (c) 2012-2014 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
+#
+###
+#
+# @file FindHeadersAndLibs.cmake
+#
+# @project MORSE
+# MORSE is a software package provided by:
+# Inria Bordeaux - Sud-Ouest,
+# Univ. of Tennessee,
+# King Abdullah Univesity of Science and Technology
+# Univ. of California Berkeley,
+# Univ. of Colorado Denver.
+#
+# @version 0.9.0
+# @author Cedric Castagnede
+# @author Emmanuel Agullo
+# @author Mathieu Faverge
+# @author Florent Pruvost
+# @date 13-07-2012
+#
+###
+
+# Some macros to print status when search for headers and libs
+include(PrintFindStatus)
+
+function(FindHeader _libname _header_to_find)
+
+ # save _libname upper and lower case
+ string(TOUPPER ${_libname} LIBNAME)
+ string(TOLOWER ${_libname} libname)
+
+ # Looking for include
+ # -------------------
+
+ # Add system include paths to search include
+ # ------------------------------------------
+ unset(_inc_env)
+ if(WIN32)
+ string(REPLACE ":" ";" _inc_env "$ENV{INCLUDE}")
+ else()
+ string(REPLACE ":" ";" _path_env "$ENV{INCLUDE}")
+ list(APPEND _inc_env "${_path_env}")
+ string(REPLACE ":" ";" _path_env "$ENV{C_INCLUDE_PATH}")
+ list(APPEND _inc_env "${_path_env}")
+ string(REPLACE ":" ";" _path_env "$ENV{CPATH}")
+ list(APPEND _inc_env "${_path_env}")
+ string(REPLACE ":" ";" _path_env "$ENV{INCLUDE_PATH}")
+ list(APPEND _inc_env "${_path_env}")
+ endif()
+ list(APPEND _inc_env "${CMAKE_C_IMPLICIT_INCLUDE_DIRECTORIES}")
+ list(REMOVE_DUPLICATES _inc_env)
+
+
+ # Try to find the _header_to_find in the given paths
+ # --------------------------------------------------
+ # call cmake macro to find the header path
+ if(${LIBNAME}_INCDIR)
+ set(${LIBNAME}_${_header_to_find}_DIRS "${LIBNAME}_${_header_to_find}_DIRS-NOTFOUND")
+ find_path(${LIBNAME}_${_header_to_find}_DIRS
+ NAMES ${_header_to_find}
+ HINTS ${${LIBNAME}_INCDIR})
+ elseif(${LIBNAME}_DIR)
+ set(${LIBNAME}_${_header_to_find}_DIRS "${LIBNAME}_${_header_to_find}_DIRS-NOTFOUND")
+ find_path(${LIBNAME}_${_header_to_find}_DIRS
+ NAMES ${_header_to_find}
+ HINTS ${${LIBNAME}_DIR}
+ PATH_SUFFIXES include)
+ else()
+ set(${LIBNAME}_${_header_to_find}_DIRS "${LIBNAME}_${_header_to_find}_DIRS-NOTFOUND")
+ find_path(${LIBNAME}_${_header_to_find}_DIRS
+ NAMES ${_header_to_find}
+ HINTS ${_inc_env})
+ endif()
+ mark_as_advanced(${LIBNAME}_${_header_to_find}_DIRS)
+
+ # Print status if not found
+ # -------------------------
+ if (NOT ${LIBNAME}_${_header_to_find}_DIRS)
+ Print_Find_Header_Status(${libname} ${_header_to_find})
+ endif ()
+
+endfunction(FindHeader)
+
+
+##
+## @end file FindHeadersAndLibs.cmake
+##
diff --git a/cmake/morse/FindInit.cmake b/cmake/morse/FindInit.cmake
new file mode 100644
index 00000000..e59d41a0
--- /dev/null
+++ b/cmake/morse/FindInit.cmake
@@ -0,0 +1,45 @@
+###
+#
+# @copyright (c) 2018 Inria. All rights reserved.
+#
+###
+#
+# @file FindInit.cmake
+#
+# @project MORSE
+# MORSE is a software package provided by:
+# Inria Bordeaux - Sud-Ouest,
+# Univ. of Tennessee,
+# King Abdullah Univesity of Science and Technology
+# Univ. of California Berkeley,
+# Univ. of Colorado Denver.
+#
+# @version 1.0.0
+# @author Florent Pruvost
+# @date 24-04-2018
+#
+###
+
+
+# This include is required to check symbols of libs
+include(CheckFunctionExists)
+
+# This include is required to check defines in headers
+include(CheckIncludeFiles)
+
+# Factorize some piece of code
+include(FindCommon)
+
+# To find headers and libs
+include(FindHeadersAndLibs)
+
+# To transform relative path into absolute for a list of libraries
+include(LibrariesAbsolutePath)
+include(FindPkgconfigLibrariesAbsolutePath)
+
+# Some macros to print status when search for headers and libs
+include(PrintFindStatus)
+
+##
+## @end file FindInit.cmake
+##
diff --git a/cmake/morse/FindPkgconfigLibrariesAbsolutePath.cmake b/cmake/morse/FindPkgconfigLibrariesAbsolutePath.cmake
new file mode 100644
index 00000000..51b08ce5
--- /dev/null
+++ b/cmake/morse/FindPkgconfigLibrariesAbsolutePath.cmake
@@ -0,0 +1,99 @@
+###
+#
+# @copyright (c) 2018 Inria. All rights reserved.
+#
+###
+#
+# @file FindPkgconfigLibrariesAbsolutePath.cmake
+#
+# @project MORSE
+# MORSE is a software package provided by:
+# Inria Bordeaux - Sud-Ouest,
+# Univ. of Tennessee,
+# King Abdullah Univesity of Science and Technology
+# Univ. of California Berkeley,
+# Univ. of Colorado Denver.
+#
+# @version 1.0.0
+# @author Florent Pruvost
+# @date 06-04-2018
+#
+###
+
+# Transform relative path into absolute path for libraries found with the
+# pkg_search_module cmake macro
+# _prefix: the name of the CMake variable used when pkg_search_module was called
+# e.g. for pkg_search_module(BLAS blas) _prefix would be BLAS
+macro(FIND_PKGCONFIG_LIBRARIES_ABSOLUTE_PATH _prefix)
+ list(APPEND _lib_env "$ENV{LIBRARY_PATH}")
+ if(WIN32)
+ string(REPLACE ":" ";" _lib_env2 "$ENV{LIB}")
+ elseif(APPLE)
+ string(REPLACE ":" ";" _lib_env2 "$ENV{DYLD_LIBRARY_PATH}")
+ else()
+ string(REPLACE ":" ";" _lib_env2 "$ENV{LD_LIBRARY_PATH}")
+ endif()
+ list(APPEND _lib_env "${_lib_env2}")
+ list(APPEND _lib_env "${CMAKE_C_IMPLICIT_LINK_DIRECTORIES}")
+ # non static case
+ set(${_prefix}_LIBRARIES_COPY "${${_prefix}_LIBRARIES}")
+ set(${_prefix}_LIBRARIES "")
+ foreach(_library ${${_prefix}_LIBRARIES_COPY})
+ if(EXISTS "${_library}")
+ list(APPEND ${_prefix}_LIBRARIES ${_library})
+ else()
+ get_filename_component(_ext "${_library}" EXT)
+ set(_lib_extensions ".so" ".a" ".dyld" ".dll")
+ list(FIND _lib_extensions "${_ext}" _index)
+ if (${_index} GREATER -1)
+ get_filename_component(_library "${_library}" NAME_WE)
+ endif()
+ find_library(_library_path NAMES ${_library}
+ HINTS ${${_prefix}_LIBDIR} ${${_prefix}_LIBRARY_DIRS} ${_lib_env})
+ if (_library_path)
+ list(APPEND ${_prefix}_LIBRARIES ${_library_path})
+ else()
+ message(FATAL_ERROR "Dependency of ${_prefix} '${_library}' NOT FOUND")
+ endif()
+ unset(_library_path CACHE)
+ endif()
+ endforeach()
+ set (${_prefix}_LIBRARIES "${${_prefix}_LIBRARIES}" CACHE INTERNAL "" FORCE)
+ ## static case
+ #set(${_prefix}_STATIC_LIBRARIES_COPY "${${_prefix}_STATIC_LIBRARIES}")
+ #set(${_prefix}_STATIC_LIBRARIES "")
+ #foreach(_library ${${_prefix}_STATIC_LIBRARIES_COPY})
+ # if(EXISTS "${_library}")
+ # list(APPEND ${_prefix}_STATIC_LIBRARIES ${_library})
+ # else()
+ # get_filename_component(_ext "${_library}" EXT)
+ # set(_lib_extensions ".so" ".a" ".dyld" ".dll")
+ # list(FIND _lib_extensions "${_ext}" _index)
+ # if (${_index} GREATER -1)
+ # get_filename_component(_library "${_library}" NAME_WE)
+ # endif()
+ # # try static first
+ # set (default_find_library_suffixes ${CMAKE_FIND_LIBRARY_SUFFIXES})
+ # set (CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_STATIC_LIBRARY_SUFFIX})
+ # find_library(_library_path NAMES ${_library}
+ # HINTS ${${_prefix}_STATIC_LIBDIR} ${${_prefix}_STATIC_LIBRARY_DIRS} ${_lib_env})
+ # set (CMAKE_FIND_LIBRARY_SUFFIXES ${default_find_library_suffixes})
+ # # if not found try dynamic
+ # if (NOT _library_path)
+ # find_library(_library_path NAMES ${_library}
+ # HINTS ${${_prefix}_STATIC_LIBDIR} ${${_prefix}_STATIC_LIBRARY_DIRS} ${_lib_env})
+ # endif()
+ # if (_library_path)
+ # list(APPEND ${_prefix}_STATIC_LIBRARIES ${_library_path})
+ # else()
+ # message(FATAL_ERROR "Dependency of ${_prefix} '${_library}' NOT FOUND")
+ # endif()
+ # unset(_library_path CACHE)
+ # endif()
+ #endforeach()
+ #set (${_prefix}_STATIC_LIBRARIES "${${_prefix}_STATIC_LIBRARIES}" CACHE INTERNAL "" FORCE)
+endmacro()
+
+##
+## @end file FindPkgconfigLibrariesAbsolutePath.cmake
+##
diff --git a/cmake/morse/LICENCE.txt b/cmake/morse/LICENCE.txt
new file mode 100644
index 00000000..b95821f3
--- /dev/null
+++ b/cmake/morse/LICENCE.txt
@@ -0,0 +1,42 @@
+###
+#
+# @copyright (c) 2009-2014 The University of Tennessee and The University
+# of Tennessee Research Foundation.
+# All rights reserved.
+# @copyright (c) 2012-2016 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria,
+# Univ. Bordeaux. All rights reserved.
+# @copyright (c) 2016 KAUST. All rights reserved.
+#
+###
+#
+# This software is a computer program whose purpose is to process
+# Matrices Over Runtime Systems @ Exascale (MORSE). More information
+# can be found on the following website: http://www.inria.fr/en/teams/morse.
+#
+# This software is governed by the CeCILL-C license under French law and
+# abiding by the rules of distribution of free software. You can use,
+# modify and/ or redistribute the software under the terms of the CeCILL-C
+# license as circulated by CEA, CNRS and INRIA at the following URL
+# "http://www.cecill.info".
+#
+# As a counterpart to the access to the source code and rights to copy,
+# modify and redistribute granted by the license, users are provided only
+# with a limited warranty and the software's author, the holder of the
+# economic rights, and the successive licensors have only limited
+# liability.
+#
+# In this respect, the user's attention is drawn to the risks associated
+# with loading, using, modifying and/or developing or reproducing the
+# software by the user in light of its specific status of free software,
+# that may mean that it is complicated to manipulate, and that also
+# therefore means that it is reserved for developers and experienced
+# professionals having in-depth computer knowledge. Users are therefore
+# encouraged to load and test the software's suitability as regards their
+# requirements in conditions enabling the security of their systems and/or
+# data to be ensured and, more generally, to use and operate it in the
+# same conditions as regards security.
+#
+# The fact that you are presently reading this means that you have had
+# knowledge of the CeCILL-C license and that you accept its terms.
+#
+###
diff --git a/cmake/morse/LibrariesAbsolutePath.cmake b/cmake/morse/LibrariesAbsolutePath.cmake
new file mode 100644
index 00000000..7aaab504
--- /dev/null
+++ b/cmake/morse/LibrariesAbsolutePath.cmake
@@ -0,0 +1,70 @@
+###
+#
+# @copyright (c) 2018 Inria. All rights reserved.
+#
+###
+#
+# @file LibrariesAbsolutePath.cmake
+#
+# @project MORSE
+# MORSE is a software package provided by:
+# Inria Bordeaux - Sud-Ouest,
+# Univ. of Tennessee,
+# King Abdullah Univesity of Science and Technology
+# Univ. of California Berkeley,
+# Univ. of Colorado Denver.
+#
+# @version 1.0.0
+# @author Florent Pruvost
+# @date 13-04-2018
+#
+###
+
+# Transform relative path into absolute path for libraries
+# lib_list (input/output): the name of the CMake variable containing libraries, e.g. BLAS_LIBRARIES
+# hints_paths (input): additional paths to add when looking for libraries
+macro(LIBRARIES_ABSOLUTE_PATH lib_list hints_paths)
+ # collect environment paths to dig
+ list(APPEND _lib_env "$ENV{LIBRARY_PATH}")
+ if(WIN32)
+ string(REPLACE ":" ";" _lib_env2 "$ENV{LIB}")
+ elseif(APPLE)
+ string(REPLACE ":" ";" _lib_env2 "$ENV{DYLD_LIBRARY_PATH}")
+ else()
+ string(REPLACE ":" ";" _lib_env2 "$ENV{LD_LIBRARY_PATH}")
+ endif()
+ list(APPEND _lib_env "${_lib_env2}")
+ list(APPEND _lib_env "${CMAKE_C_IMPLICIT_LINK_DIRECTORIES}")
+ # copy the lib list
+ set (${lib_list}_COPY "${${lib_list}}")
+ # reset the lib list to populate
+ set(${lib_list} "")
+ foreach(_library ${${lib_list}_COPY})
+ if(EXISTS "${_library}")
+ # if already an absolute path, nothing special to do
+ list(APPEND ${lib_list} ${_library})
+ else()
+ # replace pattern -lfoo -> foo
+ string(REGEX REPLACE "^-l" "" _library "${_library}")
+ # remove extensions if exist
+ get_filename_component(_ext "${_library}" EXT)
+ set(_lib_extensions ".so" ".a" ".dyld" ".dll")
+ list(FIND _lib_extensions "${_ext}" _index)
+ if (${_index} GREATER -1)
+ get_filename_component(_library "${_library}" NAME_WE)
+ endif()
+ # try to find the lib
+ find_library(_library_path NAMES ${_library} HINTS ${hints_paths} ${_lib_env})
+ if (_library_path)
+ list(APPEND ${lib_list} ${_library_path})
+ else()
+ message(FATAL_ERROR "Dependency of ${lib_list} '${_library}' NOT FOUND")
+ endif()
+ unset(_library_path CACHE)
+ endif()
+ endforeach()
+endmacro()
+
+##
+## @end file LibrariesAbsolutePath.cmake
+##
diff --git a/cmake/morse/MorseInit.cmake b/cmake/morse/MorseInit.cmake
new file mode 100644
index 00000000..fc511704
--- /dev/null
+++ b/cmake/morse/MorseInit.cmake
@@ -0,0 +1,67 @@
+###
+#
+# @copyright (c) 2009-2014 The University of Tennessee and The University
+# of Tennessee Research Foundation.
+# All rights reserved.
+# @copyright (c) 2012-2018 Inria. All rights reserved.
+# @copyright (c) 2012-2014 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
+#
+###
+#
+# @file MorseInit.cmake
+#
+# @project MORSE
+# MORSE is a software package provided by:
+# Inria Bordeaux - Sud-Ouest,
+# Univ. of Tennessee,
+# King Abdullah Univesity of Science and Technology
+# Univ. of California Berkeley,
+# Univ. of Colorado Denver.
+#
+# @version 1.0.0
+# @author Cedric Castagnede
+# @author Emmanuel Agullo
+# @author Mathieu Faverge
+# @author Florent Pruvost
+# @date 13-07-2012
+#
+###
+
+# Path to Morse modules
+get_filename_component(MORSE_CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_FILE} DIRECTORY CACHE)
+
+# Global Morse options
+option(MORSE_ENABLE_WARNING "Enable warning messages" OFF)
+option(MORSE_ENABLE_COVERAGE "Enable flags for coverage test" OFF)
+option(MORSE_ENABLE_COLOR_MESSAGE "Enable colors in messages" OFF)
+#option(MORSE_VERBOSE_FIND_PACKAGE "Add additional messages concerning packages not found" OFF)
+#message(STATUS "MORSE_VERBOSE_FIND_PACKAGE is set to OFF, turn it ON to get"
+# " information about packages not found")
+
+
+# This include is required to check symbols of libs in the main CMakeLists.txt
+include(CheckFunctionExists)
+
+# This include is required to check defines in headers
+include(CheckIncludeFiles)
+
+if (MORSE_ENABLE_COLOR_MESSAGE)
+ # colorize messages
+ include(ColorizeMessage)
+endif()
+
+# Define some auxilary flags
+include(AuxilaryFlags)
+
+# Define some variables to et info about ressources
+include(Ressources)
+
+# Add the path where we handle our FindFOO.cmake to seek for liraries
+list(APPEND CMAKE_MODULE_PATH ${MORSE_CMAKE_MODULE_PATH}/find)
+
+# To load some macros used in Finds (could be useful for other projects)
+include(FindInit)
+
+##
+## @end file MorseInit.cmake
+##
diff --git a/cmake/morse/PrintFindStatus.cmake b/cmake/morse/PrintFindStatus.cmake
new file mode 100644
index 00000000..1fdd403b
--- /dev/null
+++ b/cmake/morse/PrintFindStatus.cmake
@@ -0,0 +1,207 @@
+###
+#
+# @copyright (c) 2009-2014 The University of Tennessee and The University
+# of Tennessee Research Foundation.
+# All rights reserved.
+# @copyright (c) 2012-2014 Inria. All rights reserved.
+# @copyright (c) 2012-2014 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
+#
+###
+#
+# - Some macros to print status when search for headers and libs
+# Main parameters of macros
+# _libname: name of the lib you seek, foo for example
+# _header_to_find: name of the header you seek, foo.h for example
+# _lib_to_find: name of the library you seek, libfoo for example
+# _pc_to_find: name of the pkg-config file zyou seek, foo.pc for example
+
+
+#=============================================================================
+# Copyright 2012-2013 Inria
+# Copyright 2012-2013 Emmanuel Agullo
+# Copyright 2012-2013 Mathieu Faverge
+# Copyright 2012 Cedric Castagnede
+# Copyright 2013 Florent Pruvost
+#
+# Distributed under the OSI-approved BSD License (the "License");
+# see accompanying file MORSE-Copyright.txt for details.
+#
+# This software is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the License for more information.
+
+#=============================================================================
+# (To distribute this file outside of Morse, substitute the full
+# License text for the above reference.)
+
+
+# Set some colors
+#if(NOT WIN32)
+# string(ASCII 27 Esc)
+# set(ColourReset "${Esc}[m")
+# set(ColourBold "${Esc}[1m")
+# set(Red "${Esc}[31m")
+# set(Green "${Esc}[32m")
+# set(Yellow "${Esc}[33m")
+# set(Blue "${Esc}[34m")
+# set(Magenta "${Esc}[35m")
+# set(Cyan "${Esc}[36m")
+# set(White "${Esc}[37m")
+# set(BoldRed "${Esc}[1;31m")
+# set(BoldGreen "${Esc}[1;32m")
+# set(BoldYellow "${Esc}[1;33m")
+# set(BoldBlue "${Esc}[1;34m")
+# set(BoldMagenta "${Esc}[1;35m")
+# set(BoldCyan "${Esc}[1;36m")
+# set(BoldWhite "${Esc}[1;37m")
+#endif()
+
+
+# This macro informs why the _header_to_find file has not been found
+macro(Print_Find_Header_Status _libname _header_to_find)
+
+ # save _libname upper and lower case
+ string(TOUPPER ${_libname} LIBNAME)
+ string(TOLOWER ${_libname} libname)
+
+ # print status
+ #message(" ")
+ if(${LIBNAME}_INCDIR)
+ message("${Blue}${LIBNAME}_INCDIR is defined but ${_header_to_find}"
+ "has not been found in ${${LIBNAME}_INCDIR}${ColourReset}")
+ else()
+ if(${LIBNAME}_DIR)
+ message("${Blue}${LIBNAME}_DIR is defined but"
+ "${_header_to_find} has not been found in"
+ "${${LIBNAME}_DIR}/include${ColourReset}")
+ else()
+ message("${Blue}${_header_to_find} not found."
+ "Nor ${LIBNAME}_DIR neither ${LIBNAME}_INCDIR"
+ "are defined so that we looked for ${_header_to_find} in"
+ "system paths (INCLUDE, CPATH, C_INCLUDE_PATH,"
+ "INCLUDE_PATH, CMAKE_C_IMPLICIT_INCLUDE_DIRECTORIES)${ColourReset}")
+ if(_inc_env)
+ message("${Blue}${_header_to_find} has not been found in"
+ "${_inc_env}${ColourReset}")
+ endif()
+ endif()
+ endif()
+ message("${BoldBlue}Please indicate where to find ${_header_to_find}. You have three options:\n"
+ "- Option 1: Provide the root directory of the library with cmake option: -D${LIBNAME}_DIR=your/path/to/${libname}/\n"
+ "- Option 2: Provide the directory where to find the headers with cmake option: -D${LIBNAME}_INCDIR=your/path/to/${libname}/include/\n"
+ "- Option 3: Update your environment variable (INCLUDE or CPATH)\n"
+ "- Option 4: If your library provides a PkgConfig file, make sure pkg-config finds your library${ColourReset}")
+ #message(" ")
+
+endmacro()
+
+# This macro informs why the _lib_to_find file has not been found
+macro(Print_Find_Library_Status _libname _lib_to_find)
+
+ # save _libname upper/lower case
+ string(TOUPPER ${_libname} LIBNAME)
+ string(TOLOWER ${_libname} libname)
+
+ # print status
+ #message(" ")
+ if(${LIBNAME}_LIBDIR)
+ message("${Yellow}${LIBNAME}_LIBDIR is defined but ${_lib_to_find}"
+ "has not been found in ${${LIBNAME}_LIBDIR}${ColourReset}")
+ else()
+ if(${LIBNAME}_DIR)
+ message("${Yellow}${LIBNAME}_DIR is defined but ${_lib_to_find}"
+ "has not been found in ${${LIBNAME}_DIR}/lib(or /lib32 or"
+ "/lib64)${ColourReset}")
+ else()
+ message("${Yellow}${_lib_to_find} not found."
+ "Nor ${LIBNAME}_DIR neither ${LIBNAME}_LIBDIR"
+ "are defined so that we looked for ${_lib_to_find} in"
+ "system paths (Linux: LD_LIBRARY_PATH, Windows: LIB,"
+ "Mac: DYLD_LIBRARY_PATH,"
+ "CMAKE_C_IMPLICIT_LINK_DIRECTORIES)${ColourReset}")
+ if(_lib_env)
+ message("${Yellow}${_lib_to_find} has not been found in"
+ "${_lib_env}${ColourReset}")
+ endif()
+ endif()
+ endif()
+ message("${BoldYellow}Please indicate where to find ${_lib_to_find}. You have three options:\n"
+ "- Option 1: Provide the root directory of the library with cmake option: -D${LIBNAME}_DIR=your/path/to/${libname}/\n"
+ "- Option 2: Provide the directory where to find the library with cmake option: -D${LIBNAME}_LIBDIR=your/path/to/${libname}/lib/\n"
+ "- Option 3: Update your environment variable (Linux: LD_LIBRARY_PATH, Windows: LIB, Mac: DYLD_LIBRARY_PATH)\n"
+ "- Option 4: If your library provides a PkgConfig file, make sure pkg-config finds your library${ColourReset}")
+
+endmacro()
+
+# This macro informs why the _lib_to_find file has not been found
+macro(Print_Find_Library_Blas_Status _libname _lib_to_find)
+
+ # save _libname upper/lower case
+ string(TOUPPER ${_libname} LIBNAME)
+ string(TOLOWER ${_libname} libname)
+
+ # print status
+ #message(" ")
+ if(${LIBNAME}_LIBDIR)
+ message("${Yellow}${LIBNAME}_LIBDIR is defined but ${_lib_to_find}"
+ "has not been found in ${ARGN}${ColourReset}")
+ else()
+ if(${LIBNAME}_DIR)
+ message("${Yellow}${LIBNAME}_DIR is defined but ${_lib_to_find}"
+ "has not been found in ${ARGN}${ColourReset}")
+ else()
+ message("${Yellow}${_lib_to_find} not found."
+ "Nor ${LIBNAME}_DIR neither ${LIBNAME}_LIBDIR"
+ "are defined so that we look for ${_lib_to_find} in"
+ "system paths (Linux: LD_LIBRARY_PATH, Windows: LIB,"
+ "Mac: DYLD_LIBRARY_PATH,"
+ "CMAKE_C_IMPLICIT_LINK_DIRECTORIES)${ColourReset}")
+ if(_lib_env)
+ message("${Yellow}${_lib_to_find} has not been found in"
+ "${_lib_env}${ColourReset}")
+ endif()
+ endif()
+ endif()
+ message("${BoldYellow}Please indicate where to find ${_lib_to_find}. You have three options:\n"
+ "- Option 1: Provide the root directory of the library with cmake option: -D${LIBNAME}_DIR=your/path/to/${libname}/\n"
+ "- Option 2: Provide the directory where to find the library with cmake option: -D${LIBNAME}_LIBDIR=your/path/to/${libname}/lib/\n"
+ "- Option 3: Update your environment variable (Linux: LD_LIBRARY_PATH, Windows: LIB, Mac: DYLD_LIBRARY_PATH)\n"
+ "- Option 4: If your library provides a PkgConfig file, make sure pkg-config finds your library${ColourReset}")
+
+endmacro()
+
+# This macro informs why the _lib_to_find file has not been found
+macro(Print_Find_Library_Blas_CheckFunc_Status _name)
+
+ # save _libname upper/lower case
+ string(TOUPPER ${_name} FUNCNAME)
+ string(TOLOWER ${_name} funcname)
+
+ # print status
+ #message(" ")
+ message("${Red}Libs have been found but check of symbol ${_name} failed "
+ "with following libraries ${ARGN}${ColourReset}")
+ message("${BoldRed}Please open your error file CMakeFiles/CMakeError.log"
+ "to figure out why it fails${ColourReset}")
+ #message(" ")
+
+endmacro()
+
+# This macro informs that _pc_to_find file has not been found in the list
+# path you give as last argument (read in ${ARGN})
+# ex: Print_Find_Pkgconfig_Status(foo foo.pc ${PATHLIST}
+macro(Print_Find_Pkgconfig_Status _libname _pc_to_find)
+
+ # save _libname lower case
+ string(TOLOWER ${_libname} libname)
+
+ # print status
+ #message(" ")
+ message("${Magenta}${_pc_to_find} has not been found in"
+ "${ARGN}${ColourReset}")
+ message("${BoldMagenta}If you really want to use the pkg-config file of"
+ "${libname}, please update your PKG_CONFIG_PATH with the path"
+ "where ${_pc_to_find} states${ColourReset}")
+ #message(" ")
+
+endmacro()
--
GitLab
From 103238aaa9fbae46da86c146a87d5672db3c0517 Mon Sep 17 00:00:00 2001
From: Berenger Bramas <Berenger.Bramas@inria.fr>
Date: Mon, 11 Mar 2019 14:43:10 +0100
Subject: [PATCH 316/342] Update to remove source dir from list of include
paths
---
CMakeLists.txt | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index b7d65d0d..a04aae0b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -88,13 +88,17 @@ list(APPEND BFPS_LIBS "${FFTW_LIBRARIES}")
include_directories(${FFTW_INCLUDE_DIRS})
link_directories(${FFTW_LIBRARY_DIRS})
+#####################################################################################
+## Get the links and include from deps
+
+get_property(ALL_INCLUDE_DIRS DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES)
+get_property(ALL_LINK_DIRS DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY LINK_DIRECTORIES)
+
#####################################################################################
## Build the lib
include_directories(
${PROJECT_SOURCE_DIR}/bfps/cpp
- ${PROJECT_SOURCE_DIR}/bfps/cpp/particles
- ${PROJECT_SOURCE_DIR}/bfps/cpp/full_code
)
file(GLOB_RECURSE cpp_for_lib ${PROJECT_SOURCE_DIR}/bfps/*.cpp)
@@ -111,8 +115,6 @@ install(TARGETS bfps EXPORT BFPS_EXPORT DESTINATION lib/ PUBLIC_HEADER DESTINATI
#####################################################################################
##Â Export the configuration
-get_property(ALL_INCLUDE_DIRS DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES)
-get_property(ALL_LINK_DIRS DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY LINK_DIRECTORIES)
configure_file(${PROJECT_SOURCE_DIR}/cmake/BFPSConfig.cmake.in ${PROJECT_BINARY_DIR}/BFPSConfig.cmake @ONLY)
install(FILES "${PROJECT_BINARY_DIR}/BFPSConfig.cmake" DESTINATION lib/)
--
GitLab
From 2335190669f5403c18f200d9fcf67a940dc04126 Mon Sep 17 00:00:00 2001
From: Berenger Bramas <Berenger.Bramas@inria.fr>
Date: Mon, 11 Mar 2019 14:48:06 +0100
Subject: [PATCH 317/342] Keep hierarchy in installed headers
---
CMakeLists.txt | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index a04aae0b..a79942a5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -97,20 +97,18 @@ get_property(ALL_LINK_DIRS DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY LINK_D
#####################################################################################
## Build the lib
-include_directories(
- ${PROJECT_SOURCE_DIR}/bfps/cpp
- )
+include_directories(${PROJECT_SOURCE_DIR}/bfps/cpp)
file(GLOB_RECURSE cpp_for_lib ${PROJECT_SOURCE_DIR}/bfps/*.cpp)
file(GLOB_RECURSE hpp_for_lib ${PROJECT_SOURCE_DIR}/bfps/*.hpp)
-LIST(APPEND source_files ${cpp_for_lib})
+LIST(APPEND source_files ${hpp_for_lib} ${cpp_for_lib})
add_library(bfps ${source_files})
target_link_libraries(bfps ${BFPS_LIBS})
-set_target_properties(bfps PROPERTIES PUBLIC_HEADER "${hpp_for_lib}")
-install(TARGETS bfps EXPORT BFPS_EXPORT DESTINATION lib/ PUBLIC_HEADER DESTINATION include/bfps )
+install(TARGETS bfps EXPORT BFPS_EXPORT DESTINATION lib/ )
+install(DIRECTORY ${PROJECT_SOURCE_DIR}/bfps/cpp DESTINATION include/ FILES_MATCHING PATTERN "*.h*")
#####################################################################################
##Â Export the configuration
--
GitLab
From 3b492861a8de855130bfdab2ad62a2812a7acad8 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 11 Mar 2019 15:28:06 +0100
Subject: [PATCH 318/342] keep header files in their own subfolder
We have many, some with generic names, so it's probably safer this way.
---
CMakeLists.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index a79942a5..7bf9c04b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -108,7 +108,7 @@ add_library(bfps ${source_files})
target_link_libraries(bfps ${BFPS_LIBS})
install(TARGETS bfps EXPORT BFPS_EXPORT DESTINATION lib/ )
-install(DIRECTORY ${PROJECT_SOURCE_DIR}/bfps/cpp DESTINATION include/ FILES_MATCHING PATTERN "*.h*")
+install(DIRECTORY ${PROJECT_SOURCE_DIR}/bfps/cpp DESTINATION include/bfps/ FILES_MATCHING PATTERN "*.h*")
#####################################################################################
##Â Export the configuration
--
GitLab
From 65f0df4606b9fef5c1beaeade5e9b477ed12462a Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 11 Mar 2019 16:22:18 +0100
Subject: [PATCH 319/342] install python package from cmake
---
CMakeLists.txt | 4 ++
cpp_build.py | 94 ---------------------------
setup.py | 170 +------------------------------------------------
3 files changed, 7 insertions(+), 261 deletions(-)
delete mode 100644 cpp_build.py
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7bf9c04b..2877199e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -120,3 +120,7 @@ export(TARGETS bfps FILE "${PROJECT_BINARY_DIR}/BFPSLibraryDepends.cmake")
install(EXPORT BFPS_EXPORT DESTINATION lib/)
+#####################################################################################
+##Â Install the python wrapper
+install(CODE "execute_process(COMMAND python setup.py install --force --prefix=${CMAKE_INSTALL_PREFIX} WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})")
+
diff --git a/cpp_build.py b/cpp_build.py
deleted file mode 100644
index 39371214..00000000
--- a/cpp_build.py
+++ /dev/null
@@ -1,94 +0,0 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
-
-
-
-import os
-import subprocess
-
-src_file_list = ['hdf5_tools',
- 'full_code/get_rfields',
- 'full_code/NSVE_field_stats',
- 'full_code/native_binary_to_hdf5',
- 'full_code/postprocess',
- 'full_code/code_base',
- 'full_code/direct_numerical_simulation',
- 'full_code/NSVE',
- 'full_code/NSVEparticles',
- 'field_binary_IO',
- 'vorticity_equation',
- 'field',
- 'kspace',
- 'field_layout',
- 'field_descriptor',
- 'rFFTW_distributed_particles',
- 'distributed_particles',
- 'particles',
- 'particles_base',
- 'rFFTW_interpolator',
- 'interpolator',
- 'interpolator_base',
- 'fluid_solver',
- 'fluid_solver_base',
- 'fftw_tools',
- 'spline_n1',
- 'spline_n2',
- 'spline_n3',
- 'spline_n4',
- 'spline_n5',
- 'spline_n6',
- 'spline_n7',
- 'spline_n8',
- 'spline_n9',
- 'spline_n10',
- 'Lagrange_polys',
- 'scope_timer']
-
-def get_file_dependency_list(src_file):
- p = subprocess.Popen(
- ['g++', '-Ibfps/cpp', '-MM', 'bfps/cpp/' + src_file + '.cpp'],
- stdout = subprocess.PIPE)
- out, err = p.communicate()
- p.terminate()
- deps = str(out, 'ASCII').replace('\\\n', '')
- return deps
-
-def get_dependency_list():
- ofile = open('dependencies.txt', 'w')
- for src_file in src_file_list:
- p = subprocess.Popen(
- ['g++', '-Ibfps/cpp', '-MM', 'bfps/cpp/' + src_file + '.cpp'],
- stdout = subprocess.PIPE)
- out, err = p.communicate()
- p.terminate()
- deps = str(out, 'ASCII').replace('\\\n', '')
- print(deps.split()[0])
- ofile.write(' '.join(deps.split()[1:]) + '\n')
- ofile.close()
- return None
-
-if __name__ == '__main__':
- #pass
- get_dependency_list()
-
diff --git a/setup.py b/setup.py
index 23f9c266..2a031518 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
#######################################################################
# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
+# Copyright 2015-2019 Max Planck Institute #
+# for Dynamics and Self-Organization #
# #
# This file is part of bfps. #
# #
@@ -86,97 +86,6 @@ else:
print('This is bfps version ' + VERSION)
-
-### lists of files and MANIFEST.in
-src_file_list = [
- 'full_code/code_base',
- 'full_code/direct_numerical_simulation',
- 'full_code/NSVE',
- 'full_code/joint_acc_vel_stats',
- 'full_code/test',
- 'full_code/filter_test',
- 'full_code/field_test',
- 'full_code/symmetrize_test',
- 'full_code/field_output_test',
- 'full_code/get_rfields',
- 'full_code/field_single_to_double',
- 'full_code/resize',
- 'full_code/NSVE_field_stats',
- 'full_code/native_binary_to_hdf5',
- 'full_code/postprocess',
- 'field',
- 'kspace',
- 'field_layout',
- 'hdf5_tools',
- 'fftw_tools',
- 'vorticity_equation',
- 'field_binary_IO',
- 'spline_n1',
- 'spline_n2',
- 'spline_n3',
- 'spline_n4',
- 'spline_n5',
- 'spline_n6',
- 'spline_n7',
- 'spline_n8',
- 'spline_n9',
- 'spline_n10',
- 'Lagrange_polys',
- 'scope_timer',
- 'full_code/test_interpolation',
- 'full_code/NSVEparticles',
- 'full_code/NSVEcomplex_particles',
- 'full_code/NSVEp_extra_sampling',
- 'particles/particles_inner_computer']
-
-particle_headers = [
- 'cpp/particles/abstract_particles_input.hpp',
- 'cpp/particles/abstract_particles_output.hpp',
- 'cpp/particles/abstract_particles_system.hpp',
- 'cpp/particles/alltoall_exchanger.hpp',
- 'cpp/particles/env_utils.hpp',
- 'cpp/particles/lock_free_bool_array.hpp',
- 'cpp/particles/p2p_computer_empty.hpp',
- 'cpp/particles/p2p_computer.hpp',
- 'cpp/particles/p2p_distr_mpi.hpp',
- 'cpp/particles/p2p_tree.hpp',
- 'cpp/particles/particles_adams_bashforth.hpp',
- 'cpp/particles/particles_distr_mpi.hpp',
- 'cpp/particles/particles_field_computer.hpp',
- 'cpp/particles/particles_generic_interp.hpp',
- 'cpp/particles/particles_inner_computer_empty.hpp',
- 'cpp/particles/particles_input_hdf5.hpp',
- 'cpp/particles/particles_output_hdf5.hpp',
- 'cpp/particles/particles_output_mpiio.hpp',
- 'cpp/particles/particles_output_sampling_hdf5.hpp',
- 'cpp/particles/particles_sampling.hpp',
- 'cpp/particles/particles_system_builder.hpp',
- 'cpp/particles/particles_system.hpp',
- 'cpp/particles/particles_utils.hpp']
-
-full_code_headers = ['cpp/full_code/main_code.hpp',
- 'cpp/full_code/codes_with_no_output.hpp',
- 'cpp/full_code/NSVE_no_output.hpp',
- 'cpp/full_code/NSVEparticles_no_output.hpp']
-
-header_list = (['cpp/base.hpp'] +
- ['cpp/fftw_interface.hpp'] +
- ['cpp/bfps_timer.hpp'] +
- ['cpp/omputils.hpp'] +
- ['cpp/shared_array.hpp'] +
- ['cpp/spline.hpp'] +
- ['cpp/' + fname + '.hpp'
- for fname in src_file_list] +
- particle_headers +
- full_code_headers)
-
-with open('MANIFEST.in', 'w') as manifest_in_file:
- for fname in (['bfps/cpp/' + ff + '.cpp' for ff in src_file_list] +
- ['bfps/' + ff for ff in header_list]):
- manifest_in_file.write('include {0}\n'.format(fname))
-
-
-
### libraries
libraries = extra_libraries
@@ -204,66 +113,6 @@ class CompileLibCommand(distutils.cmd.Command):
self.disable_fftw_omp = (int(self.disable_fftw_omp) == 1)
return None
def run(self):
- if not os.path.isdir('obj'):
- os.makedirs('obj')
- need_to_compile = True
- if not os.path.isdir('obj/full_code'):
- os.makedirs('obj/full_code')
- need_to_compile = True
- if not os.path.isdir('obj/particles'):
- os.makedirs('obj/particles')
- need_to_compile = True
- if not os.path.isfile('bfps/libbfps.a'):
- need_to_compile = True
- else:
- need_to_compile = False
- ofile = 'bfps/libbfps.a'
- libtime = datetime.datetime.fromtimestamp(os.path.getctime(ofile))
- latest = libtime
- eca = extra_compile_args
- eca += ['-fPIC']
- if self.timing_output:
- eca += ['-DUSE_TIMINGOUTPUT']
- if self.split_fftw_many:
- eca += ['-DSPLIT_FFTW_MANY']
- if self.fftw_estimate:
- eca += ['-DUSE_FFTWESTIMATE']
- if self.disable_fftw_omp:
- eca += ['-DNO_FFTWOMP']
- os.makedirs('cmake_build_dir', exist_ok = True)
- os.chdir('cmake_build_dir')
- subprocess.check_call(['cmake', '..'])
- subprocess.check_call(['make', '-j4'])
- os.chdir('..')
- subprocess.check_call(['cp', 'cmake_build_dir/libbfps.a', 'bfps/'])
- #for fname in src_file_list:
- # ifile = 'bfps/cpp/' + fname + '.cpp'
- # ofile = 'obj/' + fname + '.o'
- # if not os.path.exists(ofile):
- # need_to_compile_file = True
- # else:
- # need_to_compile_file = False
- # if not need_to_compile:
- # latest = libtime
- # dependency_list = get_file_dependency_list(fname)
- # for depname in dependency_list.split()[1:]:
- # latest = max(latest,
- # datetime.datetime.fromtimestamp(os.path.getctime(depname)))
- # need_to_compile_file = (latest > libtime)
- # if need_to_compile_file:
- # command_strings = [compiler, '-c']
- # command_strings += ['bfps/cpp/' + fname + '.cpp']
- # command_strings += ['-o', 'obj/' + fname + '.o']
- # command_strings += eca
- # command_strings += ['-I' + idir for idir in include_dirs]
- # command_strings.append('-Ibfps/cpp/')
- # print(' '.join(command_strings))
- # subprocess.check_call(command_strings)
- #command_strings = ['ar', 'rvs', 'bfps/libbfps.a']
- #command_strings += ['obj/' + fname + '.o' for fname in src_file_list]
- #print(' '.join(command_strings))
- #subprocess.check_call(command_strings)
-
### save compiling information
pickle.dump(
{'include_dirs' : include_dirs,
@@ -278,26 +127,13 @@ class CompileLibCommand(distutils.cmd.Command):
protocol = 2)
return None
-def get_file_dependency_list(src_file):
- p = subprocess.Popen(
- ['g++', '-std=c++11', '-Ibfps/cpp', '-MM', 'bfps/cpp/' + src_file + '.cpp'],
- stdout = subprocess.PIPE)
- out, err = p.communicate()
- p.terminate()
- deps = str(out, 'ASCII').replace('\\\n', '')
- return deps
-
from setuptools import setup
setup(
name = 'bfps',
packages = ['bfps', 'bfps/test'],
install_requires = ['numpy>=1.8', 'h5py>=2.2.1'],
- cmdclass={'compile_library' : CompileLibCommand},
- package_data = {'bfps': header_list +
- ['libbfps.a',
- 'install_info.pickle'] +
- ['test/B32p1e4_checkpoint_0.h5']},
+ package_data = {'bfps': ['test/B32p1e4_checkpoint_0.h5']},
entry_points = {
'console_scripts': [
'bfps = bfps.__main__:main',
--
GitLab
From 01fd875a868fb6b6d436aa7bfe5e3f1163bb485e Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 11 Mar 2019 17:12:36 +0100
Subject: [PATCH 320/342] python package is built in cmake build tree
---
CMakeLists.txt | 11 ++--
{bfps/cpp => cpp}/Lagrange_polys.cpp | 0
{bfps/cpp => cpp}/Lagrange_polys.hpp | 0
{bfps/cpp => cpp}/base.hpp | 0
{bfps/cpp => cpp}/bfps_timer.hpp | 0
{bfps/cpp => cpp}/fftw_interface.hpp | 0
{bfps/cpp => cpp}/fftw_tools.cpp | 0
{bfps/cpp => cpp}/fftw_tools.hpp | 0
{bfps/cpp => cpp}/field.cpp | 0
{bfps/cpp => cpp}/field.hpp | 0
{bfps/cpp => cpp}/field_binary_IO.cpp | 0
{bfps/cpp => cpp}/field_binary_IO.hpp | 0
{bfps/cpp => cpp}/field_layout.cpp | 0
{bfps/cpp => cpp}/field_layout.hpp | 0
{bfps/cpp => cpp}/full_code/NSVE.cpp | 0
{bfps/cpp => cpp}/full_code/NSVE.hpp | 0
.../full_code/NSVE_field_stats.cpp | 0
.../full_code/NSVE_field_stats.hpp | 0
.../cpp => cpp}/full_code/NSVE_no_output.hpp | 0
.../full_code/NSVEcomplex_particles.cpp | 0
.../full_code/NSVEcomplex_particles.hpp | 0
.../full_code/NSVEp_extra_sampling.cpp | 0
.../full_code/NSVEp_extra_sampling.hpp | 0
{bfps/cpp => cpp}/full_code/NSVEparticles.cpp | 0
{bfps/cpp => cpp}/full_code/NSVEparticles.hpp | 0
.../full_code/NSVEparticles_no_output.hpp | 0
{bfps/cpp => cpp}/full_code/code_base.cpp | 0
{bfps/cpp => cpp}/full_code/code_base.hpp | 0
.../full_code/codes_with_no_output.hpp | 0
.../full_code/direct_numerical_simulation.cpp | 0
.../full_code/direct_numerical_simulation.hpp | 0
.../full_code/field_output_test.cpp | 0
.../full_code/field_output_test.hpp | 0
.../full_code/field_single_to_double.cpp | 0
.../full_code/field_single_to_double.hpp | 0
{bfps/cpp => cpp}/full_code/field_test.cpp | 0
{bfps/cpp => cpp}/full_code/field_test.hpp | 0
{bfps/cpp => cpp}/full_code/filter_test.cpp | 0
{bfps/cpp => cpp}/full_code/filter_test.hpp | 0
{bfps/cpp => cpp}/full_code/get_rfields.cpp | 0
{bfps/cpp => cpp}/full_code/get_rfields.hpp | 0
.../full_code/joint_acc_vel_stats.cpp | 0
.../full_code/joint_acc_vel_stats.hpp | 0
{bfps/cpp => cpp}/full_code/main_code.hpp | 0
.../full_code/native_binary_to_hdf5.cpp | 0
.../full_code/native_binary_to_hdf5.hpp | 0
{bfps/cpp => cpp}/full_code/postprocess.cpp | 0
{bfps/cpp => cpp}/full_code/postprocess.hpp | 0
{bfps/cpp => cpp}/full_code/resize.cpp | 0
{bfps/cpp => cpp}/full_code/resize.hpp | 0
.../cpp => cpp}/full_code/symmetrize_test.cpp | 0
.../cpp => cpp}/full_code/symmetrize_test.hpp | 0
{bfps/cpp => cpp}/full_code/test.cpp | 0
{bfps/cpp => cpp}/full_code/test.hpp | 0
.../full_code/test_interpolation.cpp | 0
.../full_code/test_interpolation.hpp | 0
{bfps/cpp => cpp}/hdf5_tools.cpp | 0
{bfps/cpp => cpp}/hdf5_tools.hpp | 0
{bfps/cpp => cpp}/kspace.cpp | 0
{bfps/cpp => cpp}/kspace.hpp | 0
{bfps/cpp => cpp}/omputils.hpp | 0
{bfps/cpp => cpp}/particles/.tocompile | 0
.../particles/abstract_particles_input.hpp | 0
.../particles/abstract_particles_output.hpp | 0
.../particles/abstract_particles_system.hpp | 0
.../particles/alltoall_exchanger.hpp | 0
{bfps/cpp => cpp}/particles/env_utils.hpp | 0
.../particles/lock_free_bool_array.hpp | 0
{bfps/cpp => cpp}/particles/p2p_computer.hpp | 0
.../particles/p2p_computer_empty.hpp | 0
{bfps/cpp => cpp}/particles/p2p_distr_mpi.hpp | 0
{bfps/cpp => cpp}/particles/p2p_tree.hpp | 0
.../particles/particles_adams_bashforth.hpp | 0
.../particles/particles_distr_mpi.hpp | 0
.../particles/particles_field_computer.hpp | 0
.../particles/particles_generic_interp.hpp | 0
.../particles/particles_inner_computer.cpp | 0
.../particles/particles_inner_computer.hpp | 0
.../particles_inner_computer_empty.hpp | 0
.../particles/particles_input_hdf5.hpp | 0
.../particles/particles_output_hdf5.hpp | 0
.../particles/particles_output_mpiio.hpp | 0
.../particles_output_sampling_hdf5.hpp | 0
.../particles/particles_sampling.hpp | 0
.../particles/particles_system.hpp | 0
.../particles/particles_system_builder.hpp | 0
.../cpp => cpp}/particles/particles_utils.hpp | 0
{bfps/cpp => cpp}/scope_timer.cpp | 0
{bfps/cpp => cpp}/scope_timer.hpp | 0
{bfps/cpp => cpp}/shared_array.hpp | 0
{bfps/cpp => cpp}/spline.hpp | 0
{bfps/cpp => cpp}/spline_n1.cpp | 0
{bfps/cpp => cpp}/spline_n1.hpp | 0
{bfps/cpp => cpp}/spline_n10.cpp | 0
{bfps/cpp => cpp}/spline_n10.hpp | 0
{bfps/cpp => cpp}/spline_n2.cpp | 0
{bfps/cpp => cpp}/spline_n2.hpp | 0
{bfps/cpp => cpp}/spline_n3.cpp | 0
{bfps/cpp => cpp}/spline_n3.hpp | 0
{bfps/cpp => cpp}/spline_n4.cpp | 0
{bfps/cpp => cpp}/spline_n4.hpp | 0
{bfps/cpp => cpp}/spline_n5.cpp | 0
{bfps/cpp => cpp}/spline_n5.hpp | 0
{bfps/cpp => cpp}/spline_n6.cpp | 0
{bfps/cpp => cpp}/spline_n6.hpp | 0
{bfps/cpp => cpp}/spline_n7.cpp | 0
{bfps/cpp => cpp}/spline_n7.hpp | 0
{bfps/cpp => cpp}/spline_n8.cpp | 0
{bfps/cpp => cpp}/spline_n8.hpp | 0
{bfps/cpp => cpp}/spline_n9.cpp | 0
{bfps/cpp => cpp}/spline_n9.hpp | 0
{bfps/cpp => cpp}/vorticity_equation.cpp | 0
{bfps/cpp => cpp}/vorticity_equation.hpp | 0
machine_settings_py.py | 63 -------------------
setup.py | 4 +-
115 files changed, 9 insertions(+), 69 deletions(-)
rename {bfps/cpp => cpp}/Lagrange_polys.cpp (100%)
rename {bfps/cpp => cpp}/Lagrange_polys.hpp (100%)
rename {bfps/cpp => cpp}/base.hpp (100%)
rename {bfps/cpp => cpp}/bfps_timer.hpp (100%)
rename {bfps/cpp => cpp}/fftw_interface.hpp (100%)
rename {bfps/cpp => cpp}/fftw_tools.cpp (100%)
rename {bfps/cpp => cpp}/fftw_tools.hpp (100%)
rename {bfps/cpp => cpp}/field.cpp (100%)
rename {bfps/cpp => cpp}/field.hpp (100%)
rename {bfps/cpp => cpp}/field_binary_IO.cpp (100%)
rename {bfps/cpp => cpp}/field_binary_IO.hpp (100%)
rename {bfps/cpp => cpp}/field_layout.cpp (100%)
rename {bfps/cpp => cpp}/field_layout.hpp (100%)
rename {bfps/cpp => cpp}/full_code/NSVE.cpp (100%)
rename {bfps/cpp => cpp}/full_code/NSVE.hpp (100%)
rename {bfps/cpp => cpp}/full_code/NSVE_field_stats.cpp (100%)
rename {bfps/cpp => cpp}/full_code/NSVE_field_stats.hpp (100%)
rename {bfps/cpp => cpp}/full_code/NSVE_no_output.hpp (100%)
rename {bfps/cpp => cpp}/full_code/NSVEcomplex_particles.cpp (100%)
rename {bfps/cpp => cpp}/full_code/NSVEcomplex_particles.hpp (100%)
rename {bfps/cpp => cpp}/full_code/NSVEp_extra_sampling.cpp (100%)
rename {bfps/cpp => cpp}/full_code/NSVEp_extra_sampling.hpp (100%)
rename {bfps/cpp => cpp}/full_code/NSVEparticles.cpp (100%)
rename {bfps/cpp => cpp}/full_code/NSVEparticles.hpp (100%)
rename {bfps/cpp => cpp}/full_code/NSVEparticles_no_output.hpp (100%)
rename {bfps/cpp => cpp}/full_code/code_base.cpp (100%)
rename {bfps/cpp => cpp}/full_code/code_base.hpp (100%)
rename {bfps/cpp => cpp}/full_code/codes_with_no_output.hpp (100%)
rename {bfps/cpp => cpp}/full_code/direct_numerical_simulation.cpp (100%)
rename {bfps/cpp => cpp}/full_code/direct_numerical_simulation.hpp (100%)
rename {bfps/cpp => cpp}/full_code/field_output_test.cpp (100%)
rename {bfps/cpp => cpp}/full_code/field_output_test.hpp (100%)
rename {bfps/cpp => cpp}/full_code/field_single_to_double.cpp (100%)
rename {bfps/cpp => cpp}/full_code/field_single_to_double.hpp (100%)
rename {bfps/cpp => cpp}/full_code/field_test.cpp (100%)
rename {bfps/cpp => cpp}/full_code/field_test.hpp (100%)
rename {bfps/cpp => cpp}/full_code/filter_test.cpp (100%)
rename {bfps/cpp => cpp}/full_code/filter_test.hpp (100%)
rename {bfps/cpp => cpp}/full_code/get_rfields.cpp (100%)
rename {bfps/cpp => cpp}/full_code/get_rfields.hpp (100%)
rename {bfps/cpp => cpp}/full_code/joint_acc_vel_stats.cpp (100%)
rename {bfps/cpp => cpp}/full_code/joint_acc_vel_stats.hpp (100%)
rename {bfps/cpp => cpp}/full_code/main_code.hpp (100%)
rename {bfps/cpp => cpp}/full_code/native_binary_to_hdf5.cpp (100%)
rename {bfps/cpp => cpp}/full_code/native_binary_to_hdf5.hpp (100%)
rename {bfps/cpp => cpp}/full_code/postprocess.cpp (100%)
rename {bfps/cpp => cpp}/full_code/postprocess.hpp (100%)
rename {bfps/cpp => cpp}/full_code/resize.cpp (100%)
rename {bfps/cpp => cpp}/full_code/resize.hpp (100%)
rename {bfps/cpp => cpp}/full_code/symmetrize_test.cpp (100%)
rename {bfps/cpp => cpp}/full_code/symmetrize_test.hpp (100%)
rename {bfps/cpp => cpp}/full_code/test.cpp (100%)
rename {bfps/cpp => cpp}/full_code/test.hpp (100%)
rename {bfps/cpp => cpp}/full_code/test_interpolation.cpp (100%)
rename {bfps/cpp => cpp}/full_code/test_interpolation.hpp (100%)
rename {bfps/cpp => cpp}/hdf5_tools.cpp (100%)
rename {bfps/cpp => cpp}/hdf5_tools.hpp (100%)
rename {bfps/cpp => cpp}/kspace.cpp (100%)
rename {bfps/cpp => cpp}/kspace.hpp (100%)
rename {bfps/cpp => cpp}/omputils.hpp (100%)
rename {bfps/cpp => cpp}/particles/.tocompile (100%)
rename {bfps/cpp => cpp}/particles/abstract_particles_input.hpp (100%)
rename {bfps/cpp => cpp}/particles/abstract_particles_output.hpp (100%)
rename {bfps/cpp => cpp}/particles/abstract_particles_system.hpp (100%)
rename {bfps/cpp => cpp}/particles/alltoall_exchanger.hpp (100%)
rename {bfps/cpp => cpp}/particles/env_utils.hpp (100%)
rename {bfps/cpp => cpp}/particles/lock_free_bool_array.hpp (100%)
rename {bfps/cpp => cpp}/particles/p2p_computer.hpp (100%)
rename {bfps/cpp => cpp}/particles/p2p_computer_empty.hpp (100%)
rename {bfps/cpp => cpp}/particles/p2p_distr_mpi.hpp (100%)
rename {bfps/cpp => cpp}/particles/p2p_tree.hpp (100%)
rename {bfps/cpp => cpp}/particles/particles_adams_bashforth.hpp (100%)
rename {bfps/cpp => cpp}/particles/particles_distr_mpi.hpp (100%)
rename {bfps/cpp => cpp}/particles/particles_field_computer.hpp (100%)
rename {bfps/cpp => cpp}/particles/particles_generic_interp.hpp (100%)
rename {bfps/cpp => cpp}/particles/particles_inner_computer.cpp (100%)
rename {bfps/cpp => cpp}/particles/particles_inner_computer.hpp (100%)
rename {bfps/cpp => cpp}/particles/particles_inner_computer_empty.hpp (100%)
rename {bfps/cpp => cpp}/particles/particles_input_hdf5.hpp (100%)
rename {bfps/cpp => cpp}/particles/particles_output_hdf5.hpp (100%)
rename {bfps/cpp => cpp}/particles/particles_output_mpiio.hpp (100%)
rename {bfps/cpp => cpp}/particles/particles_output_sampling_hdf5.hpp (100%)
rename {bfps/cpp => cpp}/particles/particles_sampling.hpp (100%)
rename {bfps/cpp => cpp}/particles/particles_system.hpp (100%)
rename {bfps/cpp => cpp}/particles/particles_system_builder.hpp (100%)
rename {bfps/cpp => cpp}/particles/particles_utils.hpp (100%)
rename {bfps/cpp => cpp}/scope_timer.cpp (100%)
rename {bfps/cpp => cpp}/scope_timer.hpp (100%)
rename {bfps/cpp => cpp}/shared_array.hpp (100%)
rename {bfps/cpp => cpp}/spline.hpp (100%)
rename {bfps/cpp => cpp}/spline_n1.cpp (100%)
rename {bfps/cpp => cpp}/spline_n1.hpp (100%)
rename {bfps/cpp => cpp}/spline_n10.cpp (100%)
rename {bfps/cpp => cpp}/spline_n10.hpp (100%)
rename {bfps/cpp => cpp}/spline_n2.cpp (100%)
rename {bfps/cpp => cpp}/spline_n2.hpp (100%)
rename {bfps/cpp => cpp}/spline_n3.cpp (100%)
rename {bfps/cpp => cpp}/spline_n3.hpp (100%)
rename {bfps/cpp => cpp}/spline_n4.cpp (100%)
rename {bfps/cpp => cpp}/spline_n4.hpp (100%)
rename {bfps/cpp => cpp}/spline_n5.cpp (100%)
rename {bfps/cpp => cpp}/spline_n5.hpp (100%)
rename {bfps/cpp => cpp}/spline_n6.cpp (100%)
rename {bfps/cpp => cpp}/spline_n6.hpp (100%)
rename {bfps/cpp => cpp}/spline_n7.cpp (100%)
rename {bfps/cpp => cpp}/spline_n7.hpp (100%)
rename {bfps/cpp => cpp}/spline_n8.cpp (100%)
rename {bfps/cpp => cpp}/spline_n8.hpp (100%)
rename {bfps/cpp => cpp}/spline_n9.cpp (100%)
rename {bfps/cpp => cpp}/spline_n9.hpp (100%)
rename {bfps/cpp => cpp}/vorticity_equation.cpp (100%)
rename {bfps/cpp => cpp}/vorticity_equation.hpp (100%)
delete mode 100644 machine_settings_py.py
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2877199e..e125f8ee 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -97,10 +97,10 @@ get_property(ALL_LINK_DIRS DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY LINK_D
#####################################################################################
## Build the lib
-include_directories(${PROJECT_SOURCE_DIR}/bfps/cpp)
+include_directories(${PROJECT_SOURCE_DIR}/cpp)
-file(GLOB_RECURSE cpp_for_lib ${PROJECT_SOURCE_DIR}/bfps/*.cpp)
-file(GLOB_RECURSE hpp_for_lib ${PROJECT_SOURCE_DIR}/bfps/*.hpp)
+file(GLOB_RECURSE cpp_for_lib ${PROJECT_SOURCE_DIR}/*.cpp)
+file(GLOB_RECURSE hpp_for_lib ${PROJECT_SOURCE_DIR}/*.hpp)
LIST(APPEND source_files ${hpp_for_lib} ${cpp_for_lib})
add_library(bfps ${source_files})
@@ -108,7 +108,7 @@ add_library(bfps ${source_files})
target_link_libraries(bfps ${BFPS_LIBS})
install(TARGETS bfps EXPORT BFPS_EXPORT DESTINATION lib/ )
-install(DIRECTORY ${PROJECT_SOURCE_DIR}/bfps/cpp DESTINATION include/bfps/ FILES_MATCHING PATTERN "*.h*")
+install(DIRECTORY ${PROJECT_SOURCE_DIR}/cpp DESTINATION include/bfps/ FILES_MATCHING PATTERN "*.h*")
#####################################################################################
##Â Export the configuration
@@ -122,5 +122,6 @@ install(EXPORT BFPS_EXPORT DESTINATION lib/)
#####################################################################################
##Â Install the python wrapper
-install(CODE "execute_process(COMMAND python setup.py install --force --prefix=${CMAKE_INSTALL_PREFIX} WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})")
+file(COPY ${PROJECT_SOURCE_DIR}/bfps DESTINATION ${PROJECT_BINARY_DIR}/python/)
+install(CODE "execute_process(COMMAND python ${PROJECT_SOURCE_DIR}/setup.py install --force --prefix=${CMAKE_INSTALL_PREFIX} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/python/)")
diff --git a/bfps/cpp/Lagrange_polys.cpp b/cpp/Lagrange_polys.cpp
similarity index 100%
rename from bfps/cpp/Lagrange_polys.cpp
rename to cpp/Lagrange_polys.cpp
diff --git a/bfps/cpp/Lagrange_polys.hpp b/cpp/Lagrange_polys.hpp
similarity index 100%
rename from bfps/cpp/Lagrange_polys.hpp
rename to cpp/Lagrange_polys.hpp
diff --git a/bfps/cpp/base.hpp b/cpp/base.hpp
similarity index 100%
rename from bfps/cpp/base.hpp
rename to cpp/base.hpp
diff --git a/bfps/cpp/bfps_timer.hpp b/cpp/bfps_timer.hpp
similarity index 100%
rename from bfps/cpp/bfps_timer.hpp
rename to cpp/bfps_timer.hpp
diff --git a/bfps/cpp/fftw_interface.hpp b/cpp/fftw_interface.hpp
similarity index 100%
rename from bfps/cpp/fftw_interface.hpp
rename to cpp/fftw_interface.hpp
diff --git a/bfps/cpp/fftw_tools.cpp b/cpp/fftw_tools.cpp
similarity index 100%
rename from bfps/cpp/fftw_tools.cpp
rename to cpp/fftw_tools.cpp
diff --git a/bfps/cpp/fftw_tools.hpp b/cpp/fftw_tools.hpp
similarity index 100%
rename from bfps/cpp/fftw_tools.hpp
rename to cpp/fftw_tools.hpp
diff --git a/bfps/cpp/field.cpp b/cpp/field.cpp
similarity index 100%
rename from bfps/cpp/field.cpp
rename to cpp/field.cpp
diff --git a/bfps/cpp/field.hpp b/cpp/field.hpp
similarity index 100%
rename from bfps/cpp/field.hpp
rename to cpp/field.hpp
diff --git a/bfps/cpp/field_binary_IO.cpp b/cpp/field_binary_IO.cpp
similarity index 100%
rename from bfps/cpp/field_binary_IO.cpp
rename to cpp/field_binary_IO.cpp
diff --git a/bfps/cpp/field_binary_IO.hpp b/cpp/field_binary_IO.hpp
similarity index 100%
rename from bfps/cpp/field_binary_IO.hpp
rename to cpp/field_binary_IO.hpp
diff --git a/bfps/cpp/field_layout.cpp b/cpp/field_layout.cpp
similarity index 100%
rename from bfps/cpp/field_layout.cpp
rename to cpp/field_layout.cpp
diff --git a/bfps/cpp/field_layout.hpp b/cpp/field_layout.hpp
similarity index 100%
rename from bfps/cpp/field_layout.hpp
rename to cpp/field_layout.hpp
diff --git a/bfps/cpp/full_code/NSVE.cpp b/cpp/full_code/NSVE.cpp
similarity index 100%
rename from bfps/cpp/full_code/NSVE.cpp
rename to cpp/full_code/NSVE.cpp
diff --git a/bfps/cpp/full_code/NSVE.hpp b/cpp/full_code/NSVE.hpp
similarity index 100%
rename from bfps/cpp/full_code/NSVE.hpp
rename to cpp/full_code/NSVE.hpp
diff --git a/bfps/cpp/full_code/NSVE_field_stats.cpp b/cpp/full_code/NSVE_field_stats.cpp
similarity index 100%
rename from bfps/cpp/full_code/NSVE_field_stats.cpp
rename to cpp/full_code/NSVE_field_stats.cpp
diff --git a/bfps/cpp/full_code/NSVE_field_stats.hpp b/cpp/full_code/NSVE_field_stats.hpp
similarity index 100%
rename from bfps/cpp/full_code/NSVE_field_stats.hpp
rename to cpp/full_code/NSVE_field_stats.hpp
diff --git a/bfps/cpp/full_code/NSVE_no_output.hpp b/cpp/full_code/NSVE_no_output.hpp
similarity index 100%
rename from bfps/cpp/full_code/NSVE_no_output.hpp
rename to cpp/full_code/NSVE_no_output.hpp
diff --git a/bfps/cpp/full_code/NSVEcomplex_particles.cpp b/cpp/full_code/NSVEcomplex_particles.cpp
similarity index 100%
rename from bfps/cpp/full_code/NSVEcomplex_particles.cpp
rename to cpp/full_code/NSVEcomplex_particles.cpp
diff --git a/bfps/cpp/full_code/NSVEcomplex_particles.hpp b/cpp/full_code/NSVEcomplex_particles.hpp
similarity index 100%
rename from bfps/cpp/full_code/NSVEcomplex_particles.hpp
rename to cpp/full_code/NSVEcomplex_particles.hpp
diff --git a/bfps/cpp/full_code/NSVEp_extra_sampling.cpp b/cpp/full_code/NSVEp_extra_sampling.cpp
similarity index 100%
rename from bfps/cpp/full_code/NSVEp_extra_sampling.cpp
rename to cpp/full_code/NSVEp_extra_sampling.cpp
diff --git a/bfps/cpp/full_code/NSVEp_extra_sampling.hpp b/cpp/full_code/NSVEp_extra_sampling.hpp
similarity index 100%
rename from bfps/cpp/full_code/NSVEp_extra_sampling.hpp
rename to cpp/full_code/NSVEp_extra_sampling.hpp
diff --git a/bfps/cpp/full_code/NSVEparticles.cpp b/cpp/full_code/NSVEparticles.cpp
similarity index 100%
rename from bfps/cpp/full_code/NSVEparticles.cpp
rename to cpp/full_code/NSVEparticles.cpp
diff --git a/bfps/cpp/full_code/NSVEparticles.hpp b/cpp/full_code/NSVEparticles.hpp
similarity index 100%
rename from bfps/cpp/full_code/NSVEparticles.hpp
rename to cpp/full_code/NSVEparticles.hpp
diff --git a/bfps/cpp/full_code/NSVEparticles_no_output.hpp b/cpp/full_code/NSVEparticles_no_output.hpp
similarity index 100%
rename from bfps/cpp/full_code/NSVEparticles_no_output.hpp
rename to cpp/full_code/NSVEparticles_no_output.hpp
diff --git a/bfps/cpp/full_code/code_base.cpp b/cpp/full_code/code_base.cpp
similarity index 100%
rename from bfps/cpp/full_code/code_base.cpp
rename to cpp/full_code/code_base.cpp
diff --git a/bfps/cpp/full_code/code_base.hpp b/cpp/full_code/code_base.hpp
similarity index 100%
rename from bfps/cpp/full_code/code_base.hpp
rename to cpp/full_code/code_base.hpp
diff --git a/bfps/cpp/full_code/codes_with_no_output.hpp b/cpp/full_code/codes_with_no_output.hpp
similarity index 100%
rename from bfps/cpp/full_code/codes_with_no_output.hpp
rename to cpp/full_code/codes_with_no_output.hpp
diff --git a/bfps/cpp/full_code/direct_numerical_simulation.cpp b/cpp/full_code/direct_numerical_simulation.cpp
similarity index 100%
rename from bfps/cpp/full_code/direct_numerical_simulation.cpp
rename to cpp/full_code/direct_numerical_simulation.cpp
diff --git a/bfps/cpp/full_code/direct_numerical_simulation.hpp b/cpp/full_code/direct_numerical_simulation.hpp
similarity index 100%
rename from bfps/cpp/full_code/direct_numerical_simulation.hpp
rename to cpp/full_code/direct_numerical_simulation.hpp
diff --git a/bfps/cpp/full_code/field_output_test.cpp b/cpp/full_code/field_output_test.cpp
similarity index 100%
rename from bfps/cpp/full_code/field_output_test.cpp
rename to cpp/full_code/field_output_test.cpp
diff --git a/bfps/cpp/full_code/field_output_test.hpp b/cpp/full_code/field_output_test.hpp
similarity index 100%
rename from bfps/cpp/full_code/field_output_test.hpp
rename to cpp/full_code/field_output_test.hpp
diff --git a/bfps/cpp/full_code/field_single_to_double.cpp b/cpp/full_code/field_single_to_double.cpp
similarity index 100%
rename from bfps/cpp/full_code/field_single_to_double.cpp
rename to cpp/full_code/field_single_to_double.cpp
diff --git a/bfps/cpp/full_code/field_single_to_double.hpp b/cpp/full_code/field_single_to_double.hpp
similarity index 100%
rename from bfps/cpp/full_code/field_single_to_double.hpp
rename to cpp/full_code/field_single_to_double.hpp
diff --git a/bfps/cpp/full_code/field_test.cpp b/cpp/full_code/field_test.cpp
similarity index 100%
rename from bfps/cpp/full_code/field_test.cpp
rename to cpp/full_code/field_test.cpp
diff --git a/bfps/cpp/full_code/field_test.hpp b/cpp/full_code/field_test.hpp
similarity index 100%
rename from bfps/cpp/full_code/field_test.hpp
rename to cpp/full_code/field_test.hpp
diff --git a/bfps/cpp/full_code/filter_test.cpp b/cpp/full_code/filter_test.cpp
similarity index 100%
rename from bfps/cpp/full_code/filter_test.cpp
rename to cpp/full_code/filter_test.cpp
diff --git a/bfps/cpp/full_code/filter_test.hpp b/cpp/full_code/filter_test.hpp
similarity index 100%
rename from bfps/cpp/full_code/filter_test.hpp
rename to cpp/full_code/filter_test.hpp
diff --git a/bfps/cpp/full_code/get_rfields.cpp b/cpp/full_code/get_rfields.cpp
similarity index 100%
rename from bfps/cpp/full_code/get_rfields.cpp
rename to cpp/full_code/get_rfields.cpp
diff --git a/bfps/cpp/full_code/get_rfields.hpp b/cpp/full_code/get_rfields.hpp
similarity index 100%
rename from bfps/cpp/full_code/get_rfields.hpp
rename to cpp/full_code/get_rfields.hpp
diff --git a/bfps/cpp/full_code/joint_acc_vel_stats.cpp b/cpp/full_code/joint_acc_vel_stats.cpp
similarity index 100%
rename from bfps/cpp/full_code/joint_acc_vel_stats.cpp
rename to cpp/full_code/joint_acc_vel_stats.cpp
diff --git a/bfps/cpp/full_code/joint_acc_vel_stats.hpp b/cpp/full_code/joint_acc_vel_stats.hpp
similarity index 100%
rename from bfps/cpp/full_code/joint_acc_vel_stats.hpp
rename to cpp/full_code/joint_acc_vel_stats.hpp
diff --git a/bfps/cpp/full_code/main_code.hpp b/cpp/full_code/main_code.hpp
similarity index 100%
rename from bfps/cpp/full_code/main_code.hpp
rename to cpp/full_code/main_code.hpp
diff --git a/bfps/cpp/full_code/native_binary_to_hdf5.cpp b/cpp/full_code/native_binary_to_hdf5.cpp
similarity index 100%
rename from bfps/cpp/full_code/native_binary_to_hdf5.cpp
rename to cpp/full_code/native_binary_to_hdf5.cpp
diff --git a/bfps/cpp/full_code/native_binary_to_hdf5.hpp b/cpp/full_code/native_binary_to_hdf5.hpp
similarity index 100%
rename from bfps/cpp/full_code/native_binary_to_hdf5.hpp
rename to cpp/full_code/native_binary_to_hdf5.hpp
diff --git a/bfps/cpp/full_code/postprocess.cpp b/cpp/full_code/postprocess.cpp
similarity index 100%
rename from bfps/cpp/full_code/postprocess.cpp
rename to cpp/full_code/postprocess.cpp
diff --git a/bfps/cpp/full_code/postprocess.hpp b/cpp/full_code/postprocess.hpp
similarity index 100%
rename from bfps/cpp/full_code/postprocess.hpp
rename to cpp/full_code/postprocess.hpp
diff --git a/bfps/cpp/full_code/resize.cpp b/cpp/full_code/resize.cpp
similarity index 100%
rename from bfps/cpp/full_code/resize.cpp
rename to cpp/full_code/resize.cpp
diff --git a/bfps/cpp/full_code/resize.hpp b/cpp/full_code/resize.hpp
similarity index 100%
rename from bfps/cpp/full_code/resize.hpp
rename to cpp/full_code/resize.hpp
diff --git a/bfps/cpp/full_code/symmetrize_test.cpp b/cpp/full_code/symmetrize_test.cpp
similarity index 100%
rename from bfps/cpp/full_code/symmetrize_test.cpp
rename to cpp/full_code/symmetrize_test.cpp
diff --git a/bfps/cpp/full_code/symmetrize_test.hpp b/cpp/full_code/symmetrize_test.hpp
similarity index 100%
rename from bfps/cpp/full_code/symmetrize_test.hpp
rename to cpp/full_code/symmetrize_test.hpp
diff --git a/bfps/cpp/full_code/test.cpp b/cpp/full_code/test.cpp
similarity index 100%
rename from bfps/cpp/full_code/test.cpp
rename to cpp/full_code/test.cpp
diff --git a/bfps/cpp/full_code/test.hpp b/cpp/full_code/test.hpp
similarity index 100%
rename from bfps/cpp/full_code/test.hpp
rename to cpp/full_code/test.hpp
diff --git a/bfps/cpp/full_code/test_interpolation.cpp b/cpp/full_code/test_interpolation.cpp
similarity index 100%
rename from bfps/cpp/full_code/test_interpolation.cpp
rename to cpp/full_code/test_interpolation.cpp
diff --git a/bfps/cpp/full_code/test_interpolation.hpp b/cpp/full_code/test_interpolation.hpp
similarity index 100%
rename from bfps/cpp/full_code/test_interpolation.hpp
rename to cpp/full_code/test_interpolation.hpp
diff --git a/bfps/cpp/hdf5_tools.cpp b/cpp/hdf5_tools.cpp
similarity index 100%
rename from bfps/cpp/hdf5_tools.cpp
rename to cpp/hdf5_tools.cpp
diff --git a/bfps/cpp/hdf5_tools.hpp b/cpp/hdf5_tools.hpp
similarity index 100%
rename from bfps/cpp/hdf5_tools.hpp
rename to cpp/hdf5_tools.hpp
diff --git a/bfps/cpp/kspace.cpp b/cpp/kspace.cpp
similarity index 100%
rename from bfps/cpp/kspace.cpp
rename to cpp/kspace.cpp
diff --git a/bfps/cpp/kspace.hpp b/cpp/kspace.hpp
similarity index 100%
rename from bfps/cpp/kspace.hpp
rename to cpp/kspace.hpp
diff --git a/bfps/cpp/omputils.hpp b/cpp/omputils.hpp
similarity index 100%
rename from bfps/cpp/omputils.hpp
rename to cpp/omputils.hpp
diff --git a/bfps/cpp/particles/.tocompile b/cpp/particles/.tocompile
similarity index 100%
rename from bfps/cpp/particles/.tocompile
rename to cpp/particles/.tocompile
diff --git a/bfps/cpp/particles/abstract_particles_input.hpp b/cpp/particles/abstract_particles_input.hpp
similarity index 100%
rename from bfps/cpp/particles/abstract_particles_input.hpp
rename to cpp/particles/abstract_particles_input.hpp
diff --git a/bfps/cpp/particles/abstract_particles_output.hpp b/cpp/particles/abstract_particles_output.hpp
similarity index 100%
rename from bfps/cpp/particles/abstract_particles_output.hpp
rename to cpp/particles/abstract_particles_output.hpp
diff --git a/bfps/cpp/particles/abstract_particles_system.hpp b/cpp/particles/abstract_particles_system.hpp
similarity index 100%
rename from bfps/cpp/particles/abstract_particles_system.hpp
rename to cpp/particles/abstract_particles_system.hpp
diff --git a/bfps/cpp/particles/alltoall_exchanger.hpp b/cpp/particles/alltoall_exchanger.hpp
similarity index 100%
rename from bfps/cpp/particles/alltoall_exchanger.hpp
rename to cpp/particles/alltoall_exchanger.hpp
diff --git a/bfps/cpp/particles/env_utils.hpp b/cpp/particles/env_utils.hpp
similarity index 100%
rename from bfps/cpp/particles/env_utils.hpp
rename to cpp/particles/env_utils.hpp
diff --git a/bfps/cpp/particles/lock_free_bool_array.hpp b/cpp/particles/lock_free_bool_array.hpp
similarity index 100%
rename from bfps/cpp/particles/lock_free_bool_array.hpp
rename to cpp/particles/lock_free_bool_array.hpp
diff --git a/bfps/cpp/particles/p2p_computer.hpp b/cpp/particles/p2p_computer.hpp
similarity index 100%
rename from bfps/cpp/particles/p2p_computer.hpp
rename to cpp/particles/p2p_computer.hpp
diff --git a/bfps/cpp/particles/p2p_computer_empty.hpp b/cpp/particles/p2p_computer_empty.hpp
similarity index 100%
rename from bfps/cpp/particles/p2p_computer_empty.hpp
rename to cpp/particles/p2p_computer_empty.hpp
diff --git a/bfps/cpp/particles/p2p_distr_mpi.hpp b/cpp/particles/p2p_distr_mpi.hpp
similarity index 100%
rename from bfps/cpp/particles/p2p_distr_mpi.hpp
rename to cpp/particles/p2p_distr_mpi.hpp
diff --git a/bfps/cpp/particles/p2p_tree.hpp b/cpp/particles/p2p_tree.hpp
similarity index 100%
rename from bfps/cpp/particles/p2p_tree.hpp
rename to cpp/particles/p2p_tree.hpp
diff --git a/bfps/cpp/particles/particles_adams_bashforth.hpp b/cpp/particles/particles_adams_bashforth.hpp
similarity index 100%
rename from bfps/cpp/particles/particles_adams_bashforth.hpp
rename to cpp/particles/particles_adams_bashforth.hpp
diff --git a/bfps/cpp/particles/particles_distr_mpi.hpp b/cpp/particles/particles_distr_mpi.hpp
similarity index 100%
rename from bfps/cpp/particles/particles_distr_mpi.hpp
rename to cpp/particles/particles_distr_mpi.hpp
diff --git a/bfps/cpp/particles/particles_field_computer.hpp b/cpp/particles/particles_field_computer.hpp
similarity index 100%
rename from bfps/cpp/particles/particles_field_computer.hpp
rename to cpp/particles/particles_field_computer.hpp
diff --git a/bfps/cpp/particles/particles_generic_interp.hpp b/cpp/particles/particles_generic_interp.hpp
similarity index 100%
rename from bfps/cpp/particles/particles_generic_interp.hpp
rename to cpp/particles/particles_generic_interp.hpp
diff --git a/bfps/cpp/particles/particles_inner_computer.cpp b/cpp/particles/particles_inner_computer.cpp
similarity index 100%
rename from bfps/cpp/particles/particles_inner_computer.cpp
rename to cpp/particles/particles_inner_computer.cpp
diff --git a/bfps/cpp/particles/particles_inner_computer.hpp b/cpp/particles/particles_inner_computer.hpp
similarity index 100%
rename from bfps/cpp/particles/particles_inner_computer.hpp
rename to cpp/particles/particles_inner_computer.hpp
diff --git a/bfps/cpp/particles/particles_inner_computer_empty.hpp b/cpp/particles/particles_inner_computer_empty.hpp
similarity index 100%
rename from bfps/cpp/particles/particles_inner_computer_empty.hpp
rename to cpp/particles/particles_inner_computer_empty.hpp
diff --git a/bfps/cpp/particles/particles_input_hdf5.hpp b/cpp/particles/particles_input_hdf5.hpp
similarity index 100%
rename from bfps/cpp/particles/particles_input_hdf5.hpp
rename to cpp/particles/particles_input_hdf5.hpp
diff --git a/bfps/cpp/particles/particles_output_hdf5.hpp b/cpp/particles/particles_output_hdf5.hpp
similarity index 100%
rename from bfps/cpp/particles/particles_output_hdf5.hpp
rename to cpp/particles/particles_output_hdf5.hpp
diff --git a/bfps/cpp/particles/particles_output_mpiio.hpp b/cpp/particles/particles_output_mpiio.hpp
similarity index 100%
rename from bfps/cpp/particles/particles_output_mpiio.hpp
rename to cpp/particles/particles_output_mpiio.hpp
diff --git a/bfps/cpp/particles/particles_output_sampling_hdf5.hpp b/cpp/particles/particles_output_sampling_hdf5.hpp
similarity index 100%
rename from bfps/cpp/particles/particles_output_sampling_hdf5.hpp
rename to cpp/particles/particles_output_sampling_hdf5.hpp
diff --git a/bfps/cpp/particles/particles_sampling.hpp b/cpp/particles/particles_sampling.hpp
similarity index 100%
rename from bfps/cpp/particles/particles_sampling.hpp
rename to cpp/particles/particles_sampling.hpp
diff --git a/bfps/cpp/particles/particles_system.hpp b/cpp/particles/particles_system.hpp
similarity index 100%
rename from bfps/cpp/particles/particles_system.hpp
rename to cpp/particles/particles_system.hpp
diff --git a/bfps/cpp/particles/particles_system_builder.hpp b/cpp/particles/particles_system_builder.hpp
similarity index 100%
rename from bfps/cpp/particles/particles_system_builder.hpp
rename to cpp/particles/particles_system_builder.hpp
diff --git a/bfps/cpp/particles/particles_utils.hpp b/cpp/particles/particles_utils.hpp
similarity index 100%
rename from bfps/cpp/particles/particles_utils.hpp
rename to cpp/particles/particles_utils.hpp
diff --git a/bfps/cpp/scope_timer.cpp b/cpp/scope_timer.cpp
similarity index 100%
rename from bfps/cpp/scope_timer.cpp
rename to cpp/scope_timer.cpp
diff --git a/bfps/cpp/scope_timer.hpp b/cpp/scope_timer.hpp
similarity index 100%
rename from bfps/cpp/scope_timer.hpp
rename to cpp/scope_timer.hpp
diff --git a/bfps/cpp/shared_array.hpp b/cpp/shared_array.hpp
similarity index 100%
rename from bfps/cpp/shared_array.hpp
rename to cpp/shared_array.hpp
diff --git a/bfps/cpp/spline.hpp b/cpp/spline.hpp
similarity index 100%
rename from bfps/cpp/spline.hpp
rename to cpp/spline.hpp
diff --git a/bfps/cpp/spline_n1.cpp b/cpp/spline_n1.cpp
similarity index 100%
rename from bfps/cpp/spline_n1.cpp
rename to cpp/spline_n1.cpp
diff --git a/bfps/cpp/spline_n1.hpp b/cpp/spline_n1.hpp
similarity index 100%
rename from bfps/cpp/spline_n1.hpp
rename to cpp/spline_n1.hpp
diff --git a/bfps/cpp/spline_n10.cpp b/cpp/spline_n10.cpp
similarity index 100%
rename from bfps/cpp/spline_n10.cpp
rename to cpp/spline_n10.cpp
diff --git a/bfps/cpp/spline_n10.hpp b/cpp/spline_n10.hpp
similarity index 100%
rename from bfps/cpp/spline_n10.hpp
rename to cpp/spline_n10.hpp
diff --git a/bfps/cpp/spline_n2.cpp b/cpp/spline_n2.cpp
similarity index 100%
rename from bfps/cpp/spline_n2.cpp
rename to cpp/spline_n2.cpp
diff --git a/bfps/cpp/spline_n2.hpp b/cpp/spline_n2.hpp
similarity index 100%
rename from bfps/cpp/spline_n2.hpp
rename to cpp/spline_n2.hpp
diff --git a/bfps/cpp/spline_n3.cpp b/cpp/spline_n3.cpp
similarity index 100%
rename from bfps/cpp/spline_n3.cpp
rename to cpp/spline_n3.cpp
diff --git a/bfps/cpp/spline_n3.hpp b/cpp/spline_n3.hpp
similarity index 100%
rename from bfps/cpp/spline_n3.hpp
rename to cpp/spline_n3.hpp
diff --git a/bfps/cpp/spline_n4.cpp b/cpp/spline_n4.cpp
similarity index 100%
rename from bfps/cpp/spline_n4.cpp
rename to cpp/spline_n4.cpp
diff --git a/bfps/cpp/spline_n4.hpp b/cpp/spline_n4.hpp
similarity index 100%
rename from bfps/cpp/spline_n4.hpp
rename to cpp/spline_n4.hpp
diff --git a/bfps/cpp/spline_n5.cpp b/cpp/spline_n5.cpp
similarity index 100%
rename from bfps/cpp/spline_n5.cpp
rename to cpp/spline_n5.cpp
diff --git a/bfps/cpp/spline_n5.hpp b/cpp/spline_n5.hpp
similarity index 100%
rename from bfps/cpp/spline_n5.hpp
rename to cpp/spline_n5.hpp
diff --git a/bfps/cpp/spline_n6.cpp b/cpp/spline_n6.cpp
similarity index 100%
rename from bfps/cpp/spline_n6.cpp
rename to cpp/spline_n6.cpp
diff --git a/bfps/cpp/spline_n6.hpp b/cpp/spline_n6.hpp
similarity index 100%
rename from bfps/cpp/spline_n6.hpp
rename to cpp/spline_n6.hpp
diff --git a/bfps/cpp/spline_n7.cpp b/cpp/spline_n7.cpp
similarity index 100%
rename from bfps/cpp/spline_n7.cpp
rename to cpp/spline_n7.cpp
diff --git a/bfps/cpp/spline_n7.hpp b/cpp/spline_n7.hpp
similarity index 100%
rename from bfps/cpp/spline_n7.hpp
rename to cpp/spline_n7.hpp
diff --git a/bfps/cpp/spline_n8.cpp b/cpp/spline_n8.cpp
similarity index 100%
rename from bfps/cpp/spline_n8.cpp
rename to cpp/spline_n8.cpp
diff --git a/bfps/cpp/spline_n8.hpp b/cpp/spline_n8.hpp
similarity index 100%
rename from bfps/cpp/spline_n8.hpp
rename to cpp/spline_n8.hpp
diff --git a/bfps/cpp/spline_n9.cpp b/cpp/spline_n9.cpp
similarity index 100%
rename from bfps/cpp/spline_n9.cpp
rename to cpp/spline_n9.cpp
diff --git a/bfps/cpp/spline_n9.hpp b/cpp/spline_n9.hpp
similarity index 100%
rename from bfps/cpp/spline_n9.hpp
rename to cpp/spline_n9.hpp
diff --git a/bfps/cpp/vorticity_equation.cpp b/cpp/vorticity_equation.cpp
similarity index 100%
rename from bfps/cpp/vorticity_equation.cpp
rename to cpp/vorticity_equation.cpp
diff --git a/bfps/cpp/vorticity_equation.hpp b/cpp/vorticity_equation.hpp
similarity index 100%
rename from bfps/cpp/vorticity_equation.hpp
rename to cpp/vorticity_equation.hpp
diff --git a/machine_settings_py.py b/machine_settings_py.py
deleted file mode 100644
index 787f1d5a..00000000
--- a/machine_settings_py.py
+++ /dev/null
@@ -1,63 +0,0 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
-
-
-
-import os
-
-########################################################################
-# these lists should be adapted for your different environment(s)
-# personally, I have access to setups where my home folder is shared
-# between different machines, including cluster and desktop, therefore
-# I check the host name when choosing libraries etc.
-# feel free to do your own thing to the copy of this file placed in
-# ./config/bfps
-########################################################################
-
-hostname = os.getenv('HOSTNAME')
-
-compiler = 'g++'
-extra_compile_args = ['-Wall', '-O2', '-g', '-mtune=native', '-ffast-math', '-std=c++11']
-extra_libraries = ['hdf5']
-include_dirs = []
-library_dirs = []
-
-if hostname == 'chichi-G':
- include_dirs = ['/usr/local/include',
- '/usr/include/mpich']
- library_dirs = ['/usr/local/lib',
- '/usr/lib/mpich']
- extra_libraries += ['mpich']
-
-if hostname in ['tolima', 'misti']:
- local_install_dir = '/scratch.local/chichi/installs'
-
- include_dirs = ['/usr/lib64/mpi/gcc/openmpi/include',
- os.path.join(local_install_dir, 'include')]
-
- library_dirs = ['/usr/lib64/mpi/gcc/openmpi/lib64',
- os.path.join(local_install_dir, 'lib'),
- os.path.join(local_install_dir, 'lib64')]
- extra_libraries += ['mpi_cxx', 'mpi']
-
diff --git a/setup.py b/setup.py
index 2a031518..3d91061d 100644
--- a/setup.py
+++ b/setup.py
@@ -146,9 +146,11 @@ setup(
version = VERSION,
########################################################################
# useless stuff folows
+# if anyone knows how to open the README when calling this script from
+# cmake, please let me know.
########################################################################
description = 'Big Fluid and Particle Simulator',
- long_description = open('README.rst', 'r').read(),
+ #long_description = open('${PROJECT_SOURCE_DIR}/README.rst', 'r').read(),
author = AUTHOR,
author_email = AUTHOR_EMAIL,
license = 'GPL version 3.0')
--
GitLab
From 6dd28874c3b1894179018f3039d91b20feaa5d84 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 11 Mar 2019 18:51:20 +0100
Subject: [PATCH 321/342] [broken] trying to compile executable
---
CMakeLists.txt | 10 ++++++++--
bfps/__init__.py | 3 ---
bfps/_base.py | 4 +---
bfps/_code.py | 31 +++----------------------------
setup.py | 29 ++---------------------------
5 files changed, 14 insertions(+), 63 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e125f8ee..6c2bf011 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,6 +1,12 @@
cmake_minimum_required(VERSION 3.10)
-project(BFPS C CXX)
+project(BFPS)
+
+execute_process(COMMAND python ${PROJECT_SOURCE_DIR}/get_version.py OUTPUT_VARIABLE BFPS_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+project(BFPS
+ VERSION ${BFPS_VERSION}
+ LANGUAGES C CXX)
set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/morse ${CMAKE_MODULE_PATH})
@@ -60,7 +66,7 @@ option(BFPS_HDF5_USE_SZIP "Set to on to also link against SZIP" OFF)
if(BFPS_HDF5_USE_SZIP)
option(BFPS_HDF5_SZIP_LIB_PATH "Additional lib path for SZIP" "")
if(BFPS_HDF5_SZIP_LIB_PATH)
- link_directories(${BFPS_HDF5_SZIP_LIB_PATH})
+ link_directories(${BFPS_HDF5_SZIP_LIB_PATH})
endif()
list(APPEND BFPS_LIBS "z")
endif()
diff --git a/bfps/__init__.py b/bfps/__init__.py
index babbc203..09e5549f 100644
--- a/bfps/__init__.py
+++ b/bfps/__init__.py
@@ -38,9 +38,6 @@ here = os.path.normcase(__file__)
header_dir = os.path.join(os.path.join(dist_loc, 'bfps'), 'cpp')
lib_dir = os.path.join(dist_loc, 'bfps')
-install_info = pickle.load(
- open(os.path.join(os.path.dirname(here), 'install_info.pickle'), 'rb'))
-
homefolder = os.path.expanduser('~')
bfpsfolder = os.path.join(homefolder, '.config/', 'bfps')
sys.path.append(bfpsfolder)
diff --git a/bfps/_base.py b/bfps/_base.py
index 3d7c747b..fdddb0f7 100644
--- a/bfps/_base.py
+++ b/bfps/_base.py
@@ -28,7 +28,6 @@ import os
import sys
import numpy as np
import h5py
-from bfps import install_info
from bfps import __version__
class _base(object):
@@ -175,8 +174,7 @@ class _base(object):
ofile['parameters/' + k] = self.parameters[k]
ofile['iteration'] = int(iter0)
ofile['bfps_info/solver_class'] = type(self).__name__
- for k in install_info.keys():
- ofile['bfps_info/' + k] = str(install_info[k])
+ ofile['bfps_info/VERSION'] = __version__
ofile.close()
return None
def rewrite_par(
diff --git a/bfps/_code.py b/bfps/_code.py
index f997d651..a8a20207 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -198,36 +198,11 @@ class _code(_base):
self,
no_debug = False):
# compile code
- if not os.path.isfile(os.path.join(bfps.header_dir, 'base.hpp')):
- raise IOError('header not there:\n' +
- '{0}\n'.format(os.path.join(bfps.header_dir, 'base.hpp')) +
- '{0}\n'.format(bfps.dist_loc))
- libraries = ['bfps']
- libraries += bfps.install_info['libraries']
-
- command_strings = [bfps.install_info['compiler']]
- command_strings += [self.name + '.cpp', '-o', self.name]
- command_strings += bfps.install_info['extra_compile_args']
- if no_debug:
- command_strings += ['-DNDEBUG']
- command_strings += ['-I' + idir for idir in bfps.install_info['include_dirs']]
- command_strings.append('-I' + bfps.header_dir)
- command_strings += ['-L' + ldir for ldir in bfps.install_info['library_dirs']]
- command_strings += ['-Wl,-rpath=' + ldir for ldir in bfps.install_info['library_dirs']]
- command_strings.append('-L' + bfps.lib_dir)
- command_strings.append('-Wl,-rpath=' + bfps.lib_dir)
-
- for libname in libraries:
- if libname[0] not in ['-', '/']:
- command_strings += ['-l' + libname]
- else:
- command_strings += [libname]
-
- command_strings += ['-fopenmp']
-
+ os.makedirs('bfps_build', exist_ok = True)
+ os.chdir('bfps_build')
self.write_src()
+
print('compiling code with command\n' + ' '.join(command_strings))
- return subprocess.check_call(command_strings)
def set_host_info(
self,
host_info = {}):
diff --git a/setup.py b/setup.py
index 3d91061d..925f6521 100644
--- a/setup.py
+++ b/setup.py
@@ -56,33 +56,8 @@ from machine_settings import compiler, include_dirs, library_dirs, extra_compile
### package versioning
-# get current time
-now = datetime.datetime.now()
-# obtain version
-try:
- git_branch = subprocess.check_output(['git',
- 'rev-parse',
- '--abbrev-ref',
- 'HEAD']).strip().split()[-1].decode()
- git_revision = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()
- git_date = datetime.datetime.fromtimestamp(int(subprocess.check_output(['git', 'log', '-1', '--format=%ct']).strip()))
-except:
- git_revision = ''
- git_branch = ''
- git_date = now
-if git_branch == '':
- # there's no git available or something
- VERSION = '{0:0>4}{1:0>2}{2:0>2}.{3:0>2}{4:0>2}{5:0>2}'.format(
- git_date.year, git_date.month, git_date.day,
- git_date.hour, git_date.minute, git_date.second)
-else:
- if (('develop' in git_branch) or
- ('feature' in git_branch) or
- ('bugfix' in git_branch)):
- VERSION = subprocess.check_output(
- ['git', 'describe', '--tags', '--dirty']).strip().decode().replace('-g', '+g').replace('-dirty', '.dirty').replace('-', '.post')
- else:
- VERSION = subprocess.check_output(['git', 'describe', '--tags']).strip().decode().split('-')[0]
+import get_version
+VERSION = get_version.main()
print('This is bfps version ' + VERSION)
--
GitLab
From 3bc3ee202997bfdae64b370e9760481473f950e2 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 11 Mar 2019 21:44:18 +0100
Subject: [PATCH 322/342] add version script
---
get_version.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 64 insertions(+)
create mode 100644 get_version.py
diff --git a/get_version.py b/get_version.py
new file mode 100644
index 00000000..896763e8
--- /dev/null
+++ b/get_version.py
@@ -0,0 +1,64 @@
+#######################################################################
+# #
+# Copyright 2015-2019 Max Planck Institute #
+# for Dynamics and Self-Organization #
+# #
+# This file is part of bfps. #
+# #
+# bfps is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# bfps is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with bfps. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+#######################################################################
+
+
+
+import datetime
+import subprocess
+
+def main():
+ # get current time
+ now = datetime.datetime.now()
+ # obtain version
+ try:
+ git_branch = subprocess.check_output(['git',
+ 'rev-parse',
+ '--abbrev-ref',
+ 'HEAD']).strip().split()[-1].decode()
+ git_revision = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()
+ git_date = datetime.datetime.fromtimestamp(int(subprocess.check_output(['git', 'log', '-1', '--format=%ct']).strip()))
+ except:
+ git_revision = ''
+ git_branch = ''
+ git_date = now
+ if git_branch == '':
+ # there's no git available or something
+ VERSION = '{0:0>4}{1:0>2}{2:0>2}.{3:0>2}{4:0>2}{5:0>2}'.format(
+ git_date.year, git_date.month, git_date.day,
+ git_date.hour, git_date.minute, git_date.second)
+ else:
+ VERSION = subprocess.check_output(['git', 'describe', '--tags']).strip().decode().split('-')[0]
+ if (('develop' in git_branch) or
+ ('feature' in git_branch) or
+ ('bugfix' in git_branch)):
+ VERSION_py = subprocess.check_output(
+ ['git', 'describe', '--tags', '--dirty']).strip().decode().replace('-g', '+g').replace('-dirty', '.dirty').replace('-', '.post')
+ else:
+ VERSION_py = VERSION
+ print(VERSION)
+ return VERSION_py
+
+if __name__ == '__main__':
+ main()
+
--
GitLab
From 32794116085aaacf18a9664121cf981a0ad9591e Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 12 Mar 2019 09:13:09 +0100
Subject: [PATCH 323/342] create temporary cmake config from _code.py
---
CMakeLists.txt | 5 +++--
bfps/_code.py | 14 ++++++++++++--
2 files changed, 15 insertions(+), 4 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6c2bf011..ba3905dc 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -114,7 +114,7 @@ add_library(bfps ${source_files})
target_link_libraries(bfps ${BFPS_LIBS})
install(TARGETS bfps EXPORT BFPS_EXPORT DESTINATION lib/ )
-install(DIRECTORY ${PROJECT_SOURCE_DIR}/cpp DESTINATION include/bfps/ FILES_MATCHING PATTERN "*.h*")
+install(DIRECTORY ${PROJECT_SOURCE_DIR}/cpp/ DESTINATION include/bfps/ FILES_MATCHING PATTERN "*.h*")
#####################################################################################
##Â Export the configuration
@@ -128,6 +128,7 @@ install(EXPORT BFPS_EXPORT DESTINATION lib/)
#####################################################################################
##Â Install the python wrapper
-file(COPY ${PROJECT_SOURCE_DIR}/bfps DESTINATION ${PROJECT_BINARY_DIR}/python/)
+# copy command
+install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E copy_directory ${PROJECT_SOURCE_DIR}/bfps ${PROJECT_BINARY_DIR}/python/bfps/)")
install(CODE "execute_process(COMMAND python ${PROJECT_SOURCE_DIR}/setup.py install --force --prefix=${CMAKE_INSTALL_PREFIX} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/python/)")
diff --git a/bfps/_code.py b/bfps/_code.py
index a8a20207..eaaa5e2a 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -201,8 +201,18 @@ class _code(_base):
os.makedirs('bfps_build', exist_ok = True)
os.chdir('bfps_build')
self.write_src()
-
- print('compiling code with command\n' + ' '.join(command_strings))
+ with open('CMakeLists.txt', 'w') as outfile:
+ outfile.write('cmake_minimum_required(VERSION 3.10)\n')
+ outfile.write('project(bla LANGUAGES CXX)\n')
+ outfile.write('set(CXX $ENV{MPICXX})\n')
+ outfile.write('find_package(BFPS REQUIRED)\n')
+ outfile.write('include_directories(${BFPS_INCLUDE_DIRECTORIES} ${BFPS_INCLUDE_DIRECTORIES}/bfps)\n')
+ outfile.write('execute_process(COMMAND echo ${BFPS_LIBS})\n')
+ outfile.write('add_executable({0} {0}.cpp)\n'.format(self.name))
+ outfile.write('target_link_libraries(' + self.name + ' ${BFPS_LIBS} "-lbfps")\n')
+ subprocess.check_call(['cmake', '.'])
+ subprocess.check_call(['make'])
+ return None
def set_host_info(
self,
host_info = {}):
--
GitLab
From 3bf50fe3aea1e9046ec05939d2074c121afcb980 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 12 Mar 2019 11:28:34 +0100
Subject: [PATCH 324/342] [still broken] tweak executable compilation
---
bfps/_code.py | 12 ++++++++----
cmake/BFPSConfig.cmake.in | 3 ++-
2 files changed, 10 insertions(+), 5 deletions(-)
diff --git a/bfps/_code.py b/bfps/_code.py
index eaaa5e2a..8a61125d 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -203,13 +203,17 @@ class _code(_base):
self.write_src()
with open('CMakeLists.txt', 'w') as outfile:
outfile.write('cmake_minimum_required(VERSION 3.10)\n')
- outfile.write('project(bla LANGUAGES CXX)\n')
outfile.write('set(CXX $ENV{MPICXX})\n')
+ outfile.write('project(bla LANGUAGES CXX)\n')
+ outfile.write('set(CMAKE_CXX_STANDARD 11)\n')
+ outfile.write('set(CMAKE_CXX_STANDARD_REQUIRED ON)\n')
+ outfile.write('set(CMAKE_CXX_COMPILE_FLAGS ${BFPS_CXX_COMPILE_FLAGS})\n')
+ outfile.write('set(CMAKE_EXE_LINKER_FLAGS "-lbfps ${BFPS_EXE_LINKER_FLAGS}")\n')
outfile.write('find_package(BFPS REQUIRED)\n')
- outfile.write('include_directories(${BFPS_INCLUDE_DIRECTORIES} ${BFPS_INCLUDE_DIRECTORIES}/bfps)\n')
- outfile.write('execute_process(COMMAND echo ${BFPS_LIBS})\n')
+ outfile.write('include_directories(${BFPS_INCLUDE_DIRECTORIES} ${BFPS_INCLUDE_DIR}/bfps)\n')
+ outfile.write('link_directories(${BFPS_LINK_DIRECTORIES} ${BFPS_LIBRARIES_DIR})\n')
outfile.write('add_executable({0} {0}.cpp)\n'.format(self.name))
- outfile.write('target_link_libraries(' + self.name + ' ${BFPS_LIBS} "-lbfps")\n')
+ outfile.write('target_link_libraries(' + self.name + ' ${BFPS_LIBS})\n')
subprocess.check_call(['cmake', '.'])
subprocess.check_call(['make'])
return None
diff --git a/cmake/BFPSConfig.cmake.in b/cmake/BFPSConfig.cmake.in
index 80660897..2a192373 100644
--- a/cmake/BFPSConfig.cmake.in
+++ b/cmake/BFPSConfig.cmake.in
@@ -21,6 +21,7 @@ SET(BFPS_INCLUDE_DIRECTORIES "@ALL_INCLUDE_DIRS@")
SET(BFPS_CXX_COMPILE_FLAGS "@CMAKE_CXX_COMPILE_FLAGS@")
SET(BFPS_EXE_LINKER_FLAGS "@CMAKE_EXE_LINKER_FLAGS@")
+SET(BFPS_LIBS "@BFPS_LIBS@")
set(BFPS_DEFINITIONS @COMPILE_DEFINITIONS@)
#
@@ -29,7 +30,7 @@ SET(BFPS_SOURCE_DIR "@BFPS_SOURCE_DIR@")
#
SET(BFPS_BUILD_TYPE "@CMAKE_BUILD_TYPE@")
-#
+#
SET(BFPS_HDF5_USE_SZIP "@BFPS_HDF5_USE_SZIP@")
SET(BFPS_HDF5_SZIP_LIB_PATH "@BFPS_HDF5_SZIP_LIB_PATH@")
--
GitLab
From 7a7196894509fb2c15bd6cd1b7f73d95eaa76f01 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 12 Mar 2019 12:52:25 +0100
Subject: [PATCH 325/342] fix multiple main definition problem
---
CMakeLists.txt | 118 ++++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 116 insertions(+), 2 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index ba3905dc..7957f456 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -105,8 +105,122 @@ get_property(ALL_LINK_DIRS DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY LINK_D
include_directories(${PROJECT_SOURCE_DIR}/cpp)
-file(GLOB_RECURSE cpp_for_lib ${PROJECT_SOURCE_DIR}/*.cpp)
-file(GLOB_RECURSE hpp_for_lib ${PROJECT_SOURCE_DIR}/*.hpp)
+#file(GLOB_RECURSE cpp_for_lib ${PROJECT_SOURCE_DIR}/*.cpp)
+set(cpp_for_lib
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/code_base.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/direct_numerical_simulation.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/NSVE.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/joint_acc_vel_stats.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/test.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/filter_test.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/field_test.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/symmetrize_test.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/field_output_test.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/get_rfields.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/field_single_to_double.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/resize.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/NSVE_field_stats.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/native_binary_to_hdf5.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/postprocess.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/field.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/kspace.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/field_layout.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/hdf5_tools.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/fftw_tools.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/vorticity_equation.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/field_binary_IO.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/spline_n1.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/spline_n2.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/spline_n3.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/spline_n4.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/spline_n5.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/spline_n6.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/spline_n7.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/spline_n8.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/spline_n9.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/spline_n10.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/Lagrange_polys.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/scope_timer.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/test_interpolation.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/NSVEparticles.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/NSVEcomplex_particles.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/NSVEp_extra_sampling.cpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/particles_inner_computer.cpp)
+set(hpp_for_lib
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/code_base.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/direct_numerical_simulation.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/NSVE.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/joint_acc_vel_stats.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/test.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/filter_test.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/field_test.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/symmetrize_test.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/field_output_test.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/get_rfields.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/field_single_to_double.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/resize.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/NSVE_field_stats.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/native_binary_to_hdf5.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/postprocess.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/field.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/kspace.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/field_layout.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/hdf5_tools.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/fftw_tools.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/vorticity_equation.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/field_binary_IO.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/spline_n1.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/spline_n2.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/spline_n3.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/spline_n4.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/spline_n5.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/spline_n6.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/spline_n7.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/spline_n8.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/spline_n9.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/spline_n10.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/Lagrange_polys.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/scope_timer.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/test_interpolation.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/NSVEparticles.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/NSVEcomplex_particles.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/NSVEp_extra_sampling.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/particles_inner_computer.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/abstract_particles_input.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/abstract_particles_output.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/abstract_particles_system.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/alltoall_exchanger.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/env_utils.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/lock_free_bool_array.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/p2p_computer_empty.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/p2p_computer.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/p2p_distr_mpi.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/p2p_tree.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/particles_adams_bashforth.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/particles_distr_mpi.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/particles_field_computer.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/particles_generic_interp.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/particles_inner_computer_empty.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/particles_input_hdf5.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/particles_output_hdf5.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/particles_output_mpiio.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/particles_output_sampling_hdf5.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/particles_sampling.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/particles_system_builder.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/particles_system.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/particles/particles_utils.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/main_code.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/codes_with_no_output.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/NSVE_no_output.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/full_code/NSVEparticles_no_output.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/base.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/fftw_interface.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/bfps_timer.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/omputils.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/shared_array.hpp
+ ${PROJECT_SOURCE_DIR}/cpp/spline.hpp
+ )
+#file(GLOB_RECURSE hpp_for_lib ${PROJECT_SOURCE_DIR}/*.hpp)
LIST(APPEND source_files ${hpp_for_lib} ${cpp_for_lib})
add_library(bfps ${source_files})
--
GitLab
From 2130b8ac1298ca4dd1d2683f3f664c8880c950b7 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 12 Mar 2019 14:57:04 +0100
Subject: [PATCH 326/342] look at verbose make error message
---
CMakeLists.txt | 4 ++++
bfps/_code.py | 6 ++++--
2 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7957f456..405c27b3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -94,6 +94,10 @@ list(APPEND BFPS_LIBS "${FFTW_LIBRARIES}")
include_directories(${FFTW_INCLUDE_DIRS})
link_directories(${FFTW_LIBRARY_DIRS})
+# hack for FFTW MPI libs
+set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lfftw3_mpi -lfftw3f_mpi")
+
+
#####################################################################################
## Get the links and include from deps
diff --git a/bfps/_code.py b/bfps/_code.py
index 8a61125d..13368744 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -208,14 +208,16 @@ class _code(_base):
outfile.write('set(CMAKE_CXX_STANDARD 11)\n')
outfile.write('set(CMAKE_CXX_STANDARD_REQUIRED ON)\n')
outfile.write('set(CMAKE_CXX_COMPILE_FLAGS ${BFPS_CXX_COMPILE_FLAGS})\n')
- outfile.write('set(CMAKE_EXE_LINKER_FLAGS "-lbfps ${BFPS_EXE_LINKER_FLAGS}")\n')
+ outfile.write('set(CMAKE_EXE_LINKER_FLAGS "${BFPS_EXE_LINKER_FLAGS} -lbfps")\n')
outfile.write('find_package(BFPS REQUIRED)\n')
outfile.write('include_directories(${BFPS_INCLUDE_DIRECTORIES} ${BFPS_INCLUDE_DIR}/bfps)\n')
outfile.write('link_directories(${BFPS_LINK_DIRECTORIES} ${BFPS_LIBRARIES_DIR})\n')
outfile.write('add_executable({0} {0}.cpp)\n'.format(self.name))
outfile.write('target_link_libraries(' + self.name + ' ${BFPS_LIBS})\n')
subprocess.check_call(['cmake', '.'])
- subprocess.check_call(['make'])
+ current_environment = os.environ
+ current_environment['VERBOSE'] = '1'
+ subprocess.check_call(['make'], env = current_environment)
return None
def set_host_info(
self,
--
GitLab
From ab1cf28c351bcfbb2be41a3502bdd02148c8efb1 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 12 Mar 2019 15:26:39 +0100
Subject: [PATCH 327/342] link bfps properly (?)
---
bfps/_code.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/bfps/_code.py b/bfps/_code.py
index 13368744..91d947a6 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -207,12 +207,14 @@ class _code(_base):
outfile.write('project(bla LANGUAGES CXX)\n')
outfile.write('set(CMAKE_CXX_STANDARD 11)\n')
outfile.write('set(CMAKE_CXX_STANDARD_REQUIRED ON)\n')
- outfile.write('set(CMAKE_CXX_COMPILE_FLAGS ${BFPS_CXX_COMPILE_FLAGS})\n')
- outfile.write('set(CMAKE_EXE_LINKER_FLAGS "${BFPS_EXE_LINKER_FLAGS} -lbfps")\n')
+ outfile.write('set(CMAKE_CXX_COMPILE_FLAGS "${CMAKE_CXX_COMPILE_FLAGS} ${BFPS_CXX_COMPILE_FLAGS}")\n')
+ outfile.write('set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${BFPS_EXE_LINKER_FLAGS}")\n')
outfile.write('find_package(BFPS REQUIRED)\n')
outfile.write('include_directories(${BFPS_INCLUDE_DIRECTORIES} ${BFPS_INCLUDE_DIR}/bfps)\n')
outfile.write('link_directories(${BFPS_LINK_DIRECTORIES} ${BFPS_LIBRARIES_DIR})\n')
+ outfile.write('find_library(BFPS_STATIC_LIBRARY bfps)\n')
outfile.write('add_executable({0} {0}.cpp)\n'.format(self.name))
+ outfile.write('target_link_libraries(' + self.name + ' ${BFPS_STATIC_LIBRARY})\n')
outfile.write('target_link_libraries(' + self.name + ' ${BFPS_LIBS})\n')
subprocess.check_call(['cmake', '.'])
current_environment = os.environ
--
GitLab
From fa2ce8f645be299aa5f5440564730746105a7a30 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 12 Mar 2019 16:36:41 +0100
Subject: [PATCH 328/342] compilation of executable works
---
CMakeLists.txt | 7 ++++++-
bfps/_code.py | 5 +++--
2 files changed, 9 insertions(+), 3 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 405c27b3..6da91ab9 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -48,6 +48,7 @@ set(CMAKE_CXX_COMPILE_FLAGS "${CMAKE_CXX_COMPILE_FLAGS} -Wall")
#####################################################################################
## HDF5
+set(HDF5_STATIC ON)
if(NOT DEFINED ENV{HDF5_ROOT})
message(WARNING "The environment variable HDF5_ROOT is undefined, this might cause trouble in finding the HDF5")
endif()
@@ -74,6 +75,7 @@ endif()
#####################################################################################
## FFTW
+set(FFTW_STATIC ON)
if(NOT DEFINED ENV{FFTW_DIR})
message(WARNING "The environment variable FFTW_DIR is undefined, this might cause trouble in finding the FFTW")
endif()
@@ -95,7 +97,10 @@ include_directories(${FFTW_INCLUDE_DIRS})
link_directories(${FFTW_LIBRARY_DIRS})
# hack for FFTW MPI libs
-set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lfftw3_mpi -lfftw3f_mpi")
+find_library(FFTW_MPI fftw3_mpi)
+list(APPEND BFPS_LIBS "${FFTW_MPI}")
+find_library(FFTWF_MPI fftw3f_mpi)
+list(APPEND BFPS_LIBS "${FFTWF_MPI}")
#####################################################################################
diff --git a/bfps/_code.py b/bfps/_code.py
index 91d947a6..ab359652 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -219,8 +219,9 @@ class _code(_base):
subprocess.check_call(['cmake', '.'])
current_environment = os.environ
current_environment['VERBOSE'] = '1'
- subprocess.check_call(['make'], env = current_environment)
- return None
+ make_result = subprocess.check_call(['make'], env = current_environment)
+ os.chdir('..')
+ return make_result
def set_host_info(
self,
host_info = {}):
--
GitLab
From 05457bec70078f16d1a6acaa9b0d79d8e73b61ee Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 12 Mar 2019 16:48:44 +0100
Subject: [PATCH 329/342] test_fftw works
---
bfps/_code.py | 40 +++++++++++++++++-----------------------
1 file changed, 17 insertions(+), 23 deletions(-)
diff --git a/bfps/_code.py b/bfps/_code.py
index ab359652..06b444ff 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -198,8 +198,9 @@ class _code(_base):
self,
no_debug = False):
# compile code
- os.makedirs('bfps_build', exist_ok = True)
- os.chdir('bfps_build')
+ build_dir = 'bfps_build_' + self.name
+ os.makedirs(build_dir, exist_ok = True)
+ os.chdir(build_dir)
self.write_src()
with open('CMakeLists.txt', 'w') as outfile:
outfile.write('cmake_minimum_required(VERSION 3.10)\n')
@@ -221,6 +222,7 @@ class _code(_base):
current_environment['VERBOSE'] = '1'
make_result = subprocess.check_call(['make'], env = current_environment)
os.chdir('..')
+ shutil.copy2(build_dir + '/' + self.name, self.name)
return make_result
def set_host_info(
self,
@@ -242,15 +244,7 @@ class _code(_base):
iter0 = data_file['iteration'].value
if not os.path.isdir(self.work_dir):
os.makedirs(self.work_dir)
- if not os.path.exists(os.path.join(self.work_dir, self.name)):
- need_to_compile = True
- else:
- need_to_compile = (datetime.fromtimestamp(os.path.getctime(os.path.join(self.work_dir, self.name))) <
- bfps.install_info['install_date'])
- if need_to_compile:
- assert self.compile_code(no_debug = no_debug) == 0
- if self.work_dir != os.path.realpath(os.getcwd()):
- shutil.copy(self.name, self.work_dir)
+ assert (self.compile_code(no_debug = no_debug) == 0)
if 'niter_todo' not in self.parameters.keys():
self.parameters['niter_todo'] = 1
current_dir = os.getcwd()
@@ -450,9 +444,9 @@ class _code(_base):
script_file.write('module li\n')
script_file.write('export OMP_NUM_THREADS={}\n'.format(nb_threads_per_process))
- script_file.write('LD_LIBRARY_PATH=' +
- ':'.join([bfps.lib_dir] + bfps.install_info['library_dirs']) +
- ':${LD_LIBRARY_PATH}\n')
+ #script_file.write('LD_LIBRARY_PATH=' +
+ # ':'.join([bfps.lib_dir] + bfps.install_info['library_dirs']) +
+ # ':${LD_LIBRARY_PATH}\n')
script_file.write('echo "Start time is `date`"\n')
script_file.write('export HTMLOUTPUT={}.html\n'.format(command_atoms[-1]))
script_file.write('cd ' + self.work_dir + '\n')
@@ -555,9 +549,9 @@ class _code(_base):
script_file.write('module li\n')
script_file.write('export OMP_NUM_THREADS={}\n'.format(nb_threads_per_process))
- script_file.write('LD_LIBRARY_PATH=' +
- ':'.join([bfps.lib_dir] + bfps.install_info['library_dirs']) +
- ':${LD_LIBRARY_PATH}\n')
+ #script_file.write('LD_LIBRARY_PATH=' +
+ # ':'.join([bfps.lib_dir] + bfps.install_info['library_dirs']) +
+ # ':${LD_LIBRARY_PATH}\n')
script_file.write('echo "Start time is `date`"\n')
script_file.write('export HTMLOUTPUT={}.html\n'.format(command_atoms[-1]))
script_file.write('cd ' + self.work_dir + '\n')
@@ -617,9 +611,9 @@ class _code(_base):
'\t-genv OMP_NUM_THREADS={0} \\\n'.format(nb_threads_per_process) +
'\t-genv OMP_PLACES=cores \\\n' +
'\t-genv LD_LIBRARY_PATH \\\n' +
- '\t"' +
- ':'.join([bfps.lib_dir] + bfps.install_info['library_dirs']) +
- '" \\\n' +
+ #'\t"' +
+ #':'.join([bfps.lib_dir] + bfps.install_info['library_dirs']) +
+ #'" \\\n' +
'\t-n {0} \\\n\t{1}\n'.format(nprocesses, ' '.join(command_atoms)))
script_file.write('echo "End time is `date`"\n')
script_file.write('exit 0\n')
@@ -688,9 +682,9 @@ class _code(_base):
script_file.write('export OMP_NUM_THREADS={0}\n'.format(nb_threads_per_process))
script_file.write('export OMP_PLACES=cores\n')
- script_file.write('LD_LIBRARY_PATH=' +
- ':'.join([bfps.lib_dir] + bfps.install_info['library_dirs']) +
- ':${LD_LIBRARY_PATH}\n')
+ #script_file.write('LD_LIBRARY_PATH=' +
+ # ':'.join([bfps.lib_dir] + bfps.install_info['library_dirs']) +
+ # ':${LD_LIBRARY_PATH}\n')
script_file.write('echo "Start time is `date`"\n')
script_file.write('cd ' + self.work_dir + '\n')
script_file.write('export HTMLOUTPUT={}.html\n'.format(command_atoms[-1]))
--
GitLab
From e2c81be725a8f1cf9b2d657cb1e9e2e5cf2c05cd Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 12 Mar 2019 17:11:16 +0100
Subject: [PATCH 330/342] remove LD_LIBRARY_PATH definitions
if code doesn't run, fix the -rpath thing in the cmake config.
---
bfps/_code.py | 21 +++------------------
1 file changed, 3 insertions(+), 18 deletions(-)
diff --git a/bfps/_code.py b/bfps/_code.py
index 06b444ff..aca00179 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -196,7 +196,7 @@ class _code(_base):
return None
def compile_code(
self,
- no_debug = False):
+ no_debug = True):
# compile code
build_dir = 'bfps_build_' + self.name
os.makedirs(build_dir, exist_ok = True)
@@ -219,7 +219,8 @@ class _code(_base):
outfile.write('target_link_libraries(' + self.name + ' ${BFPS_LIBS})\n')
subprocess.check_call(['cmake', '.'])
current_environment = os.environ
- current_environment['VERBOSE'] = '1'
+ if not no_debug:
+ current_environment['VERBOSE'] = '1'
make_result = subprocess.check_call(['make'], env = current_environment)
os.chdir('..')
shutil.copy2(build_dir + '/' + self.name, self.name)
@@ -357,9 +358,6 @@ class _code(_base):
elif self.host_info['type'] == 'pc':
os.chdir(self.work_dir)
- if os.getenv('LD_LIBRARY_PATH') != None:
- os.environ['LD_LIBRARY_PATH'] += ':{0}'.format(bfps.lib_dir)
- print('added to LD_LIBRARY_PATH the location {0}'.format(bfps.lib_dir))
for j in range(njobs):
suffix = self.simname + '_{0}'.format(iter0 + j*self.parameters['niter_todo'])
print('running code with command\n' + ' '.join(command_atoms))
@@ -444,9 +442,6 @@ class _code(_base):
script_file.write('module li\n')
script_file.write('export OMP_NUM_THREADS={}\n'.format(nb_threads_per_process))
- #script_file.write('LD_LIBRARY_PATH=' +
- # ':'.join([bfps.lib_dir] + bfps.install_info['library_dirs']) +
- # ':${LD_LIBRARY_PATH}\n')
script_file.write('echo "Start time is `date`"\n')
script_file.write('export HTMLOUTPUT={}.html\n'.format(command_atoms[-1]))
script_file.write('cd ' + self.work_dir + '\n')
@@ -549,9 +544,6 @@ class _code(_base):
script_file.write('module li\n')
script_file.write('export OMP_NUM_THREADS={}\n'.format(nb_threads_per_process))
- #script_file.write('LD_LIBRARY_PATH=' +
- # ':'.join([bfps.lib_dir] + bfps.install_info['library_dirs']) +
- # ':${LD_LIBRARY_PATH}\n')
script_file.write('echo "Start time is `date`"\n')
script_file.write('export HTMLOUTPUT={}.html\n'.format(command_atoms[-1]))
script_file.write('cd ' + self.work_dir + '\n')
@@ -610,10 +602,6 @@ class _code(_base):
'\t-machinefile $TMPDIR/machines \\\n' +
'\t-genv OMP_NUM_THREADS={0} \\\n'.format(nb_threads_per_process) +
'\t-genv OMP_PLACES=cores \\\n' +
- '\t-genv LD_LIBRARY_PATH \\\n' +
- #'\t"' +
- #':'.join([bfps.lib_dir] + bfps.install_info['library_dirs']) +
- #'" \\\n' +
'\t-n {0} \\\n\t{1}\n'.format(nprocesses, ' '.join(command_atoms)))
script_file.write('echo "End time is `date`"\n')
script_file.write('exit 0\n')
@@ -682,9 +670,6 @@ class _code(_base):
script_file.write('export OMP_NUM_THREADS={0}\n'.format(nb_threads_per_process))
script_file.write('export OMP_PLACES=cores\n')
- #script_file.write('LD_LIBRARY_PATH=' +
- # ':'.join([bfps.lib_dir] + bfps.install_info['library_dirs']) +
- # ':${LD_LIBRARY_PATH}\n')
script_file.write('echo "Start time is `date`"\n')
script_file.write('cd ' + self.work_dir + '\n')
script_file.write('export HTMLOUTPUT={}.html\n'.format(command_atoms[-1]))
--
GitLab
From ceebfaa23154282e618c0dbd2a792a5dfdf0e435 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 12 Mar 2019 17:23:05 +0100
Subject: [PATCH 331/342] fix order of FFTW libraries for laptop
---
CMakeLists.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6da91ab9..9658780c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -97,10 +97,10 @@ include_directories(${FFTW_INCLUDE_DIRS})
link_directories(${FFTW_LIBRARY_DIRS})
# hack for FFTW MPI libs
-find_library(FFTW_MPI fftw3_mpi)
-list(APPEND BFPS_LIBS "${FFTW_MPI}")
find_library(FFTWF_MPI fftw3f_mpi)
-list(APPEND BFPS_LIBS "${FFTWF_MPI}")
+set(BFPS_LIBS ${FFTWF_MPI} ${BFPS_LIBS})
+find_library(FFTW_MPI fftw3_mpi)
+set(BFPS_LIBS ${FFTW_MPI} ${BFPS_LIBS})
#####################################################################################
--
GitLab
From 962ee6096ddd40d7974ec74693fbbc5a2daaecf7 Mon Sep 17 00:00:00 2001
From: Cristian Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 13 Mar 2019 14:37:32 +0100
Subject: [PATCH 332/342] installation/compilation works on cobra
---
CMakeLists.txt | 35 ++++++++++++++++++++++++++++-------
bfps/_code.py | 21 ++++++++++++++++++---
cmake/BFPSConfig.cmake.in | 2 ++
3 files changed, 48 insertions(+), 10 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9658780c..86385736 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,4 +1,23 @@
cmake_minimum_required(VERSION 3.10)
+cmake_policy(VERSION 3.12)
+
+if (DEFINED ENV{MPICXX})
+ message(STATUS "Using CMAKE_CXX_COMPILER=MPICXX")
+ set(CMAKE_CXX_COMPILER $ENV{MPICXX})
+else()
+ message(STATUS "MPICXX environment variable undefined, trying to find MPI")
+ set(MPI_STATIC ON)
+ find_package(MPI REQUIRED)
+endif()
+
+if (DEFINED ENV{MPICC})
+ set(CMAKE_C_COMPILER $ENV{MPICC})
+ message(STATUS "Using CMAKE_C_COMPILER=MPICC")
+endif()
+
+if (DEFINED ENV{CMAKE_INSTALL_PREFIX})
+ set(CMAKE_INSTALL_PREFIX $ENV{CMAKE_INSTALL_PREFIX})
+endif()
project(BFPS)
@@ -6,7 +25,7 @@ execute_process(COMMAND python ${PROJECT_SOURCE_DIR}/get_version.py OUTPUT_VARIA
project(BFPS
VERSION ${BFPS_VERSION}
- LANGUAGES C CXX)
+ LANGUAGES CXX)
set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/morse ${CMAKE_MODULE_PATH})
@@ -15,9 +34,6 @@ set(BFPS_LIBS "")
#####################################################################################
## MPI
-set(MPI_STATIC ON)
-find_package(MPI REQUIRED)
-
set(CMAKE_CXX_COMPILE_FLAGS "${CMAKE_CXX_COMPILE_FLAGS} ${MPI_CXX_COMPILE_OPTIONS}")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${MPI_CXX_LINK_FLAGS}")
include_directories(${MPI_CXX_INCLUDE_DIRS})
@@ -43,7 +59,8 @@ list(APPEND BFPS_LIBS "${OpenMP_CXX_LIB_NAMES}")
#####################################################################################
## Extra flags
-set(CMAKE_CXX_COMPILE_FLAGS "${CMAKE_CXX_COMPILE_FLAGS} -Wall")
+set(CMAKE_CXX_COMPILE_FLAGS "${CMAKE_CXX_COMPILE_FLAGS} $ENV{BFPS_OPTIMIZATION_FLAGS} -Wall -g")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_COMPILE_FLAGS}")
#####################################################################################
## HDF5
@@ -97,9 +114,13 @@ include_directories(${FFTW_INCLUDE_DIRS})
link_directories(${FFTW_LIBRARY_DIRS})
# hack for FFTW MPI libs
-find_library(FFTWF_MPI fftw3f_mpi)
+find_library(
+ FFTWF_MPI fftw3f_mpi
+ HINTS ${FFTW_LIBRARY_DIRS})
set(BFPS_LIBS ${FFTWF_MPI} ${BFPS_LIBS})
-find_library(FFTW_MPI fftw3_mpi)
+find_library(
+ FFTW_MPI fftw3_mpi
+ HINTS ${FFTW_LIBRARY_DIRS})
set(BFPS_LIBS ${FFTW_MPI} ${BFPS_LIBS})
diff --git a/bfps/_code.py b/bfps/_code.py
index aca00179..f06e7198 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -204,13 +204,28 @@ class _code(_base):
self.write_src()
with open('CMakeLists.txt', 'w') as outfile:
outfile.write('cmake_minimum_required(VERSION 3.10)\n')
- outfile.write('set(CXX $ENV{MPICXX})\n')
- outfile.write('project(bla LANGUAGES CXX)\n')
+ outfile.write('cmake_policy(VERSION 3.12)\n')
+ outfile.write('if (DEFINED ENV{MPICXX})\n')
+ outfile.write(' message(STATUS "Using CMAKE_CXX_COMPILER=MPICXX")\n')
+ outfile.write(' set(CMAKE_CXX_COMPILER $ENV{MPICXX})\n')
+ outfile.write('else()\n')
+ outfile.write(' message(STATUS "MPICXX environment variable undefined, trying to find MPI")\n')
+ outfile.write(' set(MPI_STATIC ON)\n')
+ outfile.write(' find_package(MPI REQUIRED)\n')
+ outfile.write('endif()\n')
+ outfile.write('if (DEFINED ENV{MPICC})\n')
+ outfile.write(' set(CMAKE_C_COMPILER $ENV{MPICC})\n')
+ outfile.write(' message(STATUS "Using CMAKE_C_COMPILER=MPICC")\n')
+ outfile.write('endif()\n')
+ #ideally we should use something like the following 2 lines
+ #outfile.write('set(CMAKE_CXX_COMPILER ${BFPS_CXX_COMPILER})\n')
+ #outfile.write('set(CMAKE_C_COMPILER ${BFPS_C_COMPILER})\n')
+ outfile.write('project(project_{0} LANGUAGES CXX)\n'.format(self.name))
+ outfile.write('find_package(BFPS REQUIRED)\n')
outfile.write('set(CMAKE_CXX_STANDARD 11)\n')
outfile.write('set(CMAKE_CXX_STANDARD_REQUIRED ON)\n')
outfile.write('set(CMAKE_CXX_COMPILE_FLAGS "${CMAKE_CXX_COMPILE_FLAGS} ${BFPS_CXX_COMPILE_FLAGS}")\n')
outfile.write('set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${BFPS_EXE_LINKER_FLAGS}")\n')
- outfile.write('find_package(BFPS REQUIRED)\n')
outfile.write('include_directories(${BFPS_INCLUDE_DIRECTORIES} ${BFPS_INCLUDE_DIR}/bfps)\n')
outfile.write('link_directories(${BFPS_LINK_DIRECTORIES} ${BFPS_LIBRARIES_DIR})\n')
outfile.write('find_library(BFPS_STATIC_LIBRARY bfps)\n')
diff --git a/cmake/BFPSConfig.cmake.in b/cmake/BFPSConfig.cmake.in
index 2a192373..78e8ecf6 100644
--- a/cmake/BFPSConfig.cmake.in
+++ b/cmake/BFPSConfig.cmake.in
@@ -20,6 +20,8 @@ SET(BFPS_LINK_DIRECTORIES "@ALL_LINK_DIRS@")
SET(BFPS_INCLUDE_DIRECTORIES "@ALL_INCLUDE_DIRS@")
SET(BFPS_CXX_COMPILE_FLAGS "@CMAKE_CXX_COMPILE_FLAGS@")
+SET(BFPS_CXX_COMPILER "@CMAKE_CXX_COMPILER@")
+SET(BFPS_C_COMPILER "@CMAKE_C_COMPILER@")
SET(BFPS_EXE_LINKER_FLAGS "@CMAKE_EXE_LINKER_FLAGS@")
SET(BFPS_LIBS "@BFPS_LIBS@")
set(BFPS_DEFINITIONS @COMPILE_DEFINITIONS@)
--
GitLab
From 02e55250359d0e8a96765a0b15b8bf925ac72a2b Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 13 Mar 2019 15:29:30 +0100
Subject: [PATCH 333/342] update README
I still need to put in a `host_information.py` discussion, as well as
fixing the install location of host_information...
---
README.rst | 152 ++++++++++++++++++++++++-----------------------------
1 file changed, 70 insertions(+), 82 deletions(-)
diff --git a/README.rst b/README.rst
index b562ba94..9b804cad 100644
--- a/README.rst
+++ b/README.rst
@@ -58,36 +58,10 @@ Use a console; navigate to the ``bfps`` folder, and type:
**Full installation**
If you want to run simulations on the machine where you're installing,
-you will need to call `compile_library` before installing.
+you will need to use `cmake` to compile and install the full library.
Your machine needs to have an MPI compiler installed, the HDF5 C library
-and FFTW >= 3.4 --- a detailed prerequisite installation list is
+and FFTW >= 3.4 --- detailed instructions are
included at the end of this document.
-The file `machine_settings_py.py` should be modified
-appropriately for your machine (otherwise the `compile_library` command will most
-likely fail).
-This file will be copied the first time you run `setup.py` into
-`$HOME/.config/bfps/machine_settings.py`, **where it will be imported from
-afterwards** --- any future edits **must** be made to the new file.
-You may, obviously, edit it afterwards and rerun the `compile_library` command as
-needed.
-
-.. code:: bash
-
- python setup.py compile_library
- python setup.py install
-
-For `machine_settings.py`, please keep in mind to turn on optimizations
-for production environments.
-In particular, for clusters of unknown architecture it helps to log into
-individual nodes and run the following command:
-
-.. code:: bash
-
- gcc -march=native -Q --help=target
-
-The results can be used to then compile on the frontend node(s) without
-using `-march=native` (since the frontend node may have different
-architecture).
-------------
Documentation
@@ -96,8 +70,8 @@ Documentation
While the code is not fully documented yet, basic information is already
available, and it is recommended that you generate the manual and go
through it carefully.
-Please don't be shy about asking for specific improvements to the
-current text.
+Please do ask for specific improvements to the current text where it is
+found lacking.
In order to generate the manual, navigate to the repository folder, and
execute the following commands:
@@ -127,79 +101,93 @@ Installation with prerequisites
These installation steps assume that you have a working MPI compiler,
properly configured on your system (i.e. the various configure scripts
-are able to find it).
-If this is not the case, please consult the FFTW and HDF5 compilation
-instructions for detailed instructions (`./configure --help` should be
-enough).
+are able to find it), as well as the `cmake` tool.
+We recommend to specify the desired MPI C++ compiler by exporting the
+environment variable `MPICXX` --- the BFPS cmake configuration looks for
+this variable.
+We also recommend that an environment variable `BFPS_OPTIMIZATION_FLAGS`
+is defined appropriately.
+In particular, for clusters of unknown architecture it helps to log into
+individual nodes and run the following command:
+
+.. code:: bash
+
+ gcc -march=native -Q --help=target
+
+Detailed full installation instructions:
-1. Make directory PREFIX on local fast partition.
+1. Make directory PREFIX on a local fast partition.
2. Download, compile, install FFTW (latest version 3.x from http://www.fftw.org/).
Execute the following commands in order, feel free to customize
optimisation flags for your own computer (see http://www.fftw.org/fftw3_doc/Installation-on-Unix.html):
-.. code:: bash
+ .. code:: bash
- ./configure --prefix=PREFIX --enable-float --enable-sse --enable-sse2 --enable-avx --enable-avx2 --enable-avx-128-fma --enable-avx512 --enable-mpi --enable-openmp --enable-threads
- make
- make install
- ./configure --prefix=PREFIX --enable-sse --enable-sse2 --enable-avx --enable-avx2 --enable-avx-128-fma --enable-avx512 --enable-mpi --enable-openmp --enable-threads
- make
- make install
+ ./configure --prefix=PREFIX --enable-float --enable-sse --enable-mpi --enable-openmp --enable-threads
+ make
+ make install
+ ./configure --prefix=PREFIX --enable-sse2 --enable-avx512 --enable-mpi --enable-openmp --enable-threads
+ make
+ make install
- BFPS will try to find HDF5 using the FindFFTW from the Morse project.
- If the package is installed in a non standard location, it is recommanded
- to setup the environment variables: FFTW_DIR (or FFTW_INCDIR and FFTW_LIBDIR)
+ BFPS will try to find FFTW using the FindFFTW from the Morse project.
+ If the package is installed in a non standard location, it is recommanded
+ to setup the environment variables: FFTW_DIR (or FFTW_INCDIR and FFTW_LIBDIR).
3. Download, compile, install HDF5 (version 1.8.x, currently available
at https://portal.hdfgroup.org/display/support/HDF5+1.8.20#files).
- We are using parallel I/O, therefore we use the plain C interface of HDF5:
+ We are using parallel I/O, therefore we must use the plain C interface of HDF5:
-.. code:: bash
+ .. code:: bash
- ./configure --prefix=PREFIX --enable-parallel
- make
- make install
-
- BFPS will try to find HDF5 using the regular FindHDF5.
- Therefore, if the package is installed in a non standard location, it is recommanded
- to setup the environment variable: HDF5_ROOT
+ ./configure --prefix=PREFIX --enable-parallel
+ make
+ make install
-3. This step may be ommited.
- I recommend the creation of a virtual python3 environment (also under PREFIX) that will be used for installing bfps and dependencies.
+ BFPS will try to find HDF5 using the regular FindHDF5.
+ Therefore, if the package is installed in a non standard location, it is recommanded
+ to setup the environment variable: HDF5_ROOT.
+
+3. Optional.
+ We recommend the creation of a virtual python3 environment (also under PREFIX) that will be used for installing bfps and dependencies.
Please see https://docs.python-guide.org/dev/virtualenvs/.
4. Clone bfps repository.
-.. code:: bash
+ .. code:: bash
- git clone git@gitlab.mpcdf.mpg.de:clalescu/bfps.git
+ git clone git@gitlab.mpcdf.mpg.de:clalescu/bfps.git
5. Go into bfps repository, execute
-.. code:: bash
+ .. code:: bash
- mkdir build
- cd build
- cmake ..
- # possibly : cmake .. -DCMAKE_INSTALL_PREFIX=INSTALL_DIR
- make ..
- # to get a verbose compilation process, use VERBOSE=1 make
- # make install
-
-6. Using BFPS from an external project. Along with all the BFPS files
- (lib and headers) can be found different cmake files
- that contains all the information related to the compilation of BFPS.
- It is possible to load this file in another CMake based application
- to know the dependencies and paths.
-
- For instance, the installation will create these files:
-
-.. code:: bash
- -- Installing: install/lib/BFPSConfig.cmake
- -- Installing: install/lib/BFPS_EXPORT.cmake
- -- Installing: install/lib/BFPS_EXPORT-noconfig.cmake
-
- In case an information is not provided, it is necessary to update
- the cmake input config file: bfps/cmake/BFPSConfig.cmake.in
+ mkdir build
+ cd build
+ cmake ..
+ # possibly : cmake .. -DCMAKE_INSTALL_PREFIX=INSTALL_DIR
+ make
+ # to get a verbose compilation process, use
+ VERBOSE=1 make
+ make install
+
+6. If you used a custom install location (i.e. `CMAKE_INSTALL_PREFIX`)
+ you must include this location in the environment variable
+ `CMAKE_PREFIX_PATH`.
+ This ensures that the required `BFPSConfig.cmake` file is accessible for
+ future use by the package.
+
+7. Using BFPS from an external project.
+ BFPS creates and installs 3 files alongside the C++ headers and
+ library:
+
+ .. code:: bash
+
+ -- Installing: install/lib/BFPSConfig.cmake
+ -- Installing: install/lib/BFPS_EXPORT.cmake
+ -- Installing: install/lib/BFPS_EXPORT-noconfig.cmake
+
+ In case these files provide incomplete information, it is necessary to update
+ the cmake input config file: bfps/cmake/BFPSConfig.cmake.in.
--
GitLab
From 9266c1b3abc538a05297eba3ef95bda3c5f71de8 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Fri, 15 Mar 2019 14:03:12 +0100
Subject: [PATCH 334/342] update host_info installation
---
CMakeLists.txt | 5 +++
CMakeLists_packages.txt | 71 -----------------------------------------
bfps/DNS.py | 6 ----
bfps/PP.py | 6 ----
bfps/TEST.py | 6 ----
bfps/_code.py | 12 +++----
setup.py | 32 +------------------
7 files changed, 10 insertions(+), 128 deletions(-)
delete mode 100644 CMakeLists_packages.txt
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 86385736..2011f0e8 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -273,6 +273,11 @@ install(EXPORT BFPS_EXPORT DESTINATION lib/)
#####################################################################################
##Â Install the python wrapper
# copy command
+if(EXISTS "${PROJECT_SOURCE_DIR}/host_info.py")
+ install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_SOURCE_DIR}/host_info.py ${PROJECT_BINARY_DIR}/python/bfps/)")
+else()
+ install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_SOURCE_DIR}/pc_host_info.py ${PROJECT_BINARY_DIR}/python/bfps/host_info.py)")
+endif()
install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E copy_directory ${PROJECT_SOURCE_DIR}/bfps ${PROJECT_BINARY_DIR}/python/bfps/)")
install(CODE "execute_process(COMMAND python ${PROJECT_SOURCE_DIR}/setup.py install --force --prefix=${CMAKE_INSTALL_PREFIX} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/python/)")
diff --git a/CMakeLists_packages.txt b/CMakeLists_packages.txt
deleted file mode 100644
index 17a1099c..00000000
--- a/CMakeLists_packages.txt
+++ /dev/null
@@ -1,71 +0,0 @@
-# vim: syntax=cmake
-
-#cmake_policy(SET CMP0074 NEW)
-
-set(MPI_STATIC ON)
-find_package(MPI REQUIRED)
-
-set(HDF5_STATIC ON)
-find_package(HDF5 REQUIRED)
-
-###############################################################################
-# ugly manual setting for HDF5
-if(NOT DEFINED HDF5_INCLUDE_PATH)
- set(HDF5_INCLUDE_PATH "$ENV{HDF5_ROOT}/include")
-endif()
-###############################################################################
-
-###############################################################################
-# ugly manual settings for FFTW.
-set(FFTW_STATIC ON)
-if (NOT DEFINED ENV{FFTW_BASE})
- message(FATAL_ERROR "Required FFTW_BASE environment variable is not defined.")
-endif()
-set(FFTW_INCLUDE_FLAGS
- "$ENV{FFTW_BASE}/include"
- )
-if (NOT DEFINED ENV{FFTW_LIB})
- message(FATAL_ERROR "Required FFTW_LIB environment variable is not defined.")
-endif()
-if (NOT DEFINED ENV{FFTW_OPENMP_LIB})
- message(FATAL_ERROR "Required FFTW_OPENMP_LIB environment variable is not defined.")
-endif()
-set(FFTW_LINK_FLAGS
- "$ENV{FFTW_OPENMP_LIB} $ENV{FFTW_LIB}"
- )
-###############################################################################
-
-###############################################################################
-# ugly manual settings for SZIP.
-# needed for non-gcc environments.
-if(DEFINED env{SZIP_LIB})
- set(SZIP_LINK_FLAGS
- "$env{SZIP_LIB} -lz")
-else()
- set(SZIP_LINK_FLAGS
- "")
-endif()
-###############################################################################
-
-include_directories(
- ${MPI_INCLUDE_PATH}
- ${HDF5_INCLUDE_PATH}
- ${FFTW_INCLUDE_PATH}
- )
-
-set(CMAKE_CXX_COMPILE_FLAGS
- ${CMAKE_CXX_COMPILE_FLAGS}
- "-Wall -std=c++11 -fopenmp"
- )
-set(CMAKE_CXX_COMPILE_FLAGS
- ${CMAKE_CXX_COMPILE_FLAGS}
- ${HDF5_COMPILE_FLAGS}
- ${MPI_COMPILE_FLAGS}
- )
-set(CMAKE_CXX_LINK_FLAGS
- ${MPI_LINK_FLAGS}
- ${HDF5_LINK_FLAGS}
- ${FFTW_LINK_FLAGS}
- ${CMAKE_CXX_LINK_FLAGS}
- )
-
diff --git a/bfps/DNS.py b/bfps/DNS.py
index e0d83caf..4388a292 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -50,12 +50,6 @@ class DNS(_code):
self,
work_dir = work_dir,
simname = simname)
- self.host_info = {'type' : 'cluster',
- 'environment' : None,
- 'deltanprocs' : 1,
- 'queue' : '',
- 'mail_address': '',
- 'mail_events' : None}
self.generate_default_parameters()
self.statistics = {}
return None
diff --git a/bfps/PP.py b/bfps/PP.py
index 77bf9d6c..4a8cff4b 100644
--- a/bfps/PP.py
+++ b/bfps/PP.py
@@ -51,12 +51,6 @@ class PP(_code):
self,
work_dir = work_dir,
simname = simname)
- self.host_info = {'type' : 'cluster',
- 'environment' : None,
- 'deltanprocs' : 1,
- 'queue' : '',
- 'mail_address': '',
- 'mail_events' : None}
self.generate_default_parameters()
return None
def set_precision(
diff --git a/bfps/TEST.py b/bfps/TEST.py
index 66b2b4aa..80adf39f 100644
--- a/bfps/TEST.py
+++ b/bfps/TEST.py
@@ -51,12 +51,6 @@ class TEST(_code):
self,
work_dir = work_dir,
simname = simname)
- self.host_info = {'type' : 'cluster',
- 'environment' : None,
- 'deltanprocs' : 1,
- 'queue' : '',
- 'mail_address': '',
- 'mail_events' : None}
self.generate_default_parameters()
return None
def set_precision(
diff --git a/bfps/_code.py b/bfps/_code.py
index f06e7198..ee196fe7 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -1,7 +1,7 @@
#######################################################################
# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
+# Copyright 2015-2019 Max Planck Institute #
+# for Dynamics and Self-Organization #
# #
# This file is part of bfps. #
# #
@@ -35,6 +35,7 @@ import math
import warnings
import bfps
+from bfps.host_info import host_info
from ._base import _base
class _code(_base):
@@ -174,12 +175,7 @@ class _code(_base):
}
//endcpp
"""
- self.host_info = {'type' : 'cluster',
- 'environment' : None,
- 'deltanprocs' : 1,
- 'queue' : '',
- 'mail_address': '',
- 'mail_events' : None}
+ self.host_info = host_info
self.main = ''
return None
def write_src(self):
diff --git a/setup.py b/setup.py
index 925f6521..c9a8b70b 100644
--- a/setup.py
+++ b/setup.py
@@ -34,37 +34,12 @@ import sys
import subprocess
import pickle
-
-### compiler configuration
-# check if .config/bfps/machine_settings.py file exists, create it if not
-homefolder = os.path.expanduser('~')
-bfpsfolder = os.path.join(homefolder, '.config', 'bfps')
-if not os.path.exists(os.path.join(bfpsfolder, 'machine_settings.py')):
- if not os.path.isdir(bfpsfolder):
- os.mkdir(bfpsfolder)
- shutil.copyfile('./machine_settings_py.py', os.path.join(bfpsfolder, 'machine_settings.py'))
-# check if .config/bfps/host_information.py file exists, create it if not
-if not os.path.exists(os.path.join(bfpsfolder, 'host_information.py')):
- if not os.path.isdir(bfpsfolder):
- os.mkdir(bfpsfolder)
- open(os.path.join(bfpsfolder, 'host_information.py'),
- 'w').write('host_info = {\'type\' : \'none\'}\n')
- shutil.copyfile('./machine_settings_py.py', os.path.join(bfpsfolder, 'machine_settings.py'))
-sys.path.insert(0, bfpsfolder)
-# import stuff required for compilation of static library
-from machine_settings import compiler, include_dirs, library_dirs, extra_compile_args, extra_libraries
-
-
### package versioning
import get_version
VERSION = get_version.main()
print('This is bfps version ' + VERSION)
-### libraries
-libraries = extra_libraries
-
-
import distutils.cmd
class CompileLibCommand(distutils.cmd.Command):
@@ -90,12 +65,7 @@ class CompileLibCommand(distutils.cmd.Command):
def run(self):
### save compiling information
pickle.dump(
- {'include_dirs' : include_dirs,
- 'library_dirs' : library_dirs,
- 'compiler' : compiler,
- 'extra_compile_args' : eca,
- 'libraries' : libraries,
- 'install_date' : now,
+ {'install_date' : now,
'VERSION' : VERSION,
'git_revision' : git_revision},
open('bfps/install_info.pickle', 'wb'),
--
GitLab
From 03d23da50a6dc900fa44b0c5a939159eb25a9aaf Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 18 Mar 2019 17:08:52 +0100
Subject: [PATCH 335/342] fix host info import
---
bfps/__init__.py | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/bfps/__init__.py b/bfps/__init__.py
index 09e5549f..928e45b7 100644
--- a/bfps/__init__.py
+++ b/bfps/__init__.py
@@ -39,9 +39,7 @@ header_dir = os.path.join(os.path.join(dist_loc, 'bfps'), 'cpp')
lib_dir = os.path.join(dist_loc, 'bfps')
homefolder = os.path.expanduser('~')
-bfpsfolder = os.path.join(homefolder, '.config/', 'bfps')
-sys.path.append(bfpsfolder)
-from host_information import host_info
+from .host_info import host_info
from .DNS import DNS
from .PP import PP
--
GitLab
From 9049bb26471a2c9e4abd8ba69975f1a167115485 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 18 Mar 2019 17:13:24 +0100
Subject: [PATCH 336/342] only compile if executable does not exist
---
bfps/_code.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/bfps/_code.py b/bfps/_code.py
index ee196fe7..db5e275c 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -193,6 +193,8 @@ class _code(_base):
def compile_code(
self,
no_debug = True):
+ if os.path.exists(self.name):
+ return 0
# compile code
build_dir = 'bfps_build_' + self.name
os.makedirs(build_dir, exist_ok = True)
@@ -250,7 +252,7 @@ class _code(_base):
minutes = 10,
njobs = 1,
no_submit = False,
- no_debug = False):
+ no_debug = True):
self.read_parameters()
with h5py.File(os.path.join(self.work_dir, self.simname + '.h5'), 'r') as data_file:
iter0 = data_file['iteration'].value
--
GitLab
From c9e42dc0f501f3b10bbf6948e7a10e3e4f2839c0 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 18 Mar 2019 17:15:49 +0100
Subject: [PATCH 337/342] fix executable location
---
bfps/_code.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/bfps/_code.py b/bfps/_code.py
index db5e275c..72a102b3 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -193,7 +193,7 @@ class _code(_base):
def compile_code(
self,
no_debug = True):
- if os.path.exists(self.name):
+ if os.path.exists(os.path.join(self.work_dir, self.name)):
return 0
# compile code
build_dir = 'bfps_build_' + self.name
@@ -236,7 +236,7 @@ class _code(_base):
current_environment['VERBOSE'] = '1'
make_result = subprocess.check_call(['make'], env = current_environment)
os.chdir('..')
- shutil.copy2(build_dir + '/' + self.name, self.name)
+ shutil.copy2(os.path.join(build_dir, self.name), os.path.join(self.work_dir, self.name))
return make_result
def set_host_info(
self,
--
GitLab
From 000cfc1ade784a86738ced974a87d97453beda6c Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 18 Mar 2019 20:19:01 +0100
Subject: [PATCH 338/342] add basic host_info file
---
pc_host_info.py | 27 +++++++++++++++++++++++++++
1 file changed, 27 insertions(+)
create mode 100644 pc_host_info.py
diff --git a/pc_host_info.py b/pc_host_info.py
new file mode 100644
index 00000000..0a192dd1
--- /dev/null
+++ b/pc_host_info.py
@@ -0,0 +1,27 @@
+host_info = {'type' : 'pc'}
+
+
+# info_template = {'type' : info_template_type,
+# 'MPI' : info_template_MPI,
+# 'environment' : info_template_environment,
+# 'deltanprocs' : info_template_deltanprocs
+# 'mail_address': info_template_mail_address}
+
+# info_template_type can be one of:
+# 'pc' --- jobs run interactively
+# 'cluster' --- cluster with SGE queueing system
+# 'SLURM' --- cluster with SLURM queueing system
+# 'IBMLoadLeveler --- cluster with IBM Load Leveler queueing system
+
+# info_template_MPI can be one of:
+# 'openmpi' --- it means mpirun takes "x" as the parameter to set an environment variable
+# not defined --- use "env" instead of "x"
+
+# info_template_environment, relevant for clusters,
+# is the default queue to which jobs are submitted
+
+# info_template_deltanprocs, relevant for clusters,
+# is the number of cores per node
+
+# info_template_mail_address, relevant for clusters,
+# is the contact e-mail address placed in the job scripts.
--
GitLab
From 991e389423aaf95cb7de21ef53720de131a3d14c Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Mon, 18 Mar 2019 20:32:46 +0100
Subject: [PATCH 339/342] fix order of file copy
---
CMakeLists.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2011f0e8..82f21117 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -273,11 +273,11 @@ install(EXPORT BFPS_EXPORT DESTINATION lib/)
#####################################################################################
##Â Install the python wrapper
# copy command
+install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E copy_directory ${PROJECT_SOURCE_DIR}/bfps ${PROJECT_BINARY_DIR}/python/bfps/)")
if(EXISTS "${PROJECT_SOURCE_DIR}/host_info.py")
install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_SOURCE_DIR}/host_info.py ${PROJECT_BINARY_DIR}/python/bfps/)")
else()
install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_SOURCE_DIR}/pc_host_info.py ${PROJECT_BINARY_DIR}/python/bfps/host_info.py)")
endif()
-install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E copy_directory ${PROJECT_SOURCE_DIR}/bfps ${PROJECT_BINARY_DIR}/python/bfps/)")
install(CODE "execute_process(COMMAND python ${PROJECT_SOURCE_DIR}/setup.py install --force --prefix=${CMAKE_INSTALL_PREFIX} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/python/)")
--
GitLab
From 17aacbf09662090422942394279a9cb3b6a60c1d Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 19 Mar 2019 12:52:17 +0100
Subject: [PATCH 340/342] add/update copyright notices
---
CMakeLists.txt | 26 +++++++++++
bfps/DNS.py | 45 +++++++++----------
bfps/PP.py | 45 +++++++++----------
bfps/TEST.py | 45 +++++++++----------
bfps/__init__.py | 45 +++++++++----------
bfps/__main__.py | 45 +++++++++----------
bfps/_base.py | 45 +++++++++----------
bfps/_code.py | 45 +++++++++----------
bfps/tools.py | 45 +++++++++----------
cmake/BFPSConfig.cmake.in | 25 +++++++++++
cpp/full_code/NSVE.cpp | 25 +++++++++++
cpp/full_code/NSVE_field_stats.cpp | 25 +++++++++++
cpp/full_code/NSVEp_extra_sampling.cpp | 25 +++++++++++
cpp/full_code/NSVEp_extra_sampling.hpp | 25 +++++++++++
cpp/full_code/NSVEparticles_no_output.hpp | 25 +++++++++++
cpp/full_code/codes_with_no_output.hpp | 25 +++++++++++
cpp/full_code/direct_numerical_simulation.cpp | 25 +++++++++++
cpp/full_code/field_output_test.cpp | 25 +++++++++++
cpp/full_code/field_single_to_double.cpp | 25 +++++++++++
cpp/full_code/field_test.cpp | 25 +++++++++++
cpp/full_code/filter_test.cpp | 25 +++++++++++
cpp/full_code/get_rfields.cpp | 25 +++++++++++
cpp/full_code/joint_acc_vel_stats.cpp | 25 +++++++++++
cpp/full_code/native_binary_to_hdf5.cpp | 25 +++++++++++
cpp/full_code/postprocess.cpp | 25 +++++++++++
cpp/full_code/resize.cpp | 25 +++++++++++
cpp/full_code/symmetrize_test.cpp | 25 +++++++++++
cpp/full_code/test.cpp | 25 +++++++++++
cpp/full_code/test_interpolation.cpp | 25 +++++++++++
cpp/full_code/test_interpolation.hpp | 25 +++++++++++
cpp/particles/abstract_particles_input.hpp | 25 +++++++++++
cpp/particles/abstract_particles_output.hpp | 25 +++++++++++
cpp/particles/abstract_particles_system.hpp | 25 +++++++++++
cpp/particles/alltoall_exchanger.hpp | 25 +++++++++++
cpp/particles/env_utils.hpp | 25 +++++++++++
cpp/particles/lock_free_bool_array.hpp | 25 +++++++++++
cpp/particles/p2p_computer.hpp | 25 +++++++++++
cpp/particles/p2p_computer_empty.hpp | 25 +++++++++++
cpp/particles/p2p_distr_mpi.hpp | 25 +++++++++++
cpp/particles/p2p_tree.hpp | 25 +++++++++++
cpp/particles/particles_adams_bashforth.hpp | 25 +++++++++++
cpp/particles/particles_distr_mpi.hpp | 25 +++++++++++
cpp/particles/particles_field_computer.hpp | 25 +++++++++++
cpp/particles/particles_generic_interp.hpp | 25 +++++++++++
cpp/particles/particles_inner_computer.cpp | 25 +++++++++++
cpp/particles/particles_inner_computer.hpp | 25 +++++++++++
.../particles_inner_computer_empty.hpp | 25 +++++++++++
cpp/particles/particles_input_hdf5.hpp | 25 +++++++++++
cpp/particles/particles_output_hdf5.hpp | 25 +++++++++++
cpp/particles/particles_output_mpiio.hpp | 25 +++++++++++
.../particles_output_sampling_hdf5.hpp | 25 +++++++++++
cpp/particles/particles_sampling.hpp | 25 +++++++++++
cpp/particles/particles_system.hpp | 25 +++++++++++
cpp/particles/particles_system_builder.hpp | 25 +++++++++++
cpp/particles/particles_utils.hpp | 25 +++++++++++
cpp/shared_array.hpp | 25 +++++++++++
cpp/spline.hpp | 25 +++++++++++
get_version.py | 45 +++++++++----------
pc_host_info.py | 24 ++++++++++
setup.py | 45 +++++++++----------
60 files changed, 1470 insertions(+), 230 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 82f21117..36eacbb7 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,3 +1,29 @@
+#######################################################################
+# #
+# Copyright 2019 Max Planck Institute #
+# for Dynamics and Self-Organization #
+# #
+# This file is part of bfps. #
+# #
+# bfps is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# bfps is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with bfps. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+#######################################################################
+
+
+
cmake_minimum_required(VERSION 3.10)
cmake_policy(VERSION 3.12)
diff --git a/bfps/DNS.py b/bfps/DNS.py
index 4388a292..bb438545 100644
--- a/bfps/DNS.py
+++ b/bfps/DNS.py
@@ -1,26 +1,25 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
+################################################################################
+# #
+# Copyright 2015-2019 Max Planck Institute for Dynamics and Self-Organization #
+# #
+# This file is part of bfps. #
+# #
+# bfps is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# bfps is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with bfps. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+################################################################################
diff --git a/bfps/PP.py b/bfps/PP.py
index 4a8cff4b..914b90ef 100644
--- a/bfps/PP.py
+++ b/bfps/PP.py
@@ -1,26 +1,25 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
+################################################################################
+# #
+# Copyright 2015-2019 Max Planck Institute for Dynamics and Self-Organization #
+# #
+# This file is part of bfps. #
+# #
+# bfps is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# bfps is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with bfps. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+################################################################################
diff --git a/bfps/TEST.py b/bfps/TEST.py
index 80adf39f..2edcdfe4 100644
--- a/bfps/TEST.py
+++ b/bfps/TEST.py
@@ -1,26 +1,25 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
+################################################################################
+# #
+# Copyright 2015-2019 Max Planck Institute for Dynamics and Self-Organization #
+# #
+# This file is part of bfps. #
+# #
+# bfps is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# bfps is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with bfps. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+################################################################################
diff --git a/bfps/__init__.py b/bfps/__init__.py
index 928e45b7..9595bee4 100644
--- a/bfps/__init__.py
+++ b/bfps/__init__.py
@@ -1,26 +1,25 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
+################################################################################
+# #
+# Copyright 2015-2019 Max Planck Institute for Dynamics and Self-Organization #
+# #
+# This file is part of bfps. #
+# #
+# bfps is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# bfps is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with bfps. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+################################################################################
diff --git a/bfps/__main__.py b/bfps/__main__.py
index cf269edb..187171d0 100644
--- a/bfps/__main__.py
+++ b/bfps/__main__.py
@@ -1,26 +1,25 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
+################################################################################
+# #
+# Copyright 2015-2019 Max Planck Institute for Dynamics and Self-Organization #
+# #
+# This file is part of bfps. #
+# #
+# bfps is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# bfps is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with bfps. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+################################################################################
diff --git a/bfps/_base.py b/bfps/_base.py
index fdddb0f7..15a3c7a2 100644
--- a/bfps/_base.py
+++ b/bfps/_base.py
@@ -1,26 +1,25 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
+################################################################################
+# #
+# Copyright 2015-2019 Max Planck Institute for Dynamics and Self-Organization #
+# #
+# This file is part of bfps. #
+# #
+# bfps is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# bfps is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with bfps. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+################################################################################
diff --git a/bfps/_code.py b/bfps/_code.py
index 72a102b3..250be61d 100644
--- a/bfps/_code.py
+++ b/bfps/_code.py
@@ -1,26 +1,25 @@
-#######################################################################
-# #
-# Copyright 2015-2019 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
+################################################################################
+# #
+# Copyright 2015-2019 Max Planck Institute for Dynamics and Self-Organization #
+# #
+# This file is part of bfps. #
+# #
+# bfps is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# bfps is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with bfps. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+################################################################################
diff --git a/bfps/tools.py b/bfps/tools.py
index a3fbc9d9..0acf51b5 100644
--- a/bfps/tools.py
+++ b/bfps/tools.py
@@ -1,26 +1,25 @@
-#######################################################################
-# #
-# Copyright 2015 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
+################################################################################
+# #
+# Copyright 2015-2019 Max Planck Institute for Dynamics and Self-Organization #
+# #
+# This file is part of bfps. #
+# #
+# bfps is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# bfps is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with bfps. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+################################################################################
diff --git a/cmake/BFPSConfig.cmake.in b/cmake/BFPSConfig.cmake.in
index 78e8ecf6..bd2af716 100644
--- a/cmake/BFPSConfig.cmake.in
+++ b/cmake/BFPSConfig.cmake.in
@@ -1,3 +1,28 @@
+#######################################################################
+# #
+# Copyright 2019 Max Planck Institute #
+# for Dynamics and Self-Organization #
+# #
+# This file is part of bfps. #
+# #
+# bfps is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# bfps is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with bfps. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+#######################################################################
+
+
#-----------------------------------------------------------------------------
#
# BFPSConfig.cmake - BFPS CMake configuration file for external projects.
diff --git a/cpp/full_code/NSVE.cpp b/cpp/full_code/NSVE.cpp
index 7032d277..7b1b2d95 100644
--- a/cpp/full_code/NSVE.cpp
+++ b/cpp/full_code/NSVE.cpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#define NDEBUG
#include <string>
diff --git a/cpp/full_code/NSVE_field_stats.cpp b/cpp/full_code/NSVE_field_stats.cpp
index b1c8d567..0969175c 100644
--- a/cpp/full_code/NSVE_field_stats.cpp
+++ b/cpp/full_code/NSVE_field_stats.cpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#include <string>
#include <cmath>
#include "NSVE_field_stats.hpp"
diff --git a/cpp/full_code/NSVEp_extra_sampling.cpp b/cpp/full_code/NSVEp_extra_sampling.cpp
index 22357510..7b3e5a76 100644
--- a/cpp/full_code/NSVEp_extra_sampling.cpp
+++ b/cpp/full_code/NSVEp_extra_sampling.cpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#include "full_code/NSVEp_extra_sampling.hpp"
diff --git a/cpp/full_code/NSVEp_extra_sampling.hpp b/cpp/full_code/NSVEp_extra_sampling.hpp
index 0857cde7..d3d1c186 100644
--- a/cpp/full_code/NSVEp_extra_sampling.hpp
+++ b/cpp/full_code/NSVEp_extra_sampling.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef NSVEP_EXTRA_SAMPLING_HPP
#define NSVEP_EXTRA_SAMPLING_HPP
diff --git a/cpp/full_code/NSVEparticles_no_output.hpp b/cpp/full_code/NSVEparticles_no_output.hpp
index 5b9d5e15..4e6de379 100644
--- a/cpp/full_code/NSVEparticles_no_output.hpp
+++ b/cpp/full_code/NSVEparticles_no_output.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef NSVEPARTICLES_NO_OUTPUT_HPP
#define NSVEPARTICLES_NO_OUTPUT_HPP
diff --git a/cpp/full_code/codes_with_no_output.hpp b/cpp/full_code/codes_with_no_output.hpp
index f4cd3b54..175bed22 100644
--- a/cpp/full_code/codes_with_no_output.hpp
+++ b/cpp/full_code/codes_with_no_output.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef CODES_WITH_NO_OUTPUT_HPP
#define CODES_WITH_NO_OUTPUT_HPP
diff --git a/cpp/full_code/direct_numerical_simulation.cpp b/cpp/full_code/direct_numerical_simulation.cpp
index cacda323..5329e703 100644
--- a/cpp/full_code/direct_numerical_simulation.cpp
+++ b/cpp/full_code/direct_numerical_simulation.cpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#define NDEBUG
#include <cstdlib>
diff --git a/cpp/full_code/field_output_test.cpp b/cpp/full_code/field_output_test.cpp
index 72406099..649d8dbb 100644
--- a/cpp/full_code/field_output_test.cpp
+++ b/cpp/full_code/field_output_test.cpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#include <string>
#include <cmath>
#include <random>
diff --git a/cpp/full_code/field_single_to_double.cpp b/cpp/full_code/field_single_to_double.cpp
index 92976ecf..93a03aed 100644
--- a/cpp/full_code/field_single_to_double.cpp
+++ b/cpp/full_code/field_single_to_double.cpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#include <string>
#include <cmath>
#include "field_single_to_double.hpp"
diff --git a/cpp/full_code/field_test.cpp b/cpp/full_code/field_test.cpp
index a9d531bc..aa055a6e 100644
--- a/cpp/full_code/field_test.cpp
+++ b/cpp/full_code/field_test.cpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#include <string>
#include <cmath>
#include <random>
diff --git a/cpp/full_code/filter_test.cpp b/cpp/full_code/filter_test.cpp
index 6dbd05a9..5df45a79 100644
--- a/cpp/full_code/filter_test.cpp
+++ b/cpp/full_code/filter_test.cpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#include <string>
#include <cmath>
#include "filter_test.hpp"
diff --git a/cpp/full_code/get_rfields.cpp b/cpp/full_code/get_rfields.cpp
index 376a2659..45f6b5dc 100644
--- a/cpp/full_code/get_rfields.cpp
+++ b/cpp/full_code/get_rfields.cpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#include <string>
#include <cmath>
#include "get_rfields.hpp"
diff --git a/cpp/full_code/joint_acc_vel_stats.cpp b/cpp/full_code/joint_acc_vel_stats.cpp
index fff2e2f5..be2cd9fe 100644
--- a/cpp/full_code/joint_acc_vel_stats.cpp
+++ b/cpp/full_code/joint_acc_vel_stats.cpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#include <string>
#include <cmath>
#include "joint_acc_vel_stats.hpp"
diff --git a/cpp/full_code/native_binary_to_hdf5.cpp b/cpp/full_code/native_binary_to_hdf5.cpp
index fe8e1c41..0c2d7384 100644
--- a/cpp/full_code/native_binary_to_hdf5.cpp
+++ b/cpp/full_code/native_binary_to_hdf5.cpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#include <string>
#include <cmath>
#include "native_binary_to_hdf5.hpp"
diff --git a/cpp/full_code/postprocess.cpp b/cpp/full_code/postprocess.cpp
index 13bee700..e8c7fb27 100644
--- a/cpp/full_code/postprocess.cpp
+++ b/cpp/full_code/postprocess.cpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#include <cstdlib>
#include <sys/types.h>
#include <sys/stat.h>
diff --git a/cpp/full_code/resize.cpp b/cpp/full_code/resize.cpp
index de555a74..6d372dc4 100644
--- a/cpp/full_code/resize.cpp
+++ b/cpp/full_code/resize.cpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#include <string>
#include <cmath>
#include "resize.hpp"
diff --git a/cpp/full_code/symmetrize_test.cpp b/cpp/full_code/symmetrize_test.cpp
index 7cf96a71..111d3a83 100644
--- a/cpp/full_code/symmetrize_test.cpp
+++ b/cpp/full_code/symmetrize_test.cpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#include <string>
#include <cmath>
#include <random>
diff --git a/cpp/full_code/test.cpp b/cpp/full_code/test.cpp
index aa909362..5fd265ec 100644
--- a/cpp/full_code/test.cpp
+++ b/cpp/full_code/test.cpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#include <cstdlib>
#include <sys/types.h>
#include <sys/stat.h>
diff --git a/cpp/full_code/test_interpolation.cpp b/cpp/full_code/test_interpolation.cpp
index c3103fb4..e5722fc0 100644
--- a/cpp/full_code/test_interpolation.cpp
+++ b/cpp/full_code/test_interpolation.cpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#include "full_code/test_interpolation.hpp"
diff --git a/cpp/full_code/test_interpolation.hpp b/cpp/full_code/test_interpolation.hpp
index 2d149802..4d65d4fa 100644
--- a/cpp/full_code/test_interpolation.hpp
+++ b/cpp/full_code/test_interpolation.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef TEST_INTERPOLATION_HPP
#define TEST_INTERPOLATION_HPP
diff --git a/cpp/particles/abstract_particles_input.hpp b/cpp/particles/abstract_particles_input.hpp
index 77dcbc63..48c38bc5 100644
--- a/cpp/particles/abstract_particles_input.hpp
+++ b/cpp/particles/abstract_particles_input.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef ABSTRACT_PARTICLES_INPUT_HPP
#define ABSTRACT_PARTICLES_INPUT_HPP
diff --git a/cpp/particles/abstract_particles_output.hpp b/cpp/particles/abstract_particles_output.hpp
index a457689a..6dc85ceb 100644
--- a/cpp/particles/abstract_particles_output.hpp
+++ b/cpp/particles/abstract_particles_output.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef ABSTRACT_PARTICLES_OUTPUT
#define ABSTRACT_PARTICLES_OUTPUT
diff --git a/cpp/particles/abstract_particles_system.hpp b/cpp/particles/abstract_particles_system.hpp
index 67c46855..2f2f510f 100644
--- a/cpp/particles/abstract_particles_system.hpp
+++ b/cpp/particles/abstract_particles_system.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef ABSTRACT_PARTICLES_SYSTEM_HPP
#define ABSTRACT_PARTICLES_SYSTEM_HPP
diff --git a/cpp/particles/alltoall_exchanger.hpp b/cpp/particles/alltoall_exchanger.hpp
index 2beaf092..d3423523 100644
--- a/cpp/particles/alltoall_exchanger.hpp
+++ b/cpp/particles/alltoall_exchanger.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef ALLTOALL_EXCHANGER_HPP
#define ALLTOALL_EXCHANGER_HPP
diff --git a/cpp/particles/env_utils.hpp b/cpp/particles/env_utils.hpp
index cd6fb302..829fd5b4 100644
--- a/cpp/particles/env_utils.hpp
+++ b/cpp/particles/env_utils.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef ENV_UTILS_HPP
#define ENV_UTILS_HPP
diff --git a/cpp/particles/lock_free_bool_array.hpp b/cpp/particles/lock_free_bool_array.hpp
index 928c1753..5e32a7d4 100644
--- a/cpp/particles/lock_free_bool_array.hpp
+++ b/cpp/particles/lock_free_bool_array.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef LOCK_FREE_BOOL_ARRAY_HPP
#define LOCK_FREE_BOOL_ARRAY_HPP
diff --git a/cpp/particles/p2p_computer.hpp b/cpp/particles/p2p_computer.hpp
index 922d65d1..74d9c9eb 100644
--- a/cpp/particles/p2p_computer.hpp
+++ b/cpp/particles/p2p_computer.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef P2P_COMPUTER_HPP
#define P2P_COMPUTER_HPP
diff --git a/cpp/particles/p2p_computer_empty.hpp b/cpp/particles/p2p_computer_empty.hpp
index 5e442b86..0599dc1a 100644
--- a/cpp/particles/p2p_computer_empty.hpp
+++ b/cpp/particles/p2p_computer_empty.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef P2P_COMPUTER_EMPTY_HPP
#define P2P_COMPUTER_EMPTY_HPP
diff --git a/cpp/particles/p2p_distr_mpi.hpp b/cpp/particles/p2p_distr_mpi.hpp
index 9b8a5b13..7ab3a8b3 100644
--- a/cpp/particles/p2p_distr_mpi.hpp
+++ b/cpp/particles/p2p_distr_mpi.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef P2P_DISTR_MPI_HPP
#define P2P_DISTR_MPI_HPP
diff --git a/cpp/particles/p2p_tree.hpp b/cpp/particles/p2p_tree.hpp
index a4441543..cdb30891 100644
--- a/cpp/particles/p2p_tree.hpp
+++ b/cpp/particles/p2p_tree.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef P2P_TREE_HPP
#define P2P_TREE_HPP
diff --git a/cpp/particles/particles_adams_bashforth.hpp b/cpp/particles/particles_adams_bashforth.hpp
index e81e6022..21412e35 100644
--- a/cpp/particles/particles_adams_bashforth.hpp
+++ b/cpp/particles/particles_adams_bashforth.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef PARTICLES_ADAMS_BASHFORTH_HPP
#define PARTICLES_ADAMS_BASHFORTH_HPP
diff --git a/cpp/particles/particles_distr_mpi.hpp b/cpp/particles/particles_distr_mpi.hpp
index 8a2b77ca..43d61ca4 100644
--- a/cpp/particles/particles_distr_mpi.hpp
+++ b/cpp/particles/particles_distr_mpi.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef PARTICLES_DISTR_MPI_HPP
#define PARTICLES_DISTR_MPI_HPP
diff --git a/cpp/particles/particles_field_computer.hpp b/cpp/particles/particles_field_computer.hpp
index 50d4df78..f6494ecd 100644
--- a/cpp/particles/particles_field_computer.hpp
+++ b/cpp/particles/particles_field_computer.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef PARTICLES_FIELD_COMPUTER_HPP
#define PARTICLES_FIELD_COMPUTER_HPP
diff --git a/cpp/particles/particles_generic_interp.hpp b/cpp/particles/particles_generic_interp.hpp
index 98d0363d..da48641c 100644
--- a/cpp/particles/particles_generic_interp.hpp
+++ b/cpp/particles/particles_generic_interp.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef PARTICLES_GENERIC_INTERP_HPP
#define PARTICLES_GENERIC_INTERP_HPP
diff --git a/cpp/particles/particles_inner_computer.cpp b/cpp/particles/particles_inner_computer.cpp
index deb85dee..3a841bee 100644
--- a/cpp/particles/particles_inner_computer.cpp
+++ b/cpp/particles/particles_inner_computer.cpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#include "base.hpp"
#include "particles_utils.hpp"
#include "particles_inner_computer.hpp"
diff --git a/cpp/particles/particles_inner_computer.hpp b/cpp/particles/particles_inner_computer.hpp
index 59dbba64..7f30ad68 100644
--- a/cpp/particles/particles_inner_computer.hpp
+++ b/cpp/particles/particles_inner_computer.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef PARTICLES_INNER_COMPUTER_HPP
#define PARTICLES_INNER_COMPUTER_HPP
diff --git a/cpp/particles/particles_inner_computer_empty.hpp b/cpp/particles/particles_inner_computer_empty.hpp
index 514bcba1..a90d3aa1 100644
--- a/cpp/particles/particles_inner_computer_empty.hpp
+++ b/cpp/particles/particles_inner_computer_empty.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef PARTICLES_INNER_COMPUTER_EMPTY_HPP
#define PARTICLES_INNER_COMPUTER_EMPTY_HPP
diff --git a/cpp/particles/particles_input_hdf5.hpp b/cpp/particles/particles_input_hdf5.hpp
index 33406314..3f895be3 100644
--- a/cpp/particles/particles_input_hdf5.hpp
+++ b/cpp/particles/particles_input_hdf5.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef PARTICLES_INPUT_HDF5_HPP
#define PARTICLES_INPUT_HDF5_HPP
diff --git a/cpp/particles/particles_output_hdf5.hpp b/cpp/particles/particles_output_hdf5.hpp
index d7c987ee..6be65179 100644
--- a/cpp/particles/particles_output_hdf5.hpp
+++ b/cpp/particles/particles_output_hdf5.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef PARTICLES_OUTPUT_HDF5_HPP
#define PARTICLES_OUTPUT_HDF5_HPP
diff --git a/cpp/particles/particles_output_mpiio.hpp b/cpp/particles/particles_output_mpiio.hpp
index 5810c4a0..b1c17898 100644
--- a/cpp/particles/particles_output_mpiio.hpp
+++ b/cpp/particles/particles_output_mpiio.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef PARTICLES_OUTPUT_MPIIO
#define PARTICLES_OUTPUT_MPIIO
diff --git a/cpp/particles/particles_output_sampling_hdf5.hpp b/cpp/particles/particles_output_sampling_hdf5.hpp
index ff3782b5..ea3c94dc 100644
--- a/cpp/particles/particles_output_sampling_hdf5.hpp
+++ b/cpp/particles/particles_output_sampling_hdf5.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef PARTICLES_OUTPUT_SAMPLING_HDF5_HPP
#define PARTICLES_OUTPUT_SAMPLING_HDF5_HPP
diff --git a/cpp/particles/particles_sampling.hpp b/cpp/particles/particles_sampling.hpp
index 8baff633..672c080a 100644
--- a/cpp/particles/particles_sampling.hpp
+++ b/cpp/particles/particles_sampling.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef PARTICLES_SAMPLING_HPP
#define PARTICLES_SAMPLING_HPP
diff --git a/cpp/particles/particles_system.hpp b/cpp/particles/particles_system.hpp
index db651904..a05175ca 100644
--- a/cpp/particles/particles_system.hpp
+++ b/cpp/particles/particles_system.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef PARTICLES_SYSTEM_HPP
#define PARTICLES_SYSTEM_HPP
diff --git a/cpp/particles/particles_system_builder.hpp b/cpp/particles/particles_system_builder.hpp
index 916ab4bf..6a6f4a26 100644
--- a/cpp/particles/particles_system_builder.hpp
+++ b/cpp/particles/particles_system_builder.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef PARTICLES_SYSTEM_BUILDER_HPP
#define PARTICLES_SYSTEM_BUILDER_HPP
diff --git a/cpp/particles/particles_utils.hpp b/cpp/particles/particles_utils.hpp
index 76371e64..f1e0c790 100644
--- a/cpp/particles/particles_utils.hpp
+++ b/cpp/particles/particles_utils.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2019 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef PARTICLES_UTILS_HPP
#define PARTICLES_UTILS_HPP
diff --git a/cpp/shared_array.hpp b/cpp/shared_array.hpp
index 1951e2f9..0245dc5d 100644
--- a/cpp/shared_array.hpp
+++ b/cpp/shared_array.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2016 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef SHAREDARRAY_HPP
#define SHAREDARRAY_HPP
diff --git a/cpp/spline.hpp b/cpp/spline.hpp
index d66d2b1e..ef990088 100644
--- a/cpp/spline.hpp
+++ b/cpp/spline.hpp
@@ -1,3 +1,28 @@
+/******************************************************************************
+* *
+* Copyright 2017 Max Planck Institute for Dynamics and Self-Organization *
+* *
+* This file is part of bfps. *
+* *
+* bfps is free software: you can redistribute it and/or modify *
+* it under the terms of the GNU General Public License as published *
+* by the Free Software Foundation, either version 3 of the License, *
+* or (at your option) any later version. *
+* *
+* bfps is distributed in the hope that it will be useful, *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+* GNU General Public License for more details. *
+* *
+* You should have received a copy of the GNU General Public License *
+* along with bfps. If not, see <http://www.gnu.org/licenses/> *
+* *
+* Contact: Cristian.Lalescu@ds.mpg.de *
+* *
+******************************************************************************/
+
+
+
#ifndef SPLINE_HPP
#define SPLINE_HPP
diff --git a/get_version.py b/get_version.py
index 896763e8..fe545a67 100644
--- a/get_version.py
+++ b/get_version.py
@@ -1,26 +1,25 @@
-#######################################################################
-# #
-# Copyright 2015-2019 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
+################################################################################
+# #
+# Copyright 2019 Max Planck Institute for Dynamics and Self-Organization #
+# #
+# This file is part of bfps. #
+# #
+# bfps is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# bfps is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with bfps. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+################################################################################
diff --git a/pc_host_info.py b/pc_host_info.py
index 0a192dd1..dec9db64 100644
--- a/pc_host_info.py
+++ b/pc_host_info.py
@@ -1,3 +1,27 @@
+################################################################################
+# #
+# Copyright 2019 Max Planck Institute for Dynamics and Self-Organization #
+# #
+# This file is part of bfps. #
+# #
+# bfps is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# bfps is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with bfps. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+################################################################################
+
+
host_info = {'type' : 'pc'}
diff --git a/setup.py b/setup.py
index c9a8b70b..0b70e6d1 100644
--- a/setup.py
+++ b/setup.py
@@ -1,26 +1,25 @@
-#######################################################################
-# #
-# Copyright 2015-2019 Max Planck Institute #
-# for Dynamics and Self-Organization #
-# #
-# This file is part of bfps. #
-# #
-# bfps is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published #
-# by the Free Software Foundation, either version 3 of the License, #
-# or (at your option) any later version. #
-# #
-# bfps is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with bfps. If not, see <http://www.gnu.org/licenses/> #
-# #
-# Contact: Cristian.Lalescu@ds.mpg.de #
-# #
-#######################################################################
+################################################################################
+# #
+# Copyright 2015-2019 Max Planck Institute for Dynamics and Self-Organization #
+# #
+# This file is part of bfps. #
+# #
+# bfps is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published #
+# by the Free Software Foundation, either version 3 of the License, #
+# or (at your option) any later version. #
+# #
+# bfps is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with bfps. If not, see <http://www.gnu.org/licenses/> #
+# #
+# Contact: Cristian.Lalescu@ds.mpg.de #
+# #
+################################################################################
--
GitLab
From a06ca66fa97342543739bc9cc6624c4d4d9133a3 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Tue, 19 Mar 2019 16:50:19 +0100
Subject: [PATCH 341/342] tweak README
---
README.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/README.rst b/README.rst
index 9b804cad..9ad562c0 100644
--- a/README.rst
+++ b/README.rst
@@ -133,7 +133,7 @@ Detailed full installation instructions:
BFPS will try to find FFTW using the FindFFTW from the Morse project.
If the package is installed in a non standard location, it is recommanded
- to setup the environment variables: FFTW_DIR (or FFTW_INCDIR and FFTW_LIBDIR).
+ to setup the environment variables: `FFTW_DIR` (or `FFTW_INCDIR` and `FFTW_LIBDIR`).
3. Download, compile, install HDF5 (version 1.8.x, currently available
at https://portal.hdfgroup.org/display/support/HDF5+1.8.20#files).
--
GitLab
From ae7076189c3b4ebe172b61d6c40691b98def4db1 Mon Sep 17 00:00:00 2001
From: Cristian C Lalescu <Cristian.Lalescu@ds.mpg.de>
Date: Wed, 20 Mar 2019 16:13:54 +0100
Subject: [PATCH 342/342] mention in README the inclusion of morse project
files
I checked. They are intact since being added, so we don't need to add
extra copyright notices in them.
---
README.rst | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/README.rst b/README.rst
index 9ad562c0..7dc457a7 100644
--- a/README.rst
+++ b/README.rst
@@ -87,12 +87,18 @@ type ``make html`` instead of ``make latexpdf``.
Comments
--------
+* the `cmake` folder contains files extracted from
+ https://gitlab.inria.fr/solverstack/morse_cmake, a separate project licensed
+ under the "CeCILL-C" license, please see
+ http://www.cecill.info/licences/Licence_CeCILL-C_V1-en.html for
+ details.
+
* particles: initialization of multistep solvers is done with lower
order methods, so direct convergence tests will fail.
-* Code is only known to work with HDF5 1.8.x.
+* code is only known to work with HDF5 1.8.x.
-* Code is used mainly with Python 3.5 and later, and it is not tested at
+* code is used mainly with Python 3.5 and later, and it is not tested at
all with Python 2.x
-------------------------------
--
GitLab